X86VZeroUpper.cpp revision d51d36e3f1c83eb183625af2e3988a08bb984cb7
1//===-- X86VZeroUpper.cpp - AVX vzeroupper instruction inserter -----------===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file defines the pass which inserts x86 AVX vzeroupper instructions
11// before calls to SSE encoded functions. This avoids transition latency
12// penalty when tranfering control between AVX encoded instructions and old
13// SSE encoding mode.
14//
15//===----------------------------------------------------------------------===//
16
17#define DEBUG_TYPE "x86-vzeroupper"
18#include "X86.h"
19#include "X86InstrInfo.h"
20#include "llvm/ADT/Statistic.h"
21#include "llvm/CodeGen/MachineFunctionPass.h"
22#include "llvm/CodeGen/MachineInstrBuilder.h"
23#include "llvm/CodeGen/MachineRegisterInfo.h"
24#include "llvm/CodeGen/Passes.h"
25#include "llvm/Support/Debug.h"
26#include "llvm/Support/raw_ostream.h"
27#include "llvm/Target/TargetInstrInfo.h"
28using namespace llvm;
29
30STATISTIC(NumVZU, "Number of vzeroupper instructions inserted");
31
32namespace {
33  struct VZeroUpperInserter : public MachineFunctionPass {
34    static char ID;
35    VZeroUpperInserter() : MachineFunctionPass(ID) {}
36
37    virtual bool runOnMachineFunction(MachineFunction &MF);
38
39    bool processBasicBlock(MachineFunction &MF, MachineBasicBlock &MBB);
40
41    virtual const char *getPassName() const { return "X86 vzeroupper inserter";}
42
43  private:
44    const TargetInstrInfo *TII; // Machine instruction info.
45
46    // Any YMM register live-in to this function?
47    bool FnHasLiveInYmm;
48
49    // BBState - Contains the state of each MBB: unknown, clean, dirty
50    SmallVector<uint8_t, 8> BBState;
51
52    // BBSolved - Keep track of all MBB which had been already analyzed
53    // and there is no further processing required.
54    BitVector BBSolved;
55
56    // Machine Basic Blocks are classified according this pass:
57    //
58    //  ST_UNKNOWN - The MBB state is unknown, meaning from the entry state
59    //    until the MBB exit there isn't a instruction using YMM to change
60    //    the state to dirty, or one of the incoming predecessors is unknown
61    //    and there's not a dirty predecessor between them.
62    //
63    //  ST_CLEAN - No YMM usage in the end of the MBB. A MBB could have
64    //    instructions using YMM and be marked ST_CLEAN, as long as the state
65    //    is cleaned by a vzeroupper before any call.
66    //
67    //  ST_DIRTY - Any MBB ending with a YMM usage not cleaned up by a
68    //    vzeroupper instruction.
69    //
70    //  ST_INIT - Placeholder for an empty state set
71    //
72    enum {
73      ST_UNKNOWN = 0,
74      ST_CLEAN   = 1,
75      ST_DIRTY   = 2,
76      ST_INIT    = 3
77    };
78
79    // computeState - Given two states, compute the resulting state, in
80    // the following way
81    //
82    //  1) One dirty state yields another dirty state
83    //  2) All states must be clean for the result to be clean
84    //  3) If none above and one unknown, the result state is also unknown
85    //
86    static unsigned computeState(unsigned PrevState, unsigned CurState) {
87      if (PrevState == ST_INIT)
88        return CurState;
89
90      if (PrevState == ST_DIRTY || CurState == ST_DIRTY)
91        return ST_DIRTY;
92
93      if (PrevState == ST_CLEAN && CurState == ST_CLEAN)
94        return ST_CLEAN;
95
96      return ST_UNKNOWN;
97    }
98
99  };
100  char VZeroUpperInserter::ID = 0;
101}
102
103FunctionPass *llvm::createX86IssueVZeroUpperPass() {
104  return new VZeroUpperInserter();
105}
106
107static bool isYmmReg(unsigned Reg) {
108  return (Reg >= X86::YMM0 && Reg <= X86::YMM31);
109}
110
111static bool isZmmReg(unsigned Reg) {
112  return (Reg >= X86::ZMM0 && Reg <= X86::ZMM31);
113}
114
115static bool checkFnHasLiveInYmm(MachineRegisterInfo &MRI) {
116  for (MachineRegisterInfo::livein_iterator I = MRI.livein_begin(),
117       E = MRI.livein_end(); I != E; ++I)
118    if (isYmmReg(I->first) || isZmmReg(I->first))
119      return true;
120
121  return false;
122}
123
124static bool clobbersAllYmmRegs(const MachineOperand &MO) {
125  for (unsigned reg = X86::YMM0; reg <= X86::YMM31; ++reg) {
126    if (!MO.clobbersPhysReg(reg))
127      return false;
128  }
129  for (unsigned reg = X86::ZMM0; reg <= X86::ZMM31; ++reg) {
130    if (!MO.clobbersPhysReg(reg))
131      return false;
132  }
133  return true;
134}
135
136static bool hasYmmReg(MachineInstr *MI) {
137  for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
138    const MachineOperand &MO = MI->getOperand(i);
139    if (MI->isCall() && MO.isRegMask() && !clobbersAllYmmRegs(MO))
140      return true;
141    if (!MO.isReg())
142      continue;
143    if (MO.isDebug())
144      continue;
145    if (isYmmReg(MO.getReg()))
146      return true;
147  }
148  return false;
149}
150
151/// clobbersAnyYmmReg() - Check if any YMM register will be clobbered by this
152/// instruction.
153static bool clobbersAnyYmmReg(MachineInstr *MI) {
154  for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
155    const MachineOperand &MO = MI->getOperand(i);
156    if (!MO.isRegMask())
157      continue;
158    for (unsigned reg = X86::YMM0; reg <= X86::YMM31; ++reg) {
159      if (MO.clobbersPhysReg(reg))
160        return true;
161    }
162    for (unsigned reg = X86::ZMM0; reg <= X86::ZMM31; ++reg) {
163      if (MO.clobbersPhysReg(reg))
164        return true;
165    }
166  }
167  return false;
168}
169
170/// runOnMachineFunction - Loop over all of the basic blocks, inserting
171/// vzero upper instructions before function calls.
172bool VZeroUpperInserter::runOnMachineFunction(MachineFunction &MF) {
173  TII = MF.getTarget().getInstrInfo();
174  MachineRegisterInfo &MRI = MF.getRegInfo();
175  bool EverMadeChange = false;
176
177  // Fast check: if the function doesn't use any ymm registers, we don't need
178  // to insert any VZEROUPPER instructions.  This is constant-time, so it is
179  // cheap in the common case of no ymm use.
180  bool YMMUsed = false;
181  const TargetRegisterClass *RC = &X86::VR256RegClass;
182  for (TargetRegisterClass::iterator i = RC->begin(), e = RC->end();
183       i != e; i++) {
184    if (!MRI.reg_nodbg_empty(*i)) {
185      YMMUsed = true;
186      break;
187    }
188  }
189  if (!YMMUsed)
190    return EverMadeChange;
191
192  // Pre-compute the existence of any live-in YMM registers to this function
193  FnHasLiveInYmm = checkFnHasLiveInYmm(MRI);
194
195  assert(BBState.empty());
196  BBState.resize(MF.getNumBlockIDs(), 0);
197  BBSolved.resize(MF.getNumBlockIDs(), 0);
198
199  // Each BB state depends on all predecessors, loop over until everything
200  // converges.  (Once we converge, we can implicitly mark everything that is
201  // still ST_UNKNOWN as ST_CLEAN.)
202  while (1) {
203    bool MadeChange = false;
204
205    // Process all basic blocks.
206    for (MachineFunction::iterator I = MF.begin(), E = MF.end(); I != E; ++I)
207      MadeChange |= processBasicBlock(MF, *I);
208
209    // If this iteration over the code changed anything, keep iterating.
210    if (!MadeChange) break;
211    EverMadeChange = true;
212  }
213
214  BBState.clear();
215  BBSolved.clear();
216  return EverMadeChange;
217}
218
219/// processBasicBlock - Loop over all of the instructions in the basic block,
220/// inserting vzero upper instructions before function calls.
221bool VZeroUpperInserter::processBasicBlock(MachineFunction &MF,
222                                           MachineBasicBlock &BB) {
223  bool Changed = false;
224  unsigned BBNum = BB.getNumber();
225
226  // Don't process already solved BBs
227  if (BBSolved[BBNum])
228    return false; // No changes
229
230  // Check the state of all predecessors
231  unsigned EntryState = ST_INIT;
232  for (MachineBasicBlock::const_pred_iterator PI = BB.pred_begin(),
233       PE = BB.pred_end(); PI != PE; ++PI) {
234    EntryState = computeState(EntryState, BBState[(*PI)->getNumber()]);
235    if (EntryState == ST_DIRTY)
236      break;
237  }
238
239
240  // The entry MBB for the function may set the initial state to dirty if
241  // the function receives any YMM incoming arguments
242  if (&BB == MF.begin()) {
243    EntryState = ST_CLEAN;
244    if (FnHasLiveInYmm)
245      EntryState = ST_DIRTY;
246  }
247
248  // The current state is initialized according to the predecessors
249  unsigned CurState = EntryState;
250  bool BBHasCall = false;
251
252  for (MachineBasicBlock::iterator I = BB.begin(); I != BB.end(); ++I) {
253    DebugLoc dl = I->getDebugLoc();
254    MachineInstr *MI = I;
255
256    bool isControlFlow = MI->isCall() || MI->isReturn();
257
258    // Shortcut: don't need to check regular instructions in dirty state.
259    if (!isControlFlow && CurState == ST_DIRTY)
260      continue;
261
262    if (hasYmmReg(MI)) {
263      // We found a ymm-using instruction; this could be an AVX instruction,
264      // or it could be control flow.
265      CurState = ST_DIRTY;
266      continue;
267    }
268
269    // Check for control-flow out of the current function (which might
270    // indirectly execute SSE instructions).
271    if (!isControlFlow)
272      continue;
273
274    // If the call won't clobber any YMM register, skip it as well. It usually
275    // happens on helper function calls (such as '_chkstk', '_ftol2') where
276    // standard calling convention is not used (RegMask is not used to mark
277    // register clobbered and register usage (def/imp-def/use) is well-dfined
278    // and explicitly specified.
279    if (MI->isCall() && !clobbersAnyYmmReg(MI))
280      continue;
281
282    BBHasCall = true;
283
284    // The VZEROUPPER instruction resets the upper 128 bits of all Intel AVX
285    // registers. This instruction has zero latency. In addition, the processor
286    // changes back to Clean state, after which execution of Intel SSE
287    // instructions or Intel AVX instructions has no transition penalty. Add
288    // the VZEROUPPER instruction before any function call/return that might
289    // execute SSE code.
290    // FIXME: In some cases, we may want to move the VZEROUPPER into a
291    // predecessor block.
292    if (CurState == ST_DIRTY) {
293      // Only insert the VZEROUPPER in case the entry state isn't unknown.
294      // When unknown, only compute the information within the block to have
295      // it available in the exit if possible, but don't change the block.
296      if (EntryState != ST_UNKNOWN) {
297        BuildMI(BB, I, dl, TII->get(X86::VZEROUPPER));
298        ++NumVZU;
299      }
300
301      // After the inserted VZEROUPPER the state becomes clean again, but
302      // other YMM may appear before other subsequent calls or even before
303      // the end of the BB.
304      CurState = ST_CLEAN;
305    }
306  }
307
308  DEBUG(dbgs() << "MBB #" << BBNum
309               << ", current state: " << CurState << '\n');
310
311  // A BB can only be considered solved when we both have done all the
312  // necessary transformations, and have computed the exit state.  This happens
313  // in two cases:
314  //  1) We know the entry state: this immediately implies the exit state and
315  //     all the necessary transformations.
316  //  2) There are no calls, and and a non-call instruction marks this block:
317  //     no transformations are necessary, and we know the exit state.
318  if (EntryState != ST_UNKNOWN || (!BBHasCall && CurState != ST_UNKNOWN))
319    BBSolved[BBNum] = true;
320
321  if (CurState != BBState[BBNum])
322    Changed = true;
323
324  BBState[BBNum] = CurState;
325  return Changed;
326}
327