X86VZeroUpper.cpp revision 2990853ea8bf4888b179ac6c493836b83769e87b
1//===-- X86VZeroUpper.cpp - AVX vzeroupper instruction inserter -----------===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file defines the pass which inserts x86 AVX vzeroupper instructions 11// before calls to SSE encoded functions. This avoids transition latency 12// penalty when tranfering control between AVX encoded instructions and old 13// SSE encoding mode. 14// 15//===----------------------------------------------------------------------===// 16 17#define DEBUG_TYPE "x86-vzeroupper" 18#include "X86.h" 19#include "X86InstrInfo.h" 20#include "llvm/ADT/Statistic.h" 21#include "llvm/CodeGen/MachineFunctionPass.h" 22#include "llvm/CodeGen/MachineInstrBuilder.h" 23#include "llvm/CodeGen/MachineRegisterInfo.h" 24#include "llvm/CodeGen/Passes.h" 25#include "llvm/Support/Debug.h" 26#include "llvm/Support/raw_ostream.h" 27#include "llvm/Target/TargetInstrInfo.h" 28using namespace llvm; 29 30STATISTIC(NumVZU, "Number of vzeroupper instructions inserted"); 31 32namespace { 33 struct VZeroUpperInserter : public MachineFunctionPass { 34 static char ID; 35 VZeroUpperInserter() : MachineFunctionPass(ID) {} 36 37 virtual bool runOnMachineFunction(MachineFunction &MF); 38 39 bool processBasicBlock(MachineFunction &MF, MachineBasicBlock &MBB); 40 41 virtual const char *getPassName() const { return "X86 vzeroupper inserter";} 42 43 private: 44 const TargetInstrInfo *TII; // Machine instruction info. 45 46 // Any YMM register live-in to this function? 47 bool FnHasLiveInYmm; 48 49 // BBState - Contains the state of each MBB: unknown, clean, dirty 50 SmallVector<uint8_t, 8> BBState; 51 52 // BBSolved - Keep track of all MBB which had been already analyzed 53 // and there is no further processing required. 54 BitVector BBSolved; 55 56 // Machine Basic Blocks are classified according this pass: 57 // 58 // ST_UNKNOWN - The MBB state is unknown, meaning from the entry state 59 // until the MBB exit there isn't a instruction using YMM to change 60 // the state to dirty, or one of the incoming predecessors is unknown 61 // and there's not a dirty predecessor between them. 62 // 63 // ST_CLEAN - No YMM usage in the end of the MBB. A MBB could have 64 // instructions using YMM and be marked ST_CLEAN, as long as the state 65 // is cleaned by a vzeroupper before any call. 66 // 67 // ST_DIRTY - Any MBB ending with a YMM usage not cleaned up by a 68 // vzeroupper instruction. 69 // 70 // ST_INIT - Placeholder for an empty state set 71 // 72 enum { 73 ST_UNKNOWN = 0, 74 ST_CLEAN = 1, 75 ST_DIRTY = 2, 76 ST_INIT = 3 77 }; 78 79 // computeState - Given two states, compute the resulting state, in 80 // the following way 81 // 82 // 1) One dirty state yields another dirty state 83 // 2) All states must be clean for the result to be clean 84 // 3) If none above and one unknown, the result state is also unknown 85 // 86 static unsigned computeState(unsigned PrevState, unsigned CurState) { 87 if (PrevState == ST_INIT) 88 return CurState; 89 90 if (PrevState == ST_DIRTY || CurState == ST_DIRTY) 91 return ST_DIRTY; 92 93 if (PrevState == ST_CLEAN && CurState == ST_CLEAN) 94 return ST_CLEAN; 95 96 return ST_UNKNOWN; 97 } 98 99 }; 100 char VZeroUpperInserter::ID = 0; 101} 102 103FunctionPass *llvm::createX86IssueVZeroUpperPass() { 104 return new VZeroUpperInserter(); 105} 106 107static bool isYmmReg(unsigned Reg) { 108 return (Reg >= X86::YMM0 && Reg <= X86::YMM31); 109} 110 111static bool isZmmReg(unsigned Reg) { 112 return (Reg >= X86::ZMM0 && Reg <= X86::ZMM31); 113} 114 115static bool checkFnHasLiveInYmm(MachineRegisterInfo &MRI) { 116 for (MachineRegisterInfo::livein_iterator I = MRI.livein_begin(), 117 E = MRI.livein_end(); I != E; ++I) 118 if (isYmmReg(I->first) || isZmmReg(I->first)) 119 return true; 120 121 return false; 122} 123 124static bool clobbersAllYmmRegs(const MachineOperand &MO) { 125 for (unsigned reg = X86::YMM0; reg < X86::YMM31; ++reg) { 126 if (!MO.clobbersPhysReg(reg)) 127 return false; 128 } 129 for (unsigned reg = X86::ZMM0; reg < X86::ZMM31; ++reg) { 130 if (!MO.clobbersPhysReg(reg)) 131 return false; 132 } 133 return true; 134} 135 136static bool hasYmmReg(MachineInstr *MI) { 137 for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) { 138 const MachineOperand &MO = MI->getOperand(i); 139 if (MI->isCall() && MO.isRegMask() && !clobbersAllYmmRegs(MO)) 140 return true; 141 if (!MO.isReg()) 142 continue; 143 if (MO.isDebug()) 144 continue; 145 if (isYmmReg(MO.getReg())) 146 return true; 147 } 148 return false; 149} 150 151/// clobbersAnyYmmReg() - Check if any YMM register will be clobbered by this 152/// instruction. 153static bool clobbersAnyYmmReg(MachineInstr *MI) { 154 for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) { 155 const MachineOperand &MO = MI->getOperand(i); 156 if (!MO.isRegMask()) 157 continue; 158 for (unsigned reg = X86::YMM0; reg < X86::YMM31; ++reg) { 159 if (MO.clobbersPhysReg(reg)) 160 return true; 161 } 162 for (unsigned reg = X86::ZMM0; reg < X86::ZMM31; ++reg) { 163 if (MO.clobbersPhysReg(reg)) 164 return true; 165 } 166 } 167 return false; 168} 169 170/// runOnMachineFunction - Loop over all of the basic blocks, inserting 171/// vzero upper instructions before function calls. 172bool VZeroUpperInserter::runOnMachineFunction(MachineFunction &MF) { 173 TII = MF.getTarget().getInstrInfo(); 174 MachineRegisterInfo &MRI = MF.getRegInfo(); 175 bool EverMadeChange = false; 176 177 // Fast check: if the function doesn't use any ymm registers, we don't need 178 // to insert any VZEROUPPER instructions. This is constant-time, so it is 179 // cheap in the common case of no ymm use. 180 bool YMMUsed = false; 181 const TargetRegisterClass *RC = &X86::VR256RegClass; 182 for (TargetRegisterClass::iterator i = RC->begin(), e = RC->end(); 183 i != e; i++) { 184 if (!MRI.reg_nodbg_empty(*i)) { 185 YMMUsed = true; 186 break; 187 } 188 } 189 if (!YMMUsed) 190 return EverMadeChange; 191 192 // Pre-compute the existence of any live-in YMM registers to this function 193 FnHasLiveInYmm = checkFnHasLiveInYmm(MRI); 194 195 assert(BBState.empty()); 196 BBState.resize(MF.getNumBlockIDs(), 0); 197 BBSolved.resize(MF.getNumBlockIDs(), 0); 198 199 // Each BB state depends on all predecessors, loop over until everything 200 // converges. (Once we converge, we can implicitly mark everything that is 201 // still ST_UNKNOWN as ST_CLEAN.) 202 while (1) { 203 bool MadeChange = false; 204 205 // Process all basic blocks. 206 for (MachineFunction::iterator I = MF.begin(), E = MF.end(); I != E; ++I) 207 MadeChange |= processBasicBlock(MF, *I); 208 209 // If this iteration over the code changed anything, keep iterating. 210 if (!MadeChange) break; 211 EverMadeChange = true; 212 } 213 214 BBState.clear(); 215 BBSolved.clear(); 216 return EverMadeChange; 217} 218 219/// processBasicBlock - Loop over all of the instructions in the basic block, 220/// inserting vzero upper instructions before function calls. 221bool VZeroUpperInserter::processBasicBlock(MachineFunction &MF, 222 MachineBasicBlock &BB) { 223 bool Changed = false; 224 unsigned BBNum = BB.getNumber(); 225 226 // Don't process already solved BBs 227 if (BBSolved[BBNum]) 228 return false; // No changes 229 230 // Check the state of all predecessors 231 unsigned EntryState = ST_INIT; 232 for (MachineBasicBlock::const_pred_iterator PI = BB.pred_begin(), 233 PE = BB.pred_end(); PI != PE; ++PI) { 234 EntryState = computeState(EntryState, BBState[(*PI)->getNumber()]); 235 if (EntryState == ST_DIRTY) 236 break; 237 } 238 239 240 // The entry MBB for the function may set the initial state to dirty if 241 // the function receives any YMM incoming arguments 242 if (&BB == MF.begin()) { 243 EntryState = ST_CLEAN; 244 if (FnHasLiveInYmm) 245 EntryState = ST_DIRTY; 246 } 247 248 // The current state is initialized according to the predecessors 249 unsigned CurState = EntryState; 250 bool BBHasCall = false; 251 252 for (MachineBasicBlock::iterator I = BB.begin(); I != BB.end(); ++I) { 253 DebugLoc dl = I->getDebugLoc(); 254 MachineInstr *MI = I; 255 256 bool isControlFlow = MI->isCall() || MI->isReturn(); 257 258 // Shortcut: don't need to check regular instructions in dirty state. 259 if (!isControlFlow && CurState == ST_DIRTY) 260 continue; 261 262 if (hasYmmReg(MI)) { 263 // We found a ymm-using instruction; this could be an AVX instruction, 264 // or it could be control flow. 265 CurState = ST_DIRTY; 266 continue; 267 } 268 269 // Check for control-flow out of the current function (which might 270 // indirectly execute SSE instructions). 271 if (!isControlFlow) 272 continue; 273 274 // If the call won't clobber any YMM register, skip it as well. It usually 275 // happens on helper function calls (such as '_chkstk', '_ftol2') where 276 // standard calling convention is not used (RegMask is not used to mark 277 // register clobbered and register usage (def/imp-def/use) is well-dfined 278 // and explicitly specified. 279 if (MI->isCall() && !clobbersAnyYmmReg(MI)) 280 continue; 281 282 BBHasCall = true; 283 284 // The VZEROUPPER instruction resets the upper 128 bits of all Intel AVX 285 // registers. This instruction has zero latency. In addition, the processor 286 // changes back to Clean state, after which execution of Intel SSE 287 // instructions or Intel AVX instructions has no transition penalty. Add 288 // the VZEROUPPER instruction before any function call/return that might 289 // execute SSE code. 290 // FIXME: In some cases, we may want to move the VZEROUPPER into a 291 // predecessor block. 292 if (CurState == ST_DIRTY) { 293 // Only insert the VZEROUPPER in case the entry state isn't unknown. 294 // When unknown, only compute the information within the block to have 295 // it available in the exit if possible, but don't change the block. 296 if (EntryState != ST_UNKNOWN) { 297 BuildMI(BB, I, dl, TII->get(X86::VZEROUPPER)); 298 ++NumVZU; 299 } 300 301 // After the inserted VZEROUPPER the state becomes clean again, but 302 // other YMM may appear before other subsequent calls or even before 303 // the end of the BB. 304 CurState = ST_CLEAN; 305 } 306 } 307 308 DEBUG(dbgs() << "MBB #" << BBNum 309 << ", current state: " << CurState << '\n'); 310 311 // A BB can only be considered solved when we both have done all the 312 // necessary transformations, and have computed the exit state. This happens 313 // in two cases: 314 // 1) We know the entry state: this immediately implies the exit state and 315 // all the necessary transformations. 316 // 2) There are no calls, and and a non-call instruction marks this block: 317 // no transformations are necessary, and we know the exit state. 318 if (EntryState != ST_UNKNOWN || (!BBHasCall && CurState != ST_UNKNOWN)) 319 BBSolved[BBNum] = true; 320 321 if (CurState != BBState[BBNum]) 322 Changed = true; 323 324 BBState[BBNum] = CurState; 325 return Changed; 326} 327