MLxExpansionPass.cpp revision 36b56886974eae4f9c5ebc96befd3e7bfe5de338 
1//===-- MLxExpansionPass.cpp - Expand MLx instrs to avoid hazards ---------===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// Expand VFP / NEON floating point MLA / MLS instructions (each to a pair of
11// multiple and add / sub instructions) when special VMLx hazards are detected.
12//
13//===----------------------------------------------------------------------===//
14
15#define DEBUG_TYPE "mlx-expansion"
16#include "ARM.h"
17#include "ARMBaseInstrInfo.h"
18#include "ARMSubtarget.h"
19#include "llvm/ADT/SmallPtrSet.h"
20#include "llvm/ADT/Statistic.h"
21#include "llvm/CodeGen/MachineFunctionPass.h"
22#include "llvm/CodeGen/MachineInstr.h"
23#include "llvm/CodeGen/MachineInstrBuilder.h"
24#include "llvm/CodeGen/MachineRegisterInfo.h"
25#include "llvm/Support/CommandLine.h"
26#include "llvm/Support/Debug.h"
27#include "llvm/Support/raw_ostream.h"
28#include "llvm/Target/TargetRegisterInfo.h"
29using namespace llvm;
30
31static cl::opt<bool>
32ForceExapnd("expand-all-fp-mlx", cl::init(false), cl::Hidden);
33static cl::opt<unsigned>
34ExpandLimit("expand-limit", cl::init(~0U), cl::Hidden);
35
36STATISTIC(NumExpand, "Number of fp MLA / MLS instructions expanded");
37
38namespace {
39  struct MLxExpansion : public MachineFunctionPass {
40    static char ID;
41    MLxExpansion() : MachineFunctionPass(ID) {}
42
43    bool runOnMachineFunction(MachineFunction &Fn) override;
44
45    const char *getPassName() const override {
46      return "ARM MLA / MLS expansion pass";
47    }
48
49  private:
50    const ARMBaseInstrInfo *TII;
51    const TargetRegisterInfo *TRI;
52    MachineRegisterInfo *MRI;
53
54    bool isLikeA9;
55    bool isSwift;
56    unsigned MIIdx;
57    MachineInstr* LastMIs[4];
58    SmallPtrSet<MachineInstr*, 4> IgnoreStall;
59
60    void clearStack();
61    void pushStack(MachineInstr *MI);
62    MachineInstr *getAccDefMI(MachineInstr *MI) const;
63    unsigned getDefReg(MachineInstr *MI) const;
64    bool hasLoopHazard(MachineInstr *MI) const;
65    bool hasRAWHazard(unsigned Reg, MachineInstr *MI) const;
66    bool FindMLxHazard(MachineInstr *MI);
67    void ExpandFPMLxInstruction(MachineBasicBlock &MBB, MachineInstr *MI,
68                                unsigned MulOpc, unsigned AddSubOpc,
69                                bool NegAcc, bool HasLane);
70    bool ExpandFPMLxInstructions(MachineBasicBlock &MBB);
71  };
72  char MLxExpansion::ID = 0;
73}
74
75void MLxExpansion::clearStack() {
76  std::fill(LastMIs, LastMIs + 4, (MachineInstr*)0);
77  MIIdx = 0;
78}
79
80void MLxExpansion::pushStack(MachineInstr *MI) {
81  LastMIs[MIIdx] = MI;
82  if (++MIIdx == 4)
83    MIIdx = 0;
84}
85
86MachineInstr *MLxExpansion::getAccDefMI(MachineInstr *MI) const {
87  // Look past COPY and INSERT_SUBREG instructions to find the
88  // real definition MI. This is important for _sfp instructions.
89  unsigned Reg = MI->getOperand(1).getReg();
90  if (TargetRegisterInfo::isPhysicalRegister(Reg))
91    return 0;
92
93  MachineBasicBlock *MBB = MI->getParent();
94  MachineInstr *DefMI = MRI->getVRegDef(Reg);
95  while (true) {
96    if (DefMI->getParent() != MBB)
97      break;
98    if (DefMI->isCopyLike()) {
99      Reg = DefMI->getOperand(1).getReg();
100      if (TargetRegisterInfo::isVirtualRegister(Reg)) {
101        DefMI = MRI->getVRegDef(Reg);
102        continue;
103      }
104    } else if (DefMI->isInsertSubreg()) {
105      Reg = DefMI->getOperand(2).getReg();
106      if (TargetRegisterInfo::isVirtualRegister(Reg)) {
107        DefMI = MRI->getVRegDef(Reg);
108        continue;
109      }
110    }
111    break;
112  }
113  return DefMI;
114}
115
116unsigned MLxExpansion::getDefReg(MachineInstr *MI) const {
117  unsigned Reg = MI->getOperand(0).getReg();
118  if (TargetRegisterInfo::isPhysicalRegister(Reg) ||
119      !MRI->hasOneNonDBGUse(Reg))
120    return Reg;
121
122  MachineBasicBlock *MBB = MI->getParent();
123  MachineInstr *UseMI = &*MRI->use_instr_nodbg_begin(Reg);
124  if (UseMI->getParent() != MBB)
125    return Reg;
126
127  while (UseMI->isCopy() || UseMI->isInsertSubreg()) {
128    Reg = UseMI->getOperand(0).getReg();
129    if (TargetRegisterInfo::isPhysicalRegister(Reg) ||
130        !MRI->hasOneNonDBGUse(Reg))
131      return Reg;
132    UseMI = &*MRI->use_instr_nodbg_begin(Reg);
133    if (UseMI->getParent() != MBB)
134      return Reg;
135  }
136
137  return Reg;
138}
139
140/// hasLoopHazard - Check whether an MLx instruction is chained to itself across
141/// a single-MBB loop.
142bool MLxExpansion::hasLoopHazard(MachineInstr *MI) const {
143  unsigned Reg = MI->getOperand(1).getReg();
144  if (TargetRegisterInfo::isPhysicalRegister(Reg))
145    return false;
146
147  MachineBasicBlock *MBB = MI->getParent();
148  MachineInstr *DefMI = MRI->getVRegDef(Reg);
149  while (true) {
150outer_continue:
151    if (DefMI->getParent() != MBB)
152      break;
153
154    if (DefMI->isPHI()) {
155      for (unsigned i = 1, e = DefMI->getNumOperands(); i < e; i += 2) {
156        if (DefMI->getOperand(i + 1).getMBB() == MBB) {
157          unsigned SrcReg = DefMI->getOperand(i).getReg();
158          if (TargetRegisterInfo::isVirtualRegister(SrcReg)) {
159            DefMI = MRI->getVRegDef(SrcReg);
160            goto outer_continue;
161          }
162        }
163      }
164    } else if (DefMI->isCopyLike()) {
165      Reg = DefMI->getOperand(1).getReg();
166      if (TargetRegisterInfo::isVirtualRegister(Reg)) {
167        DefMI = MRI->getVRegDef(Reg);
168        continue;
169      }
170    } else if (DefMI->isInsertSubreg()) {
171      Reg = DefMI->getOperand(2).getReg();
172      if (TargetRegisterInfo::isVirtualRegister(Reg)) {
173        DefMI = MRI->getVRegDef(Reg);
174        continue;
175      }
176    }
177
178    break;
179  }
180
181  return DefMI == MI;
182}
183
184bool MLxExpansion::hasRAWHazard(unsigned Reg, MachineInstr *MI) const {
185  // FIXME: Detect integer instructions properly.
186  const MCInstrDesc &MCID = MI->getDesc();
187  unsigned Domain = MCID.TSFlags & ARMII::DomainMask;
188  if (MI->mayStore())
189    return false;
190  unsigned Opcode = MCID.getOpcode();
191  if (Opcode == ARM::VMOVRS || Opcode == ARM::VMOVRRD)
192    return false;
193  if ((Domain & ARMII::DomainVFP) || (Domain & ARMII::DomainNEON))
194    return MI->readsRegister(Reg, TRI);
195  return false;
196}
197
198static bool isFpMulInstruction(unsigned Opcode) {
199  switch (Opcode) {
200  case ARM::VMULS:
201  case ARM::VMULfd:
202  case ARM::VMULfq:
203  case ARM::VMULD:
204  case ARM::VMULslfd:
205  case ARM::VMULslfq:
206    return true;
207  default:
208    return false;
209  }
210}
211
212bool MLxExpansion::FindMLxHazard(MachineInstr *MI) {
213  if (NumExpand >= ExpandLimit)
214    return false;
215
216  if (ForceExapnd)
217    return true;
218
219  MachineInstr *DefMI = getAccDefMI(MI);
220  if (TII->isFpMLxInstruction(DefMI->getOpcode())) {
221    // r0 = vmla
222    // r3 = vmla r0, r1, r2
223    // takes 16 - 17 cycles
224    //
225    // r0 = vmla
226    // r4 = vmul r1, r2
227    // r3 = vadd r0, r4
228    // takes about 14 - 15 cycles even with vmul stalling for 4 cycles.
229    IgnoreStall.insert(DefMI);
230    return true;
231  }
232
233  // On Swift, we mostly care about hazards from multiplication instructions
234  // writing the accumulator and the pipelining of loop iterations by out-of-
235  // order execution.
236  if (isSwift)
237    return isFpMulInstruction(DefMI->getOpcode()) || hasLoopHazard(MI);
238
239  if (IgnoreStall.count(MI))
240    return false;
241
242  // If a VMLA.F is followed by an VADD.F or VMUL.F with no RAW hazard, the
243  // VADD.F or VMUL.F will stall 4 cycles before issue. The 4 cycle stall
244  // preserves the in-order retirement of the instructions.
245  // Look at the next few instructions, if *most* of them can cause hazards,
246  // then the scheduler can't *fix* this, we'd better break up the VMLA.
247  unsigned Limit1 = isLikeA9 ? 1 : 4;
248  unsigned Limit2 = isLikeA9 ? 1 : 4;
249  for (unsigned i = 1; i <= 4; ++i) {
250    int Idx = ((int)MIIdx - i + 4) % 4;
251    MachineInstr *NextMI = LastMIs[Idx];
252    if (!NextMI)
253      continue;
254
255    if (TII->canCauseFpMLxStall(NextMI->getOpcode())) {
256      if (i <= Limit1)
257        return true;
258    }
259
260    // Look for VMLx RAW hazard.
261    if (i <= Limit2 && hasRAWHazard(getDefReg(MI), NextMI))
262      return true;
263  }
264
265  return false;
266}
267
268/// ExpandFPMLxInstructions - Expand a MLA / MLS instruction into a pair
269/// of MUL + ADD / SUB instructions.
270void
271MLxExpansion::ExpandFPMLxInstruction(MachineBasicBlock &MBB, MachineInstr *MI,
272                                     unsigned MulOpc, unsigned AddSubOpc,
273                                     bool NegAcc, bool HasLane) {
274  unsigned DstReg = MI->getOperand(0).getReg();
275  bool DstDead = MI->getOperand(0).isDead();
276  unsigned AccReg = MI->getOperand(1).getReg();
277  unsigned Src1Reg = MI->getOperand(2).getReg();
278  unsigned Src2Reg = MI->getOperand(3).getReg();
279  bool Src1Kill = MI->getOperand(2).isKill();
280  bool Src2Kill = MI->getOperand(3).isKill();
281  unsigned LaneImm = HasLane ? MI->getOperand(4).getImm() : 0;
282  unsigned NextOp = HasLane ? 5 : 4;
283  ARMCC::CondCodes Pred = (ARMCC::CondCodes)MI->getOperand(NextOp).getImm();
284  unsigned PredReg = MI->getOperand(++NextOp).getReg();
285
286  const MCInstrDesc &MCID1 = TII->get(MulOpc);
287  const MCInstrDesc &MCID2 = TII->get(AddSubOpc);
288  const MachineFunction &MF = *MI->getParent()->getParent();
289  unsigned TmpReg = MRI->createVirtualRegister(
290                      TII->getRegClass(MCID1, 0, TRI, MF));
291
292  MachineInstrBuilder MIB = BuildMI(MBB, MI, MI->getDebugLoc(), MCID1, TmpReg)
293    .addReg(Src1Reg, getKillRegState(Src1Kill))
294    .addReg(Src2Reg, getKillRegState(Src2Kill));
295  if (HasLane)
296    MIB.addImm(LaneImm);
297  MIB.addImm(Pred).addReg(PredReg);
298
299  MIB = BuildMI(MBB, MI, MI->getDebugLoc(), MCID2)
300    .addReg(DstReg, getDefRegState(true) | getDeadRegState(DstDead));
301
302  if (NegAcc) {
303    bool AccKill = MRI->hasOneNonDBGUse(AccReg);
304    MIB.addReg(TmpReg, getKillRegState(true))
305       .addReg(AccReg, getKillRegState(AccKill));
306  } else {
307    MIB.addReg(AccReg).addReg(TmpReg, getKillRegState(true));
308  }
309  MIB.addImm(Pred).addReg(PredReg);
310
311  DEBUG({
312      dbgs() << "Expanding: " << *MI;
313      dbgs() << "  to:\n";
314      MachineBasicBlock::iterator MII = MI;
315      MII = std::prev(MII);
316      MachineInstr &MI2 = *MII;
317      MII = std::prev(MII);
318      MachineInstr &MI1 = *MII;
319      dbgs() << "    " << MI1;
320      dbgs() << "    " << MI2;
321   });
322
323  MI->eraseFromParent();
324  ++NumExpand;
325}
326
327bool MLxExpansion::ExpandFPMLxInstructions(MachineBasicBlock &MBB) {
328  bool Changed = false;
329
330  clearStack();
331  IgnoreStall.clear();
332
333  unsigned Skip = 0;
334  MachineBasicBlock::reverse_iterator MII = MBB.rbegin(), E = MBB.rend();
335  while (MII != E) {
336    MachineInstr *MI = &*MII;
337
338    if (MI->isPosition() || MI->isImplicitDef() || MI->isCopy()) {
339      ++MII;
340      continue;
341    }
342
343    const MCInstrDesc &MCID = MI->getDesc();
344    if (MI->isBarrier()) {
345      clearStack();
346      Skip = 0;
347      ++MII;
348      continue;
349    }
350
351    unsigned Domain = MCID.TSFlags & ARMII::DomainMask;
352    if (Domain == ARMII::DomainGeneral) {
353      if (++Skip == 2)
354        // Assume dual issues of non-VFP / NEON instructions.
355        pushStack(0);
356    } else {
357      Skip = 0;
358
359      unsigned MulOpc, AddSubOpc;
360      bool NegAcc, HasLane;
361      if (!TII->isFpMLxInstruction(MCID.getOpcode(),
362                                   MulOpc, AddSubOpc, NegAcc, HasLane) ||
363          !FindMLxHazard(MI))
364        pushStack(MI);
365      else {
366        ExpandFPMLxInstruction(MBB, MI, MulOpc, AddSubOpc, NegAcc, HasLane);
367        E = MBB.rend(); // May have changed if MI was the 1st instruction.
368        Changed = true;
369        continue;
370      }
371    }
372
373    ++MII;
374  }
375
376  return Changed;
377}
378
379bool MLxExpansion::runOnMachineFunction(MachineFunction &Fn) {
380  TII = static_cast<const ARMBaseInstrInfo*>(Fn.getTarget().getInstrInfo());
381  TRI = Fn.getTarget().getRegisterInfo();
382  MRI = &Fn.getRegInfo();
383  const ARMSubtarget *STI = &Fn.getTarget().getSubtarget<ARMSubtarget>();
384  isLikeA9 = STI->isLikeA9() || STI->isSwift();
385  isSwift = STI->isSwift();
386
387  bool Modified = false;
388  for (MachineBasicBlock &MBB : Fn)
389    Modified |= ExpandFPMLxInstructions(MBB);
390
391  return Modified;
392}
393
394FunctionPass *llvm::createMLxExpansionPass() {
395  return new MLxExpansion();
396}
397