MLxExpansionPass.cpp revision dce4a407a24b04eebc6a376f8e62b41aaa7b071f 
1//===-- MLxExpansionPass.cpp - Expand MLx instrs to avoid hazards ---------===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// Expand VFP / NEON floating point MLA / MLS instructions (each to a pair of
11// multiple and add / sub instructions) when special VMLx hazards are detected.
12//
13//===----------------------------------------------------------------------===//
14
15#include "ARM.h"
16#include "ARMBaseInstrInfo.h"
17#include "ARMSubtarget.h"
18#include "llvm/ADT/SmallPtrSet.h"
19#include "llvm/ADT/Statistic.h"
20#include "llvm/CodeGen/MachineFunctionPass.h"
21#include "llvm/CodeGen/MachineInstr.h"
22#include "llvm/CodeGen/MachineInstrBuilder.h"
23#include "llvm/CodeGen/MachineRegisterInfo.h"
24#include "llvm/Support/CommandLine.h"
25#include "llvm/Support/Debug.h"
26#include "llvm/Support/raw_ostream.h"
27#include "llvm/Target/TargetRegisterInfo.h"
28using namespace llvm;
29
30#define DEBUG_TYPE "mlx-expansion"
31
32static cl::opt<bool>
33ForceExapnd("expand-all-fp-mlx", cl::init(false), cl::Hidden);
34static cl::opt<unsigned>
35ExpandLimit("expand-limit", cl::init(~0U), cl::Hidden);
36
37STATISTIC(NumExpand, "Number of fp MLA / MLS instructions expanded");
38
39namespace {
40  struct MLxExpansion : public MachineFunctionPass {
41    static char ID;
42    MLxExpansion() : MachineFunctionPass(ID) {}
43
44    bool runOnMachineFunction(MachineFunction &Fn) override;
45
46    const char *getPassName() const override {
47      return "ARM MLA / MLS expansion pass";
48    }
49
50  private:
51    const ARMBaseInstrInfo *TII;
52    const TargetRegisterInfo *TRI;
53    MachineRegisterInfo *MRI;
54
55    bool isLikeA9;
56    bool isSwift;
57    unsigned MIIdx;
58    MachineInstr* LastMIs[4];
59    SmallPtrSet<MachineInstr*, 4> IgnoreStall;
60
61    void clearStack();
62    void pushStack(MachineInstr *MI);
63    MachineInstr *getAccDefMI(MachineInstr *MI) const;
64    unsigned getDefReg(MachineInstr *MI) const;
65    bool hasLoopHazard(MachineInstr *MI) const;
66    bool hasRAWHazard(unsigned Reg, MachineInstr *MI) const;
67    bool FindMLxHazard(MachineInstr *MI);
68    void ExpandFPMLxInstruction(MachineBasicBlock &MBB, MachineInstr *MI,
69                                unsigned MulOpc, unsigned AddSubOpc,
70                                bool NegAcc, bool HasLane);
71    bool ExpandFPMLxInstructions(MachineBasicBlock &MBB);
72  };
73  char MLxExpansion::ID = 0;
74}
75
76void MLxExpansion::clearStack() {
77  std::fill(LastMIs, LastMIs + 4, nullptr);
78  MIIdx = 0;
79}
80
81void MLxExpansion::pushStack(MachineInstr *MI) {
82  LastMIs[MIIdx] = MI;
83  if (++MIIdx == 4)
84    MIIdx = 0;
85}
86
87MachineInstr *MLxExpansion::getAccDefMI(MachineInstr *MI) const {
88  // Look past COPY and INSERT_SUBREG instructions to find the
89  // real definition MI. This is important for _sfp instructions.
90  unsigned Reg = MI->getOperand(1).getReg();
91  if (TargetRegisterInfo::isPhysicalRegister(Reg))
92    return nullptr;
93
94  MachineBasicBlock *MBB = MI->getParent();
95  MachineInstr *DefMI = MRI->getVRegDef(Reg);
96  while (true) {
97    if (DefMI->getParent() != MBB)
98      break;
99    if (DefMI->isCopyLike()) {
100      Reg = DefMI->getOperand(1).getReg();
101      if (TargetRegisterInfo::isVirtualRegister(Reg)) {
102        DefMI = MRI->getVRegDef(Reg);
103        continue;
104      }
105    } else if (DefMI->isInsertSubreg()) {
106      Reg = DefMI->getOperand(2).getReg();
107      if (TargetRegisterInfo::isVirtualRegister(Reg)) {
108        DefMI = MRI->getVRegDef(Reg);
109        continue;
110      }
111    }
112    break;
113  }
114  return DefMI;
115}
116
117unsigned MLxExpansion::getDefReg(MachineInstr *MI) const {
118  unsigned Reg = MI->getOperand(0).getReg();
119  if (TargetRegisterInfo::isPhysicalRegister(Reg) ||
120      !MRI->hasOneNonDBGUse(Reg))
121    return Reg;
122
123  MachineBasicBlock *MBB = MI->getParent();
124  MachineInstr *UseMI = &*MRI->use_instr_nodbg_begin(Reg);
125  if (UseMI->getParent() != MBB)
126    return Reg;
127
128  while (UseMI->isCopy() || UseMI->isInsertSubreg()) {
129    Reg = UseMI->getOperand(0).getReg();
130    if (TargetRegisterInfo::isPhysicalRegister(Reg) ||
131        !MRI->hasOneNonDBGUse(Reg))
132      return Reg;
133    UseMI = &*MRI->use_instr_nodbg_begin(Reg);
134    if (UseMI->getParent() != MBB)
135      return Reg;
136  }
137
138  return Reg;
139}
140
141/// hasLoopHazard - Check whether an MLx instruction is chained to itself across
142/// a single-MBB loop.
143bool MLxExpansion::hasLoopHazard(MachineInstr *MI) const {
144  unsigned Reg = MI->getOperand(1).getReg();
145  if (TargetRegisterInfo::isPhysicalRegister(Reg))
146    return false;
147
148  MachineBasicBlock *MBB = MI->getParent();
149  MachineInstr *DefMI = MRI->getVRegDef(Reg);
150  while (true) {
151outer_continue:
152    if (DefMI->getParent() != MBB)
153      break;
154
155    if (DefMI->isPHI()) {
156      for (unsigned i = 1, e = DefMI->getNumOperands(); i < e; i += 2) {
157        if (DefMI->getOperand(i + 1).getMBB() == MBB) {
158          unsigned SrcReg = DefMI->getOperand(i).getReg();
159          if (TargetRegisterInfo::isVirtualRegister(SrcReg)) {
160            DefMI = MRI->getVRegDef(SrcReg);
161            goto outer_continue;
162          }
163        }
164      }
165    } else if (DefMI->isCopyLike()) {
166      Reg = DefMI->getOperand(1).getReg();
167      if (TargetRegisterInfo::isVirtualRegister(Reg)) {
168        DefMI = MRI->getVRegDef(Reg);
169        continue;
170      }
171    } else if (DefMI->isInsertSubreg()) {
172      Reg = DefMI->getOperand(2).getReg();
173      if (TargetRegisterInfo::isVirtualRegister(Reg)) {
174        DefMI = MRI->getVRegDef(Reg);
175        continue;
176      }
177    }
178
179    break;
180  }
181
182  return DefMI == MI;
183}
184
185bool MLxExpansion::hasRAWHazard(unsigned Reg, MachineInstr *MI) const {
186  // FIXME: Detect integer instructions properly.
187  const MCInstrDesc &MCID = MI->getDesc();
188  unsigned Domain = MCID.TSFlags & ARMII::DomainMask;
189  if (MI->mayStore())
190    return false;
191  unsigned Opcode = MCID.getOpcode();
192  if (Opcode == ARM::VMOVRS || Opcode == ARM::VMOVRRD)
193    return false;
194  if ((Domain & ARMII::DomainVFP) || (Domain & ARMII::DomainNEON))
195    return MI->readsRegister(Reg, TRI);
196  return false;
197}
198
199static bool isFpMulInstruction(unsigned Opcode) {
200  switch (Opcode) {
201  case ARM::VMULS:
202  case ARM::VMULfd:
203  case ARM::VMULfq:
204  case ARM::VMULD:
205  case ARM::VMULslfd:
206  case ARM::VMULslfq:
207    return true;
208  default:
209    return false;
210  }
211}
212
213bool MLxExpansion::FindMLxHazard(MachineInstr *MI) {
214  if (NumExpand >= ExpandLimit)
215    return false;
216
217  if (ForceExapnd)
218    return true;
219
220  MachineInstr *DefMI = getAccDefMI(MI);
221  if (TII->isFpMLxInstruction(DefMI->getOpcode())) {
222    // r0 = vmla
223    // r3 = vmla r0, r1, r2
224    // takes 16 - 17 cycles
225    //
226    // r0 = vmla
227    // r4 = vmul r1, r2
228    // r3 = vadd r0, r4
229    // takes about 14 - 15 cycles even with vmul stalling for 4 cycles.
230    IgnoreStall.insert(DefMI);
231    return true;
232  }
233
234  // On Swift, we mostly care about hazards from multiplication instructions
235  // writing the accumulator and the pipelining of loop iterations by out-of-
236  // order execution.
237  if (isSwift)
238    return isFpMulInstruction(DefMI->getOpcode()) || hasLoopHazard(MI);
239
240  if (IgnoreStall.count(MI))
241    return false;
242
243  // If a VMLA.F is followed by an VADD.F or VMUL.F with no RAW hazard, the
244  // VADD.F or VMUL.F will stall 4 cycles before issue. The 4 cycle stall
245  // preserves the in-order retirement of the instructions.
246  // Look at the next few instructions, if *most* of them can cause hazards,
247  // then the scheduler can't *fix* this, we'd better break up the VMLA.
248  unsigned Limit1 = isLikeA9 ? 1 : 4;
249  unsigned Limit2 = isLikeA9 ? 1 : 4;
250  for (unsigned i = 1; i <= 4; ++i) {
251    int Idx = ((int)MIIdx - i + 4) % 4;
252    MachineInstr *NextMI = LastMIs[Idx];
253    if (!NextMI)
254      continue;
255
256    if (TII->canCauseFpMLxStall(NextMI->getOpcode())) {
257      if (i <= Limit1)
258        return true;
259    }
260
261    // Look for VMLx RAW hazard.
262    if (i <= Limit2 && hasRAWHazard(getDefReg(MI), NextMI))
263      return true;
264  }
265
266  return false;
267}
268
269/// ExpandFPMLxInstructions - Expand a MLA / MLS instruction into a pair
270/// of MUL + ADD / SUB instructions.
271void
272MLxExpansion::ExpandFPMLxInstruction(MachineBasicBlock &MBB, MachineInstr *MI,
273                                     unsigned MulOpc, unsigned AddSubOpc,
274                                     bool NegAcc, bool HasLane) {
275  unsigned DstReg = MI->getOperand(0).getReg();
276  bool DstDead = MI->getOperand(0).isDead();
277  unsigned AccReg = MI->getOperand(1).getReg();
278  unsigned Src1Reg = MI->getOperand(2).getReg();
279  unsigned Src2Reg = MI->getOperand(3).getReg();
280  bool Src1Kill = MI->getOperand(2).isKill();
281  bool Src2Kill = MI->getOperand(3).isKill();
282  unsigned LaneImm = HasLane ? MI->getOperand(4).getImm() : 0;
283  unsigned NextOp = HasLane ? 5 : 4;
284  ARMCC::CondCodes Pred = (ARMCC::CondCodes)MI->getOperand(NextOp).getImm();
285  unsigned PredReg = MI->getOperand(++NextOp).getReg();
286
287  const MCInstrDesc &MCID1 = TII->get(MulOpc);
288  const MCInstrDesc &MCID2 = TII->get(AddSubOpc);
289  const MachineFunction &MF = *MI->getParent()->getParent();
290  unsigned TmpReg = MRI->createVirtualRegister(
291                      TII->getRegClass(MCID1, 0, TRI, MF));
292
293  MachineInstrBuilder MIB = BuildMI(MBB, MI, MI->getDebugLoc(), MCID1, TmpReg)
294    .addReg(Src1Reg, getKillRegState(Src1Kill))
295    .addReg(Src2Reg, getKillRegState(Src2Kill));
296  if (HasLane)
297    MIB.addImm(LaneImm);
298  MIB.addImm(Pred).addReg(PredReg);
299
300  MIB = BuildMI(MBB, MI, MI->getDebugLoc(), MCID2)
301    .addReg(DstReg, getDefRegState(true) | getDeadRegState(DstDead));
302
303  if (NegAcc) {
304    bool AccKill = MRI->hasOneNonDBGUse(AccReg);
305    MIB.addReg(TmpReg, getKillRegState(true))
306       .addReg(AccReg, getKillRegState(AccKill));
307  } else {
308    MIB.addReg(AccReg).addReg(TmpReg, getKillRegState(true));
309  }
310  MIB.addImm(Pred).addReg(PredReg);
311
312  DEBUG({
313      dbgs() << "Expanding: " << *MI;
314      dbgs() << "  to:\n";
315      MachineBasicBlock::iterator MII = MI;
316      MII = std::prev(MII);
317      MachineInstr &MI2 = *MII;
318      MII = std::prev(MII);
319      MachineInstr &MI1 = *MII;
320      dbgs() << "    " << MI1;
321      dbgs() << "    " << MI2;
322   });
323
324  MI->eraseFromParent();
325  ++NumExpand;
326}
327
328bool MLxExpansion::ExpandFPMLxInstructions(MachineBasicBlock &MBB) {
329  bool Changed = false;
330
331  clearStack();
332  IgnoreStall.clear();
333
334  unsigned Skip = 0;
335  MachineBasicBlock::reverse_iterator MII = MBB.rbegin(), E = MBB.rend();
336  while (MII != E) {
337    MachineInstr *MI = &*MII;
338
339    if (MI->isPosition() || MI->isImplicitDef() || MI->isCopy()) {
340      ++MII;
341      continue;
342    }
343
344    const MCInstrDesc &MCID = MI->getDesc();
345    if (MI->isBarrier()) {
346      clearStack();
347      Skip = 0;
348      ++MII;
349      continue;
350    }
351
352    unsigned Domain = MCID.TSFlags & ARMII::DomainMask;
353    if (Domain == ARMII::DomainGeneral) {
354      if (++Skip == 2)
355        // Assume dual issues of non-VFP / NEON instructions.
356        pushStack(nullptr);
357    } else {
358      Skip = 0;
359
360      unsigned MulOpc, AddSubOpc;
361      bool NegAcc, HasLane;
362      if (!TII->isFpMLxInstruction(MCID.getOpcode(),
363                                   MulOpc, AddSubOpc, NegAcc, HasLane) ||
364          !FindMLxHazard(MI))
365        pushStack(MI);
366      else {
367        ExpandFPMLxInstruction(MBB, MI, MulOpc, AddSubOpc, NegAcc, HasLane);
368        E = MBB.rend(); // May have changed if MI was the 1st instruction.
369        Changed = true;
370        continue;
371      }
372    }
373
374    ++MII;
375  }
376
377  return Changed;
378}
379
380bool MLxExpansion::runOnMachineFunction(MachineFunction &Fn) {
381  TII = static_cast<const ARMBaseInstrInfo*>(Fn.getTarget().getInstrInfo());
382  TRI = Fn.getTarget().getRegisterInfo();
383  MRI = &Fn.getRegInfo();
384  const ARMSubtarget *STI = &Fn.getTarget().getSubtarget<ARMSubtarget>();
385  isLikeA9 = STI->isLikeA9() || STI->isSwift();
386  isSwift = STI->isSwift();
387
388  bool Modified = false;
389  for (MachineBasicBlock &MBB : Fn)
390    Modified |= ExpandFPMLxInstructions(MBB);
391
392  return Modified;
393}
394
395FunctionPass *llvm::createMLxExpansionPass() {
396  return new MLxExpansion();
397}
398