StackMaps.cpp revision 36b56886974eae4f9c5ebc96befd3e7bfe5de338 
1//===---------------------------- StackMaps.cpp ---------------------------===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9
10#define DEBUG_TYPE "stackmaps"
11
12#include "llvm/CodeGen/StackMaps.h"
13#include "llvm/CodeGen/AsmPrinter.h"
14#include "llvm/CodeGen/MachineFrameInfo.h"
15#include "llvm/CodeGen/MachineFunction.h"
16#include "llvm/CodeGen/MachineInstr.h"
17#include "llvm/IR/DataLayout.h"
18#include "llvm/MC/MCContext.h"
19#include "llvm/MC/MCExpr.h"
20#include "llvm/MC/MCObjectFileInfo.h"
21#include "llvm/MC/MCSectionMachO.h"
22#include "llvm/MC/MCStreamer.h"
23#include "llvm/Support/Debug.h"
24#include "llvm/Support/raw_ostream.h"
25#include "llvm/Target/TargetMachine.h"
26#include "llvm/Target/TargetOpcodes.h"
27#include "llvm/Target/TargetRegisterInfo.h"
28#include <iterator>
29
30using namespace llvm;
31
32PatchPointOpers::PatchPointOpers(const MachineInstr *MI)
33  : MI(MI),
34    HasDef(MI->getOperand(0).isReg() && MI->getOperand(0).isDef() &&
35           !MI->getOperand(0).isImplicit()),
36    IsAnyReg(MI->getOperand(getMetaIdx(CCPos)).getImm() == CallingConv::AnyReg)
37{
38#ifndef NDEBUG
39  unsigned CheckStartIdx = 0, e = MI->getNumOperands();
40  while (CheckStartIdx < e && MI->getOperand(CheckStartIdx).isReg() &&
41         MI->getOperand(CheckStartIdx).isDef() &&
42         !MI->getOperand(CheckStartIdx).isImplicit())
43    ++CheckStartIdx;
44
45  assert(getMetaIdx() == CheckStartIdx &&
46         "Unexpected additional definition in Patchpoint intrinsic.");
47#endif
48}
49
50unsigned PatchPointOpers::getNextScratchIdx(unsigned StartIdx) const {
51  if (!StartIdx)
52    StartIdx = getVarIdx();
53
54  // Find the next scratch register (implicit def and early clobber)
55  unsigned ScratchIdx = StartIdx, e = MI->getNumOperands();
56  while (ScratchIdx < e &&
57         !(MI->getOperand(ScratchIdx).isReg() &&
58           MI->getOperand(ScratchIdx).isDef() &&
59           MI->getOperand(ScratchIdx).isImplicit() &&
60           MI->getOperand(ScratchIdx).isEarlyClobber()))
61    ++ScratchIdx;
62
63  assert(ScratchIdx != e && "No scratch register available");
64  return ScratchIdx;
65}
66
67MachineInstr::const_mop_iterator
68StackMaps::parseOperand(MachineInstr::const_mop_iterator MOI,
69                        MachineInstr::const_mop_iterator MOE,
70                        LocationVec &Locs, LiveOutVec &LiveOuts) const {
71  if (MOI->isImm()) {
72    switch (MOI->getImm()) {
73    default: llvm_unreachable("Unrecognized operand type.");
74    case StackMaps::DirectMemRefOp: {
75      unsigned Size = AP.TM.getDataLayout()->getPointerSizeInBits();
76      assert((Size % 8) == 0 && "Need pointer size in bytes.");
77      Size /= 8;
78      unsigned Reg = (++MOI)->getReg();
79      int64_t Imm = (++MOI)->getImm();
80      Locs.push_back(Location(StackMaps::Location::Direct, Size, Reg, Imm));
81      break;
82    }
83    case StackMaps::IndirectMemRefOp: {
84      int64_t Size = (++MOI)->getImm();
85      assert(Size > 0 && "Need a valid size for indirect memory locations.");
86      unsigned Reg = (++MOI)->getReg();
87      int64_t Imm = (++MOI)->getImm();
88      Locs.push_back(Location(StackMaps::Location::Indirect, Size, Reg, Imm));
89      break;
90    }
91    case StackMaps::ConstantOp: {
92      ++MOI;
93      assert(MOI->isImm() && "Expected constant operand.");
94      int64_t Imm = MOI->getImm();
95      Locs.push_back(Location(Location::Constant, sizeof(int64_t), 0, Imm));
96      break;
97    }
98    }
99    return ++MOI;
100  }
101
102  // The physical register number will ultimately be encoded as a DWARF regno.
103  // The stack map also records the size of a spill slot that can hold the
104  // register content. (The runtime can track the actual size of the data type
105  // if it needs to.)
106  if (MOI->isReg()) {
107    // Skip implicit registers (this includes our scratch registers)
108    if (MOI->isImplicit())
109      return ++MOI;
110
111    assert(TargetRegisterInfo::isPhysicalRegister(MOI->getReg()) &&
112           "Virtreg operands should have been rewritten before now.");
113    const TargetRegisterClass *RC =
114      AP.TM.getRegisterInfo()->getMinimalPhysRegClass(MOI->getReg());
115    assert(!MOI->getSubReg() && "Physical subreg still around.");
116    Locs.push_back(
117      Location(Location::Register, RC->getSize(), MOI->getReg(), 0));
118    return ++MOI;
119  }
120
121  if (MOI->isRegLiveOut())
122    LiveOuts = parseRegisterLiveOutMask(MOI->getRegLiveOut());
123
124  return ++MOI;
125}
126
127/// Go up the super-register chain until we hit a valid dwarf register number.
128static unsigned getDwarfRegNum(unsigned Reg, const TargetRegisterInfo *TRI) {
129  int RegNo = TRI->getDwarfRegNum(Reg, false);
130  for (MCSuperRegIterator SR(Reg, TRI); SR.isValid() && RegNo < 0; ++SR)
131    RegNo = TRI->getDwarfRegNum(*SR, false);
132
133  assert(RegNo >= 0 && "Invalid Dwarf register number.");
134  return (unsigned) RegNo;
135}
136
137/// Create a live-out register record for the given register Reg.
138StackMaps::LiveOutReg
139StackMaps::createLiveOutReg(unsigned Reg, const TargetRegisterInfo *TRI) const {
140  unsigned RegNo = getDwarfRegNum(Reg, TRI);
141  unsigned Size = TRI->getMinimalPhysRegClass(Reg)->getSize();
142  return LiveOutReg(Reg, RegNo, Size);
143}
144
145/// Parse the register live-out mask and return a vector of live-out registers
146/// that need to be recorded in the stackmap.
147StackMaps::LiveOutVec
148StackMaps::parseRegisterLiveOutMask(const uint32_t *Mask) const {
149  assert(Mask && "No register mask specified");
150  const TargetRegisterInfo *TRI = AP.TM.getRegisterInfo();
151  LiveOutVec LiveOuts;
152
153  // Create a LiveOutReg for each bit that is set in the register mask.
154  for (unsigned Reg = 0, NumRegs = TRI->getNumRegs(); Reg != NumRegs; ++Reg)
155    if ((Mask[Reg / 32] >> Reg % 32) & 1)
156      LiveOuts.push_back(createLiveOutReg(Reg, TRI));
157
158  // We don't need to keep track of a register if its super-register is already
159  // in the list. Merge entries that refer to the same dwarf register and use
160  // the maximum size that needs to be spilled.
161  std::sort(LiveOuts.begin(), LiveOuts.end());
162  for (LiveOutVec::iterator I = LiveOuts.begin(), E = LiveOuts.end();
163       I != E; ++I) {
164    for (LiveOutVec::iterator II = std::next(I); II != E; ++II) {
165      if (I->RegNo != II->RegNo) {
166        // Skip all the now invalid entries.
167        I = --II;
168        break;
169      }
170      I->Size = std::max(I->Size, II->Size);
171      if (TRI->isSuperRegister(I->Reg, II->Reg))
172        I->Reg = II->Reg;
173      II->MarkInvalid();
174    }
175  }
176  LiveOuts.erase(std::remove_if(LiveOuts.begin(), LiveOuts.end(),
177                                LiveOutReg::IsInvalid), LiveOuts.end());
178  return LiveOuts;
179}
180
181void StackMaps::recordStackMapOpers(const MachineInstr &MI, uint64_t ID,
182                                    MachineInstr::const_mop_iterator MOI,
183                                    MachineInstr::const_mop_iterator MOE,
184                                    bool recordResult) {
185
186  MCContext &OutContext = AP.OutStreamer.getContext();
187  MCSymbol *MILabel = OutContext.CreateTempSymbol();
188  AP.OutStreamer.EmitLabel(MILabel);
189
190  LocationVec Locations;
191  LiveOutVec LiveOuts;
192
193  if (recordResult) {
194    assert(PatchPointOpers(&MI).hasDef() && "Stackmap has no return value.");
195    parseOperand(MI.operands_begin(), std::next(MI.operands_begin()),
196                 Locations, LiveOuts);
197  }
198
199  // Parse operands.
200  while (MOI != MOE) {
201    MOI = parseOperand(MOI, MOE, Locations, LiveOuts);
202  }
203
204  // Move large constants into the constant pool.
205  for (LocationVec::iterator I = Locations.begin(), E = Locations.end();
206       I != E; ++I) {
207    // Constants are encoded as sign-extended integers.
208    // -1 is directly encoded as .long 0xFFFFFFFF with no constant pool.
209    if (I->LocType == Location::Constant &&
210        ((I->Offset + (int64_t(1)<<31)) >> 32) != 0) {
211      I->LocType = Location::ConstantIndex;
212      I->Offset = ConstPool.getConstantIndex(I->Offset);
213    }
214  }
215
216  // Create an expression to calculate the offset of the callsite from function
217  // entry.
218  const MCExpr *CSOffsetExpr = MCBinaryExpr::CreateSub(
219    MCSymbolRefExpr::Create(MILabel, OutContext),
220    MCSymbolRefExpr::Create(AP.CurrentFnSym, OutContext),
221    OutContext);
222
223  CSInfos.push_back(CallsiteInfo(CSOffsetExpr, ID, Locations, LiveOuts));
224
225  // Record the stack size of the current function.
226  const MachineFrameInfo *MFI = AP.MF->getFrameInfo();
227  FnStackSize[AP.CurrentFnSym] =
228    MFI->hasVarSizedObjects() ? UINT64_MAX : MFI->getStackSize();
229}
230
231void StackMaps::recordStackMap(const MachineInstr &MI) {
232  assert(MI.getOpcode() == TargetOpcode::STACKMAP && "expected stackmap");
233
234  int64_t ID = MI.getOperand(0).getImm();
235  recordStackMapOpers(MI, ID, std::next(MI.operands_begin(), 2),
236                      MI.operands_end());
237}
238
239void StackMaps::recordPatchPoint(const MachineInstr &MI) {
240  assert(MI.getOpcode() == TargetOpcode::PATCHPOINT && "expected patchpoint");
241
242  PatchPointOpers opers(&MI);
243  int64_t ID = opers.getMetaOper(PatchPointOpers::IDPos).getImm();
244
245  MachineInstr::const_mop_iterator MOI =
246    std::next(MI.operands_begin(), opers.getStackMapStartIdx());
247  recordStackMapOpers(MI, ID, MOI, MI.operands_end(),
248                      opers.isAnyReg() && opers.hasDef());
249
250#ifndef NDEBUG
251  // verify anyregcc
252  LocationVec &Locations = CSInfos.back().Locations;
253  if (opers.isAnyReg()) {
254    unsigned NArgs = opers.getMetaOper(PatchPointOpers::NArgPos).getImm();
255    for (unsigned i = 0, e = (opers.hasDef() ? NArgs+1 : NArgs); i != e; ++i)
256      assert(Locations[i].LocType == Location::Register &&
257             "anyreg arg must be in reg.");
258  }
259#endif
260}
261
262/// serializeToStackMapSection conceptually populates the following fields:
263///
264/// Header {
265///   uint8  : Stack Map Version (currently 1)
266///   uint8  : Reserved (expected to be 0)
267///   uint16 : Reserved (expected to be 0)
268/// }
269/// uint32 : NumFunctions
270/// uint32 : NumConstants
271/// uint32 : NumRecords
272/// StkSizeRecord[NumFunctions] {
273///   uint64 : Function Address
274///   uint64 : Stack Size
275/// }
276/// int64  : Constants[NumConstants]
277/// StkMapRecord[NumRecords] {
278///   uint64 : PatchPoint ID
279///   uint32 : Instruction Offset
280///   uint16 : Reserved (record flags)
281///   uint16 : NumLocations
282///   Location[NumLocations] {
283///     uint8  : Register | Direct | Indirect | Constant | ConstantIndex
284///     uint8  : Size in Bytes
285///     uint16 : Dwarf RegNum
286///     int32  : Offset
287///   }
288///   uint16 : Padding
289///   uint16 : NumLiveOuts
290///   LiveOuts[NumLiveOuts] {
291///     uint16 : Dwarf RegNum
292///     uint8  : Reserved
293///     uint8  : Size in Bytes
294///   }
295///   uint32 : Padding (only if required to align to 8 byte)
296/// }
297///
298/// Location Encoding, Type, Value:
299///   0x1, Register, Reg                 (value in register)
300///   0x2, Direct, Reg + Offset          (frame index)
301///   0x3, Indirect, [Reg + Offset]      (spilled value)
302///   0x4, Constant, Offset              (small constant)
303///   0x5, ConstIndex, Constants[Offset] (large constant)
304///
305void StackMaps::serializeToStackMapSection() {
306  // Bail out if there's no stack map data.
307  if (CSInfos.empty())
308    return;
309
310  MCContext &OutContext = AP.OutStreamer.getContext();
311  const TargetRegisterInfo *TRI = AP.TM.getRegisterInfo();
312
313  // Create the section.
314  const MCSection *StackMapSection =
315    OutContext.getObjectFileInfo()->getStackMapSection();
316  AP.OutStreamer.SwitchSection(StackMapSection);
317
318  // Emit a dummy symbol to force section inclusion.
319  AP.OutStreamer.EmitLabel(
320    OutContext.GetOrCreateSymbol(Twine("__LLVM_StackMaps")));
321
322  // Serialize data.
323  const char *WSMP = "Stack Maps: ";
324  (void)WSMP;
325
326  DEBUG(dbgs() << "********** Stack Map Output **********\n");
327
328  // Header.
329  AP.OutStreamer.EmitIntValue(1, 1); // Version.
330  AP.OutStreamer.EmitIntValue(0, 1); // Reserved.
331  AP.OutStreamer.EmitIntValue(0, 2); // Reserved.
332
333  // Num functions.
334  DEBUG(dbgs() << WSMP << "#functions = " << FnStackSize.size() << '\n');
335  AP.OutStreamer.EmitIntValue(FnStackSize.size(), 4);
336  // Num constants.
337  DEBUG(dbgs() << WSMP << "#constants = " << ConstPool.getNumConstants()
338               << '\n');
339  AP.OutStreamer.EmitIntValue(ConstPool.getNumConstants(), 4);
340  // Num callsites.
341  DEBUG(dbgs() << WSMP << "#callsites = " << CSInfos.size() << '\n');
342  AP.OutStreamer.EmitIntValue(CSInfos.size(), 4);
343
344  // Function stack size entries.
345  for (FnStackSizeMap::iterator I = FnStackSize.begin(), E = FnStackSize.end();
346       I != E; ++I) {
347    AP.OutStreamer.EmitSymbolValue(I->first, 8);
348    AP.OutStreamer.EmitIntValue(I->second, 8);
349  }
350
351  // Constant pool entries.
352  for (unsigned i = 0; i < ConstPool.getNumConstants(); ++i)
353    AP.OutStreamer.EmitIntValue(ConstPool.getConstant(i), 8);
354
355  // Callsite entries.
356  for (CallsiteInfoList::const_iterator CSII = CSInfos.begin(),
357       CSIE = CSInfos.end(); CSII != CSIE; ++CSII) {
358    uint64_t CallsiteID = CSII->ID;
359    const LocationVec &CSLocs = CSII->Locations;
360    const LiveOutVec &LiveOuts = CSII->LiveOuts;
361
362    DEBUG(dbgs() << WSMP << "callsite " << CallsiteID << "\n");
363
364    // Verify stack map entry. It's better to communicate a problem to the
365    // runtime than crash in case of in-process compilation. Currently, we do
366    // simple overflow checks, but we may eventually communicate other
367    // compilation errors this way.
368    if (CSLocs.size() > UINT16_MAX || LiveOuts.size() > UINT16_MAX) {
369      AP.OutStreamer.EmitIntValue(UINT64_MAX, 8); // Invalid ID.
370      AP.OutStreamer.EmitValue(CSII->CSOffsetExpr, 4);
371      AP.OutStreamer.EmitIntValue(0, 2); // Reserved.
372      AP.OutStreamer.EmitIntValue(0, 2); // 0 locations.
373      AP.OutStreamer.EmitIntValue(0, 2); // padding.
374      AP.OutStreamer.EmitIntValue(0, 2); // 0 live-out registers.
375      AP.OutStreamer.EmitIntValue(0, 4); // padding.
376      continue;
377    }
378
379    AP.OutStreamer.EmitIntValue(CallsiteID, 8);
380    AP.OutStreamer.EmitValue(CSII->CSOffsetExpr, 4);
381
382    // Reserved for flags.
383    AP.OutStreamer.EmitIntValue(0, 2);
384
385    DEBUG(dbgs() << WSMP << "  has " << CSLocs.size() << " locations\n");
386
387    AP.OutStreamer.EmitIntValue(CSLocs.size(), 2);
388
389    unsigned operIdx = 0;
390    for (LocationVec::const_iterator LocI = CSLocs.begin(), LocE = CSLocs.end();
391         LocI != LocE; ++LocI, ++operIdx) {
392      const Location &Loc = *LocI;
393      unsigned RegNo = 0;
394      int Offset = Loc.Offset;
395      if(Loc.Reg) {
396        RegNo = getDwarfRegNum(Loc.Reg, TRI);
397
398        // If this is a register location, put the subregister byte offset in
399        // the location offset.
400        if (Loc.LocType == Location::Register) {
401          assert(!Loc.Offset && "Register location should have zero offset");
402          unsigned LLVMRegNo = TRI->getLLVMRegNum(RegNo, false);
403          unsigned SubRegIdx = TRI->getSubRegIndex(LLVMRegNo, Loc.Reg);
404          if (SubRegIdx)
405            Offset = TRI->getSubRegIdxOffset(SubRegIdx);
406        }
407      }
408      else {
409        assert(Loc.LocType != Location::Register &&
410               "Missing location register");
411      }
412
413      DEBUG(
414        dbgs() << WSMP << "  Loc " << operIdx << ": ";
415        switch (Loc.LocType) {
416        case Location::Unprocessed:
417          dbgs() << "<Unprocessed operand>";
418          break;
419        case Location::Register:
420          dbgs() << "Register " << TRI->getName(Loc.Reg);
421          break;
422        case Location::Direct:
423          dbgs() << "Direct " << TRI->getName(Loc.Reg);
424          if (Loc.Offset)
425            dbgs() << " + " << Loc.Offset;
426          break;
427        case Location::Indirect:
428          dbgs() << "Indirect " << TRI->getName(Loc.Reg)
429                 << " + " << Loc.Offset;
430          break;
431        case Location::Constant:
432          dbgs() << "Constant " << Loc.Offset;
433          break;
434        case Location::ConstantIndex:
435          dbgs() << "Constant Index " << Loc.Offset;
436          break;
437        }
438        dbgs() << "     [encoding: .byte " << Loc.LocType
439               << ", .byte " << Loc.Size
440               << ", .short " << RegNo
441               << ", .int " << Offset << "]\n";
442      );
443
444      AP.OutStreamer.EmitIntValue(Loc.LocType, 1);
445      AP.OutStreamer.EmitIntValue(Loc.Size, 1);
446      AP.OutStreamer.EmitIntValue(RegNo, 2);
447      AP.OutStreamer.EmitIntValue(Offset, 4);
448    }
449
450    DEBUG(dbgs() << WSMP << "  has " << LiveOuts.size()
451                 << " live-out registers\n");
452
453    // Num live-out registers and padding to align to 4 byte.
454    AP.OutStreamer.EmitIntValue(0, 2);
455    AP.OutStreamer.EmitIntValue(LiveOuts.size(), 2);
456
457    operIdx = 0;
458    for (LiveOutVec::const_iterator LI = LiveOuts.begin(), LE = LiveOuts.end();
459         LI != LE; ++LI, ++operIdx) {
460      DEBUG(dbgs() << WSMP << "  LO " << operIdx << ": "
461                   << TRI->getName(LI->Reg)
462                   << "     [encoding: .short " << LI->RegNo
463                   << ", .byte 0, .byte " << LI->Size << "]\n");
464
465      AP.OutStreamer.EmitIntValue(LI->RegNo, 2);
466      AP.OutStreamer.EmitIntValue(0, 1);
467      AP.OutStreamer.EmitIntValue(LI->Size, 1);
468    }
469    // Emit alignment to 8 byte.
470    AP.OutStreamer.EmitValueToAlignment(8);
471  }
472
473  AP.OutStreamer.AddBlankLine();
474
475  CSInfos.clear();
476}
477