StackMaps.cpp revision dce4a407a24b04eebc6a376f8e62b41aaa7b071f 
1//===---------------------------- StackMaps.cpp ---------------------------===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9
10#include "llvm/CodeGen/StackMaps.h"
11#include "llvm/CodeGen/AsmPrinter.h"
12#include "llvm/CodeGen/MachineFrameInfo.h"
13#include "llvm/CodeGen/MachineFunction.h"
14#include "llvm/CodeGen/MachineInstr.h"
15#include "llvm/IR/DataLayout.h"
16#include "llvm/MC/MCContext.h"
17#include "llvm/MC/MCExpr.h"
18#include "llvm/MC/MCObjectFileInfo.h"
19#include "llvm/MC/MCSectionMachO.h"
20#include "llvm/MC/MCStreamer.h"
21#include "llvm/Support/CommandLine.h"
22#include "llvm/Support/Debug.h"
23#include "llvm/Support/raw_ostream.h"
24#include "llvm/Target/TargetMachine.h"
25#include "llvm/Target/TargetOpcodes.h"
26#include "llvm/Target/TargetRegisterInfo.h"
27#include <iterator>
28
29using namespace llvm;
30
31#define DEBUG_TYPE "stackmaps"
32
33static cl::opt<int> StackMapVersion("stackmap-version", cl::init(1),
34  cl::desc("Specify the stackmap encoding version (default = 1)"));
35
36const char *StackMaps::WSMP = "Stack Maps: ";
37
38PatchPointOpers::PatchPointOpers(const MachineInstr *MI)
39  : MI(MI),
40    HasDef(MI->getOperand(0).isReg() && MI->getOperand(0).isDef() &&
41           !MI->getOperand(0).isImplicit()),
42    IsAnyReg(MI->getOperand(getMetaIdx(CCPos)).getImm() == CallingConv::AnyReg)
43{
44#ifndef NDEBUG
45  unsigned CheckStartIdx = 0, e = MI->getNumOperands();
46  while (CheckStartIdx < e && MI->getOperand(CheckStartIdx).isReg() &&
47         MI->getOperand(CheckStartIdx).isDef() &&
48         !MI->getOperand(CheckStartIdx).isImplicit())
49    ++CheckStartIdx;
50
51  assert(getMetaIdx() == CheckStartIdx &&
52         "Unexpected additional definition in Patchpoint intrinsic.");
53#endif
54}
55
56unsigned PatchPointOpers::getNextScratchIdx(unsigned StartIdx) const {
57  if (!StartIdx)
58    StartIdx = getVarIdx();
59
60  // Find the next scratch register (implicit def and early clobber)
61  unsigned ScratchIdx = StartIdx, e = MI->getNumOperands();
62  while (ScratchIdx < e &&
63         !(MI->getOperand(ScratchIdx).isReg() &&
64           MI->getOperand(ScratchIdx).isDef() &&
65           MI->getOperand(ScratchIdx).isImplicit() &&
66           MI->getOperand(ScratchIdx).isEarlyClobber()))
67    ++ScratchIdx;
68
69  assert(ScratchIdx != e && "No scratch register available");
70  return ScratchIdx;
71}
72
73StackMaps::StackMaps(AsmPrinter &AP) : AP(AP) {
74  if (StackMapVersion != 1)
75    llvm_unreachable("Unsupported stackmap version!");
76}
77
78MachineInstr::const_mop_iterator
79StackMaps::parseOperand(MachineInstr::const_mop_iterator MOI,
80                        MachineInstr::const_mop_iterator MOE,
81                        LocationVec &Locs, LiveOutVec &LiveOuts) const {
82  if (MOI->isImm()) {
83    switch (MOI->getImm()) {
84    default: llvm_unreachable("Unrecognized operand type.");
85    case StackMaps::DirectMemRefOp: {
86      unsigned Size = AP.TM.getDataLayout()->getPointerSizeInBits();
87      assert((Size % 8) == 0 && "Need pointer size in bytes.");
88      Size /= 8;
89      unsigned Reg = (++MOI)->getReg();
90      int64_t Imm = (++MOI)->getImm();
91      Locs.push_back(Location(StackMaps::Location::Direct, Size, Reg, Imm));
92      break;
93    }
94    case StackMaps::IndirectMemRefOp: {
95      int64_t Size = (++MOI)->getImm();
96      assert(Size > 0 && "Need a valid size for indirect memory locations.");
97      unsigned Reg = (++MOI)->getReg();
98      int64_t Imm = (++MOI)->getImm();
99      Locs.push_back(Location(StackMaps::Location::Indirect, Size, Reg, Imm));
100      break;
101    }
102    case StackMaps::ConstantOp: {
103      ++MOI;
104      assert(MOI->isImm() && "Expected constant operand.");
105      int64_t Imm = MOI->getImm();
106      Locs.push_back(Location(Location::Constant, sizeof(int64_t), 0, Imm));
107      break;
108    }
109    }
110    return ++MOI;
111  }
112
113  // The physical register number will ultimately be encoded as a DWARF regno.
114  // The stack map also records the size of a spill slot that can hold the
115  // register content. (The runtime can track the actual size of the data type
116  // if it needs to.)
117  if (MOI->isReg()) {
118    // Skip implicit registers (this includes our scratch registers)
119    if (MOI->isImplicit())
120      return ++MOI;
121
122    assert(TargetRegisterInfo::isPhysicalRegister(MOI->getReg()) &&
123           "Virtreg operands should have been rewritten before now.");
124    const TargetRegisterClass *RC =
125      AP.TM.getRegisterInfo()->getMinimalPhysRegClass(MOI->getReg());
126    assert(!MOI->getSubReg() && "Physical subreg still around.");
127    Locs.push_back(
128      Location(Location::Register, RC->getSize(), MOI->getReg(), 0));
129    return ++MOI;
130  }
131
132  if (MOI->isRegLiveOut())
133    LiveOuts = parseRegisterLiveOutMask(MOI->getRegLiveOut());
134
135  return ++MOI;
136}
137
138/// Go up the super-register chain until we hit a valid dwarf register number.
139static unsigned getDwarfRegNum(unsigned Reg, const TargetRegisterInfo *TRI) {
140  int RegNo = TRI->getDwarfRegNum(Reg, false);
141  for (MCSuperRegIterator SR(Reg, TRI); SR.isValid() && RegNo < 0; ++SR)
142    RegNo = TRI->getDwarfRegNum(*SR, false);
143
144  assert(RegNo >= 0 && "Invalid Dwarf register number.");
145  return (unsigned) RegNo;
146}
147
148/// Create a live-out register record for the given register Reg.
149StackMaps::LiveOutReg
150StackMaps::createLiveOutReg(unsigned Reg, const TargetRegisterInfo *TRI) const {
151  unsigned RegNo = getDwarfRegNum(Reg, TRI);
152  unsigned Size = TRI->getMinimalPhysRegClass(Reg)->getSize();
153  return LiveOutReg(Reg, RegNo, Size);
154}
155
156/// Parse the register live-out mask and return a vector of live-out registers
157/// that need to be recorded in the stackmap.
158StackMaps::LiveOutVec
159StackMaps::parseRegisterLiveOutMask(const uint32_t *Mask) const {
160  assert(Mask && "No register mask specified");
161  const TargetRegisterInfo *TRI = AP.TM.getRegisterInfo();
162  LiveOutVec LiveOuts;
163
164  // Create a LiveOutReg for each bit that is set in the register mask.
165  for (unsigned Reg = 0, NumRegs = TRI->getNumRegs(); Reg != NumRegs; ++Reg)
166    if ((Mask[Reg / 32] >> Reg % 32) & 1)
167      LiveOuts.push_back(createLiveOutReg(Reg, TRI));
168
169  // We don't need to keep track of a register if its super-register is already
170  // in the list. Merge entries that refer to the same dwarf register and use
171  // the maximum size that needs to be spilled.
172  std::sort(LiveOuts.begin(), LiveOuts.end());
173  for (LiveOutVec::iterator I = LiveOuts.begin(), E = LiveOuts.end();
174       I != E; ++I) {
175    for (LiveOutVec::iterator II = std::next(I); II != E; ++II) {
176      if (I->RegNo != II->RegNo) {
177        // Skip all the now invalid entries.
178        I = --II;
179        break;
180      }
181      I->Size = std::max(I->Size, II->Size);
182      if (TRI->isSuperRegister(I->Reg, II->Reg))
183        I->Reg = II->Reg;
184      II->MarkInvalid();
185    }
186  }
187  LiveOuts.erase(std::remove_if(LiveOuts.begin(), LiveOuts.end(),
188                                LiveOutReg::IsInvalid), LiveOuts.end());
189  return LiveOuts;
190}
191
192void StackMaps::recordStackMapOpers(const MachineInstr &MI, uint64_t ID,
193                                    MachineInstr::const_mop_iterator MOI,
194                                    MachineInstr::const_mop_iterator MOE,
195                                    bool recordResult) {
196
197  MCContext &OutContext = AP.OutStreamer.getContext();
198  MCSymbol *MILabel = OutContext.CreateTempSymbol();
199  AP.OutStreamer.EmitLabel(MILabel);
200
201  LocationVec Locations;
202  LiveOutVec LiveOuts;
203
204  if (recordResult) {
205    assert(PatchPointOpers(&MI).hasDef() && "Stackmap has no return value.");
206    parseOperand(MI.operands_begin(), std::next(MI.operands_begin()),
207                 Locations, LiveOuts);
208  }
209
210  // Parse operands.
211  while (MOI != MOE) {
212    MOI = parseOperand(MOI, MOE, Locations, LiveOuts);
213  }
214
215  // Move large constants into the constant pool.
216  for (LocationVec::iterator I = Locations.begin(), E = Locations.end();
217       I != E; ++I) {
218    // Constants are encoded as sign-extended integers.
219    // -1 is directly encoded as .long 0xFFFFFFFF with no constant pool.
220    if (I->LocType == Location::Constant &&
221        ((I->Offset + (int64_t(1)<<31)) >> 32) != 0) {
222      I->LocType = Location::ConstantIndex;
223      auto Result = ConstPool.insert(std::make_pair(I->Offset, I->Offset));
224      I->Offset = Result.first - ConstPool.begin();
225    }
226  }
227
228  // Create an expression to calculate the offset of the callsite from function
229  // entry.
230  const MCExpr *CSOffsetExpr = MCBinaryExpr::CreateSub(
231    MCSymbolRefExpr::Create(MILabel, OutContext),
232    MCSymbolRefExpr::Create(AP.CurrentFnSym, OutContext),
233    OutContext);
234
235  CSInfos.push_back(CallsiteInfo(CSOffsetExpr, ID, Locations, LiveOuts));
236
237  // Record the stack size of the current function.
238  const MachineFrameInfo *MFI = AP.MF->getFrameInfo();
239  FnStackSize[AP.CurrentFnSym] =
240    MFI->hasVarSizedObjects() ? UINT64_MAX : MFI->getStackSize();
241}
242
243void StackMaps::recordStackMap(const MachineInstr &MI) {
244  assert(MI.getOpcode() == TargetOpcode::STACKMAP && "expected stackmap");
245
246  int64_t ID = MI.getOperand(0).getImm();
247  recordStackMapOpers(MI, ID, std::next(MI.operands_begin(), 2),
248                      MI.operands_end());
249}
250
251void StackMaps::recordPatchPoint(const MachineInstr &MI) {
252  assert(MI.getOpcode() == TargetOpcode::PATCHPOINT && "expected patchpoint");
253
254  PatchPointOpers opers(&MI);
255  int64_t ID = opers.getMetaOper(PatchPointOpers::IDPos).getImm();
256
257  MachineInstr::const_mop_iterator MOI =
258    std::next(MI.operands_begin(), opers.getStackMapStartIdx());
259  recordStackMapOpers(MI, ID, MOI, MI.operands_end(),
260                      opers.isAnyReg() && opers.hasDef());
261
262#ifndef NDEBUG
263  // verify anyregcc
264  LocationVec &Locations = CSInfos.back().Locations;
265  if (opers.isAnyReg()) {
266    unsigned NArgs = opers.getMetaOper(PatchPointOpers::NArgPos).getImm();
267    for (unsigned i = 0, e = (opers.hasDef() ? NArgs+1 : NArgs); i != e; ++i)
268      assert(Locations[i].LocType == Location::Register &&
269             "anyreg arg must be in reg.");
270  }
271#endif
272}
273
274/// Emit the stackmap header.
275///
276/// Header {
277///   uint8  : Stack Map Version (currently 1)
278///   uint8  : Reserved (expected to be 0)
279///   uint16 : Reserved (expected to be 0)
280/// }
281/// uint32 : NumFunctions
282/// uint32 : NumConstants
283/// uint32 : NumRecords
284void StackMaps::emitStackmapHeader(MCStreamer &OS) {
285  // Header.
286  OS.EmitIntValue(StackMapVersion, 1); // Version.
287  OS.EmitIntValue(0, 1); // Reserved.
288  OS.EmitIntValue(0, 2); // Reserved.
289
290  // Num functions.
291  DEBUG(dbgs() << WSMP << "#functions = " << FnStackSize.size() << '\n');
292  OS.EmitIntValue(FnStackSize.size(), 4);
293  // Num constants.
294  DEBUG(dbgs() << WSMP << "#constants = " << ConstPool.size() << '\n');
295  OS.EmitIntValue(ConstPool.size(), 4);
296  // Num callsites.
297  DEBUG(dbgs() << WSMP << "#callsites = " << CSInfos.size() << '\n');
298  OS.EmitIntValue(CSInfos.size(), 4);
299}
300
301/// Emit the function frame record for each function.
302///
303/// StkSizeRecord[NumFunctions] {
304///   uint64 : Function Address
305///   uint64 : Stack Size
306/// }
307void StackMaps::emitFunctionFrameRecords(MCStreamer &OS) {
308  // Function Frame records.
309  DEBUG(dbgs() << WSMP << "functions:\n");
310  for (auto const &FR : FnStackSize) {
311    DEBUG(dbgs() << WSMP << "function addr: " << FR.first
312                         << " frame size: " << FR.second);
313    OS.EmitSymbolValue(FR.first, 8);
314    OS.EmitIntValue(FR.second, 8);
315  }
316}
317
318/// Emit the constant pool.
319///
320/// int64  : Constants[NumConstants]
321void StackMaps::emitConstantPoolEntries(MCStreamer &OS) {
322  // Constant pool entries.
323  DEBUG(dbgs() << WSMP << "constants:\n");
324  for (auto ConstEntry : ConstPool) {
325    DEBUG(dbgs() << WSMP << ConstEntry.second << '\n');
326    OS.EmitIntValue(ConstEntry.second, 8);
327  }
328}
329
330/// Emit the callsite info for each callsite.
331///
332/// StkMapRecord[NumRecords] {
333///   uint64 : PatchPoint ID
334///   uint32 : Instruction Offset
335///   uint16 : Reserved (record flags)
336///   uint16 : NumLocations
337///   Location[NumLocations] {
338///     uint8  : Register | Direct | Indirect | Constant | ConstantIndex
339///     uint8  : Size in Bytes
340///     uint16 : Dwarf RegNum
341///     int32  : Offset
342///   }
343///   uint16 : Padding
344///   uint16 : NumLiveOuts
345///   LiveOuts[NumLiveOuts] {
346///     uint16 : Dwarf RegNum
347///     uint8  : Reserved
348///     uint8  : Size in Bytes
349///   }
350///   uint32 : Padding (only if required to align to 8 byte)
351/// }
352///
353/// Location Encoding, Type, Value:
354///   0x1, Register, Reg                 (value in register)
355///   0x2, Direct, Reg + Offset          (frame index)
356///   0x3, Indirect, [Reg + Offset]      (spilled value)
357///   0x4, Constant, Offset              (small constant)
358///   0x5, ConstIndex, Constants[Offset] (large constant)
359void StackMaps::emitCallsiteEntries(MCStreamer &OS,
360                                    const TargetRegisterInfo *TRI) {
361  // Callsite entries.
362  DEBUG(dbgs() << WSMP << "callsites:\n");
363  for (const auto &CSI : CSInfos) {
364    const LocationVec &CSLocs = CSI.Locations;
365    const LiveOutVec &LiveOuts = CSI.LiveOuts;
366
367    DEBUG(dbgs() << WSMP << "callsite " << CSI.ID << "\n");
368
369    // Verify stack map entry. It's better to communicate a problem to the
370    // runtime than crash in case of in-process compilation. Currently, we do
371    // simple overflow checks, but we may eventually communicate other
372    // compilation errors this way.
373    if (CSLocs.size() > UINT16_MAX || LiveOuts.size() > UINT16_MAX) {
374      OS.EmitIntValue(UINT64_MAX, 8); // Invalid ID.
375      OS.EmitValue(CSI.CSOffsetExpr, 4);
376      OS.EmitIntValue(0, 2); // Reserved.
377      OS.EmitIntValue(0, 2); // 0 locations.
378      OS.EmitIntValue(0, 2); // padding.
379      OS.EmitIntValue(0, 2); // 0 live-out registers.
380      OS.EmitIntValue(0, 4); // padding.
381      continue;
382    }
383
384    OS.EmitIntValue(CSI.ID, 8);
385    OS.EmitValue(CSI.CSOffsetExpr, 4);
386
387    // Reserved for flags.
388    OS.EmitIntValue(0, 2);
389
390    DEBUG(dbgs() << WSMP << "  has " << CSLocs.size() << " locations\n");
391
392    OS.EmitIntValue(CSLocs.size(), 2);
393
394    unsigned OperIdx = 0;
395    for (const auto &Loc : CSLocs) {
396      unsigned RegNo = 0;
397      int Offset = Loc.Offset;
398      if(Loc.Reg) {
399        RegNo = getDwarfRegNum(Loc.Reg, TRI);
400
401        // If this is a register location, put the subregister byte offset in
402        // the location offset.
403        if (Loc.LocType == Location::Register) {
404          assert(!Loc.Offset && "Register location should have zero offset");
405          unsigned LLVMRegNo = TRI->getLLVMRegNum(RegNo, false);
406          unsigned SubRegIdx = TRI->getSubRegIndex(LLVMRegNo, Loc.Reg);
407          if (SubRegIdx)
408            Offset = TRI->getSubRegIdxOffset(SubRegIdx);
409        }
410      }
411      else {
412        assert(Loc.LocType != Location::Register &&
413               "Missing location register");
414      }
415
416      DEBUG(dbgs() << WSMP << "  Loc " << OperIdx << ": ";
417            switch (Loc.LocType) {
418            case Location::Unprocessed:
419              dbgs() << "<Unprocessed operand>";
420              break;
421            case Location::Register:
422              dbgs() << "Register " << TRI->getName(Loc.Reg);
423              break;
424            case Location::Direct:
425              dbgs() << "Direct " << TRI->getName(Loc.Reg);
426              if (Loc.Offset)
427              dbgs() << " + " << Loc.Offset;
428              break;
429            case Location::Indirect:
430              dbgs() << "Indirect " << TRI->getName(Loc.Reg)
431              << " + " << Loc.Offset;
432              break;
433            case Location::Constant:
434              dbgs() << "Constant " << Loc.Offset;
435              break;
436            case Location::ConstantIndex:
437              dbgs() << "Constant Index " << Loc.Offset;
438              break;
439              }
440            dbgs() << "     [encoding: .byte " << Loc.LocType
441            << ", .byte " << Loc.Size
442            << ", .short " << RegNo
443            << ", .int " << Offset << "]\n";
444            );
445
446      OS.EmitIntValue(Loc.LocType, 1);
447      OS.EmitIntValue(Loc.Size, 1);
448      OS.EmitIntValue(RegNo, 2);
449      OS.EmitIntValue(Offset, 4);
450      OperIdx++;
451    }
452
453    DEBUG(dbgs() << WSMP << "  has " << LiveOuts.size()
454                         << " live-out registers\n");
455
456    // Num live-out registers and padding to align to 4 byte.
457    OS.EmitIntValue(0, 2);
458    OS.EmitIntValue(LiveOuts.size(), 2);
459
460    OperIdx = 0;
461    for (const auto &LO : LiveOuts) {
462      DEBUG(dbgs() << WSMP << "  LO " << OperIdx << ": "
463                           << TRI->getName(LO.Reg)
464                           << "     [encoding: .short " << LO.RegNo
465                           << ", .byte 0, .byte " << LO.Size << "]\n");
466      OS.EmitIntValue(LO.RegNo, 2);
467      OS.EmitIntValue(0, 1);
468      OS.EmitIntValue(LO.Size, 1);
469    }
470    // Emit alignment to 8 byte.
471    OS.EmitValueToAlignment(8);
472  }
473}
474
475/// Serialize the stackmap data.
476void StackMaps::serializeToStackMapSection() {
477  (void) WSMP;
478  // Bail out if there's no stack map data.
479  assert((!CSInfos.empty() || (CSInfos.empty() && ConstPool.empty())) &&
480         "Expected empty constant pool too!");
481  assert((!CSInfos.empty() || (CSInfos.empty() && FnStackSize.empty())) &&
482         "Expected empty function record too!");
483  if (CSInfos.empty())
484    return;
485
486  MCContext &OutContext = AP.OutStreamer.getContext();
487  MCStreamer &OS = AP.OutStreamer;
488  const TargetRegisterInfo *TRI = AP.TM.getRegisterInfo();
489
490  // Create the section.
491  const MCSection *StackMapSection =
492    OutContext.getObjectFileInfo()->getStackMapSection();
493  OS.SwitchSection(StackMapSection);
494
495  // Emit a dummy symbol to force section inclusion.
496  OS.EmitLabel(OutContext.GetOrCreateSymbol(Twine("__LLVM_StackMaps")));
497
498  // Serialize data.
499  DEBUG(dbgs() << "********** Stack Map Output **********\n");
500  emitStackmapHeader(OS);
501  emitFunctionFrameRecords(OS);
502  emitConstantPoolEntries(OS);
503  emitCallsiteEntries(OS, TRI);
504  OS.AddBlankLine();
505
506  // Clean up.
507  CSInfos.clear();
508  ConstPool.clear();
509}
510