PTHLexer.cpp revision 32a8ad526f9bc00539f000a2dd1ac3e167db61c1 
1//===--- PTHLexer.cpp - Lex from a token stream ---------------------------===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file implements the PTHLexer interface.
11//
12//===----------------------------------------------------------------------===//
13
14#include "clang/Basic/TokenKinds.h"
15#include "clang/Basic/FileManager.h"
16#include "clang/Basic/IdentifierTable.h"
17#include "clang/Lex/PTHLexer.h"
18#include "clang/Lex/Preprocessor.h"
19#include "clang/Lex/PTHManager.h"
20#include "clang/Lex/Token.h"
21#include "clang/Lex/Preprocessor.h"
22#include "llvm/Support/Compiler.h"
23#include "llvm/Support/MemoryBuffer.h"
24#include "llvm/ADT/StringMap.h"
25#include "llvm/ADT/OwningPtr.h"
26
27using namespace clang;
28
29#define DISK_TOKEN_SIZE (1+1+3+4+2)
30
31//===----------------------------------------------------------------------===//
32// Utility methods for reading from the mmap'ed PTH file.
33//===----------------------------------------------------------------------===//
34
35static inline uint8_t Read8(const char*& data) {
36  return (uint8_t) *(data++);
37}
38
39static inline uint32_t Read32(const char*& data) {
40  uint32_t V = (uint32_t) Read8(data);
41  V |= (((uint32_t) Read8(data)) << 8);
42  V |= (((uint32_t) Read8(data)) << 16);
43  V |= (((uint32_t) Read8(data)) << 24);
44  return V;
45}
46
47//===----------------------------------------------------------------------===//
48// PTHLexer methods.
49//===----------------------------------------------------------------------===//
50
51PTHLexer::PTHLexer(Preprocessor& pp, SourceLocation fileloc, const char* D,
52                   const char* ppcond,
53                   const char* spellingTable, unsigned NumSpellings,
54                   PTHManager& PM)
55  : PreprocessorLexer(&pp, fileloc), TokBuf(D), CurPtr(D), LastHashTokPtr(0),
56    PPCond(ppcond), CurPPCondPtr(ppcond),
57    SpellingTable(spellingTable), SpellingsLeft(NumSpellings),
58    PTHMgr(PM) {}
59
60void PTHLexer::Lex(Token& Tok) {
61LexNextToken:
62
63  //===--------------------------------------==//
64  // Read the raw token data.
65  //===--------------------------------------==//
66
67  // Shadow CurPtr into an automatic variable.
68  const unsigned char *CurPtrShadow = (const unsigned char*) CurPtr;
69
70  // Read in the data for the token.  14 bytes in total.
71  tok::TokenKind k = (tok::TokenKind) CurPtrShadow[0];
72  Token::TokenFlags flags = (Token::TokenFlags) CurPtrShadow[1];
73
74  uint32_t perID = ((uint32_t) CurPtrShadow[2])
75      | (((uint32_t) CurPtrShadow[3]) << 8)
76      | (((uint32_t) CurPtrShadow[4]) << 16);
77
78  uint32_t FileOffset = ((uint32_t) CurPtrShadow[5])
79      | (((uint32_t) CurPtrShadow[6]) << 8)
80      | (((uint32_t) CurPtrShadow[7]) << 16)
81      | (((uint32_t) CurPtrShadow[8]) << 24);
82
83  uint32_t Len = ((uint32_t) CurPtrShadow[9])
84      | (((uint32_t) CurPtrShadow[10]) << 8);
85
86  CurPtr = (const char*) (CurPtrShadow + DISK_TOKEN_SIZE);
87
88  //===--------------------------------------==//
89  // Construct the token itself.
90  //===--------------------------------------==//
91
92  Tok.startToken();
93  Tok.setKind(k);
94  Tok.setFlag(flags);
95  assert(!LexingRawMode);
96  Tok.setIdentifierInfo(perID ? PTHMgr.GetIdentifierInfo(perID-1) : 0);
97  Tok.setLocation(SourceLocation::getFileLoc(FileID, FileOffset));
98  Tok.setLength(Len);
99
100  //===--------------------------------------==//
101  // Process the token.
102  //===--------------------------------------==//
103
104  if (k == tok::identifier) {
105    MIOpt.ReadToken();
106    return PP->HandleIdentifier(Tok);
107  }
108
109  if (k == tok::eof) {
110    // Save the end-of-file token.
111    EofToken = Tok;
112
113    Preprocessor *PPCache = PP;
114
115    assert(!ParsingPreprocessorDirective);
116    assert(!LexingRawMode);
117
118    // FIXME: Issue diagnostics similar to Lexer.
119    if (PP->HandleEndOfFile(Tok, false))
120      return;
121
122    assert(PPCache && "Raw buffer::LexEndOfFile should return a token");
123    return PPCache->Lex(Tok);
124  }
125
126  if (k == tok::hash && Tok.isAtStartOfLine()) {
127    LastHashTokPtr = CurPtr - DISK_TOKEN_SIZE;
128    assert(!LexingRawMode);
129    PP->HandleDirective(Tok);
130
131    if (PP->isCurrentLexer(this))
132      goto LexNextToken;
133
134    return PP->Lex(Tok);
135  }
136
137  if (k == tok::eom) {
138    assert(ParsingPreprocessorDirective);
139    ParsingPreprocessorDirective = false;
140    return;
141  }
142
143  MIOpt.ReadToken();
144}
145
146// FIXME: We can just grab the last token instead of storing a copy
147// into EofToken.
148void PTHLexer::getEOF(Token& Tok) {
149  assert(!EofToken.is(tok::eof));
150  Tok = EofToken;
151}
152
153void PTHLexer::DiscardToEndOfLine() {
154  assert(ParsingPreprocessorDirective && ParsingFilename == false &&
155         "Must be in a preprocessing directive!");
156
157  // We assume that if the preprocessor wishes to discard to the end of
158  // the line that it also means to end the current preprocessor directive.
159  ParsingPreprocessorDirective = false;
160
161  // Skip tokens by only peeking at their token kind and the flags.
162  // We don't need to actually reconstruct full tokens from the token buffer.
163  // This saves some copies and it also reduces IdentifierInfo* lookup.
164  const char* p = CurPtr;
165  while (1) {
166    // Read the token kind.  Are we at the end of the file?
167    tok::TokenKind x = (tok::TokenKind) (uint8_t) *p;
168    if (x == tok::eof) break;
169
170    // Read the token flags.  Are we at the start of the next line?
171    Token::TokenFlags y = (Token::TokenFlags) (uint8_t) p[1];
172    if (y & Token::StartOfLine) break;
173
174    // Skip to the next token.
175    p += DISK_TOKEN_SIZE;
176  }
177
178  CurPtr = p;
179}
180
181/// SkipBlock - Used by Preprocessor to skip the current conditional block.
182bool PTHLexer::SkipBlock() {
183  assert(CurPPCondPtr && "No cached PP conditional information.");
184  assert(LastHashTokPtr && "No known '#' token.");
185
186  const char* HashEntryI = 0;
187  uint32_t Offset;
188  uint32_t TableIdx;
189
190  do {
191    // Read the token offset from the side-table.
192    Offset = Read32(CurPPCondPtr);
193
194    // Read the target table index from the side-table.
195    TableIdx = Read32(CurPPCondPtr);
196
197    // Compute the actual memory address of the '#' token data for this entry.
198    HashEntryI = TokBuf + Offset;
199
200    // Optmization: "Sibling jumping".  #if...#else...#endif blocks can
201    //  contain nested blocks.  In the side-table we can jump over these
202    //  nested blocks instead of doing a linear search if the next "sibling"
203    //  entry is not at a location greater than LastHashTokPtr.
204    if (HashEntryI < LastHashTokPtr && TableIdx) {
205      // In the side-table we are still at an entry for a '#' token that
206      // is earlier than the last one we saw.  Check if the location we would
207      // stride gets us closer.
208      const char* NextPPCondPtr = PPCond + TableIdx*(sizeof(uint32_t)*2);
209      assert(NextPPCondPtr >= CurPPCondPtr);
210      // Read where we should jump to.
211      uint32_t TmpOffset = Read32(NextPPCondPtr);
212      const char* HashEntryJ = TokBuf + TmpOffset;
213
214      if (HashEntryJ <= LastHashTokPtr) {
215        // Jump directly to the next entry in the side table.
216        HashEntryI = HashEntryJ;
217        Offset = TmpOffset;
218        TableIdx = Read32(NextPPCondPtr);
219        CurPPCondPtr = NextPPCondPtr;
220      }
221    }
222  }
223  while (HashEntryI < LastHashTokPtr);
224  assert(HashEntryI == LastHashTokPtr && "No PP-cond entry found for '#'");
225  assert(TableIdx && "No jumping from #endifs.");
226
227  // Update our side-table iterator.
228  const char* NextPPCondPtr = PPCond + TableIdx*(sizeof(uint32_t)*2);
229  assert(NextPPCondPtr >= CurPPCondPtr);
230  CurPPCondPtr = NextPPCondPtr;
231
232  // Read where we should jump to.
233  HashEntryI = TokBuf + Read32(NextPPCondPtr);
234  uint32_t NextIdx = Read32(NextPPCondPtr);
235
236  // By construction NextIdx will be zero if this is a #endif.  This is useful
237  // to know to obviate lexing another token.
238  bool isEndif = NextIdx == 0;
239
240  // This case can occur when we see something like this:
241  //
242  //  #if ...
243  //   /* a comment or nothing */
244  //  #elif
245  //
246  // If we are skipping the first #if block it will be the case that CurPtr
247  // already points 'elif'.  Just return.
248
249  if (CurPtr > HashEntryI) {
250    assert(CurPtr == HashEntryI + DISK_TOKEN_SIZE);
251    // Did we reach a #endif?  If so, go ahead and consume that token as well.
252    if (isEndif)
253      CurPtr += DISK_TOKEN_SIZE*2;
254    else
255      LastHashTokPtr = HashEntryI;
256
257    return isEndif;
258  }
259
260  // Otherwise, we need to advance.  Update CurPtr to point to the '#' token.
261  CurPtr = HashEntryI;
262
263  // Update the location of the last observed '#'.  This is useful if we
264  // are skipping multiple blocks.
265  LastHashTokPtr = CurPtr;
266
267  // Skip the '#' token.
268  assert(((tok::TokenKind) (unsigned char) *CurPtr) == tok::hash);
269  CurPtr += DISK_TOKEN_SIZE;
270
271  // Did we reach a #endif?  If so, go ahead and consume that token as well.
272  if (isEndif) { CurPtr += DISK_TOKEN_SIZE*2; }
273
274  return isEndif;
275}
276
277SourceLocation PTHLexer::getSourceLocation() {
278  // getLocation is not on the hot path.  It is used to get the location of
279  // the next token when transitioning back to this lexer when done
280  // handling a #included file.  Just read the necessary data from the token
281  // data buffer to construct the SourceLocation object.
282  // NOTE: This is a virtual function; hence it is defined out-of-line.
283  const char* p = CurPtr + (1 + 1 + 3);
284  uint32_t offset =
285       ((uint32_t) ((uint8_t) p[0]))
286    | (((uint32_t) ((uint8_t) p[1])) << 8)
287    | (((uint32_t) ((uint8_t) p[2])) << 16)
288    | (((uint32_t) ((uint8_t) p[3])) << 24);
289  return SourceLocation::getFileLoc(FileID, offset);
290}
291
292unsigned PTHManager::GetSpelling(unsigned PTHOffset, const char *& Buffer) {
293  const char* p = Buf->getBufferStart() + PTHOffset;
294  assert(p < Buf->getBufferEnd());
295
296  // The string is prefixed by 16 bits for its length, followed by the string
297  // itself.
298  unsigned len = ((unsigned) ((uint8_t) p[0]))
299    | (((unsigned) ((uint8_t) p[1])) << 8);
300
301  Buffer = p + 2;
302  return len;
303}
304
305unsigned PTHLexer::getSpelling(SourceLocation sloc, const char *&Buffer) {
306  const char* p = SpellingTable;
307  SourceManager& SM = PP->getSourceManager();
308  unsigned fpos = SM.getFullFilePos(SM.getPhysicalLoc(sloc));
309  unsigned len = 0;
310
311  while (SpellingsLeft) {
312    uint32_t TokOffset =
313      ((uint32_t) ((uint8_t) p[0]))
314      | (((uint32_t) ((uint8_t) p[1])) << 8)
315      | (((uint32_t) ((uint8_t) p[2])) << 16)
316      | (((uint32_t) ((uint8_t) p[3])) << 24);
317
318    if (TokOffset > fpos)
319      break;
320
321    --SpellingsLeft;
322
323    // Did we find a matching token offset for this spelling?
324    if (TokOffset == fpos) {
325      uint32_t SpellingPTHOffset =
326        ((uint32_t) ((uint8_t) p[4]))
327        | (((uint32_t) ((uint8_t) p[5])) << 8)
328        | (((uint32_t) ((uint8_t) p[6])) << 16)
329        | (((uint32_t) ((uint8_t) p[7])) << 24);
330
331      len = PTHMgr.GetSpelling(SpellingPTHOffset, Buffer);
332      break;
333    }
334
335    // No match.  Keep on looking.
336    p += sizeof(uint32_t)*2;
337  }
338
339  SpellingTable = p;
340  return len;
341}
342
343//===----------------------------------------------------------------------===//
344// Internal Data Structures for PTH file lookup and resolving identifiers.
345//===----------------------------------------------------------------------===//
346
347
348/// PTHFileLookup - This internal data structure is used by the PTHManager
349///  to map from FileEntry objects managed by FileManager to offsets within
350///  the PTH file.
351namespace {
352class VISIBILITY_HIDDEN PTHFileLookup {
353public:
354  class Val {
355    uint32_t TokenOff;
356    uint32_t PPCondOff;
357    uint32_t SpellingOff;
358
359  public:
360    Val() : TokenOff(~0) {}
361    Val(uint32_t toff, uint32_t poff, uint32_t soff)
362      : TokenOff(toff), PPCondOff(poff), SpellingOff(soff) {}
363
364    uint32_t getTokenOffset() const {
365      assert(TokenOff != ~((uint32_t)0) && "PTHFileLookup entry initialized.");
366      return TokenOff;
367    }
368
369    uint32_t getPPCondOffset() const {
370      assert(TokenOff != ~((uint32_t)0) && "PTHFileLookup entry initialized.");
371      return PPCondOff;
372    }
373
374    uint32_t getSpellingOffset() const {
375      assert(TokenOff != ~((uint32_t)0) && "PTHFileLookup entry initialized.");
376      return SpellingOff;
377    }
378
379    bool isValid() const { return TokenOff != ~((uint32_t)0); }
380  };
381
382private:
383  llvm::StringMap<Val> FileMap;
384
385public:
386  PTHFileLookup() {};
387
388  Val Lookup(const FileEntry* FE) {
389    const char* s = FE->getName();
390    unsigned size = strlen(s);
391    return FileMap.GetOrCreateValue(s, s+size).getValue();
392  }
393
394  void ReadTable(const char* D) {
395    uint32_t N = Read32(D);     // Read the length of the table.
396
397    for ( ; N > 0; --N) {       // The rest of the data is the table itself.
398      uint32_t len = Read32(D);
399      const char* s = D;
400      D += len;
401
402      uint32_t TokenOff = Read32(D);
403      uint32_t PPCondOff = Read32(D);
404      uint32_t SpellingOff = Read32(D);
405
406      FileMap.GetOrCreateValue(s, s+len).getValue() =
407        Val(TokenOff, PPCondOff, SpellingOff);
408    }
409  }
410};
411} // end anonymous namespace
412
413//===----------------------------------------------------------------------===//
414// PTHManager methods.
415//===----------------------------------------------------------------------===//
416
417PTHManager::PTHManager(const llvm::MemoryBuffer* buf, void* fileLookup,
418                       const char* idDataTable, IdentifierInfo** perIDCache,
419                       Preprocessor& pp)
420: Buf(buf), PerIDCache(perIDCache), FileLookup(fileLookup),
421  IdDataTable(idDataTable), ITable(pp.getIdentifierTable()), PP(pp) {}
422
423PTHManager::~PTHManager() {
424  delete Buf;
425  delete (PTHFileLookup*) FileLookup;
426  free(PerIDCache);
427}
428
429PTHManager* PTHManager::Create(const std::string& file, Preprocessor& PP) {
430
431  // Memory map the PTH file.
432  llvm::OwningPtr<llvm::MemoryBuffer>
433  File(llvm::MemoryBuffer::getFile(file.c_str()));
434
435  if (!File)
436    return 0;
437
438  // Get the buffer ranges and check if there are at least three 32-bit
439  // words at the end of the file.
440  const char* BufBeg = File->getBufferStart();
441  const char* BufEnd = File->getBufferEnd();
442
443  if(!(BufEnd > BufBeg + sizeof(uint32_t)*3)) {
444    assert(false && "Invalid PTH file.");
445    return 0; // FIXME: Proper error diagnostic?
446  }
447
448  // Compute the address of the index table at the end of the PTH file.
449  // This table contains the offset of the file lookup table, the
450  // persistent ID -> identifer data table.
451  const char* EndTable = BufEnd - sizeof(uint32_t)*3;
452
453  // Construct the file lookup table.  This will be used for mapping from
454  // FileEntry*'s to cached tokens.
455  const char* FileTableOffset = EndTable + sizeof(uint32_t)*2;
456  const char* FileTable = BufBeg + Read32(FileTableOffset);
457
458  if (!(FileTable > BufBeg && FileTable < BufEnd)) {
459    assert(false && "Invalid PTH file.");
460    return 0; // FIXME: Proper error diagnostic?
461  }
462
463  llvm::OwningPtr<PTHFileLookup> FL(new PTHFileLookup());
464  FL->ReadTable(FileTable);
465
466  // Get the location of the table mapping from persistent ids to the
467  // data needed to reconstruct identifiers.
468  const char* IDTableOffset = EndTable + sizeof(uint32_t)*1;
469  const char* IData = BufBeg + Read32(IDTableOffset);
470  if (!(IData > BufBeg && IData < BufEnd)) {
471    assert(false && "Invalid PTH file.");
472    return 0; // FIXME: Proper error diagnostic?
473  }
474
475  // Get the number of IdentifierInfos and pre-allocate the identifier cache.
476  uint32_t NumIds = Read32(IData);
477
478  // Pre-allocate the peristent ID -> IdentifierInfo* cache.  We use calloc()
479  // so that we in the best case only zero out memory once when the OS returns
480  // us new pages.
481  IdentifierInfo** PerIDCache =
482    (IdentifierInfo**) calloc(NumIds, sizeof(*PerIDCache));
483
484  if (!PerIDCache) {
485    assert(false && "Could not allocate Persistent ID cache.");
486    return 0;
487  }
488
489  // Create the new lexer.
490  return new PTHManager(File.take(), FL.take(), IData, PerIDCache, PP);
491}
492
493IdentifierInfo* PTHManager::GetIdentifierInfo(unsigned persistentID) {
494
495  // Check if the IdentifierInfo has already been resolved.
496  IdentifierInfo*& II = PerIDCache[persistentID];
497  if (II) return II;
498
499  // Look in the PTH file for the string data for the IdentifierInfo object.
500  const char* TableEntry = IdDataTable + sizeof(uint32_t) * persistentID;
501  const char* IDData = Buf->getBufferStart() + Read32(TableEntry);
502  assert(IDData < Buf->getBufferEnd());
503
504  // Read the length of the string.
505  uint32_t len = Read32(IDData);
506
507  // Get the IdentifierInfo* with the specified string.
508  II = &ITable.get(IDData, IDData+len);
509  return II;
510}
511
512PTHLexer* PTHManager::CreateLexer(unsigned FileID, const FileEntry* FE) {
513
514  if (!FE)
515    return 0;
516
517  // Lookup the FileEntry object in our file lookup data structure.  It will
518  // return a variant that indicates whether or not there is an offset within
519  // the PTH file that contains cached tokens.
520  PTHFileLookup::Val FileData = ((PTHFileLookup*) FileLookup)->Lookup(FE);
521
522  if (!FileData.isValid()) // No tokens available.
523    return 0;
524
525  // Compute the offset of the token data within the buffer.
526  const char* data = Buf->getBufferStart() + FileData.getTokenOffset();
527
528  // Get the location of pp-conditional table.
529  const char* ppcond = Buf->getBufferStart() + FileData.getPPCondOffset();
530  uint32_t len = Read32(ppcond);
531  if (len == 0) ppcond = 0;
532
533  // Get the location of the spelling table.
534  const char* spellingTable = Buf->getBufferStart() +
535                              FileData.getSpellingOffset();
536
537  len = Read32(spellingTable);
538  if (len == 0) spellingTable = 0;
539
540  assert(data < Buf->getBufferEnd());
541  return new PTHLexer(PP, SourceLocation::getFileLoc(FileID, 0), data, ppcond,
542                      spellingTable, len, *this);
543}
544