PTHLexer.cpp revision 6f78c3b8b9343e7e9fbf2d457cccf00df6da5d47 
1//===--- PTHLexer.cpp - Lex from a token stream ---------------------------===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file implements the PTHLexer interface.
11//
12//===----------------------------------------------------------------------===//
13
14#include "clang/Basic/TokenKinds.h"
15#include "clang/Basic/FileManager.h"
16#include "clang/Basic/IdentifierTable.h"
17#include "clang/Lex/PTHLexer.h"
18#include "clang/Lex/Preprocessor.h"
19#include "clang/Lex/PTHManager.h"
20#include "clang/Lex/Token.h"
21#include "clang/Lex/Preprocessor.h"
22#include "llvm/ADT/StringMap.h"
23#include "llvm/ADT/OwningPtr.h"
24#include "llvm/Support/Compiler.h"
25#include "llvm/Support/MathExtras.h"
26#include "llvm/Support/MemoryBuffer.h"
27#include "llvm/System/Host.h"
28using namespace clang;
29
30#define DISK_TOKEN_SIZE (1+1+2+4+4)
31
32//===----------------------------------------------------------------------===//
33// Utility methods for reading from the mmap'ed PTH file.
34//===----------------------------------------------------------------------===//
35
36static inline uint16_t ReadUnalignedLE16(const unsigned char *&Data) {
37  uint16_t V = ((uint16_t)Data[0] <<  0) |
38               ((uint16_t)Data[1] <<  8);
39  Data += 2;
40  return V;
41}
42
43static inline uint32_t ReadLE32(const unsigned char *&Data) {
44  // Hosts that directly support unaligned little-endian 32-bit loads can just
45  // use them.
46  uint32_t V = *((uint32_t*)Data);
47  if (llvm::sys::isBigEndianHost())
48    V = llvm::ByteSwap_32(V);
49  Data += 4;
50  return V;
51}
52
53
54//===----------------------------------------------------------------------===//
55// PTHLexer methods.
56//===----------------------------------------------------------------------===//
57
58PTHLexer::PTHLexer(Preprocessor &PP, FileID FID, const unsigned char *D,
59                   const unsigned char *ppcond,
60                   PTHSpellingSearch &mySpellingSrch, PTHManager &PM)
61  : PreprocessorLexer(&PP, FID), TokBuf(D), CurPtr(D), LastHashTokPtr(0),
62    PPCond(ppcond), CurPPCondPtr(ppcond), MySpellingSrch(mySpellingSrch),
63    PTHMgr(PM) {
64
65  FileStartLoc = PP.getSourceManager().getLocForStartOfFile(FID);
66}
67
68void PTHLexer::Lex(Token& Tok) {
69LexNextToken:
70
71  //===--------------------------------------==//
72  // Read the raw token data.
73  //===--------------------------------------==//
74
75  // Shadow CurPtr into an automatic variable.
76  const unsigned char *CurPtrShadow = CurPtr;
77
78  // Read in the data for the token.
79  unsigned Word0 = ReadLE32(CurPtrShadow);
80  uint32_t IdentifierID = ReadLE32(CurPtrShadow);
81  uint32_t FileOffset = ReadLE32(CurPtrShadow);
82
83  tok::TokenKind TKind = (tok::TokenKind) (Word0 & 0xFF);
84  Token::TokenFlags TFlags = (Token::TokenFlags) ((Word0 >> 8) & 0xFF);
85  uint32_t Len = Word0 >> 16;
86
87  CurPtr = CurPtrShadow;
88
89  //===--------------------------------------==//
90  // Construct the token itself.
91  //===--------------------------------------==//
92
93  Tok.startToken();
94  Tok.setKind(TKind);
95  Tok.setFlag(TFlags);
96  assert(!LexingRawMode);
97  Tok.setLocation(FileStartLoc.getFileLocWithOffset(FileOffset));
98  Tok.setLength(Len);
99
100  // Handle identifiers.
101  if (IdentifierID) {
102    MIOpt.ReadToken();
103    IdentifierInfo *II = PTHMgr.GetIdentifierInfo(IdentifierID-1);
104    Tok.setIdentifierInfo(II);
105    if (II->isHandleIdentifierCase())
106      PP->HandleIdentifier(Tok);
107    return;
108  }
109
110  //===--------------------------------------==//
111  // Process the token.
112  //===--------------------------------------==//
113#if 0
114  SourceManager& SM = PP->getSourceManager();
115  llvm::cerr << SM.getFileEntryForID(FileID)->getName()
116    << ':' << SM.getLogicalLineNumber(Tok.getLocation())
117    << ':' << SM.getLogicalColumnNumber(Tok.getLocation())
118    << '\n';
119#endif
120
121  if (TKind == tok::eof) {
122    // Save the end-of-file token.
123    EofToken = Tok;
124
125    Preprocessor *PPCache = PP;
126
127    assert(!ParsingPreprocessorDirective);
128    assert(!LexingRawMode);
129
130    // FIXME: Issue diagnostics similar to Lexer.
131    if (PP->HandleEndOfFile(Tok, false))
132      return;
133
134    assert(PPCache && "Raw buffer::LexEndOfFile should return a token");
135    return PPCache->Lex(Tok);
136  }
137
138  if (TKind == tok::hash && Tok.isAtStartOfLine()) {
139    LastHashTokPtr = CurPtr - DISK_TOKEN_SIZE;
140    assert(!LexingRawMode);
141    PP->HandleDirective(Tok);
142
143    if (PP->isCurrentLexer(this))
144      goto LexNextToken;
145
146    return PP->Lex(Tok);
147  }
148
149  if (TKind == tok::eom) {
150    assert(ParsingPreprocessorDirective);
151    ParsingPreprocessorDirective = false;
152    return;
153  }
154
155  MIOpt.ReadToken();
156}
157
158// FIXME: We can just grab the last token instead of storing a copy
159// into EofToken.
160void PTHLexer::getEOF(Token& Tok) {
161  assert(EofToken.is(tok::eof));
162  Tok = EofToken;
163}
164
165void PTHLexer::DiscardToEndOfLine() {
166  assert(ParsingPreprocessorDirective && ParsingFilename == false &&
167         "Must be in a preprocessing directive!");
168
169  // We assume that if the preprocessor wishes to discard to the end of
170  // the line that it also means to end the current preprocessor directive.
171  ParsingPreprocessorDirective = false;
172
173  // Skip tokens by only peeking at their token kind and the flags.
174  // We don't need to actually reconstruct full tokens from the token buffer.
175  // This saves some copies and it also reduces IdentifierInfo* lookup.
176  const unsigned char* p = CurPtr;
177  while (1) {
178    // Read the token kind.  Are we at the end of the file?
179    tok::TokenKind x = (tok::TokenKind) (uint8_t) *p;
180    if (x == tok::eof) break;
181
182    // Read the token flags.  Are we at the start of the next line?
183    Token::TokenFlags y = (Token::TokenFlags) (uint8_t) p[1];
184    if (y & Token::StartOfLine) break;
185
186    // Skip to the next token.
187    p += DISK_TOKEN_SIZE;
188  }
189
190  CurPtr = p;
191}
192
193/// SkipBlock - Used by Preprocessor to skip the current conditional block.
194bool PTHLexer::SkipBlock() {
195  assert(CurPPCondPtr && "No cached PP conditional information.");
196  assert(LastHashTokPtr && "No known '#' token.");
197
198  const unsigned char* HashEntryI = 0;
199  uint32_t Offset;
200  uint32_t TableIdx;
201
202  do {
203    // Read the token offset from the side-table.
204    Offset = ReadLE32(CurPPCondPtr);
205
206    // Read the target table index from the side-table.
207    TableIdx = ReadLE32(CurPPCondPtr);
208
209    // Compute the actual memory address of the '#' token data for this entry.
210    HashEntryI = TokBuf + Offset;
211
212    // Optmization: "Sibling jumping".  #if...#else...#endif blocks can
213    //  contain nested blocks.  In the side-table we can jump over these
214    //  nested blocks instead of doing a linear search if the next "sibling"
215    //  entry is not at a location greater than LastHashTokPtr.
216    if (HashEntryI < LastHashTokPtr && TableIdx) {
217      // In the side-table we are still at an entry for a '#' token that
218      // is earlier than the last one we saw.  Check if the location we would
219      // stride gets us closer.
220      const unsigned char* NextPPCondPtr =
221        PPCond + TableIdx*(sizeof(uint32_t)*2);
222      assert(NextPPCondPtr >= CurPPCondPtr);
223      // Read where we should jump to.
224      uint32_t TmpOffset = ReadLE32(NextPPCondPtr);
225      const unsigned char* HashEntryJ = TokBuf + TmpOffset;
226
227      if (HashEntryJ <= LastHashTokPtr) {
228        // Jump directly to the next entry in the side table.
229        HashEntryI = HashEntryJ;
230        Offset = TmpOffset;
231        TableIdx = ReadLE32(NextPPCondPtr);
232        CurPPCondPtr = NextPPCondPtr;
233      }
234    }
235  }
236  while (HashEntryI < LastHashTokPtr);
237  assert(HashEntryI == LastHashTokPtr && "No PP-cond entry found for '#'");
238  assert(TableIdx && "No jumping from #endifs.");
239
240  // Update our side-table iterator.
241  const unsigned char* NextPPCondPtr = PPCond + TableIdx*(sizeof(uint32_t)*2);
242  assert(NextPPCondPtr >= CurPPCondPtr);
243  CurPPCondPtr = NextPPCondPtr;
244
245  // Read where we should jump to.
246  HashEntryI = TokBuf + ReadLE32(NextPPCondPtr);
247  uint32_t NextIdx = ReadLE32(NextPPCondPtr);
248
249  // By construction NextIdx will be zero if this is a #endif.  This is useful
250  // to know to obviate lexing another token.
251  bool isEndif = NextIdx == 0;
252
253  // This case can occur when we see something like this:
254  //
255  //  #if ...
256  //   /* a comment or nothing */
257  //  #elif
258  //
259  // If we are skipping the first #if block it will be the case that CurPtr
260  // already points 'elif'.  Just return.
261
262  if (CurPtr > HashEntryI) {
263    assert(CurPtr == HashEntryI + DISK_TOKEN_SIZE);
264    // Did we reach a #endif?  If so, go ahead and consume that token as well.
265    if (isEndif)
266      CurPtr += DISK_TOKEN_SIZE*2;
267    else
268      LastHashTokPtr = HashEntryI;
269
270    return isEndif;
271  }
272
273  // Otherwise, we need to advance.  Update CurPtr to point to the '#' token.
274  CurPtr = HashEntryI;
275
276  // Update the location of the last observed '#'.  This is useful if we
277  // are skipping multiple blocks.
278  LastHashTokPtr = CurPtr;
279
280  // Skip the '#' token.
281  assert(((tok::TokenKind)*CurPtr) == tok::hash);
282  CurPtr += DISK_TOKEN_SIZE;
283
284  // Did we reach a #endif?  If so, go ahead and consume that token as well.
285  if (isEndif) { CurPtr += DISK_TOKEN_SIZE*2; }
286
287  return isEndif;
288}
289
290SourceLocation PTHLexer::getSourceLocation() {
291  // getSourceLocation is not on the hot path.  It is used to get the location
292  // of the next token when transitioning back to this lexer when done
293  // handling a #included file.  Just read the necessary data from the token
294  // data buffer to construct the SourceLocation object.
295  // NOTE: This is a virtual function; hence it is defined out-of-line.
296  const unsigned char *OffsetPtr = CurPtr + (DISK_TOKEN_SIZE - 4);
297  uint32_t Offset = ReadLE32(OffsetPtr);
298  return FileStartLoc.getFileLocWithOffset(Offset);
299}
300
301//===----------------------------------------------------------------------===//
302// getSpelling() - Use cached data in PTH files for getSpelling().
303//===----------------------------------------------------------------------===//
304
305unsigned PTHManager::getSpelling(FileID FID, unsigned FPos,
306                                 const char *&Buffer) {
307  llvm::DenseMap<FileID, PTHSpellingSearch*>::iterator I =SpellingMap.find(FID);
308
309  if (I == SpellingMap.end())
310    return 0;
311
312  return I->second->getSpellingBinarySearch(FPos, Buffer);
313}
314
315unsigned PTHManager::getSpelling(SourceLocation Loc, const char *&Buffer) {
316  SourceManager &SM = PP->getSourceManager();
317  Loc = SM.getSpellingLoc(Loc);
318  std::pair<FileID, unsigned> LocInfo = SM.getDecomposedFileLoc(Loc);
319  return getSpelling(LocInfo.first, LocInfo.second, Buffer);
320}
321
322unsigned PTHManager::getSpellingAtPTHOffset(unsigned PTHOffset,
323                                            const char *&Buffer) {
324  assert(PTHOffset < Buf->getBufferSize());
325  const unsigned char* Ptr =
326    (const unsigned char*)Buf->getBufferStart() + PTHOffset;
327
328  // The string is prefixed by 16 bits for its length, followed by the string
329  // itself.
330  unsigned Len = ReadUnalignedLE16(Ptr);
331  Buffer = (const char *)Ptr;
332  return Len;
333}
334
335unsigned PTHSpellingSearch::getSpellingLinearSearch(unsigned FPos,
336                                                    const char *&Buffer) {
337  const unsigned char *Ptr = LinearItr;
338  unsigned Len = 0;
339
340  if (Ptr == TableEnd)
341    return getSpellingBinarySearch(FPos, Buffer);
342
343  do {
344    uint32_t TokOffset = ReadLE32(Ptr);
345
346    if (TokOffset > FPos)
347      return getSpellingBinarySearch(FPos, Buffer);
348
349    // Did we find a matching token offset for this spelling?
350    if (TokOffset == FPos) {
351      uint32_t SpellingPTHOffset = ReadLE32(Ptr);
352      Len = PTHMgr.getSpellingAtPTHOffset(SpellingPTHOffset, Buffer);
353      break;
354    }
355  } while (Ptr != TableEnd);
356
357  LinearItr = Ptr;
358  return Len;
359}
360
361
362unsigned PTHSpellingSearch::getSpellingBinarySearch(unsigned FPos,
363                                                    const char *&Buffer) {
364
365  assert((TableEnd - TableBeg) % SpellingEntrySize == 0);
366  assert(TableEnd >= TableBeg);
367
368  if (TableEnd == TableBeg)
369    return 0;
370
371  unsigned min = 0;
372  const unsigned char *tb = TableBeg;
373  unsigned max = NumSpellings;
374
375  do {
376    unsigned i = (max - min) / 2 + min;
377    const unsigned char *Ptr = tb + (i * SpellingEntrySize);
378
379    uint32_t TokOffset = ReadLE32(Ptr);
380    if (TokOffset > FPos) {
381      max = i;
382      assert(!(max == min) || (min == i));
383      continue;
384    }
385
386    if (TokOffset < FPos) {
387      if (i == min)
388        break;
389
390      min = i;
391      continue;
392    }
393
394    uint32_t SpellingPTHOffset = ReadLE32(Ptr);
395    return PTHMgr.getSpellingAtPTHOffset(SpellingPTHOffset, Buffer);
396  }
397  while (min != max);
398
399  return 0;
400}
401
402unsigned PTHLexer::getSpelling(SourceLocation Loc, const char *&Buffer) {
403  SourceManager &SM = PP->getSourceManager();
404  Loc = SM.getSpellingLoc(Loc);
405  std::pair<FileID, unsigned> LocInfo = SM.getDecomposedFileLoc(Loc);
406
407  FileID FID = LocInfo.first;
408  unsigned FPos = LocInfo.second;
409
410  if (FID == getFileID())
411    return MySpellingSrch.getSpellingLinearSearch(FPos, Buffer);
412  return PTHMgr.getSpelling(FID, FPos, Buffer);
413}
414
415//===----------------------------------------------------------------------===//
416// Internal Data Structures for PTH file lookup and resolving identifiers.
417//===----------------------------------------------------------------------===//
418
419
420/// PTHFileLookup - This internal data structure is used by the PTHManager
421///  to map from FileEntry objects managed by FileManager to offsets within
422///  the PTH file.
423namespace {
424class VISIBILITY_HIDDEN PTHFileLookup {
425public:
426  class Val {
427    uint32_t TokenOff;
428    uint32_t PPCondOff;
429    uint32_t SpellingOff;
430  public:
431    Val() : TokenOff(~0) {}
432    Val(uint32_t toff, uint32_t poff, uint32_t soff)
433      : TokenOff(toff), PPCondOff(poff), SpellingOff(soff) {}
434
435    bool isValid() const { return TokenOff != ~((uint32_t)0); }
436
437    uint32_t getTokenOffset() const {
438      assert(isValid() && "PTHFileLookup entry initialized.");
439      return TokenOff;
440    }
441
442    uint32_t getPPCondOffset() const {
443      assert(isValid() && "PTHFileLookup entry initialized.");
444      return PPCondOff;
445    }
446
447    uint32_t getSpellingOffset() const {
448      assert(isValid() && "PTHFileLookup entry initialized.");
449      return SpellingOff;
450    }
451  };
452
453private:
454  llvm::StringMap<Val> FileMap;
455
456public:
457  PTHFileLookup() {};
458
459  bool isEmpty() const {
460    return FileMap.empty();
461  }
462
463  Val Lookup(const FileEntry* FE) {
464    const char* s = FE->getName();
465    unsigned size = strlen(s);
466    return FileMap.GetOrCreateValue(s, s+size).getValue();
467  }
468
469  void ReadTable(const unsigned char* D) {
470    uint32_t N = ReadLE32(D);     // Read the length of the table.
471
472    for ( ; N > 0; --N) {       // The rest of the data is the table itself.
473      uint32_t Len = ReadLE32(D);
474      const char* s = (const char *)D;
475      D += Len;
476
477      uint32_t TokenOff = ReadLE32(D);
478      uint32_t PPCondOff = ReadLE32(D);
479      uint32_t SpellingOff = ReadLE32(D);
480
481      FileMap.GetOrCreateValue(s, s+Len).getValue() =
482        Val(TokenOff, PPCondOff, SpellingOff);
483    }
484  }
485};
486} // end anonymous namespace
487
488//===----------------------------------------------------------------------===//
489// PTHManager methods.
490//===----------------------------------------------------------------------===//
491
492PTHManager::PTHManager(const llvm::MemoryBuffer* buf, void* fileLookup,
493                       const unsigned char* idDataTable,
494                       IdentifierInfo** perIDCache,
495                       const unsigned char* sortedIdTable, unsigned numIds)
496: Buf(buf), PerIDCache(perIDCache), FileLookup(fileLookup),
497  IdDataTable(idDataTable), SortedIdTable(sortedIdTable),
498  NumIds(numIds), PP(0) {}
499
500PTHManager::~PTHManager() {
501  delete Buf;
502  delete (PTHFileLookup*) FileLookup;
503  free(PerIDCache);
504}
505
506PTHManager* PTHManager::Create(const std::string& file) {
507  // Memory map the PTH file.
508  llvm::OwningPtr<llvm::MemoryBuffer>
509  File(llvm::MemoryBuffer::getFile(file.c_str()));
510
511  if (!File)
512    return 0;
513
514  // Get the buffer ranges and check if there are at least three 32-bit
515  // words at the end of the file.
516  const unsigned char* BufBeg = (unsigned char*)File->getBufferStart();
517  const unsigned char* BufEnd = (unsigned char*)File->getBufferEnd();
518
519  if(!(BufEnd > BufBeg + sizeof(uint32_t)*3)) {
520    assert(false && "Invalid PTH file.");
521    return 0; // FIXME: Proper error diagnostic?
522  }
523
524  // Compute the address of the index table at the end of the PTH file.
525  // This table contains the offset of the file lookup table, the
526  // persistent ID -> identifer data table.
527  // FIXME: We should just embed this offset in the PTH file.
528  const unsigned char* EndTable = BufEnd - sizeof(uint32_t)*4;
529
530  // Construct the file lookup table.  This will be used for mapping from
531  // FileEntry*'s to cached tokens.
532  const unsigned char* FileTableOffset = EndTable + sizeof(uint32_t)*3;
533  const unsigned char* FileTable = BufBeg + ReadLE32(FileTableOffset);
534
535  if (!(FileTable > BufBeg && FileTable < BufEnd)) {
536    assert(false && "Invalid PTH file.");
537    return 0; // FIXME: Proper error diagnostic?
538  }
539
540  llvm::OwningPtr<PTHFileLookup> FL(new PTHFileLookup());
541  FL->ReadTable(FileTable);
542
543  if (FL->isEmpty())
544    return 0;
545
546  // Get the location of the table mapping from persistent ids to the
547  // data needed to reconstruct identifiers.
548  const unsigned char* IDTableOffset = EndTable + sizeof(uint32_t)*1;
549  const unsigned char* IData = BufBeg + ReadLE32(IDTableOffset);
550
551  if (!(IData >= BufBeg && IData < BufEnd)) {
552    assert(false && "Invalid PTH file.");
553    return 0; // FIXME: Proper error diagnostic?
554  }
555
556  // Get the location of the lexigraphically-sorted table of persistent IDs.
557  const unsigned char* SortedIdTableOffset = EndTable + sizeof(uint32_t)*2;
558  const unsigned char* SortedIdTable = BufBeg + ReadLE32(SortedIdTableOffset);
559  if (!(SortedIdTable >= BufBeg && SortedIdTable < BufEnd)) {
560    assert(false && "Invalid PTH file.");
561    return 0; // FIXME: Proper error diagnostic?
562  }
563
564  // Get the number of IdentifierInfos and pre-allocate the identifier cache.
565  uint32_t NumIds = ReadLE32(IData);
566
567  // Pre-allocate the peristent ID -> IdentifierInfo* cache.  We use calloc()
568  // so that we in the best case only zero out memory once when the OS returns
569  // us new pages.
570  IdentifierInfo** PerIDCache = 0;
571
572  if (NumIds) {
573    PerIDCache = (IdentifierInfo**)calloc(NumIds, sizeof(*PerIDCache));
574    if (!PerIDCache) {
575      assert(false && "Could not allocate Persistent ID cache.");
576      return 0;
577    }
578  }
579
580  // Create the new PTHManager.
581  return new PTHManager(File.take(), FL.take(), IData, PerIDCache,
582                        SortedIdTable, NumIds);
583}
584IdentifierInfo* PTHManager::LazilyCreateIdentifierInfo(unsigned PersistentID) {
585  // Look in the PTH file for the string data for the IdentifierInfo object.
586  const unsigned char* TableEntry = IdDataTable + sizeof(uint32_t)*PersistentID;
587  const unsigned char* IDData =
588    (const unsigned char*)Buf->getBufferStart() + ReadLE32(TableEntry);
589  assert(IDData < (const unsigned char*)Buf->getBufferEnd());
590
591  // Allocate the object.
592  std::pair<IdentifierInfo,const unsigned char*> *Mem =
593    Alloc.Allocate<std::pair<IdentifierInfo,const unsigned char*> >();
594
595  Mem->second = IDData;
596  IdentifierInfo *II = new ((void*) Mem) IdentifierInfo();
597
598  // Store the new IdentifierInfo in the cache.
599  PerIDCache[PersistentID] = II;
600  return II;
601}
602
603IdentifierInfo* PTHManager::get(const char *NameStart, const char *NameEnd) {
604  unsigned min = 0;
605  unsigned max = NumIds;
606  unsigned Len = NameEnd - NameStart;
607
608  do {
609    unsigned i = (max - min) / 2 + min;
610    const unsigned char *Ptr = SortedIdTable + (i * 4);
611
612    // Read the persistentID.
613    unsigned perID = ReadLE32(Ptr);
614
615    // Get the IdentifierInfo.
616    IdentifierInfo* II = GetIdentifierInfo(perID);
617
618    // First compare the lengths.
619    unsigned IILen = II->getLength();
620    if (Len < IILen) goto IsLess;
621    if (Len > IILen) goto IsGreater;
622
623    // Now compare the strings!
624    {
625      signed comp = strncmp(NameStart, II->getName(), Len);
626      if (comp < 0) goto IsLess;
627      if (comp > 0) goto IsGreater;
628    }
629    // We found a match!
630    return II;
631
632  IsGreater:
633    if (i == min) break;
634    min = i;
635    continue;
636
637  IsLess:
638    max = i;
639    assert(!(max == min) || (min == i));
640  }
641  while (min != max);
642
643  return 0;
644}
645
646
647PTHLexer *PTHManager::CreateLexer(FileID FID) {
648  const FileEntry *FE = PP->getSourceManager().getFileEntryForID(FID);
649  if (!FE)
650    return 0;
651
652  // Lookup the FileEntry object in our file lookup data structure.  It will
653  // return a variant that indicates whether or not there is an offset within
654  // the PTH file that contains cached tokens.
655  PTHFileLookup::Val FileData = ((PTHFileLookup*)FileLookup)->Lookup(FE);
656
657  if (!FileData.isValid()) // No tokens available.
658    return 0;
659
660  const unsigned char *BufStart = (const unsigned char *)Buf->getBufferStart();
661  // Compute the offset of the token data within the buffer.
662  const unsigned char* data = BufStart + FileData.getTokenOffset();
663
664  // Get the location of pp-conditional table.
665  const unsigned char* ppcond = BufStart + FileData.getPPCondOffset();
666  uint32_t Len = ReadLE32(ppcond);
667  if (Len == 0) ppcond = 0;
668
669  // Get the location of the spelling table.
670  const unsigned char* spellingTable = BufStart + FileData.getSpellingOffset();
671
672  Len = ReadLE32(spellingTable);
673  if (Len == 0) spellingTable = 0;
674
675  assert(data < (const unsigned char*)Buf->getBufferEnd());
676
677  // Create the SpellingSearch object for this FileID.
678  PTHSpellingSearch* ss = new PTHSpellingSearch(*this, Len, spellingTable);
679  SpellingMap[FID] = ss;
680
681  assert(PP && "No preprocessor set yet!");
682  return new PTHLexer(*PP, FID, data, ppcond, *ss, *this);
683}
684