Preprocessor.h revision ad0fe03b897f9486191e75c8d90c3ffa9b4fd6a5
1//===--- Preprocessor.h - C Language Family Preprocessor --------*- C++ -*-===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10//  This file defines the Preprocessor interface.
11//
12//===----------------------------------------------------------------------===//
13
14#ifndef LLVM_CLANG_LEX_PREPROCESSOR_H
15#define LLVM_CLANG_LEX_PREPROCESSOR_H
16
17#include "clang/Lex/MacroInfo.h"
18#include "clang/Lex/Lexer.h"
19#include "clang/Lex/PTHLexer.h"
20#include "clang/Lex/PPCallbacks.h"
21#include "clang/Lex/TokenLexer.h"
22#include "clang/Lex/PTHManager.h"
23#include "clang/Basic/Builtins.h"
24#include "clang/Basic/Diagnostic.h"
25#include "clang/Basic/IdentifierTable.h"
26#include "clang/Basic/SourceLocation.h"
27#include "llvm/ADT/DenseMap.h"
28#include "llvm/ADT/IntrusiveRefCntPtr.h"
29#include "llvm/ADT/SmallPtrSet.h"
30#include "llvm/ADT/OwningPtr.h"
31#include "llvm/ADT/SmallVector.h"
32#include "llvm/ADT/ArrayRef.h"
33#include "llvm/Support/Allocator.h"
34#include <vector>
35
36namespace llvm {
37  template<unsigned InternalLen> class SmallString;
38}
39
40namespace clang {
41
42class SourceManager;
43class ExternalPreprocessorSource;
44class FileManager;
45class FileEntry;
46class HeaderSearch;
47class PragmaNamespace;
48class PragmaHandler;
49class CommentHandler;
50class ScratchBuffer;
51class TargetInfo;
52class PPCallbacks;
53class CodeCompletionHandler;
54class DirectoryLookup;
55class PreprocessingRecord;
56class ModuleLoader;
57
58/// Preprocessor - This object engages in a tight little dance with the lexer to
59/// efficiently preprocess tokens.  Lexers know only about tokens within a
60/// single source file, and don't know anything about preprocessor-level issues
61/// like the \#include stack, token expansion, etc.
62///
63class Preprocessor : public RefCountedBase<Preprocessor> {
64  DiagnosticsEngine        *Diags;
65  LangOptions       &LangOpts;
66  const TargetInfo  *Target;
67  FileManager       &FileMgr;
68  SourceManager     &SourceMgr;
69  ScratchBuffer     *ScratchBuf;
70  HeaderSearch      &HeaderInfo;
71  ModuleLoader      &TheModuleLoader;
72
73  /// \brief External source of macros.
74  ExternalPreprocessorSource *ExternalSource;
75
76
77  /// PTH - An optional PTHManager object used for getting tokens from
78  ///  a token cache rather than lexing the original source file.
79  OwningPtr<PTHManager> PTH;
80
81  /// BP - A BumpPtrAllocator object used to quickly allocate and release
82  ///  objects internal to the Preprocessor.
83  llvm::BumpPtrAllocator BP;
84
85  /// Identifiers for builtin macros and other builtins.
86  IdentifierInfo *Ident__LINE__, *Ident__FILE__;   // __LINE__, __FILE__
87  IdentifierInfo *Ident__DATE__, *Ident__TIME__;   // __DATE__, __TIME__
88  IdentifierInfo *Ident__INCLUDE_LEVEL__;          // __INCLUDE_LEVEL__
89  IdentifierInfo *Ident__BASE_FILE__;              // __BASE_FILE__
90  IdentifierInfo *Ident__TIMESTAMP__;              // __TIMESTAMP__
91  IdentifierInfo *Ident__COUNTER__;                // __COUNTER__
92  IdentifierInfo *Ident_Pragma, *Ident__pragma;    // _Pragma, __pragma
93  IdentifierInfo *Ident__VA_ARGS__;                // __VA_ARGS__
94  IdentifierInfo *Ident__has_feature;              // __has_feature
95  IdentifierInfo *Ident__has_extension;            // __has_extension
96  IdentifierInfo *Ident__has_builtin;              // __has_builtin
97  IdentifierInfo *Ident__has_attribute;            // __has_attribute
98  IdentifierInfo *Ident__has_include;              // __has_include
99  IdentifierInfo *Ident__has_include_next;         // __has_include_next
100  IdentifierInfo *Ident__has_warning;              // __has_warning
101
102  SourceLocation DATELoc, TIMELoc;
103  unsigned CounterValue;  // Next __COUNTER__ value.
104
105  enum {
106    /// MaxIncludeStackDepth - Maximum depth of \#includes.
107    MaxAllowedIncludeStackDepth = 200
108  };
109
110  // State that is set before the preprocessor begins.
111  bool KeepComments : 1;
112  bool KeepMacroComments : 1;
113  bool SuppressIncludeNotFoundError : 1;
114
115  // State that changes while the preprocessor runs:
116  bool InMacroArgs : 1;            // True if parsing fn macro invocation args.
117
118  /// Whether the preprocessor owns the header search object.
119  bool OwnsHeaderSearch : 1;
120
121  /// DisableMacroExpansion - True if macro expansion is disabled.
122  bool DisableMacroExpansion : 1;
123
124  /// MacroExpansionInDirectivesOverride - Temporarily disables
125  /// DisableMacroExpansion (i.e. enables expansion) when parsing preprocessor
126  /// directives.
127  bool MacroExpansionInDirectivesOverride : 1;
128
129  class ResetMacroExpansionHelper;
130
131  /// \brief Whether we have already loaded macros from the external source.
132  mutable bool ReadMacrosFromExternalSource : 1;
133
134  /// \brief True if pragmas are enabled.
135  bool PragmasEnabled : 1;
136
137  /// \brief True if we are pre-expanding macro arguments.
138  bool InMacroArgPreExpansion;
139
140  /// Identifiers - This is mapping/lookup information for all identifiers in
141  /// the program, including program keywords.
142  mutable IdentifierTable Identifiers;
143
144  /// Selectors - This table contains all the selectors in the program. Unlike
145  /// IdentifierTable above, this table *isn't* populated by the preprocessor.
146  /// It is declared/expanded here because it's role/lifetime is
147  /// conceptually similar the IdentifierTable. In addition, the current control
148  /// flow (in clang::ParseAST()), make it convenient to put here.
149  /// FIXME: Make sure the lifetime of Identifiers/Selectors *isn't* tied to
150  /// the lifetime of the preprocessor.
151  SelectorTable Selectors;
152
153  /// BuiltinInfo - Information about builtins.
154  Builtin::Context BuiltinInfo;
155
156  /// PragmaHandlers - This tracks all of the pragmas that the client registered
157  /// with this preprocessor.
158  PragmaNamespace *PragmaHandlers;
159
160  /// \brief Tracks all of the comment handlers that the client registered
161  /// with this preprocessor.
162  std::vector<CommentHandler *> CommentHandlers;
163
164  /// \brief True if we want to ignore EOF token and continue later on (thus
165  /// avoid tearing the Lexer and etc. down).
166  bool IncrementalProcessing;
167
168  /// \brief The code-completion handler.
169  CodeCompletionHandler *CodeComplete;
170
171  /// \brief The file that we're performing code-completion for, if any.
172  const FileEntry *CodeCompletionFile;
173
174  /// \brief The offset in file for the code-completion point.
175  unsigned CodeCompletionOffset;
176
177  /// \brief The location for the code-completion point. This gets instantiated
178  /// when the CodeCompletionFile gets \#include'ed for preprocessing.
179  SourceLocation CodeCompletionLoc;
180
181  /// \brief The start location for the file of the code-completion point.
182  ///
183  /// This gets instantiated when the CodeCompletionFile gets \#include'ed
184  /// for preprocessing.
185  SourceLocation CodeCompletionFileLoc;
186
187  /// \brief The source location of the 'import' contextual keyword we just
188  /// lexed, if any.
189  SourceLocation ModuleImportLoc;
190
191  /// \brief The module import path that we're currently processing.
192  llvm::SmallVector<std::pair<IdentifierInfo *, SourceLocation>, 2>
193    ModuleImportPath;
194
195  /// \brief Whether the module import expectes an identifier next. Otherwise,
196  /// it expects a '.' or ';'.
197  bool ModuleImportExpectsIdentifier;
198
199  /// \brief The source location of the currently-active
200  /// #pragma clang arc_cf_code_audited begin.
201  SourceLocation PragmaARCCFCodeAuditedLoc;
202
203  /// \brief True if we hit the code-completion point.
204  bool CodeCompletionReached;
205
206  /// \brief The number of bytes that we will initially skip when entering the
207  /// main file, which is used when loading a precompiled preamble, along
208  /// with a flag that indicates whether skipping this number of bytes will
209  /// place the lexer at the start of a line.
210  std::pair<unsigned, bool> SkipMainFilePreamble;
211
212  /// CurLexer - This is the current top of the stack that we're lexing from if
213  /// not expanding a macro and we are lexing directly from source code.
214  ///  Only one of CurLexer, CurPTHLexer, or CurTokenLexer will be non-null.
215  OwningPtr<Lexer> CurLexer;
216
217  /// CurPTHLexer - This is the current top of stack that we're lexing from if
218  ///  not expanding from a macro and we are lexing from a PTH cache.
219  ///  Only one of CurLexer, CurPTHLexer, or CurTokenLexer will be non-null.
220  OwningPtr<PTHLexer> CurPTHLexer;
221
222  /// CurPPLexer - This is the current top of the stack what we're lexing from
223  ///  if not expanding a macro.  This is an alias for either CurLexer or
224  ///  CurPTHLexer.
225  PreprocessorLexer *CurPPLexer;
226
227  /// CurLookup - The DirectoryLookup structure used to find the current
228  /// FileEntry, if CurLexer is non-null and if applicable.  This allows us to
229  /// implement \#include_next and find directory-specific properties.
230  const DirectoryLookup *CurDirLookup;
231
232  /// CurTokenLexer - This is the current macro we are expanding, if we are
233  /// expanding a macro.  One of CurLexer and CurTokenLexer must be null.
234  OwningPtr<TokenLexer> CurTokenLexer;
235
236  /// \brief The kind of lexer we're currently working with.
237  enum CurLexerKind {
238    CLK_Lexer,
239    CLK_PTHLexer,
240    CLK_TokenLexer,
241    CLK_CachingLexer,
242    CLK_LexAfterModuleImport
243  } CurLexerKind;
244
245  /// IncludeMacroStack - This keeps track of the stack of files currently
246  /// \#included, and macros currently being expanded from, not counting
247  /// CurLexer/CurTokenLexer.
248  struct IncludeStackInfo {
249    enum CurLexerKind     CurLexerKind;
250    Lexer                 *TheLexer;
251    PTHLexer              *ThePTHLexer;
252    PreprocessorLexer     *ThePPLexer;
253    TokenLexer            *TheTokenLexer;
254    const DirectoryLookup *TheDirLookup;
255
256    IncludeStackInfo(enum CurLexerKind K, Lexer *L, PTHLexer* P,
257                     PreprocessorLexer* PPL,
258                     TokenLexer* TL, const DirectoryLookup *D)
259      : CurLexerKind(K), TheLexer(L), ThePTHLexer(P), ThePPLexer(PPL),
260        TheTokenLexer(TL), TheDirLookup(D) {}
261  };
262  std::vector<IncludeStackInfo> IncludeMacroStack;
263
264  /// Callbacks - These are actions invoked when some preprocessor activity is
265  /// encountered (e.g. a file is \#included, etc).
266  PPCallbacks *Callbacks;
267
268  struct MacroExpandsInfo {
269    Token Tok;
270    MacroInfo *MI;
271    SourceRange Range;
272    MacroExpandsInfo(Token Tok, MacroInfo *MI, SourceRange Range)
273      : Tok(Tok), MI(MI), Range(Range) { }
274  };
275  SmallVector<MacroExpandsInfo, 2> DelayedMacroExpandsCallbacks;
276
277  /// Macros - For each IdentifierInfo with 'HasMacro' set, we keep a mapping
278  /// to the actual definition of the macro.
279  llvm::DenseMap<IdentifierInfo*, MacroInfo*> Macros;
280
281  /// \brief Macros that we want to warn because they are not used at the end
282  /// of the translation unit; we store just their SourceLocations instead
283  /// something like MacroInfo*. The benefit of this is that when we are
284  /// deserializing from PCH, we don't need to deserialize identifier & macros
285  /// just so that we can report that they are unused, we just warn using
286  /// the SourceLocations of this set (that will be filled by the ASTReader).
287  /// We are using SmallPtrSet instead of a vector for faster removal.
288  typedef llvm::SmallPtrSet<SourceLocation, 32> WarnUnusedMacroLocsTy;
289  WarnUnusedMacroLocsTy WarnUnusedMacroLocs;
290
291  /// MacroArgCache - This is a "freelist" of MacroArg objects that can be
292  /// reused for quick allocation.
293  MacroArgs *MacroArgCache;
294  friend class MacroArgs;
295
296  /// PragmaPushMacroInfo - For each IdentifierInfo used in a #pragma
297  /// push_macro directive, we keep a MacroInfo stack used to restore
298  /// previous macro value.
299  llvm::DenseMap<IdentifierInfo*, std::vector<MacroInfo*> > PragmaPushMacroInfo;
300
301  // Various statistics we track for performance analysis.
302  unsigned NumDirectives, NumIncluded, NumDefined, NumUndefined, NumPragma;
303  unsigned NumIf, NumElse, NumEndif;
304  unsigned NumEnteredSourceFiles, MaxIncludeStackDepth;
305  unsigned NumMacroExpanded, NumFnMacroExpanded, NumBuiltinMacroExpanded;
306  unsigned NumFastMacroExpanded, NumTokenPaste, NumFastTokenPaste;
307  unsigned NumSkipped;
308
309  /// Predefines - This string is the predefined macros that preprocessor
310  /// should use from the command line etc.
311  std::string Predefines;
312
313  /// TokenLexerCache - Cache macro expanders to reduce malloc traffic.
314  enum { TokenLexerCacheSize = 8 };
315  unsigned NumCachedTokenLexers;
316  TokenLexer *TokenLexerCache[TokenLexerCacheSize];
317
318  /// \brief Keeps macro expanded tokens for TokenLexers.
319  //
320  /// Works like a stack; a TokenLexer adds the macro expanded tokens that is
321  /// going to lex in the cache and when it finishes the tokens are removed
322  /// from the end of the cache.
323  SmallVector<Token, 16> MacroExpandedTokens;
324  std::vector<std::pair<TokenLexer *, size_t> > MacroExpandingLexersStack;
325
326  /// \brief A record of the macro definitions and expansions that
327  /// occurred during preprocessing.
328  ///
329  /// This is an optional side structure that can be enabled with
330  /// \c createPreprocessingRecord() prior to preprocessing.
331  PreprocessingRecord *Record;
332
333private:  // Cached tokens state.
334  typedef SmallVector<Token, 1> CachedTokensTy;
335
336  /// CachedTokens - Cached tokens are stored here when we do backtracking or
337  /// lookahead. They are "lexed" by the CachingLex() method.
338  CachedTokensTy CachedTokens;
339
340  /// CachedLexPos - The position of the cached token that CachingLex() should
341  /// "lex" next. If it points beyond the CachedTokens vector, it means that
342  /// a normal Lex() should be invoked.
343  CachedTokensTy::size_type CachedLexPos;
344
345  /// BacktrackPositions - Stack of backtrack positions, allowing nested
346  /// backtracks. The EnableBacktrackAtThisPos() method pushes a position to
347  /// indicate where CachedLexPos should be set when the BackTrack() method is
348  /// invoked (at which point the last position is popped).
349  std::vector<CachedTokensTy::size_type> BacktrackPositions;
350
351  struct MacroInfoChain {
352    MacroInfo MI;
353    MacroInfoChain *Next;
354    MacroInfoChain *Prev;
355  };
356
357  /// MacroInfos are managed as a chain for easy disposal.  This is the head
358  /// of that list.
359  MacroInfoChain *MIChainHead;
360
361  /// MICache - A "freelist" of MacroInfo objects that can be reused for quick
362  /// allocation.
363  MacroInfoChain *MICache;
364
365  MacroInfo *getInfoForMacro(IdentifierInfo *II) const;
366
367public:
368  Preprocessor(DiagnosticsEngine &diags, LangOptions &opts,
369               const TargetInfo *target,
370               SourceManager &SM, HeaderSearch &Headers,
371               ModuleLoader &TheModuleLoader,
372               IdentifierInfoLookup *IILookup = 0,
373               bool OwnsHeaderSearch = false,
374               bool DelayInitialization = false,
375               bool IncrProcessing = false);
376
377  ~Preprocessor();
378
379  /// \brief Initialize the preprocessor, if the constructor did not already
380  /// perform the initialization.
381  ///
382  /// \param Target Information about the target.
383  void Initialize(const TargetInfo &Target);
384
385  DiagnosticsEngine &getDiagnostics() const { return *Diags; }
386  void setDiagnostics(DiagnosticsEngine &D) { Diags = &D; }
387
388  const LangOptions &getLangOpts() const { return LangOpts; }
389  const TargetInfo &getTargetInfo() const { return *Target; }
390  FileManager &getFileManager() const { return FileMgr; }
391  SourceManager &getSourceManager() const { return SourceMgr; }
392  HeaderSearch &getHeaderSearchInfo() const { return HeaderInfo; }
393
394  IdentifierTable &getIdentifierTable() { return Identifiers; }
395  SelectorTable &getSelectorTable() { return Selectors; }
396  Builtin::Context &getBuiltinInfo() { return BuiltinInfo; }
397  llvm::BumpPtrAllocator &getPreprocessorAllocator() { return BP; }
398
399  void setPTHManager(PTHManager* pm);
400
401  PTHManager *getPTHManager() { return PTH.get(); }
402
403  void setExternalSource(ExternalPreprocessorSource *Source) {
404    ExternalSource = Source;
405  }
406
407  ExternalPreprocessorSource *getExternalSource() const {
408    return ExternalSource;
409  }
410
411  /// \brief Retrieve the module loader associated with this preprocessor.
412  ModuleLoader &getModuleLoader() const { return TheModuleLoader; }
413
414  /// SetCommentRetentionState - Control whether or not the preprocessor retains
415  /// comments in output.
416  void SetCommentRetentionState(bool KeepComments, bool KeepMacroComments) {
417    this->KeepComments = KeepComments | KeepMacroComments;
418    this->KeepMacroComments = KeepMacroComments;
419  }
420
421  bool getCommentRetentionState() const { return KeepComments; }
422
423  void setPragmasEnabled(bool Enabled) { PragmasEnabled = Enabled; }
424  bool getPragmasEnabled() const { return PragmasEnabled; }
425
426  void SetSuppressIncludeNotFoundError(bool Suppress) {
427    SuppressIncludeNotFoundError = Suppress;
428  }
429
430  bool GetSuppressIncludeNotFoundError() {
431    return SuppressIncludeNotFoundError;
432  }
433
434  /// isCurrentLexer - Return true if we are lexing directly from the specified
435  /// lexer.
436  bool isCurrentLexer(const PreprocessorLexer *L) const {
437    return CurPPLexer == L;
438  }
439
440  /// getCurrentLexer - Return the current lexer being lexed from.  Note
441  /// that this ignores any potentially active macro expansions and _Pragma
442  /// expansions going on at the time.
443  PreprocessorLexer *getCurrentLexer() const { return CurPPLexer; }
444
445  /// getCurrentFileLexer - Return the current file lexer being lexed from.
446  /// Note that this ignores any potentially active macro expansions and _Pragma
447  /// expansions going on at the time.
448  PreprocessorLexer *getCurrentFileLexer() const;
449
450  /// getPPCallbacks/addPPCallbacks - Accessors for preprocessor callbacks.
451  /// Note that this class takes ownership of any PPCallbacks object given to
452  /// it.
453  PPCallbacks *getPPCallbacks() const { return Callbacks; }
454  void addPPCallbacks(PPCallbacks *C) {
455    if (Callbacks)
456      C = new PPChainedCallbacks(C, Callbacks);
457    Callbacks = C;
458  }
459
460  /// \brief Given an identifier, return the MacroInfo it is \#defined to
461  /// or null if it isn't \#define'd.
462  MacroInfo *getMacroInfo(IdentifierInfo *II) const {
463    if (!II->hasMacroDefinition())
464      return 0;
465
466    return getInfoForMacro(II);
467  }
468
469  /// \brief Specify a macro for this identifier.
470  void setMacroInfo(IdentifierInfo *II, MacroInfo *MI,
471                    bool LoadedFromAST = false);
472
473  /// macro_iterator/macro_begin/macro_end - This allows you to walk the current
474  /// state of the macro table.  This visits every currently-defined macro.
475  typedef llvm::DenseMap<IdentifierInfo*,
476                         MacroInfo*>::const_iterator macro_iterator;
477  macro_iterator macro_begin(bool IncludeExternalMacros = true) const;
478  macro_iterator macro_end(bool IncludeExternalMacros = true) const;
479
480  const std::string &getPredefines() const { return Predefines; }
481  /// setPredefines - Set the predefines for this Preprocessor.  These
482  /// predefines are automatically injected when parsing the main file.
483  void setPredefines(const char *P) { Predefines = P; }
484  void setPredefines(const std::string &P) { Predefines = P; }
485
486  /// getIdentifierInfo - Return information about the specified preprocessor
487  /// identifier token.  The version of this method that takes two character
488  /// pointers is preferred unless the identifier is already available as a
489  /// string (this avoids allocation and copying of memory to construct an
490  /// std::string).
491  IdentifierInfo *getIdentifierInfo(StringRef Name) const {
492    return &Identifiers.get(Name);
493  }
494
495  /// AddPragmaHandler - Add the specified pragma handler to the preprocessor.
496  /// If 'Namespace' is non-null, then it is a token required to exist on the
497  /// pragma line before the pragma string starts, e.g. "STDC" or "GCC".
498  void AddPragmaHandler(StringRef Namespace, PragmaHandler *Handler);
499  void AddPragmaHandler(PragmaHandler *Handler) {
500    AddPragmaHandler(StringRef(), Handler);
501  }
502
503  /// RemovePragmaHandler - Remove the specific pragma handler from
504  /// the preprocessor. If \arg Namespace is non-null, then it should
505  /// be the namespace that \arg Handler was added to. It is an error
506  /// to remove a handler that has not been registered.
507  void RemovePragmaHandler(StringRef Namespace, PragmaHandler *Handler);
508  void RemovePragmaHandler(PragmaHandler *Handler) {
509    RemovePragmaHandler(StringRef(), Handler);
510  }
511
512  /// \brief Add the specified comment handler to the preprocessor.
513  void addCommentHandler(CommentHandler *Handler);
514
515  /// \brief Remove the specified comment handler.
516  ///
517  /// It is an error to remove a handler that has not been registered.
518  void removeCommentHandler(CommentHandler *Handler);
519
520  /// \brief Set the code completion handler to the given object.
521  void setCodeCompletionHandler(CodeCompletionHandler &Handler) {
522    CodeComplete = &Handler;
523  }
524
525  /// \brief Retrieve the current code-completion handler.
526  CodeCompletionHandler *getCodeCompletionHandler() const {
527    return CodeComplete;
528  }
529
530  /// \brief Clear out the code completion handler.
531  void clearCodeCompletionHandler() {
532    CodeComplete = 0;
533  }
534
535  /// \brief Hook used by the lexer to invoke the "natural language" code
536  /// completion point.
537  void CodeCompleteNaturalLanguage();
538
539  /// \brief Retrieve the preprocessing record, or NULL if there is no
540  /// preprocessing record.
541  PreprocessingRecord *getPreprocessingRecord() const { return Record; }
542
543  /// \brief Create a new preprocessing record, which will keep track of
544  /// all macro expansions, macro definitions, etc.
545  void createPreprocessingRecord(bool RecordConditionalDirectives);
546
547  /// EnterMainSourceFile - Enter the specified FileID as the main source file,
548  /// which implicitly adds the builtin defines etc.
549  void EnterMainSourceFile();
550
551  /// EndSourceFile - Inform the preprocessor callbacks that processing is
552  /// complete.
553  void EndSourceFile();
554
555  /// EnterSourceFile - Add a source file to the top of the include stack and
556  /// start lexing tokens from it instead of the current buffer.  Emit an error
557  /// and don't enter the file on error.
558  void EnterSourceFile(FileID CurFileID, const DirectoryLookup *Dir,
559                       SourceLocation Loc);
560
561  /// EnterMacro - Add a Macro to the top of the include stack and start lexing
562  /// tokens from it instead of the current buffer.  Args specifies the
563  /// tokens input to a function-like macro.
564  ///
565  /// ILEnd specifies the location of the ')' for a function-like macro or the
566  /// identifier for an object-like macro.
567  void EnterMacro(Token &Identifier, SourceLocation ILEnd, MacroArgs *Args);
568
569  /// EnterTokenStream - Add a "macro" context to the top of the include stack,
570  /// which will cause the lexer to start returning the specified tokens.
571  ///
572  /// If DisableMacroExpansion is true, tokens lexed from the token stream will
573  /// not be subject to further macro expansion.  Otherwise, these tokens will
574  /// be re-macro-expanded when/if expansion is enabled.
575  ///
576  /// If OwnsTokens is false, this method assumes that the specified stream of
577  /// tokens has a permanent owner somewhere, so they do not need to be copied.
578  /// If it is true, it assumes the array of tokens is allocated with new[] and
579  /// must be freed.
580  ///
581  void EnterTokenStream(const Token *Toks, unsigned NumToks,
582                        bool DisableMacroExpansion, bool OwnsTokens);
583
584  /// RemoveTopOfLexerStack - Pop the current lexer/macro exp off the top of the
585  /// lexer stack.  This should only be used in situations where the current
586  /// state of the top-of-stack lexer is known.
587  void RemoveTopOfLexerStack();
588
589  /// EnableBacktrackAtThisPos - From the point that this method is called, and
590  /// until CommitBacktrackedTokens() or Backtrack() is called, the Preprocessor
591  /// keeps track of the lexed tokens so that a subsequent Backtrack() call will
592  /// make the Preprocessor re-lex the same tokens.
593  ///
594  /// Nested backtracks are allowed, meaning that EnableBacktrackAtThisPos can
595  /// be called multiple times and CommitBacktrackedTokens/Backtrack calls will
596  /// be combined with the EnableBacktrackAtThisPos calls in reverse order.
597  ///
598  /// NOTE: *DO NOT* forget to call either CommitBacktrackedTokens or Backtrack
599  /// at some point after EnableBacktrackAtThisPos. If you don't, caching of
600  /// tokens will continue indefinitely.
601  ///
602  void EnableBacktrackAtThisPos();
603
604  /// CommitBacktrackedTokens - Disable the last EnableBacktrackAtThisPos call.
605  void CommitBacktrackedTokens();
606
607  /// Backtrack - Make Preprocessor re-lex the tokens that were lexed since
608  /// EnableBacktrackAtThisPos() was previously called.
609  void Backtrack();
610
611  /// isBacktrackEnabled - True if EnableBacktrackAtThisPos() was called and
612  /// caching of tokens is on.
613  bool isBacktrackEnabled() const { return !BacktrackPositions.empty(); }
614
615  /// Lex - To lex a token from the preprocessor, just pull a token from the
616  /// current lexer or macro object.
617  void Lex(Token &Result) {
618    switch (CurLexerKind) {
619    case CLK_Lexer: CurLexer->Lex(Result); break;
620    case CLK_PTHLexer: CurPTHLexer->Lex(Result); break;
621    case CLK_TokenLexer: CurTokenLexer->Lex(Result); break;
622    case CLK_CachingLexer: CachingLex(Result); break;
623    case CLK_LexAfterModuleImport: LexAfterModuleImport(Result); break;
624    }
625  }
626
627  void LexAfterModuleImport(Token &Result);
628
629  /// LexNonComment - Lex a token.  If it's a comment, keep lexing until we get
630  /// something not a comment.  This is useful in -E -C mode where comments
631  /// would foul up preprocessor directive handling.
632  void LexNonComment(Token &Result) {
633    do
634      Lex(Result);
635    while (Result.getKind() == tok::comment);
636  }
637
638  /// LexUnexpandedToken - This is just like Lex, but this disables macro
639  /// expansion of identifier tokens.
640  void LexUnexpandedToken(Token &Result) {
641    // Disable macro expansion.
642    bool OldVal = DisableMacroExpansion;
643    DisableMacroExpansion = true;
644    // Lex the token.
645    Lex(Result);
646
647    // Reenable it.
648    DisableMacroExpansion = OldVal;
649  }
650
651  /// LexUnexpandedNonComment - Like LexNonComment, but this disables macro
652  /// expansion of identifier tokens.
653  void LexUnexpandedNonComment(Token &Result) {
654    do
655      LexUnexpandedToken(Result);
656    while (Result.getKind() == tok::comment);
657  }
658
659  /// Disables macro expansion everywhere except for preprocessor directives.
660  void SetMacroExpansionOnlyInDirectives() {
661    DisableMacroExpansion = true;
662    MacroExpansionInDirectivesOverride = true;
663  }
664
665  /// LookAhead - This peeks ahead N tokens and returns that token without
666  /// consuming any tokens.  LookAhead(0) returns the next token that would be
667  /// returned by Lex(), LookAhead(1) returns the token after it, etc.  This
668  /// returns normal tokens after phase 5.  As such, it is equivalent to using
669  /// 'Lex', not 'LexUnexpandedToken'.
670  const Token &LookAhead(unsigned N) {
671    if (CachedLexPos + N < CachedTokens.size())
672      return CachedTokens[CachedLexPos+N];
673    else
674      return PeekAhead(N+1);
675  }
676
677  /// RevertCachedTokens - When backtracking is enabled and tokens are cached,
678  /// this allows to revert a specific number of tokens.
679  /// Note that the number of tokens being reverted should be up to the last
680  /// backtrack position, not more.
681  void RevertCachedTokens(unsigned N) {
682    assert(isBacktrackEnabled() &&
683           "Should only be called when tokens are cached for backtracking");
684    assert(signed(CachedLexPos) - signed(N) >= signed(BacktrackPositions.back())
685         && "Should revert tokens up to the last backtrack position, not more");
686    assert(signed(CachedLexPos) - signed(N) >= 0 &&
687           "Corrupted backtrack positions ?");
688    CachedLexPos -= N;
689  }
690
691  /// EnterToken - Enters a token in the token stream to be lexed next. If
692  /// BackTrack() is called afterwards, the token will remain at the insertion
693  /// point.
694  void EnterToken(const Token &Tok) {
695    EnterCachingLexMode();
696    CachedTokens.insert(CachedTokens.begin()+CachedLexPos, Tok);
697  }
698
699  /// AnnotateCachedTokens - We notify the Preprocessor that if it is caching
700  /// tokens (because backtrack is enabled) it should replace the most recent
701  /// cached tokens with the given annotation token. This function has no effect
702  /// if backtracking is not enabled.
703  ///
704  /// Note that the use of this function is just for optimization; so that the
705  /// cached tokens doesn't get re-parsed and re-resolved after a backtrack is
706  /// invoked.
707  void AnnotateCachedTokens(const Token &Tok) {
708    assert(Tok.isAnnotation() && "Expected annotation token");
709    if (CachedLexPos != 0 && isBacktrackEnabled())
710      AnnotatePreviousCachedTokens(Tok);
711  }
712
713  /// \brief Replace the last token with an annotation token.
714  ///
715  /// Like AnnotateCachedTokens(), this routine replaces an
716  /// already-parsed (and resolved) token with an annotation
717  /// token. However, this routine only replaces the last token with
718  /// the annotation token; it does not affect any other cached
719  /// tokens. This function has no effect if backtracking is not
720  /// enabled.
721  void ReplaceLastTokenWithAnnotation(const Token &Tok) {
722    assert(Tok.isAnnotation() && "Expected annotation token");
723    if (CachedLexPos != 0 && isBacktrackEnabled())
724      CachedTokens[CachedLexPos-1] = Tok;
725  }
726
727  /// TypoCorrectToken - Update the current token to represent the provided
728  /// identifier, in order to cache an action performed by typo correction.
729  void TypoCorrectToken(const Token &Tok) {
730    assert(Tok.getIdentifierInfo() && "Expected identifier token");
731    if (CachedLexPos != 0 && isBacktrackEnabled())
732      CachedTokens[CachedLexPos-1] = Tok;
733  }
734
735  /// \brief Recompute the current lexer kind based on the CurLexer/CurPTHLexer/
736  /// CurTokenLexer pointers.
737  void recomputeCurLexerKind();
738
739  /// \brief Returns true if incremental processing is enabled
740  bool isIncrementalProcessingEnabled() const { return IncrementalProcessing; }
741
742  /// \brief Enables the incremental processing
743  void enableIncrementalProcessing(bool value = true) {
744    IncrementalProcessing = value;
745  }
746
747  /// \brief Specify the point at which code-completion will be performed.
748  ///
749  /// \param File the file in which code completion should occur. If
750  /// this file is included multiple times, code-completion will
751  /// perform completion the first time it is included. If NULL, this
752  /// function clears out the code-completion point.
753  ///
754  /// \param Line the line at which code completion should occur
755  /// (1-based).
756  ///
757  /// \param Column the column at which code completion should occur
758  /// (1-based).
759  ///
760  /// \returns true if an error occurred, false otherwise.
761  bool SetCodeCompletionPoint(const FileEntry *File,
762                              unsigned Line, unsigned Column);
763
764  /// \brief Determine if we are performing code completion.
765  bool isCodeCompletionEnabled() const { return CodeCompletionFile != 0; }
766
767  /// \brief Returns the location of the code-completion point.
768  /// Returns an invalid location if code-completion is not enabled or the file
769  /// containing the code-completion point has not been lexed yet.
770  SourceLocation getCodeCompletionLoc() const { return CodeCompletionLoc; }
771
772  /// \brief Returns the start location of the file of code-completion point.
773  /// Returns an invalid location if code-completion is not enabled or the file
774  /// containing the code-completion point has not been lexed yet.
775  SourceLocation getCodeCompletionFileLoc() const {
776    return CodeCompletionFileLoc;
777  }
778
779  /// \brief Returns true if code-completion is enabled and we have hit the
780  /// code-completion point.
781  bool isCodeCompletionReached() const { return CodeCompletionReached; }
782
783  /// \brief Note that we hit the code-completion point.
784  void setCodeCompletionReached() {
785    assert(isCodeCompletionEnabled() && "Code-completion not enabled!");
786    CodeCompletionReached = true;
787    // Silence any diagnostics that occur after we hit the code-completion.
788    getDiagnostics().setSuppressAllDiagnostics(true);
789  }
790
791  /// \brief The location of the currently-active \#pragma clang
792  /// arc_cf_code_audited begin.  Returns an invalid location if there
793  /// is no such pragma active.
794  SourceLocation getPragmaARCCFCodeAuditedLoc() const {
795    return PragmaARCCFCodeAuditedLoc;
796  }
797
798  /// \brief Set the location of the currently-active \#pragma clang
799  /// arc_cf_code_audited begin.  An invalid location ends the pragma.
800  void setPragmaARCCFCodeAuditedLoc(SourceLocation Loc) {
801    PragmaARCCFCodeAuditedLoc = Loc;
802  }
803
804  /// \brief Instruct the preprocessor to skip part of the main source file.
805  ///
806  /// \param Bytes The number of bytes in the preamble to skip.
807  ///
808  /// \param StartOfLine Whether skipping these bytes puts the lexer at the
809  /// start of a line.
810  void setSkipMainFilePreamble(unsigned Bytes, bool StartOfLine) {
811    SkipMainFilePreamble.first = Bytes;
812    SkipMainFilePreamble.second = StartOfLine;
813  }
814
815  /// Diag - Forwarding function for diagnostics.  This emits a diagnostic at
816  /// the specified Token's location, translating the token's start
817  /// position in the current buffer into a SourcePosition object for rendering.
818  DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const {
819    return Diags->Report(Loc, DiagID);
820  }
821
822  DiagnosticBuilder Diag(const Token &Tok, unsigned DiagID) const {
823    return Diags->Report(Tok.getLocation(), DiagID);
824  }
825
826  /// getSpelling() - Return the 'spelling' of the token at the given
827  /// location; does not go up to the spelling location or down to the
828  /// expansion location.
829  ///
830  /// \param buffer A buffer which will be used only if the token requires
831  ///   "cleaning", e.g. if it contains trigraphs or escaped newlines
832  /// \param invalid If non-null, will be set \c true if an error occurs.
833  StringRef getSpelling(SourceLocation loc,
834                              SmallVectorImpl<char> &buffer,
835                              bool *invalid = 0) const {
836    return Lexer::getSpelling(loc, buffer, SourceMgr, LangOpts, invalid);
837  }
838
839  /// getSpelling() - Return the 'spelling' of the Tok token.  The spelling of a
840  /// token is the characters used to represent the token in the source file
841  /// after trigraph expansion and escaped-newline folding.  In particular, this
842  /// wants to get the true, uncanonicalized, spelling of things like digraphs
843  /// UCNs, etc.
844  ///
845  /// \param Invalid If non-null, will be set \c true if an error occurs.
846  std::string getSpelling(const Token &Tok, bool *Invalid = 0) const {
847    return Lexer::getSpelling(Tok, SourceMgr, LangOpts, Invalid);
848  }
849
850  /// getSpelling - This method is used to get the spelling of a token into a
851  /// preallocated buffer, instead of as an std::string.  The caller is required
852  /// to allocate enough space for the token, which is guaranteed to be at least
853  /// Tok.getLength() bytes long.  The length of the actual result is returned.
854  ///
855  /// Note that this method may do two possible things: it may either fill in
856  /// the buffer specified with characters, or it may *change the input pointer*
857  /// to point to a constant buffer with the data already in it (avoiding a
858  /// copy).  The caller is not allowed to modify the returned buffer pointer
859  /// if an internal buffer is returned.
860  unsigned getSpelling(const Token &Tok, const char *&Buffer,
861                       bool *Invalid = 0) const {
862    return Lexer::getSpelling(Tok, Buffer, SourceMgr, LangOpts, Invalid);
863  }
864
865  /// getSpelling - This method is used to get the spelling of a token into a
866  /// SmallVector. Note that the returned StringRef may not point to the
867  /// supplied buffer if a copy can be avoided.
868  StringRef getSpelling(const Token &Tok,
869                        SmallVectorImpl<char> &Buffer,
870                        bool *Invalid = 0) const;
871
872  /// getSpellingOfSingleCharacterNumericConstant - Tok is a numeric constant
873  /// with length 1, return the character.
874  char getSpellingOfSingleCharacterNumericConstant(const Token &Tok,
875                                                   bool *Invalid = 0) const {
876    assert(Tok.is(tok::numeric_constant) &&
877           Tok.getLength() == 1 && "Called on unsupported token");
878    assert(!Tok.needsCleaning() && "Token can't need cleaning with length 1");
879
880    // If the token is carrying a literal data pointer, just use it.
881    if (const char *D = Tok.getLiteralData())
882      return *D;
883
884    // Otherwise, fall back on getCharacterData, which is slower, but always
885    // works.
886    return *SourceMgr.getCharacterData(Tok.getLocation(), Invalid);
887  }
888
889  /// \brief Retrieve the name of the immediate macro expansion.
890  ///
891  /// This routine starts from a source location, and finds the name of the macro
892  /// responsible for its immediate expansion. It looks through any intervening
893  /// macro argument expansions to compute this. It returns a StringRef which
894  /// refers to the SourceManager-owned buffer of the source where that macro
895  /// name is spelled. Thus, the result shouldn't out-live the SourceManager.
896  StringRef getImmediateMacroName(SourceLocation Loc) {
897    return Lexer::getImmediateMacroName(Loc, SourceMgr, getLangOpts());
898  }
899
900  /// CreateString - Plop the specified string into a scratch buffer and set the
901  /// specified token's location and length to it.  If specified, the source
902  /// location provides a location of the expansion point of the token.
903  void CreateString(const char *Buf, unsigned Len, Token &Tok,
904                    SourceLocation ExpansionLocStart = SourceLocation(),
905                    SourceLocation ExpansionLocEnd = SourceLocation());
906
907  /// \brief Computes the source location just past the end of the
908  /// token at this source location.
909  ///
910  /// This routine can be used to produce a source location that
911  /// points just past the end of the token referenced by \p Loc, and
912  /// is generally used when a diagnostic needs to point just after a
913  /// token where it expected something different that it received. If
914  /// the returned source location would not be meaningful (e.g., if
915  /// it points into a macro), this routine returns an invalid
916  /// source location.
917  ///
918  /// \param Offset an offset from the end of the token, where the source
919  /// location should refer to. The default offset (0) produces a source
920  /// location pointing just past the end of the token; an offset of 1 produces
921  /// a source location pointing to the last character in the token, etc.
922  SourceLocation getLocForEndOfToken(SourceLocation Loc, unsigned Offset = 0) {
923    return Lexer::getLocForEndOfToken(Loc, Offset, SourceMgr, LangOpts);
924  }
925
926  /// \brief Returns true if the given MacroID location points at the first
927  /// token of the macro expansion.
928  ///
929  /// \param MacroBegin If non-null and function returns true, it is set to
930  /// begin location of the macro.
931  bool isAtStartOfMacroExpansion(SourceLocation loc,
932                                 SourceLocation *MacroBegin = 0) const {
933    return Lexer::isAtStartOfMacroExpansion(loc, SourceMgr, LangOpts,
934                                            MacroBegin);
935  }
936
937  /// \brief Returns true if the given MacroID location points at the last
938  /// token of the macro expansion.
939  ///
940  /// \param MacroEnd If non-null and function returns true, it is set to
941  /// end location of the macro.
942  bool isAtEndOfMacroExpansion(SourceLocation loc,
943                               SourceLocation *MacroEnd = 0) const {
944    return Lexer::isAtEndOfMacroExpansion(loc, SourceMgr, LangOpts, MacroEnd);
945  }
946
947  /// DumpToken - Print the token to stderr, used for debugging.
948  ///
949  void DumpToken(const Token &Tok, bool DumpFlags = false) const;
950  void DumpLocation(SourceLocation Loc) const;
951  void DumpMacro(const MacroInfo &MI) const;
952
953  /// AdvanceToTokenCharacter - Given a location that specifies the start of a
954  /// token, return a new location that specifies a character within the token.
955  SourceLocation AdvanceToTokenCharacter(SourceLocation TokStart,
956                                         unsigned Char) const {
957    return Lexer::AdvanceToTokenCharacter(TokStart, Char, SourceMgr, LangOpts);
958  }
959
960  /// IncrementPasteCounter - Increment the counters for the number of token
961  /// paste operations performed.  If fast was specified, this is a 'fast paste'
962  /// case we handled.
963  ///
964  void IncrementPasteCounter(bool isFast) {
965    if (isFast)
966      ++NumFastTokenPaste;
967    else
968      ++NumTokenPaste;
969  }
970
971  void PrintStats();
972
973  size_t getTotalMemory() const;
974
975  /// HandleMicrosoftCommentPaste - When the macro expander pastes together a
976  /// comment (/##/) in microsoft mode, this method handles updating the current
977  /// state, returning the token on the next source line.
978  void HandleMicrosoftCommentPaste(Token &Tok);
979
980  //===--------------------------------------------------------------------===//
981  // Preprocessor callback methods.  These are invoked by a lexer as various
982  // directives and events are found.
983
984  /// LookUpIdentifierInfo - Given a tok::raw_identifier token, look up the
985  /// identifier information for the token and install it into the token,
986  /// updating the token kind accordingly.
987  IdentifierInfo *LookUpIdentifierInfo(Token &Identifier) const;
988
989private:
990  llvm::DenseMap<IdentifierInfo*,unsigned> PoisonReasons;
991
992public:
993
994  // SetPoisonReason - Call this function to indicate the reason for
995  // poisoning an identifier. If that identifier is accessed while
996  // poisoned, then this reason will be used instead of the default
997  // "poisoned" diagnostic.
998  void SetPoisonReason(IdentifierInfo *II, unsigned DiagID);
999
1000  // HandlePoisonedIdentifier - Display reason for poisoned
1001  // identifier.
1002  void HandlePoisonedIdentifier(Token & Tok);
1003
1004  void MaybeHandlePoisonedIdentifier(Token & Identifier) {
1005    if(IdentifierInfo * II = Identifier.getIdentifierInfo()) {
1006      if(II->isPoisoned()) {
1007        HandlePoisonedIdentifier(Identifier);
1008      }
1009    }
1010  }
1011
1012private:
1013  /// Identifiers used for SEH handling in Borland. These are only
1014  /// allowed in particular circumstances
1015  // __except block
1016  IdentifierInfo *Ident__exception_code,
1017                 *Ident___exception_code,
1018                 *Ident_GetExceptionCode;
1019  // __except filter expression
1020  IdentifierInfo *Ident__exception_info,
1021                 *Ident___exception_info,
1022                 *Ident_GetExceptionInfo;
1023  // __finally
1024  IdentifierInfo *Ident__abnormal_termination,
1025                 *Ident___abnormal_termination,
1026                 *Ident_AbnormalTermination;
1027public:
1028  void PoisonSEHIdentifiers(bool Poison = true); // Borland
1029
1030  /// HandleIdentifier - This callback is invoked when the lexer reads an
1031  /// identifier and has filled in the tokens IdentifierInfo member.  This
1032  /// callback potentially macro expands it or turns it into a named token (like
1033  /// 'for').
1034  void HandleIdentifier(Token &Identifier);
1035
1036
1037  /// HandleEndOfFile - This callback is invoked when the lexer hits the end of
1038  /// the current file.  This either returns the EOF token and returns true, or
1039  /// pops a level off the include stack and returns false, at which point the
1040  /// client should call lex again.
1041  bool HandleEndOfFile(Token &Result, bool isEndOfMacro = false);
1042
1043  /// HandleEndOfTokenLexer - This callback is invoked when the current
1044  /// TokenLexer hits the end of its token stream.
1045  bool HandleEndOfTokenLexer(Token &Result);
1046
1047  /// HandleDirective - This callback is invoked when the lexer sees a # token
1048  /// at the start of a line.  This consumes the directive, modifies the
1049  /// lexer/preprocessor state, and advances the lexer(s) so that the next token
1050  /// read is the correct one.
1051  void HandleDirective(Token &Result);
1052
1053  /// CheckEndOfDirective - Ensure that the next token is a tok::eod token.  If
1054  /// not, emit a diagnostic and consume up until the eod.  If EnableMacros is
1055  /// true, then we consider macros that expand to zero tokens as being ok.
1056  void CheckEndOfDirective(const char *Directive, bool EnableMacros = false);
1057
1058  /// DiscardUntilEndOfDirective - Read and discard all tokens remaining on the
1059  /// current line until the tok::eod token is found.
1060  void DiscardUntilEndOfDirective();
1061
1062  /// SawDateOrTime - This returns true if the preprocessor has seen a use of
1063  /// __DATE__ or __TIME__ in the file so far.
1064  bool SawDateOrTime() const {
1065    return DATELoc != SourceLocation() || TIMELoc != SourceLocation();
1066  }
1067  unsigned getCounterValue() const { return CounterValue; }
1068  void setCounterValue(unsigned V) { CounterValue = V; }
1069
1070  /// \brief Retrieves the module that we're currently building, if any.
1071  Module *getCurrentModule();
1072
1073  /// \brief Allocate a new MacroInfo object with the provided SourceLocation.
1074  MacroInfo *AllocateMacroInfo(SourceLocation L);
1075
1076  /// \brief Allocate a new MacroInfo object which is clone of \p MI.
1077  MacroInfo *CloneMacroInfo(const MacroInfo &MI);
1078
1079  /// \brief Turn the specified lexer token into a fully checked and spelled
1080  /// filename, e.g. as an operand of \#include.
1081  ///
1082  /// The caller is expected to provide a buffer that is large enough to hold
1083  /// the spelling of the filename, but is also expected to handle the case
1084  /// when this method decides to use a different buffer.
1085  ///
1086  /// \returns true if the input filename was in <>'s or false if it was
1087  /// in ""'s.
1088  bool GetIncludeFilenameSpelling(SourceLocation Loc,StringRef &Filename);
1089
1090  /// \brief Given a "foo" or \<foo> reference, look up the indicated file.
1091  ///
1092  /// Returns null on failure.  \p isAngled indicates whether the file
1093  /// reference is for system \#include's or not (i.e. using <> instead of "").
1094  const FileEntry *LookupFile(StringRef Filename,
1095                              bool isAngled, const DirectoryLookup *FromDir,
1096                              const DirectoryLookup *&CurDir,
1097                              SmallVectorImpl<char> *SearchPath,
1098                              SmallVectorImpl<char> *RelativePath,
1099                              Module **SuggestedModule,
1100                              bool SkipCache = false);
1101
1102  /// GetCurLookup - The DirectoryLookup structure used to find the current
1103  /// FileEntry, if CurLexer is non-null and if applicable.  This allows us to
1104  /// implement \#include_next and find directory-specific properties.
1105  const DirectoryLookup *GetCurDirLookup() { return CurDirLookup; }
1106
1107  /// \brief Return true if we're in the top-level file, not in a \#include.
1108  bool isInPrimaryFile() const;
1109
1110  /// ConcatenateIncludeName - Handle cases where the \#include name is expanded
1111  /// from a macro as multiple tokens, which need to be glued together.  This
1112  /// occurs for code like:
1113  /// \code
1114  ///    \#define FOO <a/b.h>
1115  ///    \#include FOO
1116  /// \endcode
1117  /// because in this case, "<a/b.h>" is returned as 7 tokens, not one.
1118  ///
1119  /// This code concatenates and consumes tokens up to the '>' token.  It
1120  /// returns false if the > was found, otherwise it returns true if it finds
1121  /// and consumes the EOD marker.
1122  bool ConcatenateIncludeName(SmallString<128> &FilenameBuffer,
1123                              SourceLocation &End);
1124
1125  /// LexOnOffSwitch - Lex an on-off-switch (C99 6.10.6p2) and verify that it is
1126  /// followed by EOD.  Return true if the token is not a valid on-off-switch.
1127  bool LexOnOffSwitch(tok::OnOffSwitch &OOS);
1128
1129private:
1130
1131  void PushIncludeMacroStack() {
1132    IncludeMacroStack.push_back(IncludeStackInfo(CurLexerKind,
1133                                                 CurLexer.take(),
1134                                                 CurPTHLexer.take(),
1135                                                 CurPPLexer,
1136                                                 CurTokenLexer.take(),
1137                                                 CurDirLookup));
1138    CurPPLexer = 0;
1139  }
1140
1141  void PopIncludeMacroStack() {
1142    CurLexer.reset(IncludeMacroStack.back().TheLexer);
1143    CurPTHLexer.reset(IncludeMacroStack.back().ThePTHLexer);
1144    CurPPLexer = IncludeMacroStack.back().ThePPLexer;
1145    CurTokenLexer.reset(IncludeMacroStack.back().TheTokenLexer);
1146    CurDirLookup  = IncludeMacroStack.back().TheDirLookup;
1147    CurLexerKind = IncludeMacroStack.back().CurLexerKind;
1148    IncludeMacroStack.pop_back();
1149  }
1150
1151  /// \brief Allocate a new MacroInfo object.
1152  MacroInfo *AllocateMacroInfo();
1153
1154  /// \brief Release the specified MacroInfo for re-use.
1155  ///
1156  /// This memory will  be reused for allocating new MacroInfo objects.
1157  void ReleaseMacroInfo(MacroInfo* MI);
1158
1159  /// ReadMacroName - Lex and validate a macro name, which occurs after a
1160  /// \#define or \#undef.  This emits a diagnostic, sets the token kind to eod,
1161  /// and discards the rest of the macro line if the macro name is invalid.
1162  void ReadMacroName(Token &MacroNameTok, char isDefineUndef = 0);
1163
1164  /// ReadMacroDefinitionArgList - The ( starting an argument list of a macro
1165  /// definition has just been read.  Lex the rest of the arguments and the
1166  /// closing ), updating MI with what we learn and saving in LastTok the
1167  /// last token read.
1168  /// Return true if an error occurs parsing the arg list.
1169  bool ReadMacroDefinitionArgList(MacroInfo *MI, Token& LastTok);
1170
1171  /// We just read a \#if or related directive and decided that the
1172  /// subsequent tokens are in the \#if'd out portion of the
1173  /// file.  Lex the rest of the file, until we see an \#endif.  If \p
1174  /// FoundNonSkipPortion is true, then we have already emitted code for part of
1175  /// this \#if directive, so \#else/\#elif blocks should never be entered. If
1176  /// \p FoundElse is false, then \#else directives are ok, if not, then we have
1177  /// already seen one so a \#else directive is a duplicate.  When this returns,
1178  /// the caller can lex the first valid token.
1179  void SkipExcludedConditionalBlock(SourceLocation IfTokenLoc,
1180                                    bool FoundNonSkipPortion, bool FoundElse,
1181                                    SourceLocation ElseLoc = SourceLocation());
1182
1183  /// \brief A fast PTH version of SkipExcludedConditionalBlock.
1184  void PTHSkipExcludedConditionalBlock();
1185
1186  /// EvaluateDirectiveExpression - Evaluate an integer constant expression that
1187  /// may occur after a #if or #elif directive and return it as a bool.  If the
1188  /// expression is equivalent to "!defined(X)" return X in IfNDefMacro.
1189  bool EvaluateDirectiveExpression(IdentifierInfo *&IfNDefMacro);
1190
1191  /// RegisterBuiltinPragmas - Install the standard preprocessor pragmas:
1192  /// \#pragma GCC poison/system_header/dependency and \#pragma once.
1193  void RegisterBuiltinPragmas();
1194
1195  /// \brief Register builtin macros such as __LINE__ with the identifier table.
1196  void RegisterBuiltinMacros();
1197
1198  /// HandleMacroExpandedIdentifier - If an identifier token is read that is to
1199  /// be expanded as a macro, handle it and return the next token as 'Tok'.  If
1200  /// the macro should not be expanded return true, otherwise return false.
1201  bool HandleMacroExpandedIdentifier(Token &Tok, MacroInfo *MI);
1202
1203  /// \brief Cache macro expanded tokens for TokenLexers.
1204  //
1205  /// Works like a stack; a TokenLexer adds the macro expanded tokens that is
1206  /// going to lex in the cache and when it finishes the tokens are removed
1207  /// from the end of the cache.
1208  Token *cacheMacroExpandedTokens(TokenLexer *tokLexer,
1209                                  ArrayRef<Token> tokens);
1210  void removeCachedMacroExpandedTokensOfLastLexer();
1211  friend void TokenLexer::ExpandFunctionArguments();
1212
1213  /// isNextPPTokenLParen - Determine whether the next preprocessor token to be
1214  /// lexed is a '('.  If so, consume the token and return true, if not, this
1215  /// method should have no observable side-effect on the lexed tokens.
1216  bool isNextPPTokenLParen();
1217
1218  /// ReadFunctionLikeMacroArgs - After reading "MACRO(", this method is
1219  /// invoked to read all of the formal arguments specified for the macro
1220  /// invocation.  This returns null on error.
1221  MacroArgs *ReadFunctionLikeMacroArgs(Token &MacroName, MacroInfo *MI,
1222                                       SourceLocation &ExpansionEnd);
1223
1224  /// ExpandBuiltinMacro - If an identifier token is read that is to be expanded
1225  /// as a builtin macro, handle it and return the next token as 'Tok'.
1226  void ExpandBuiltinMacro(Token &Tok);
1227
1228  /// Handle_Pragma - Read a _Pragma directive, slice it up, process it, then
1229  /// return the first token after the directive.  The _Pragma token has just
1230  /// been read into 'Tok'.
1231  void Handle_Pragma(Token &Tok);
1232
1233  /// HandleMicrosoft__pragma - Like Handle_Pragma except the pragma text
1234  /// is not enclosed within a string literal.
1235  void HandleMicrosoft__pragma(Token &Tok);
1236
1237  /// EnterSourceFileWithLexer - Add a lexer to the top of the include stack and
1238  /// start lexing tokens from it instead of the current buffer.
1239  void EnterSourceFileWithLexer(Lexer *TheLexer, const DirectoryLookup *Dir);
1240
1241  /// EnterSourceFileWithPTH - Add a lexer to the top of the include stack and
1242  /// start getting tokens from it using the PTH cache.
1243  void EnterSourceFileWithPTH(PTHLexer *PL, const DirectoryLookup *Dir);
1244
1245  /// IsFileLexer - Returns true if we are lexing from a file and not a
1246  ///  pragma or a macro.
1247  static bool IsFileLexer(const Lexer* L, const PreprocessorLexer* P) {
1248    return L ? !L->isPragmaLexer() : P != 0;
1249  }
1250
1251  static bool IsFileLexer(const IncludeStackInfo& I) {
1252    return IsFileLexer(I.TheLexer, I.ThePPLexer);
1253  }
1254
1255  bool IsFileLexer() const {
1256    return IsFileLexer(CurLexer.get(), CurPPLexer);
1257  }
1258
1259  //===--------------------------------------------------------------------===//
1260  // Caching stuff.
1261  void CachingLex(Token &Result);
1262  bool InCachingLexMode() const {
1263    // If the Lexer pointers are 0 and IncludeMacroStack is empty, it means
1264    // that we are past EOF, not that we are in CachingLex mode.
1265    return CurPPLexer == 0 && CurTokenLexer == 0 && CurPTHLexer == 0 &&
1266           !IncludeMacroStack.empty();
1267  }
1268  void EnterCachingLexMode();
1269  void ExitCachingLexMode() {
1270    if (InCachingLexMode())
1271      RemoveTopOfLexerStack();
1272  }
1273  const Token &PeekAhead(unsigned N);
1274  void AnnotatePreviousCachedTokens(const Token &Tok);
1275
1276  //===--------------------------------------------------------------------===//
1277  /// Handle*Directive - implement the various preprocessor directives.  These
1278  /// should side-effect the current preprocessor object so that the next call
1279  /// to Lex() will return the appropriate token next.
1280  void HandleLineDirective(Token &Tok);
1281  void HandleDigitDirective(Token &Tok);
1282  void HandleUserDiagnosticDirective(Token &Tok, bool isWarning);
1283  void HandleIdentSCCSDirective(Token &Tok);
1284  void HandleMacroPublicDirective(Token &Tok);
1285  void HandleMacroPrivateDirective(Token &Tok);
1286
1287  // File inclusion.
1288  void HandleIncludeDirective(SourceLocation HashLoc,
1289                              Token &Tok,
1290                              const DirectoryLookup *LookupFrom = 0,
1291                              bool isImport = false);
1292  void HandleIncludeNextDirective(SourceLocation HashLoc, Token &Tok);
1293  void HandleIncludeMacrosDirective(SourceLocation HashLoc, Token &Tok);
1294  void HandleImportDirective(SourceLocation HashLoc, Token &Tok);
1295  void HandleMicrosoftImportDirective(Token &Tok);
1296
1297  // Macro handling.
1298  void HandleDefineDirective(Token &Tok);
1299  void HandleUndefDirective(Token &Tok);
1300
1301  // Conditional Inclusion.
1302  void HandleIfdefDirective(Token &Tok, bool isIfndef,
1303                            bool ReadAnyTokensBeforeDirective);
1304  void HandleIfDirective(Token &Tok, bool ReadAnyTokensBeforeDirective);
1305  void HandleEndifDirective(Token &Tok);
1306  void HandleElseDirective(Token &Tok);
1307  void HandleElifDirective(Token &Tok);
1308
1309  // Pragmas.
1310  void HandlePragmaDirective(unsigned Introducer);
1311public:
1312  void HandlePragmaOnce(Token &OnceTok);
1313  void HandlePragmaMark();
1314  void HandlePragmaPoison(Token &PoisonTok);
1315  void HandlePragmaSystemHeader(Token &SysHeaderTok);
1316  void HandlePragmaDependency(Token &DependencyTok);
1317  void HandlePragmaComment(Token &CommentTok);
1318  void HandlePragmaMessage(Token &MessageTok);
1319  void HandlePragmaPushMacro(Token &Tok);
1320  void HandlePragmaPopMacro(Token &Tok);
1321  void HandlePragmaIncludeAlias(Token &Tok);
1322  IdentifierInfo *ParsePragmaPushOrPopMacro(Token &Tok);
1323
1324  // Return true and store the first token only if any CommentHandler
1325  // has inserted some tokens and getCommentRetentionState() is false.
1326  bool HandleComment(Token &Token, SourceRange Comment);
1327
1328  /// \brief A macro is used, update information about macros that need unused
1329  /// warnings.
1330  void markMacroAsUsed(MacroInfo *MI);
1331};
1332
1333/// \brief Abstract base class that describes a handler that will receive
1334/// source ranges for each of the comments encountered in the source file.
1335class CommentHandler {
1336public:
1337  virtual ~CommentHandler();
1338
1339  // The handler shall return true if it has pushed any tokens
1340  // to be read using e.g. EnterToken or EnterTokenStream.
1341  virtual bool HandleComment(Preprocessor &PP, SourceRange Comment) = 0;
1342};
1343
1344}  // end namespace clang
1345
1346#endif
1347