15f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer//===--- Preprocessor.h - C Language Family Preprocessor --------*- C++ -*-===//
25f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer//
35f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer//                     The LLVM Compiler Infrastructure
45f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer//
50bc735ffcfb223c0186419547abaa5c84482663eChris Lattner// This file is distributed under the University of Illinois Open Source
60bc735ffcfb223c0186419547abaa5c84482663eChris Lattner// License. See LICENSE.TXT for details.
75f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer//
85f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer//===----------------------------------------------------------------------===//
9651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines///
10651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines/// \file
11651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines/// \brief Defines the clang::Preprocessor interface.
12651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines///
135f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer//===----------------------------------------------------------------------===//
145f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer
155f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer#ifndef LLVM_CLANG_LEX_PREPROCESSOR_H
165f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer#define LLVM_CLANG_LEX_PREPROCESSOR_H
175f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer
181b63e4f732dbc73d90abf886b4d21f8e3a165f6dChris Lattner#include "clang/Basic/Builtins.h"
19d7a3e2c5f61cd4893f95b69a424fe4def3aa0f69Benjamin Kramer#include "clang/Basic/Diagnostic.h"
20c7229c338c21ef26b01ef3ecf9eec4fd373fa9ecChris Lattner#include "clang/Basic/IdentifierTable.h"
215f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer#include "clang/Basic/SourceLocation.h"
2230a2e16f6c27f888dd11eba6bbbae1e980078fcbChandler Carruth#include "clang/Lex/Lexer.h"
2330a2e16f6c27f888dd11eba6bbbae1e980078fcbChandler Carruth#include "clang/Lex/MacroInfo.h"
24bc3f628815b3841dc99109e7f67f9afa7793bc94Lawrence Crowl#include "clang/Lex/ModuleMap.h"
2530a2e16f6c27f888dd11eba6bbbae1e980078fcbChandler Carruth#include "clang/Lex/PPCallbacks.h"
2630a2e16f6c27f888dd11eba6bbbae1e980078fcbChandler Carruth#include "clang/Lex/PTHLexer.h"
2730a2e16f6c27f888dd11eba6bbbae1e980078fcbChandler Carruth#include "clang/Lex/PTHManager.h"
2830a2e16f6c27f888dd11eba6bbbae1e980078fcbChandler Carruth#include "clang/Lex/TokenLexer.h"
2930a2e16f6c27f888dd11eba6bbbae1e980078fcbChandler Carruth#include "llvm/ADT/ArrayRef.h"
30cc1a875f94630e58d24a55577ffbf0e89b7da8c7Chris Lattner#include "llvm/ADT/DenseMap.h"
314f32786ac45210143654390177105eb749b614e9Ted Kremenek#include "llvm/ADT/IntrusiveRefCntPtr.h"
3230a2e16f6c27f888dd11eba6bbbae1e980078fcbChandler Carruth#include "llvm/ADT/SmallPtrSet.h"
33f4d5eb4866a27d497f0bb75b12c2ffd48ad4d9c0Benjamin Kramer#include "llvm/ADT/SmallVector.h"
340ea76727ae91bca918a8414ed85b530eddcfedebTed Kremenek#include "llvm/Support/Allocator.h"
35651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines#include <memory>
362e22253e03e175144aeb9d13350a12fd83f858beDouglas Gregor#include <vector>
375f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer
388fe83e1df954d72c0f4ffc15d20a5222ec151c21Benjamin Kramernamespace llvm {
398fe83e1df954d72c0f4ffc15d20a5222ec151c21Benjamin Kramer  template<unsigned InternalLen> class SmallString;
408fe83e1df954d72c0f4ffc15d20a5222ec151c21Benjamin Kramer}
418fe83e1df954d72c0f4ffc15d20a5222ec151c21Benjamin Kramer
425f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencernamespace clang {
431eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump
445f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencerclass SourceManager;
4588a35862fbe473f2a4f0c19f24dbe536937e1dc6Douglas Gregorclass ExternalPreprocessorSource;
465f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencerclass FileManager;
475f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencerclass FileEntry;
485f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencerclass HeaderSearch;
495f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencerclass PragmaNamespace;
505f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencerclass PragmaHandler;
512e22253e03e175144aeb9d13350a12fd83f858beDouglas Gregorclass CommentHandler;
525f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencerclass ScratchBuffer;
535f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencerclass TargetInfo;
545f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencerclass PPCallbacks;
55f44e854ed1e3aa86d2ed6d615ccd109d50ddcff9Douglas Gregorclass CodeCompletionHandler;
565f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencerclass DirectoryLookup;
5794dc8f640ebea52241412512ed48601626edbc58Douglas Gregorclass PreprocessingRecord;
586aa52ec6b969faabf3764baf79d89810b8249a7eDouglas Gregorclass ModuleLoader;
5936a16498ff911a218f26c7955376bbe99ddb16dfDouglas Gregorclass PreprocessorOptions;
60ba243b59a1074e0962f6abfa3bb9aa984eac1245David Blaikie
611952354bd376062c3ab3d328c0fc6c36530c9309Dmitri Gribenko/// \brief Stores token information for comparing actual tokens with
621952354bd376062c3ab3d328c0fc6c36530c9309Dmitri Gribenko/// predefined values.  Only handles simple tokens and identifiers.
631952354bd376062c3ab3d328c0fc6c36530c9309Dmitri Gribenkoclass TokenValue {
641952354bd376062c3ab3d328c0fc6c36530c9309Dmitri Gribenko  tok::TokenKind Kind;
651952354bd376062c3ab3d328c0fc6c36530c9309Dmitri Gribenko  IdentifierInfo *II;
661952354bd376062c3ab3d328c0fc6c36530c9309Dmitri Gribenko
671952354bd376062c3ab3d328c0fc6c36530c9309Dmitri Gribenkopublic:
686bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines  TokenValue(tok::TokenKind Kind) : Kind(Kind), II(nullptr) {
691952354bd376062c3ab3d328c0fc6c36530c9309Dmitri Gribenko    assert(Kind != tok::raw_identifier && "Raw identifiers are not supported.");
701952354bd376062c3ab3d328c0fc6c36530c9309Dmitri Gribenko    assert(Kind != tok::identifier &&
711952354bd376062c3ab3d328c0fc6c36530c9309Dmitri Gribenko           "Identifiers should be created by TokenValue(IdentifierInfo *)");
721952354bd376062c3ab3d328c0fc6c36530c9309Dmitri Gribenko    assert(!tok::isLiteral(Kind) && "Literals are not supported.");
731952354bd376062c3ab3d328c0fc6c36530c9309Dmitri Gribenko    assert(!tok::isAnnotation(Kind) && "Annotations are not supported.");
741952354bd376062c3ab3d328c0fc6c36530c9309Dmitri Gribenko  }
751952354bd376062c3ab3d328c0fc6c36530c9309Dmitri Gribenko  TokenValue(IdentifierInfo *II) : Kind(tok::identifier), II(II) {}
761952354bd376062c3ab3d328c0fc6c36530c9309Dmitri Gribenko  bool operator==(const Token &Tok) const {
771952354bd376062c3ab3d328c0fc6c36530c9309Dmitri Gribenko    return Tok.getKind() == Kind &&
781952354bd376062c3ab3d328c0fc6c36530c9309Dmitri Gribenko        (!II || II == Tok.getIdentifierInfo());
791952354bd376062c3ab3d328c0fc6c36530c9309Dmitri Gribenko  }
801952354bd376062c3ab3d328c0fc6c36530c9309Dmitri Gribenko};
811952354bd376062c3ab3d328c0fc6c36530c9309Dmitri Gribenko
82651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines/// \brief Engages in a tight little dance with the lexer to efficiently
83651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines/// preprocess tokens.
845f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer///
85651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines/// Lexers know only about tokens within a single source file, and don't
86651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines/// know anything about preprocessor-level issues like the \#include stack,
87651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines/// token expansion, etc.
88c93dc7889644293e318e19d82830ea2acc45b678Dylan Noblesmithclass Preprocessor : public RefCountedBase<Preprocessor> {
89cfa88f893915ceb8ae4ce2f17c46c24a4d67502fDmitri Gribenko  IntrusiveRefCntPtr<PreprocessorOptions> PPOpts;
90d7a3e2c5f61cd4893f95b69a424fe4def3aa0f69Benjamin Kramer  DiagnosticsEngine        *Diags;
914e4d08403ca5cfd4d558fa2936215d3a4e5a528dDavid Blaikie  LangOptions       &LangOpts;
92998b3d3e8528ebd9d2c5d78d3a82edd90a8953a4Douglas Gregor  const TargetInfo  *Target;
935f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer  FileManager       &FileMgr;
945f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer  SourceManager     &SourceMgr;
955f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer  ScratchBuffer     *ScratchBuf;
965f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer  HeaderSearch      &HeaderInfo;
976aa52ec6b969faabf3764baf79d89810b8249a7eDouglas Gregor  ModuleLoader      &TheModuleLoader;
981eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump
9988a35862fbe473f2a4f0c19f24dbe536937e1dc6Douglas Gregor  /// \brief External source of macros.
10088a35862fbe473f2a4f0c19f24dbe536937e1dc6Douglas Gregor  ExternalPreprocessorSource *ExternalSource;
101a5d10c4df435964600e104ebef6a96b106e416b7Kovarththanan Rajaratnam
102ba243b59a1074e0962f6abfa3bb9aa984eac1245David Blaikie
103651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// An optional PTHManager object used for getting tokens from
104651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// a token cache rather than lexing the original source file.
105651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  std::unique_ptr<PTHManager> PTH;
1061eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump
107651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// A BumpPtrAllocator object used to quickly allocate and release
108651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// objects internal to the Preprocessor.
1090ea76727ae91bca918a8414ed85b530eddcfedebTed Kremenek  llvm::BumpPtrAllocator BP;
1101eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump
1115f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer  /// Identifiers for builtin macros and other builtins.
1125f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer  IdentifierInfo *Ident__LINE__, *Ident__FILE__;   // __LINE__, __FILE__
1135f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer  IdentifierInfo *Ident__DATE__, *Ident__TIME__;   // __DATE__, __TIME__
1145f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer  IdentifierInfo *Ident__INCLUDE_LEVEL__;          // __INCLUDE_LEVEL__
1155f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer  IdentifierInfo *Ident__BASE_FILE__;              // __BASE_FILE__
1165f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer  IdentifierInfo *Ident__TIMESTAMP__;              // __TIMESTAMP__
117c1f9d828c733ec1eba06d01070735d1f36fda733Chris Lattner  IdentifierInfo *Ident__COUNTER__;                // __COUNTER__
1181ef8a2e7675f3d8b6e8d9963b00378086e1dcdc7John McCall  IdentifierInfo *Ident_Pragma, *Ident__pragma;    // _Pragma, __pragma
119651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  IdentifierInfo *Ident__identifier;               // __identifier
1201ef8a2e7675f3d8b6e8d9963b00378086e1dcdc7John McCall  IdentifierInfo *Ident__VA_ARGS__;                // __VA_ARGS__
121148772a841cae6f32db16d890e788b92a763bb3fChris Lattner  IdentifierInfo *Ident__has_feature;              // __has_feature
122c1b5fa41f09512c74030b9a2a0d1564535e22a76Peter Collingbourne  IdentifierInfo *Ident__has_extension;            // __has_extension
123148772a841cae6f32db16d890e788b92a763bb3fChris Lattner  IdentifierInfo *Ident__has_builtin;              // __has_builtin
124cae5095c116c29a4e52c91fa0ad88df09dd2b45fAnders Carlsson  IdentifierInfo *Ident__has_attribute;            // __has_attribute
12592bd8c70a6837b647a6c55964f8d0a50bf561dbcJohn Thompson  IdentifierInfo *Ident__has_include;              // __has_include
12692bd8c70a6837b647a6c55964f8d0a50bf561dbcJohn Thompson  IdentifierInfo *Ident__has_include_next;         // __has_include_next
127d768150ef57f617c8d9fef48f3c92e8f21698024Ted Kremenek  IdentifierInfo *Ident__has_warning;              // __has_warning
1286bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines  IdentifierInfo *Ident__is_identifier;            // __is_identifier
129b09de5177ee8101818a59dcd0038c75b190a2509Douglas Gregor  IdentifierInfo *Ident__building_module;          // __building_module
130b09de5177ee8101818a59dcd0038c75b190a2509Douglas Gregor  IdentifierInfo *Ident__MODULE__;                 // __MODULE__
1311eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump
1325f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer  SourceLocation DATELoc, TIMELoc;
133c1f9d828c733ec1eba06d01070735d1f36fda733Chris Lattner  unsigned CounterValue;  // Next __COUNTER__ value.
1345f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer
1355f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer  enum {
136651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines    /// \brief Maximum depth of \#includes.
1375f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer    MaxAllowedIncludeStackDepth = 200
1385f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer  };
1395f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer
1405f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer  // State that is set before the preprocessor begins.
1415f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer  bool KeepComments : 1;
1425f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer  bool KeepMacroComments : 1;
143f84139a1331c63c998e8b7d54148c75ac0b48ccdEli Friedman  bool SuppressIncludeNotFoundError : 1;
144ba243b59a1074e0962f6abfa3bb9aa984eac1245David Blaikie
1455f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer  // State that changes while the preprocessor runs:
1465f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer  bool InMacroArgs : 1;            // True if parsing fn macro invocation args.
1475f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer
1485814e657c9ad9ef6049a2a4af0d2aad248a8a15cDaniel Dunbar  /// Whether the preprocessor owns the header search object.
1495814e657c9ad9ef6049a2a4af0d2aad248a8a15cDaniel Dunbar  bool OwnsHeaderSearch : 1;
1505814e657c9ad9ef6049a2a4af0d2aad248a8a15cDaniel Dunbar
151651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// True if macro expansion is disabled.
1521d9c54df56391ac4740db27d551782e81189cb51Chris Lattner  bool DisableMacroExpansion : 1;
1531d9c54df56391ac4740db27d551782e81189cb51Chris Lattner
154651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// Temporarily disables DisableMacroExpansion (i.e. enables expansion)
155651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// when parsing preprocessor directives.
1568c0b3787e7ccc7978b42dfbb84da2b802c743a5dDavid Blaikie  bool MacroExpansionInDirectivesOverride : 1;
1578c0b3787e7ccc7978b42dfbb84da2b802c743a5dDavid Blaikie
1588c0b3787e7ccc7978b42dfbb84da2b802c743a5dDavid Blaikie  class ResetMacroExpansionHelper;
1598c0b3787e7ccc7978b42dfbb84da2b802c743a5dDavid Blaikie
16088a35862fbe473f2a4f0c19f24dbe536937e1dc6Douglas Gregor  /// \brief Whether we have already loaded macros from the external source.
16188a35862fbe473f2a4f0c19f24dbe536937e1dc6Douglas Gregor  mutable bool ReadMacrosFromExternalSource : 1;
162a5d10c4df435964600e104ebef6a96b106e416b7Kovarththanan Rajaratnam
1636fe6a49c4058211ff4489023c78615ec0266c5ffJordan Rose  /// \brief True if pragmas are enabled.
1646fe6a49c4058211ff4489023c78615ec0266c5ffJordan Rose  bool PragmasEnabled : 1;
1656fe6a49c4058211ff4489023c78615ec0266c5ffJordan Rose
16674c2498bb9e339345ee32bdd095e76157cec3b86Jordan Rose  /// \brief True if the current build action is a preprocessing action.
16774c2498bb9e339345ee32bdd095e76157cec3b86Jordan Rose  bool PreprocessedOutput : 1;
16874c2498bb9e339345ee32bdd095e76157cec3b86Jordan Rose
16931672b1745195d709b641f1c0f44e203742fa73bAaron Ballman  /// \brief True if we are currently preprocessing a #if or #elif directive
17031672b1745195d709b641f1c0f44e203742fa73bAaron Ballman  bool ParsingIfOrElifDirective;
17131672b1745195d709b641f1c0f44e203742fa73bAaron Ballman
17214e645557ae91c6770d62beb00a1c522e0bfd5d6Argyrios Kyrtzidis  /// \brief True if we are pre-expanding macro arguments.
17314e645557ae91c6770d62beb00a1c522e0bfd5d6Argyrios Kyrtzidis  bool InMacroArgPreExpansion;
17414e645557ae91c6770d62beb00a1c522e0bfd5d6Argyrios Kyrtzidis
175651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// \brief Mapping/lookup information for all identifiers in
1765f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer  /// the program, including program keywords.
177c3222091e1ffa35d0264ca6b680a88c9dc84ede2Daniel Dunbar  mutable IdentifierTable Identifiers;
1781eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump
179651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// \brief This table contains all the selectors in the program.
180651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  ///
181651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// Unlike IdentifierTable above, this table *isn't* populated by the
182651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// preprocessor. It is declared/expanded here because its role/lifetime is
183651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// conceptually similar to the IdentifierTable. In addition, the current
184651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// control flow (in clang::ParseAST()), make it convenient to put here.
185651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  ///
18668d331a78e655d97294e94fcfa63f92cc1f40578Steve Naroff  /// FIXME: Make sure the lifetime of Identifiers/Selectors *isn't* tied to
187083abdf67f157e9d2ab5a8c9d5e71240479d3c99Sebastian Redl  /// the lifetime of the preprocessor.
18829238a0bf7cbf5b396efb451a0adb5fe4aa037caSteve Naroff  SelectorTable Selectors;
1892e1cd4264d363ca869bf37ef160902f211d21b8cDouglas Gregor
190651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// \brief Information about builtins.
1911b63e4f732dbc73d90abf886b4d21f8e3a165f6dChris Lattner  Builtin::Context BuiltinInfo;
1921eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump
193651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// \brief Tracks all of the pragmas that the client registered
1945f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer  /// with this preprocessor.
1955f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer  PragmaNamespace *PragmaHandlers;
1961eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump
1971eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump  /// \brief Tracks all of the comment handlers that the client registered
1982e22253e03e175144aeb9d13350a12fd83f858beDouglas Gregor  /// with this preprocessor.
1992e22253e03e175144aeb9d13350a12fd83f858beDouglas Gregor  std::vector<CommentHandler *> CommentHandlers;
2001eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump
201e55329d6834647ba0e06f8a319e5d84c77310035Axel Naumann  /// \brief True if we want to ignore EOF token and continue later on (thus
202e55329d6834647ba0e06f8a319e5d84c77310035Axel Naumann  /// avoid tearing the Lexer and etc. down).
203e55329d6834647ba0e06f8a319e5d84c77310035Axel Naumann  bool IncrementalProcessing;
204e55329d6834647ba0e06f8a319e5d84c77310035Axel Naumann
205651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// The kind of translation unit we are processing.
206651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  TranslationUnitKind TUKind;
207651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines
208f44e854ed1e3aa86d2ed6d615ccd109d50ddcff9Douglas Gregor  /// \brief The code-completion handler.
209f44e854ed1e3aa86d2ed6d615ccd109d50ddcff9Douglas Gregor  CodeCompletionHandler *CodeComplete;
210ba243b59a1074e0962f6abfa3bb9aa984eac1245David Blaikie
2112968442603b029949246467253eeac8139a5b6d8Douglas Gregor  /// \brief The file that we're performing code-completion for, if any.
2122968442603b029949246467253eeac8139a5b6d8Douglas Gregor  const FileEntry *CodeCompletionFile;
2132968442603b029949246467253eeac8139a5b6d8Douglas Gregor
2147d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis  /// \brief The offset in file for the code-completion point.
2157d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis  unsigned CodeCompletionOffset;
2167d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis
2177d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis  /// \brief The location for the code-completion point. This gets instantiated
21858bf98725b6d4588338e191d2ab981b104471dabJames Dennett  /// when the CodeCompletionFile gets \#include'ed for preprocessing.
2197d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis  SourceLocation CodeCompletionLoc;
2207d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis
2217d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis  /// \brief The start location for the file of the code-completion point.
22258bf98725b6d4588338e191d2ab981b104471dabJames Dennett  ///
22358bf98725b6d4588338e191d2ab981b104471dabJames Dennett  /// This gets instantiated when the CodeCompletionFile gets \#include'ed
2247d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis  /// for preprocessing.
2257d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis  SourceLocation CodeCompletionFileLoc;
2267d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis
227651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// \brief The source location of the \c import contextual keyword we just
228c13a34b690d2dc2a03c2fea75a0a1438636c19ceDouglas Gregor  /// lexed, if any.
229b8db7cd9ac05c522855631670ec2e97255384f5aDouglas Gregor  SourceLocation ModuleImportLoc;
2308dfac0baaf0f81d3945bcb306480e358ba8d1f08John McCall
231b514c792821a8f053027d88444e13bfaa8efef76Douglas Gregor  /// \brief The module import path that we're currently processing.
232cfa88f893915ceb8ae4ce2f17c46c24a4d67502fDmitri Gribenko  SmallVector<std::pair<IdentifierInfo *, SourceLocation>, 2> ModuleImportPath;
233a5ba7b405dff48a5a0a63fff32c99eedb044842bDouglas Gregor
234a5ba7b405dff48a5a0a63fff32c99eedb044842bDouglas Gregor  /// \brief Whether the last token we lexed was an '@'.
235a5ba7b405dff48a5a0a63fff32c99eedb044842bDouglas Gregor  bool LastTokenWasAt;
236a5ba7b405dff48a5a0a63fff32c99eedb044842bDouglas Gregor
237651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// \brief Whether the module import expects an identifier next. Otherwise,
238b514c792821a8f053027d88444e13bfaa8efef76Douglas Gregor  /// it expects a '.' or ';'.
239b514c792821a8f053027d88444e13bfaa8efef76Douglas Gregor  bool ModuleImportExpectsIdentifier;
240b514c792821a8f053027d88444e13bfaa8efef76Douglas Gregor
2418dfac0baaf0f81d3945bcb306480e358ba8d1f08John McCall  /// \brief The source location of the currently-active
242651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// \#pragma clang arc_cf_code_audited begin.
2438dfac0baaf0f81d3945bcb306480e358ba8d1f08John McCall  SourceLocation PragmaARCCFCodeAuditedLoc;
244ba243b59a1074e0962f6abfa3bb9aa984eac1245David Blaikie
2457d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis  /// \brief True if we hit the code-completion point.
2467d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis  bool CodeCompletionReached;
2477d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis
248f4f6c9db68465b886ec2e596feaa6ecc782395a4Douglas Gregor  /// \brief The number of bytes that we will initially skip when entering the
249651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// main file, along with a flag that indicates whether skipping this number
250651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// of bytes will place the lexer at the start of a line.
251651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  ///
252651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// This is used when loading a precompiled preamble.
253f4f6c9db68465b886ec2e596feaa6ecc782395a4Douglas Gregor  std::pair<unsigned, bool> SkipMainFilePreamble;
254ba243b59a1074e0962f6abfa3bb9aa984eac1245David Blaikie
255651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// \brief The current top of the stack that we're lexing from if
2569c1b750c59d510e6c9eccb1f37bccc46ccfe6844Ted Kremenek  /// not expanding a macro and we are lexing directly from source code.
257651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  ///
258651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// Only one of CurLexer, CurPTHLexer, or CurTokenLexer will be non-null.
259651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  std::unique_ptr<Lexer> CurLexer;
2601eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump
261651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// \brief The current top of stack that we're lexing from if
262651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// not expanding from a macro and we are lexing from a PTH cache.
263651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  ///
264651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// Only one of CurLexer, CurPTHLexer, or CurTokenLexer will be non-null.
265651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  std::unique_ptr<PTHLexer> CurPTHLexer;
2661eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump
267651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// \brief The current top of the stack what we're lexing from
268651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// if not expanding a macro.
269651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  ///
270651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// This is an alias for either CurLexer or  CurPTHLexer.
27123f77e59718385512984d4e2a021bef52b9f6ddfChris Lattner  PreprocessorLexer *CurPPLexer;
272ba243b59a1074e0962f6abfa3bb9aa984eac1245David Blaikie
273651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// \brief Used to find the current FileEntry, if CurLexer is non-null
274651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// and if applicable.
275651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  ///
276651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// This allows us to implement \#include_next and find directory-specific
277651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// properties.
2785f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer  const DirectoryLookup *CurDirLookup;
2795f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer
280651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// \brief The current macro we are expanding, if we are expanding a macro.
281651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  ///
282651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// One of CurLexer and CurTokenLexer must be null.
283651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  std::unique_ptr<TokenLexer> CurTokenLexer;
2841eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump
285b8db7cd9ac05c522855631670ec2e97255384f5aDouglas Gregor  /// \brief The kind of lexer we're currently working with.
286ba243b59a1074e0962f6abfa3bb9aa984eac1245David Blaikie  enum CurLexerKind {
287ba243b59a1074e0962f6abfa3bb9aa984eac1245David Blaikie    CLK_Lexer,
288ba243b59a1074e0962f6abfa3bb9aa984eac1245David Blaikie    CLK_PTHLexer,
289ba243b59a1074e0962f6abfa3bb9aa984eac1245David Blaikie    CLK_TokenLexer,
290b8db7cd9ac05c522855631670ec2e97255384f5aDouglas Gregor    CLK_CachingLexer,
291b8db7cd9ac05c522855631670ec2e97255384f5aDouglas Gregor    CLK_LexAfterModuleImport
292b8db7cd9ac05c522855631670ec2e97255384f5aDouglas Gregor  } CurLexerKind;
293b8db7cd9ac05c522855631670ec2e97255384f5aDouglas Gregor
294651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// \brief If the current lexer is for a submodule that is being built, this
295651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// is that submodule.
296651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  Module *CurSubmodule;
297651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines
298651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// \brief Keeps track of the stack of files currently
29958bf98725b6d4588338e191d2ab981b104471dabJames Dennett  /// \#included, and macros currently being expanded from, not counting
3006cfe7594a46b5d270142cfcb688a9c1a3a487a48Chris Lattner  /// CurLexer/CurTokenLexer.
3015f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer  struct IncludeStackInfo {
302651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines    enum CurLexerKind           CurLexerKind;
303651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines    Module                     *TheSubmodule;
304651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines    std::unique_ptr<Lexer>      TheLexer;
305651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines    std::unique_ptr<PTHLexer>   ThePTHLexer;
306651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines    PreprocessorLexer          *ThePPLexer;
307651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines    std::unique_ptr<TokenLexer> TheTokenLexer;
308651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines    const DirectoryLookup      *TheDirLookup;
309651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines
310651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines    // The following constructors are completely useless copies of the default
311651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines    // versions, only needed to pacify MSVC.
312651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines    IncludeStackInfo(enum CurLexerKind CurLexerKind, Module *TheSubmodule,
313651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines                     std::unique_ptr<Lexer> &&TheLexer,
314651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines                     std::unique_ptr<PTHLexer> &&ThePTHLexer,
315651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines                     PreprocessorLexer *ThePPLexer,
316651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines                     std::unique_ptr<TokenLexer> &&TheTokenLexer,
317651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines                     const DirectoryLookup *TheDirLookup)
318651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines        : CurLexerKind(std::move(CurLexerKind)),
319651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines          TheSubmodule(std::move(TheSubmodule)), TheLexer(std::move(TheLexer)),
320651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines          ThePTHLexer(std::move(ThePTHLexer)),
321651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines          ThePPLexer(std::move(ThePPLexer)),
322651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines          TheTokenLexer(std::move(TheTokenLexer)),
323651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines          TheDirLookup(std::move(TheDirLookup)) {}
324651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines    IncludeStackInfo(IncludeStackInfo &&RHS)
325651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines        : CurLexerKind(std::move(RHS.CurLexerKind)),
326651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines          TheSubmodule(std::move(RHS.TheSubmodule)),
327651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines          TheLexer(std::move(RHS.TheLexer)),
328651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines          ThePTHLexer(std::move(RHS.ThePTHLexer)),
329651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines          ThePPLexer(std::move(RHS.ThePPLexer)),
330651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines          TheTokenLexer(std::move(RHS.TheTokenLexer)),
331651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines          TheDirLookup(std::move(RHS.TheDirLookup)) {}
3325f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer  };
3335f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer  std::vector<IncludeStackInfo> IncludeMacroStack;
3341eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump
335651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// \brief Actions invoked when some preprocessor activity is
33658bf98725b6d4588338e191d2ab981b104471dabJames Dennett  /// encountered (e.g. a file is \#included, etc).
3375f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer  PPCallbacks *Callbacks;
3381eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump
33966c44e700fb3f244b2c443bfbd6a05b5f1843ec0Argyrios Kyrtzidis  struct MacroExpandsInfo {
34066c44e700fb3f244b2c443bfbd6a05b5f1843ec0Argyrios Kyrtzidis    Token Tok;
341c515978bd3a703aa733f846a0094ffa84d149074Argyrios Kyrtzidis    MacroDirective *MD;
34266c44e700fb3f244b2c443bfbd6a05b5f1843ec0Argyrios Kyrtzidis    SourceRange Range;
343c515978bd3a703aa733f846a0094ffa84d149074Argyrios Kyrtzidis    MacroExpandsInfo(Token Tok, MacroDirective *MD, SourceRange Range)
344c515978bd3a703aa733f846a0094ffa84d149074Argyrios Kyrtzidis      : Tok(Tok), MD(MD), Range(Range) { }
34566c44e700fb3f244b2c443bfbd6a05b5f1843ec0Argyrios Kyrtzidis  };
34666c44e700fb3f244b2c443bfbd6a05b5f1843ec0Argyrios Kyrtzidis  SmallVector<MacroExpandsInfo, 2> DelayedMacroExpandsCallbacks;
34766c44e700fb3f244b2c443bfbd6a05b5f1843ec0Argyrios Kyrtzidis
348651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// For each IdentifierInfo that was associated with a macro, we
3498a64bb58c3b24d7d97895e435bbc0965c99bd3beAlexander Kornienko  /// keep a mapping to the history of all macro definitions and #undefs in
3508a64bb58c3b24d7d97895e435bbc0965c99bd3beAlexander Kornienko  /// the reverse order (the latest one is in the head of the list).
3519818a1d443e97677dd3422305de9cc2b1fb2a8c1Argyrios Kyrtzidis  llvm::DenseMap<const IdentifierInfo*, MacroDirective*> Macros;
352c12906ee78e006df6a71acf52ca9dd3171ddbf07Douglas Gregor  friend class ASTReader;
353c12906ee78e006df6a71acf52ca9dd3171ddbf07Douglas Gregor
3540827408865e32789e0ec4b8113a302ccdc531423Argyrios Kyrtzidis  /// \brief Macros that we want to warn because they are not used at the end
355651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// of the translation unit.
356651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  ///
357651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// We store just their SourceLocations instead of
3580827408865e32789e0ec4b8113a302ccdc531423Argyrios Kyrtzidis  /// something like MacroInfo*. The benefit of this is that when we are
3590827408865e32789e0ec4b8113a302ccdc531423Argyrios Kyrtzidis  /// deserializing from PCH, we don't need to deserialize identifier & macros
3600827408865e32789e0ec4b8113a302ccdc531423Argyrios Kyrtzidis  /// just so that we can report that they are unused, we just warn using
3610827408865e32789e0ec4b8113a302ccdc531423Argyrios Kyrtzidis  /// the SourceLocations of this set (that will be filled by the ASTReader).
3620827408865e32789e0ec4b8113a302ccdc531423Argyrios Kyrtzidis  /// We are using SmallPtrSet instead of a vector for faster removal.
3630827408865e32789e0ec4b8113a302ccdc531423Argyrios Kyrtzidis  typedef llvm::SmallPtrSet<SourceLocation, 32> WarnUnusedMacroLocsTy;
3640827408865e32789e0ec4b8113a302ccdc531423Argyrios Kyrtzidis  WarnUnusedMacroLocsTy WarnUnusedMacroLocs;
3650827408865e32789e0ec4b8113a302ccdc531423Argyrios Kyrtzidis
366651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// \brief A "freelist" of MacroArg objects that can be
36723f77e59718385512984d4e2a021bef52b9f6ddfChris Lattner  /// reused for quick allocation.
36823f77e59718385512984d4e2a021bef52b9f6ddfChris Lattner  MacroArgs *MacroArgCache;
36923f77e59718385512984d4e2a021bef52b9f6ddfChris Lattner  friend class MacroArgs;
370ba243b59a1074e0962f6abfa3bb9aa984eac1245David Blaikie
371651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// For each IdentifierInfo used in a \#pragma push_macro directive,
372651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// we keep a MacroInfo stack used to restore the previous macro value.
373f47724bf78299c7a50f008e0443c5f9f9f279ddcChris Lattner  llvm::DenseMap<IdentifierInfo*, std::vector<MacroInfo*> > PragmaPushMacroInfo;
3741eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump
3755f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer  // Various statistics we track for performance analysis.
3766bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines  unsigned NumDirectives, NumDefined, NumUndefined, NumPragma;
3775f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer  unsigned NumIf, NumElse, NumEndif;
3785f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer  unsigned NumEnteredSourceFiles, MaxIncludeStackDepth;
3795f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer  unsigned NumMacroExpanded, NumFnMacroExpanded, NumBuiltinMacroExpanded;
3805f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer  unsigned NumFastMacroExpanded, NumTokenPaste, NumFastTokenPaste;
3815f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer  unsigned NumSkipped;
3821eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump
383651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// \brief The predefined macros that preprocessor should use from the
384651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// command line etc.
385aa39197431a0a0b1326ecf6b3be6a11f6e2f8503Chris Lattner  std::string Predefines;
3861eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump
38795d912c994832333a998bc7d4ae0e8e830a2d38bArgyrios Kyrtzidis  /// \brief The file ID for the preprocessor predefines.
38895d912c994832333a998bc7d4ae0e8e830a2d38bArgyrios Kyrtzidis  FileID PredefinesFileID;
38995d912c994832333a998bc7d4ae0e8e830a2d38bArgyrios Kyrtzidis
390651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// \{
391651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// \brief Cache of macro expanders to reduce malloc traffic.
3926cfe7594a46b5d270142cfcb688a9c1a3a487a48Chris Lattner  enum { TokenLexerCacheSize = 8 };
3936cfe7594a46b5d270142cfcb688a9c1a3a487a48Chris Lattner  unsigned NumCachedTokenLexers;
3946cfe7594a46b5d270142cfcb688a9c1a3a487a48Chris Lattner  TokenLexer *TokenLexerCache[TokenLexerCacheSize];
395651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// \}
3969e0ed0bd5a3a7bac73973980ff32132a7724e674Argyrios Kyrtzidis
3975b3284a9275a27f5c4410e25eb8933be540601d6Argyrios Kyrtzidis  /// \brief Keeps macro expanded tokens for TokenLexers.
3985b3284a9275a27f5c4410e25eb8933be540601d6Argyrios Kyrtzidis  //
3995b3284a9275a27f5c4410e25eb8933be540601d6Argyrios Kyrtzidis  /// Works like a stack; a TokenLexer adds the macro expanded tokens that is
4005b3284a9275a27f5c4410e25eb8933be540601d6Argyrios Kyrtzidis  /// going to lex in the cache and when it finishes the tokens are removed
4015b3284a9275a27f5c4410e25eb8933be540601d6Argyrios Kyrtzidis  /// from the end of the cache.
402686775deca8b8685eb90801495880e3abdd844c2Chris Lattner  SmallVector<Token, 16> MacroExpandedTokens;
4035b3284a9275a27f5c4410e25eb8933be540601d6Argyrios Kyrtzidis  std::vector<std::pair<TokenLexer *, size_t> > MacroExpandingLexersStack;
4045b3284a9275a27f5c4410e25eb8933be540601d6Argyrios Kyrtzidis
4059e5bb85ac899eeab7c21b5ff9030c3da6ff4837bChandler Carruth  /// \brief A record of the macro definitions and expansions that
406ba243b59a1074e0962f6abfa3bb9aa984eac1245David Blaikie  /// occurred during preprocessing.
40794dc8f640ebea52241412512ed48601626edbc58Douglas Gregor  ///
40894dc8f640ebea52241412512ed48601626edbc58Douglas Gregor  /// This is an optional side structure that can be enabled with
40994dc8f640ebea52241412512ed48601626edbc58Douglas Gregor  /// \c createPreprocessingRecord() prior to preprocessing.
410b9e1b75772db2c7db566c6034ba90a07f22e35ebDouglas Gregor  PreprocessingRecord *Record;
411ba243b59a1074e0962f6abfa3bb9aa984eac1245David Blaikie
412e671e1bc73615eda155059a772266ed2882d758cChris Lattnerprivate:  // Cached tokens state.
413686775deca8b8685eb90801495880e3abdd844c2Chris Lattner  typedef SmallVector<Token, 1> CachedTokensTy;
41403db1b31dd926409b7defc1c90b66549464652c0Argyrios Kyrtzidis
415651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// \brief Cached tokens are stored here when we do backtracking or
41603db1b31dd926409b7defc1c90b66549464652c0Argyrios Kyrtzidis  /// lookahead. They are "lexed" by the CachingLex() method.
41703db1b31dd926409b7defc1c90b66549464652c0Argyrios Kyrtzidis  CachedTokensTy CachedTokens;
41803db1b31dd926409b7defc1c90b66549464652c0Argyrios Kyrtzidis
419651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// \brief The position of the cached token that CachingLex() should
420651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// "lex" next.
421651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  ///
422651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// If it points beyond the CachedTokens vector, it means that a normal
423651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// Lex() should be invoked.
42403db1b31dd926409b7defc1c90b66549464652c0Argyrios Kyrtzidis  CachedTokensTy::size_type CachedLexPos;
42503db1b31dd926409b7defc1c90b66549464652c0Argyrios Kyrtzidis
426651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// \brief Stack of backtrack positions, allowing nested backtracks.
427651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  ///
428651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// The EnableBacktrackAtThisPos() method pushes a position to
429a9e274c01ebae45629d93aaa07be450fb77dd3cbArgyrios Kyrtzidis  /// indicate where CachedLexPos should be set when the BackTrack() method is
430a9e274c01ebae45629d93aaa07be450fb77dd3cbArgyrios Kyrtzidis  /// invoked (at which point the last position is popped).
431a9e274c01ebae45629d93aaa07be450fb77dd3cbArgyrios Kyrtzidis  std::vector<CachedTokensTy::size_type> BacktrackPositions;
43203db1b31dd926409b7defc1c90b66549464652c0Argyrios Kyrtzidis
433af8fa25c0d4e0540952a50bbd06dc1558954ccd9Ted Kremenek  struct MacroInfoChain {
434af8fa25c0d4e0540952a50bbd06dc1558954ccd9Ted Kremenek    MacroInfo MI;
435af8fa25c0d4e0540952a50bbd06dc1558954ccd9Ted Kremenek    MacroInfoChain *Next;
4369714a2385cb66b6efa373fc668641de602dd9adbTed Kremenek    MacroInfoChain *Prev;
437af8fa25c0d4e0540952a50bbd06dc1558954ccd9Ted Kremenek  };
438af8fa25c0d4e0540952a50bbd06dc1558954ccd9Ted Kremenek
439af8fa25c0d4e0540952a50bbd06dc1558954ccd9Ted Kremenek  /// MacroInfos are managed as a chain for easy disposal.  This is the head
440af8fa25c0d4e0540952a50bbd06dc1558954ccd9Ted Kremenek  /// of that list.
441af8fa25c0d4e0540952a50bbd06dc1558954ccd9Ted Kremenek  MacroInfoChain *MIChainHead;
442af8fa25c0d4e0540952a50bbd06dc1558954ccd9Ted Kremenek
443651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// A "freelist" of MacroInfo objects that can be reused for quick
4449714a2385cb66b6efa373fc668641de602dd9adbTed Kremenek  /// allocation.
4459714a2385cb66b6efa373fc668641de602dd9adbTed Kremenek  MacroInfoChain *MICache;
4469714a2385cb66b6efa373fc668641de602dd9adbTed Kremenek
4473e25b990f1e2ba2a9a63dde5fd111492a7f6194aArgyrios Kyrtzidis  struct DeserializedMacroInfoChain {
4483e25b990f1e2ba2a9a63dde5fd111492a7f6194aArgyrios Kyrtzidis    MacroInfo MI;
4493e25b990f1e2ba2a9a63dde5fd111492a7f6194aArgyrios Kyrtzidis    unsigned OwningModuleID; // MUST be immediately after the MacroInfo object
4503e25b990f1e2ba2a9a63dde5fd111492a7f6194aArgyrios Kyrtzidis                     // so it can be accessed by MacroInfo::getOwningModuleID().
4513e25b990f1e2ba2a9a63dde5fd111492a7f6194aArgyrios Kyrtzidis    DeserializedMacroInfoChain *Next;
4523e25b990f1e2ba2a9a63dde5fd111492a7f6194aArgyrios Kyrtzidis  };
4533e25b990f1e2ba2a9a63dde5fd111492a7f6194aArgyrios Kyrtzidis  DeserializedMacroInfoChain *DeserialMIChainHead;
4543e25b990f1e2ba2a9a63dde5fd111492a7f6194aArgyrios Kyrtzidis
4555f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencerpublic:
456cfa88f893915ceb8ae4ce2f17c46c24a4d67502fDmitri Gribenko  Preprocessor(IntrusiveRefCntPtr<PreprocessorOptions> PPOpts,
45736a16498ff911a218f26c7955376bbe99ddb16dfDouglas Gregor               DiagnosticsEngine &diags, LangOptions &opts,
45872b1b15ee88aac0a63e2c1dc53fe22f5ab297b20Ted Kremenek               SourceManager &SM, HeaderSearch &Headers,
4596aa52ec6b969faabf3764baf79d89810b8249a7eDouglas Gregor               ModuleLoader &TheModuleLoader,
4606bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines               IdentifierInfoLookup *IILookup = nullptr,
461998b3d3e8528ebd9d2c5d78d3a82edd90a8953a4Douglas Gregor               bool OwnsHeaderSearch = false,
462651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines               TranslationUnitKind TUKind = TU_Complete);
46372b1b15ee88aac0a63e2c1dc53fe22f5ab297b20Ted Kremenek
4645f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer  ~Preprocessor();
4655f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer
4666bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines  /// \brief Initialize the preprocessor using information about the target.
467998b3d3e8528ebd9d2c5d78d3a82edd90a8953a4Douglas Gregor  ///
4686bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines  /// \param Target is owned by the caller and must remain valid for the
4696bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines  /// lifetime of the preprocessor.
470998b3d3e8528ebd9d2c5d78d3a82edd90a8953a4Douglas Gregor  void Initialize(const TargetInfo &Target);
471ba243b59a1074e0962f6abfa3bb9aa984eac1245David Blaikie
472a71a7d8a1ce4474e7bdb680658fb58b6caf391d3Douglas Gregor  /// \brief Retrieve the preprocessor options used to initialize this
473a71a7d8a1ce4474e7bdb680658fb58b6caf391d3Douglas Gregor  /// preprocessor.
474a71a7d8a1ce4474e7bdb680658fb58b6caf391d3Douglas Gregor  PreprocessorOptions &getPreprocessorOpts() const { return *PPOpts; }
475a71a7d8a1ce4474e7bdb680658fb58b6caf391d3Douglas Gregor
476d6471f7c1921c7802804ce3ff6fe9768310f72b9David Blaikie  DiagnosticsEngine &getDiagnostics() const { return *Diags; }
477d6471f7c1921c7802804ce3ff6fe9768310f72b9David Blaikie  void setDiagnostics(DiagnosticsEngine &D) { Diags = &D; }
4787c175fb196a2bc3dbc86ea3865c713e1875f3f6dChris Lattner
4794e4d08403ca5cfd4d558fa2936215d3a4e5a528dDavid Blaikie  const LangOptions &getLangOpts() const { return LangOpts; }
480998b3d3e8528ebd9d2c5d78d3a82edd90a8953a4Douglas Gregor  const TargetInfo &getTargetInfo() const { return *Target; }
4815f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer  FileManager &getFileManager() const { return FileMgr; }
4825f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer  SourceManager &getSourceManager() const { return SourceMgr; }
4835f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer  HeaderSearch &getHeaderSearchInfo() const { return HeaderInfo; }
4845f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer
4855f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer  IdentifierTable &getIdentifierTable() { return Identifiers; }
48629238a0bf7cbf5b396efb451a0adb5fe4aa037caSteve Naroff  SelectorTable &getSelectorTable() { return Selectors; }
4871b63e4f732dbc73d90abf886b4d21f8e3a165f6dChris Lattner  Builtin::Context &getBuiltinInfo() { return BuiltinInfo; }
48832fca722dd974b8202d0fb9c71b6c185c0767da6Chris Lattner  llvm::BumpPtrAllocator &getPreprocessorAllocator() { return BP; }
4891eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump
490337edcdbec05316b407d0d64865c88ff8597d910Ted Kremenek  void setPTHManager(PTHManager* pm);
4911eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump
492880dcf21dfdb3ff763c60195b6794bab0d913095Ted Kremenek  PTHManager *getPTHManager() { return PTH.get(); }
4932e1cd4264d363ca869bf37ef160902f211d21b8cDouglas Gregor
49488a35862fbe473f2a4f0c19f24dbe536937e1dc6Douglas Gregor  void setExternalSource(ExternalPreprocessorSource *Source) {
49588a35862fbe473f2a4f0c19f24dbe536937e1dc6Douglas Gregor    ExternalSource = Source;
49688a35862fbe473f2a4f0c19f24dbe536937e1dc6Douglas Gregor  }
497a5d10c4df435964600e104ebef6a96b106e416b7Kovarththanan Rajaratnam
49888a35862fbe473f2a4f0c19f24dbe536937e1dc6Douglas Gregor  ExternalPreprocessorSource *getExternalSource() const {
49988a35862fbe473f2a4f0c19f24dbe536937e1dc6Douglas Gregor    return ExternalSource;
50088a35862fbe473f2a4f0c19f24dbe536937e1dc6Douglas Gregor  }
501a5d10c4df435964600e104ebef6a96b106e416b7Kovarththanan Rajaratnam
5026aa52ec6b969faabf3764baf79d89810b8249a7eDouglas Gregor  /// \brief Retrieve the module loader associated with this preprocessor.
5036aa52ec6b969faabf3764baf79d89810b8249a7eDouglas Gregor  ModuleLoader &getModuleLoader() const { return TheModuleLoader; }
504ba243b59a1074e0962f6abfa3bb9aa984eac1245David Blaikie
5053b7deda7137e62810a810ce25b062927a9fc7c71Argyrios Kyrtzidis  bool hadModuleLoaderFatalFailure() const {
5063b7deda7137e62810a810ce25b062927a9fc7c71Argyrios Kyrtzidis    return TheModuleLoader.HadFatalFailure;
5073b7deda7137e62810a810ce25b062927a9fc7c71Argyrios Kyrtzidis  }
5083b7deda7137e62810a810ce25b062927a9fc7c71Argyrios Kyrtzidis
50931672b1745195d709b641f1c0f44e203742fa73bAaron Ballman  /// \brief True if we are currently preprocessing a #if or #elif directive
51031672b1745195d709b641f1c0f44e203742fa73bAaron Ballman  bool isParsingIfOrElifDirective() const {
51131672b1745195d709b641f1c0f44e203742fa73bAaron Ballman    return ParsingIfOrElifDirective;
51231672b1745195d709b641f1c0f44e203742fa73bAaron Ballman  }
51331672b1745195d709b641f1c0f44e203742fa73bAaron Ballman
514651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// \brief Control whether the preprocessor retains comments in output.
5155f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer  void SetCommentRetentionState(bool KeepComments, bool KeepMacroComments) {
5165f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer    this->KeepComments = KeepComments | KeepMacroComments;
5175f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer    this->KeepMacroComments = KeepMacroComments;
5185f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer  }
5191eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump
5205f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer  bool getCommentRetentionState() const { return KeepComments; }
5211eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump
5226fe6a49c4058211ff4489023c78615ec0266c5ffJordan Rose  void setPragmasEnabled(bool Enabled) { PragmasEnabled = Enabled; }
5236fe6a49c4058211ff4489023c78615ec0266c5ffJordan Rose  bool getPragmasEnabled() const { return PragmasEnabled; }
5246fe6a49c4058211ff4489023c78615ec0266c5ffJordan Rose
525f84139a1331c63c998e8b7d54148c75ac0b48ccdEli Friedman  void SetSuppressIncludeNotFoundError(bool Suppress) {
526f84139a1331c63c998e8b7d54148c75ac0b48ccdEli Friedman    SuppressIncludeNotFoundError = Suppress;
527f84139a1331c63c998e8b7d54148c75ac0b48ccdEli Friedman  }
528f84139a1331c63c998e8b7d54148c75ac0b48ccdEli Friedman
529f84139a1331c63c998e8b7d54148c75ac0b48ccdEli Friedman  bool GetSuppressIncludeNotFoundError() {
530f84139a1331c63c998e8b7d54148c75ac0b48ccdEli Friedman    return SuppressIncludeNotFoundError;
531f84139a1331c63c998e8b7d54148c75ac0b48ccdEli Friedman  }
532f84139a1331c63c998e8b7d54148c75ac0b48ccdEli Friedman
53374c2498bb9e339345ee32bdd095e76157cec3b86Jordan Rose  /// Sets whether the preprocessor is responsible for producing output or if
53474c2498bb9e339345ee32bdd095e76157cec3b86Jordan Rose  /// it is producing tokens to be consumed by Parse and Sema.
53574c2498bb9e339345ee32bdd095e76157cec3b86Jordan Rose  void setPreprocessedOutput(bool IsPreprocessedOutput) {
53674c2498bb9e339345ee32bdd095e76157cec3b86Jordan Rose    PreprocessedOutput = IsPreprocessedOutput;
53774c2498bb9e339345ee32bdd095e76157cec3b86Jordan Rose  }
53874c2498bb9e339345ee32bdd095e76157cec3b86Jordan Rose
53974c2498bb9e339345ee32bdd095e76157cec3b86Jordan Rose  /// Returns true if the preprocessor is responsible for generating output,
54074c2498bb9e339345ee32bdd095e76157cec3b86Jordan Rose  /// false if it is producing tokens to be consumed by Parse and Sema.
54174c2498bb9e339345ee32bdd095e76157cec3b86Jordan Rose  bool isPreprocessedOutput() const { return PreprocessedOutput; }
54274c2498bb9e339345ee32bdd095e76157cec3b86Jordan Rose
543651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// \brief Return true if we are lexing directly from the specified lexer.
5447062d9e9fc3ed89c4e4ea7055efea6585b7bac8dTed Kremenek  bool isCurrentLexer(const PreprocessorLexer *L) const {
5457062d9e9fc3ed89c4e4ea7055efea6585b7bac8dTed Kremenek    return CurPPLexer == L;
5465f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer  }
5471eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump
548651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// \brief Return the current lexer being lexed from.
549651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  ///
550651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// Note that this ignores any potentially active macro expansions and _Pragma
551a28cc090f265d5bdaef9ebb24fb9d579928d3a18John Thompson  /// expansions going on at the time.
552a28cc090f265d5bdaef9ebb24fb9d579928d3a18John Thompson  PreprocessorLexer *getCurrentLexer() const { return CurPPLexer; }
553a28cc090f265d5bdaef9ebb24fb9d579928d3a18John Thompson
554651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// \brief Return the current file lexer being lexed from.
555651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  ///
556321c2688198f6f1a4086964e505f239683423cecChris Lattner  /// Note that this ignores any potentially active macro expansions and _Pragma
5575f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer  /// expansions going on at the time.
55868e48e4a81767997ef0231e47eca4f665102c95eTed Kremenek  PreprocessorLexer *getCurrentFileLexer() const;
5591eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump
560651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// \brief Returns the FileID for the preprocessor predefines.
5619484c1dcf084452228aae439edcd00e1b4ce7470Argyrios Kyrtzidis  FileID getPredefinesFileID() const { return PredefinesFileID; }
56295d912c994832333a998bc7d4ae0e8e830a2d38bArgyrios Kyrtzidis
563651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// \{
564651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// \brief Accessors for preprocessor callbacks.
565651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  ///
566eb50ed88c2aa040fac08bf2a50bde4dd3da6eb19Chris Lattner  /// Note that this class takes ownership of any PPCallbacks object given to
567eb50ed88c2aa040fac08bf2a50bde4dd3da6eb19Chris Lattner  /// it.
5685f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer  PPCallbacks *getPPCallbacks() const { return Callbacks; }
569a5d10c4df435964600e104ebef6a96b106e416b7Kovarththanan Rajaratnam  void addPPCallbacks(PPCallbacks *C) {
570e5393fb93eb879d9ebbef102ae9311fa77e023ccDaniel Dunbar    if (Callbacks)
571e5393fb93eb879d9ebbef102ae9311fa77e023ccDaniel Dunbar      C = new PPChainedCallbacks(C, Callbacks);
5725f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer    Callbacks = C;
5735f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer  }
574651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// \}
5751eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump
576c56fff7fd231aebf4b152f60f8f11ef91835c48aArgyrios Kyrtzidis  /// \brief Given an identifier, return its latest MacroDirective if it is
577651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// \#defined or null if it isn't \#define'd.
5789818a1d443e97677dd3422305de9cc2b1fb2a8c1Argyrios Kyrtzidis  MacroDirective *getMacroDirective(IdentifierInfo *II) const {
579295a2a617ac335f590e430ab7fcd98f8ce109251Douglas Gregor    if (!II->hasMacroDefinition())
5806bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines      return nullptr;
581ba243b59a1074e0962f6abfa3bb9aa984eac1245David Blaikie
5829818a1d443e97677dd3422305de9cc2b1fb2a8c1Argyrios Kyrtzidis    MacroDirective *MD = getMacroDirectiveHistory(II);
583c56fff7fd231aebf4b152f60f8f11ef91835c48aArgyrios Kyrtzidis    assert(MD->isDefined() && "Macro is undefined!");
5849818a1d443e97677dd3422305de9cc2b1fb2a8c1Argyrios Kyrtzidis    return MD;
5859818a1d443e97677dd3422305de9cc2b1fb2a8c1Argyrios Kyrtzidis  }
5869818a1d443e97677dd3422305de9cc2b1fb2a8c1Argyrios Kyrtzidis
5879818a1d443e97677dd3422305de9cc2b1fb2a8c1Argyrios Kyrtzidis  const MacroInfo *getMacroInfo(IdentifierInfo *II) const {
5889818a1d443e97677dd3422305de9cc2b1fb2a8c1Argyrios Kyrtzidis    return const_cast<Preprocessor*>(this)->getMacroInfo(II);
5899818a1d443e97677dd3422305de9cc2b1fb2a8c1Argyrios Kyrtzidis  }
5909818a1d443e97677dd3422305de9cc2b1fb2a8c1Argyrios Kyrtzidis
5919818a1d443e97677dd3422305de9cc2b1fb2a8c1Argyrios Kyrtzidis  MacroInfo *getMacroInfo(IdentifierInfo *II) {
5929818a1d443e97677dd3422305de9cc2b1fb2a8c1Argyrios Kyrtzidis    if (MacroDirective *MD = getMacroDirective(II))
593c56fff7fd231aebf4b152f60f8f11ef91835c48aArgyrios Kyrtzidis      return MD->getMacroInfo();
5946bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines    return nullptr;
595cc1a875f94630e58d24a55577ffbf0e89b7da8c7Chris Lattner  }
5961eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump
5974d7e0ced7f16a04aabe2d8d91cbbb52fb1162810Alexander Kornienko  /// \brief Given an identifier, return the (probably #undef'd) MacroInfo
598651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// representing the most recent macro definition.
599651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  ///
600651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// One can iterate over all previous macro definitions from the most recent
601651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// one. This should only be called for identifiers that hadMacroDefinition().
6029818a1d443e97677dd3422305de9cc2b1fb2a8c1Argyrios Kyrtzidis  MacroDirective *getMacroDirectiveHistory(const IdentifierInfo *II) const;
6034d7e0ced7f16a04aabe2d8d91cbbb52fb1162810Alexander Kornienko
604c56fff7fd231aebf4b152f60f8f11ef91835c48aArgyrios Kyrtzidis  /// \brief Add a directive to the macro directive history for this identifier.
605c56fff7fd231aebf4b152f60f8f11ef91835c48aArgyrios Kyrtzidis  void appendMacroDirective(IdentifierInfo *II, MacroDirective *MD);
606c56fff7fd231aebf4b152f60f8f11ef91835c48aArgyrios Kyrtzidis  DefMacroDirective *appendDefMacroDirective(IdentifierInfo *II, MacroInfo *MI,
607c56fff7fd231aebf4b152f60f8f11ef91835c48aArgyrios Kyrtzidis                                             SourceLocation Loc,
608c56fff7fd231aebf4b152f60f8f11ef91835c48aArgyrios Kyrtzidis                                             bool isImported) {
609c56fff7fd231aebf4b152f60f8f11ef91835c48aArgyrios Kyrtzidis    DefMacroDirective *MD = AllocateDefMacroDirective(MI, Loc, isImported);
610c56fff7fd231aebf4b152f60f8f11ef91835c48aArgyrios Kyrtzidis    appendMacroDirective(II, MD);
6119317ab94bb68122ba6fc728eb73c1308fb913cd1Argyrios Kyrtzidis    return MD;
6129317ab94bb68122ba6fc728eb73c1308fb913cd1Argyrios Kyrtzidis  }
613c56fff7fd231aebf4b152f60f8f11ef91835c48aArgyrios Kyrtzidis  DefMacroDirective *appendDefMacroDirective(IdentifierInfo *II, MacroInfo *MI){
614c56fff7fd231aebf4b152f60f8f11ef91835c48aArgyrios Kyrtzidis    return appendDefMacroDirective(II, MI, MI->getDefinitionLoc(), false);
6159818a1d443e97677dd3422305de9cc2b1fb2a8c1Argyrios Kyrtzidis  }
6169317ab94bb68122ba6fc728eb73c1308fb913cd1Argyrios Kyrtzidis  /// \brief Set a MacroDirective that was loaded from a PCH file.
6179317ab94bb68122ba6fc728eb73c1308fb913cd1Argyrios Kyrtzidis  void setLoadedMacroDirective(IdentifierInfo *II, MacroDirective *MD);
6181eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump
619651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// \{
620651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// Iterators for the macro history table. Currently defined macros have
6218a64bb58c3b24d7d97895e435bbc0965c99bd3beAlexander Kornienko  /// IdentifierInfo::hasMacroDefinition() set and an empty
6228a64bb58c3b24d7d97895e435bbc0965c99bd3beAlexander Kornienko  /// MacroInfo::getUndefLoc() at the head of the list.
623b3958476b21702a15f64f09d2862506d03dffe7bDmitri Gribenko  typedef llvm::DenseMap<const IdentifierInfo *,
6249818a1d443e97677dd3422305de9cc2b1fb2a8c1Argyrios Kyrtzidis                         MacroDirective*>::const_iterator macro_iterator;
62588a35862fbe473f2a4f0c19f24dbe536937e1dc6Douglas Gregor  macro_iterator macro_begin(bool IncludeExternalMacros = true) const;
62688a35862fbe473f2a4f0c19f24dbe536937e1dc6Douglas Gregor  macro_iterator macro_end(bool IncludeExternalMacros = true) const;
627651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// \}
628a5d10c4df435964600e104ebef6a96b106e416b7Kovarththanan Rajaratnam
6291952354bd376062c3ab3d328c0fc6c36530c9309Dmitri Gribenko  /// \brief Return the name of the macro defined before \p Loc that has
6301952354bd376062c3ab3d328c0fc6c36530c9309Dmitri Gribenko  /// spelling \p Tokens.  If there are multiple macros with same spelling,
6311952354bd376062c3ab3d328c0fc6c36530c9309Dmitri Gribenko  /// return the last one defined.
6321952354bd376062c3ab3d328c0fc6c36530c9309Dmitri Gribenko  StringRef getLastMacroWithSpelling(SourceLocation Loc,
6331952354bd376062c3ab3d328c0fc6c36530c9309Dmitri Gribenko                                     ArrayRef<TokenValue> Tokens) const;
6341952354bd376062c3ab3d328c0fc6c36530c9309Dmitri Gribenko
635e3d5e3ae5bd8028774f07d7c3751d4db82118942Chris Lattner  const std::string &getPredefines() const { return Predefines; }
636651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// \brief Set the predefines for this Preprocessor.
637651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  ///
638651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// These predefines are automatically injected when parsing the main file.
639aa39197431a0a0b1326ecf6b3be6a11f6e2f8503Chris Lattner  void setPredefines(const char *P) { Predefines = P; }
640aa39197431a0a0b1326ecf6b3be6a11f6e2f8503Chris Lattner  void setPredefines(const std::string &P) { Predefines = P; }
6411eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump
64203e67fbf7a2646447972742b6bb97c82ce4698fbNico Weber  /// Return information about the specified preprocessor
64303e67fbf7a2646447972742b6bb97c82ce4698fbNico Weber  /// identifier token.
644686775deca8b8685eb90801495880e3abdd844c2Chris Lattner  IdentifierInfo *getIdentifierInfo(StringRef Name) const {
6453da736c1143126be19b253804b3b135ebcd3d6ffDaniel Dunbar    return &Identifiers.get(Name);
6465f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer  }
6471eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump
648651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// \brief Add the specified pragma handler to this preprocessor.
649651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  ///
650651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// If \p Namespace is non-null, then it is a token required to exist on the
6515f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer  /// pragma line before the pragma string starts, e.g. "STDC" or "GCC".
652686775deca8b8685eb90801495880e3abdd844c2Chris Lattner  void AddPragmaHandler(StringRef Namespace, PragmaHandler *Handler);
6539b36c3f0de0105e903130bbda3c4aea7d792c0afArgyrios Kyrtzidis  void AddPragmaHandler(PragmaHandler *Handler) {
654686775deca8b8685eb90801495880e3abdd844c2Chris Lattner    AddPragmaHandler(StringRef(), Handler);
6559b36c3f0de0105e903130bbda3c4aea7d792c0afArgyrios Kyrtzidis  }
6565f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer
657651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// \brief Remove the specific pragma handler from this preprocessor.
658651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  ///
659651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// If \p Namespace is non-null, then it should be the namespace that
660651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// \p Handler was added to. It is an error to remove a handler that
661651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// has not been registered.
662686775deca8b8685eb90801495880e3abdd844c2Chris Lattner  void RemovePragmaHandler(StringRef Namespace, PragmaHandler *Handler);
6639b36c3f0de0105e903130bbda3c4aea7d792c0afArgyrios Kyrtzidis  void RemovePragmaHandler(PragmaHandler *Handler) {
664686775deca8b8685eb90801495880e3abdd844c2Chris Lattner    RemovePragmaHandler(StringRef(), Handler);
6659b36c3f0de0105e903130bbda3c4aea7d792c0afArgyrios Kyrtzidis  }
6664095080aff204008eefb26b100906c6ca2bc4bb6Daniel Dunbar
6676bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines  /// Install empty handlers for all pragmas (making them ignored).
6686bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines  void IgnorePragmas();
6696bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines
6702e22253e03e175144aeb9d13350a12fd83f858beDouglas Gregor  /// \brief Add the specified comment handler to the preprocessor.
671aa0cd85838f2a024e589ea4e8c2094130065af21Dmitri Gribenko  void addCommentHandler(CommentHandler *Handler);
6721eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump
6732e22253e03e175144aeb9d13350a12fd83f858beDouglas Gregor  /// \brief Remove the specified comment handler.
6742e22253e03e175144aeb9d13350a12fd83f858beDouglas Gregor  ///
6752e22253e03e175144aeb9d13350a12fd83f858beDouglas Gregor  /// It is an error to remove a handler that has not been registered.
676aa0cd85838f2a024e589ea4e8c2094130065af21Dmitri Gribenko  void removeCommentHandler(CommentHandler *Handler);
6771eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump
678f44e854ed1e3aa86d2ed6d615ccd109d50ddcff9Douglas Gregor  /// \brief Set the code completion handler to the given object.
679f44e854ed1e3aa86d2ed6d615ccd109d50ddcff9Douglas Gregor  void setCodeCompletionHandler(CodeCompletionHandler &Handler) {
680f44e854ed1e3aa86d2ed6d615ccd109d50ddcff9Douglas Gregor    CodeComplete = &Handler;
681f44e854ed1e3aa86d2ed6d615ccd109d50ddcff9Douglas Gregor  }
682ba243b59a1074e0962f6abfa3bb9aa984eac1245David Blaikie
6831fbb447e9d43c2c676e94081fbfee7eb6cbe933bDouglas Gregor  /// \brief Retrieve the current code-completion handler.
6841fbb447e9d43c2c676e94081fbfee7eb6cbe933bDouglas Gregor  CodeCompletionHandler *getCodeCompletionHandler() const {
6851fbb447e9d43c2c676e94081fbfee7eb6cbe933bDouglas Gregor    return CodeComplete;
6861fbb447e9d43c2c676e94081fbfee7eb6cbe933bDouglas Gregor  }
687ba243b59a1074e0962f6abfa3bb9aa984eac1245David Blaikie
688f44e854ed1e3aa86d2ed6d615ccd109d50ddcff9Douglas Gregor  /// \brief Clear out the code completion handler.
689f44e854ed1e3aa86d2ed6d615ccd109d50ddcff9Douglas Gregor  void clearCodeCompletionHandler() {
6906bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines    CodeComplete = nullptr;
691f44e854ed1e3aa86d2ed6d615ccd109d50ddcff9Douglas Gregor  }
692ba243b59a1074e0962f6abfa3bb9aa984eac1245David Blaikie
69355817afdf9d453a443262a733f6caf6692dca118Douglas Gregor  /// \brief Hook used by the lexer to invoke the "natural language" code
69455817afdf9d453a443262a733f6caf6692dca118Douglas Gregor  /// completion point.
69555817afdf9d453a443262a733f6caf6692dca118Douglas Gregor  void CodeCompleteNaturalLanguage();
696ba243b59a1074e0962f6abfa3bb9aa984eac1245David Blaikie
69794dc8f640ebea52241412512ed48601626edbc58Douglas Gregor  /// \brief Retrieve the preprocessing record, or NULL if there is no
69894dc8f640ebea52241412512ed48601626edbc58Douglas Gregor  /// preprocessing record.
699b9e1b75772db2c7db566c6034ba90a07f22e35ebDouglas Gregor  PreprocessingRecord *getPreprocessingRecord() const { return Record; }
700ba243b59a1074e0962f6abfa3bb9aa984eac1245David Blaikie
701ba243b59a1074e0962f6abfa3bb9aa984eac1245David Blaikie  /// \brief Create a new preprocessing record, which will keep track of
70294dc8f640ebea52241412512ed48601626edbc58Douglas Gregor  /// all macro expansions, macro definitions, etc.
70337ed12720a35b7bfa1c4de73ad6f1c6c1c88ee17Argyrios Kyrtzidis  void createPreprocessingRecord();
704ba243b59a1074e0962f6abfa3bb9aa984eac1245David Blaikie
705651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// \brief Enter the specified FileID as the main source file,
7067dcc968f17a6ff9088c9651dddccc8d4025a1271Ted Kremenek  /// which implicitly adds the builtin defines etc.
707e127a0d80155b45dafe77f2b4380e5fa111a3345Chris Lattner  void EnterMainSourceFile();
7081eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump
709651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// \brief Inform the preprocessor callbacks that processing is complete.
710dbd8209b33e6c9f151e4913a9c095d64a95439c4Daniel Dunbar  void EndSourceFile();
711dbd8209b33e6c9f151e4913a9c095d64a95439c4Daniel Dunbar
712651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// \brief Add a source file to the top of the include stack and
713651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// start lexing tokens from it instead of the current buffer.
714651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  ///
715651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// Emits a diagnostic, doesn't enter the file, and returns true on error.
716651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  bool EnterSourceFile(FileID CurFileID, const DirectoryLookup *Dir,
717e127a0d80155b45dafe77f2b4380e5fa111a3345Chris Lattner                       SourceLocation Loc);
7185f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer
719651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// \brief Add a Macro to the top of the include stack and start lexing
720651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// tokens from it instead of the current buffer.
721e7fb48466afcbf2c4ccdfa658824282fdc3c512cChris Lattner  ///
722651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// \param Args specifies the tokens input to a function-like macro.
723651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// \param ILEnd specifies the location of the ')' for a function-like macro
724651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// or the identifier for an object-like macro.
725c30981a563a8947cb26b1e308d122fa2ef90fcebRichard Smith  void EnterMacro(Token &Identifier, SourceLocation ILEnd, MacroInfo *Macro,
726c30981a563a8947cb26b1e308d122fa2ef90fcebRichard Smith                  MacroArgs *Args);
7271eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump
728651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// \brief Add a "macro" context to the top of the include stack,
7296b884508c3bc97cc9df9516adb92fbf88dd0a2e4Chris Lattner  /// which will cause the lexer to start returning the specified tokens.
7306b884508c3bc97cc9df9516adb92fbf88dd0a2e4Chris Lattner  ///
731651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// If \p DisableMacroExpansion is true, tokens lexed from the token stream
732651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// will not be subject to further macro expansion. Otherwise, these tokens
733651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// will be re-macro-expanded when/if expansion is enabled.
7346b884508c3bc97cc9df9516adb92fbf88dd0a2e4Chris Lattner  ///
735651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// If \p OwnsTokens is false, this method assumes that the specified stream
736651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// of tokens has a permanent owner somewhere, so they do not need to be
737651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// copied. If it is true, it assumes the array of tokens is allocated with
738651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// \c new[] and must be freed.
7396b884508c3bc97cc9df9516adb92fbf88dd0a2e4Chris Lattner  void EnterTokenStream(const Token *Toks, unsigned NumToks,
7406b884508c3bc97cc9df9516adb92fbf88dd0a2e4Chris Lattner                        bool DisableMacroExpansion, bool OwnsTokens);
7411eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump
742651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// \brief Pop the current lexer/macro exp off the top of the lexer stack.
743651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  ///
744651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// This should only be used in situations where the current state of the
745651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// top-of-stack lexer is known.
7465f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer  void RemoveTopOfLexerStack();
74703db1b31dd926409b7defc1c90b66549464652c0Argyrios Kyrtzidis
748651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// From the point that this method is called, and until
749651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines  /// CommitBacktrackedTokens() or Backtrack() is called, the Preprocessor
750ed5c38682c056c147c8a4abb748b4f285de206ddArgyrios Kyrtzidis  /// keeps track of the lexed tokens so that a subsequent Backtrack() call will
751ed5c38682c056c147c8a4abb748b4f285de206ddArgyrios Kyrtzidis  /// make the Preprocessor re-lex the same tokens.
75203db1b31dd926409b7defc1c90b66549464652c0Argyrios Kyrtzidis  ///
753a9e274c01ebae45629d93aaa07be450fb77dd3cbArgyrios Kyrtzidis  /// Nested backtracks are allowed, meaning that EnableBacktrackAtThisPos can
754ed5c38682c056c147c8a4abb748b4f285de206ddArgyrios Kyrtzidis  /// be called multiple times and CommitBacktrackedTokens/Backtrack calls will
755ed5c38682c056c147c8a4abb748b4f285de206ddArgyrios Kyrtzidis  /// be combined with the EnableBacktrackAtThisPos calls in reverse order.
75603db1b31dd926409b7defc1c90b66549464652c0Argyrios Kyrtzidis  ///
757ed5c38682c056c147c8a4abb748b4f285de206ddArgyrios Kyrtzidis  /// NOTE: *DO NOT* forget to call either CommitBacktrackedTokens or Backtrack
758ed5c38682c056c147c8a4abb748b4f285de206ddArgyrios Kyrtzidis  /// at some point after EnableBacktrackAtThisPos. If you don't, caching of
759ed5c38682c056c147c8a4abb748b4f285de206ddArgyrios Kyrtzidis  /// tokens will continue indefinitely.
76003db1b31dd926409b7defc1c90b66549464652c0Argyrios Kyrtzidis  ///
761a9e274c01ebae45629d93aaa07be450fb77dd3cbArgyrios Kyrtzidis  void EnableBacktrackAtThisPos();
762