TokenLexer.h revision d60a34a4e514ec0dfddd05ef2744be104e111f45
11543e9c69202b8e128c7fe34784ae7aa90964889Chris Lattner//===--- TokenLexer.h - Lex from a token buffer -----------------*- C++ -*-===// 25f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// 35f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// The LLVM Compiler Infrastructure 45f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// 50bc735ffcfb223c0186419547abaa5c84482663eChris Lattner// This file is distributed under the University of Illinois Open Source 60bc735ffcfb223c0186419547abaa5c84482663eChris Lattner// License. See LICENSE.TXT for details. 75f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// 85f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer//===----------------------------------------------------------------------===// 95f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// 10e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner// This file defines the TokenLexer interface. 115f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// 125f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer//===----------------------------------------------------------------------===// 135f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 141543e9c69202b8e128c7fe34784ae7aa90964889Chris Lattner#ifndef LLVM_CLANG_TOKENLEXER_H 151543e9c69202b8e128c7fe34784ae7aa90964889Chris Lattner#define LLVM_CLANG_TOKENLEXER_H 165f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 175f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer#include "clang/Basic/SourceLocation.h" 185f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 195f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencernamespace clang { 205f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer class MacroInfo; 215f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer class Preprocessor; 22d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner class Token; 23e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner class MacroArgs; 241eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 251543e9c69202b8e128c7fe34784ae7aa90964889Chris Lattner/// TokenLexer - This implements a lexer that returns token from a macro body 265d75de0f821023f4ed4815825bf3aea8a0b5e40dChris Lattner/// or token stream instead of lexing from a character buffer. This is used for 275d75de0f821023f4ed4815825bf3aea8a0b5e40dChris Lattner/// macro expansion and _Pragma handling, for example. 285f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// 291543e9c69202b8e128c7fe34784ae7aa90964889Chris Lattnerclass TokenLexer { 305f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer /// Macro - The macro we are expanding from. This is null if expanding a 315f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer /// token stream. 325f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer /// 335f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer MacroInfo *Macro; 345f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 355f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer /// ActualArgs - The actual arguments specified for a function-like macro, or 361543e9c69202b8e128c7fe34784ae7aa90964889Chris Lattner /// null. The TokenLexer owns the pointed-to object. 375f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer MacroArgs *ActualArgs; 385f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 395f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer /// PP - The current preprocessor object we are expanding for. 405f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer /// 415f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Preprocessor &PP; 425f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 438d896438031f5990cb42eb95a7bae233db2eb333Chris Lattner /// Tokens - This is the pointer to an array of tokens that the macro is 445f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer /// defined to, with arguments expanded for function-like macros. If this is 4532fca722dd974b8202d0fb9c71b6c185c0767da6Chris Lattner /// a token stream, these are the tokens we are returning. This points into 465b3284a9275a27f5c4410e25eb8933be540601d6Argyrios Kyrtzidis /// the macro definition we are lexing from, a cache buffer that is owned by 475b3284a9275a27f5c4410e25eb8933be540601d6Argyrios Kyrtzidis /// the preprocessor, or some other buffer that we may or may not own 485b3284a9275a27f5c4410e25eb8933be540601d6Argyrios Kyrtzidis /// (depending on OwnsTokens). 495b3284a9275a27f5c4410e25eb8933be540601d6Argyrios Kyrtzidis /// Note that if it points into Preprocessor's cache buffer, the Preprocessor 505b3284a9275a27f5c4410e25eb8933be540601d6Argyrios Kyrtzidis /// may update the pointer as needed. 518d896438031f5990cb42eb95a7bae233db2eb333Chris Lattner const Token *Tokens; 525b3284a9275a27f5c4410e25eb8933be540601d6Argyrios Kyrtzidis friend class Preprocessor; 531eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 548d896438031f5990cb42eb95a7bae233db2eb333Chris Lattner /// NumTokens - This is the length of the Tokens array. 555f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer /// 568d896438031f5990cb42eb95a7bae233db2eb333Chris Lattner unsigned NumTokens; 571eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 585f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer /// CurToken - This is the next token that Lex will return. 595f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer /// 605f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer unsigned CurToken; 611eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 62d0f5e443ceed37e1a72015bd109db8e5fc6206cfChandler Carruth /// ExpandLocStart/End - The source location range where this macro was 63d0f5e443ceed37e1a72015bd109db8e5fc6206cfChandler Carruth /// expanded. 64d0f5e443ceed37e1a72015bd109db8e5fc6206cfChandler Carruth SourceLocation ExpandLocStart, ExpandLocEnd; 651eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 66b73377eeb3eff76be134203aebb6068244b177f3Argyrios Kyrtzidis /// \brief Source location pointing at the source location entry chunk that 67d0f5e443ceed37e1a72015bd109db8e5fc6206cfChandler Carruth /// was reserved for the current macro expansion. 68b73377eeb3eff76be134203aebb6068244b177f3Argyrios Kyrtzidis SourceLocation MacroExpansionStart; 69b73377eeb3eff76be134203aebb6068244b177f3Argyrios Kyrtzidis 70d0f5e443ceed37e1a72015bd109db8e5fc6206cfChandler Carruth /// \brief The offset of the macro expansion in the 71b73377eeb3eff76be134203aebb6068244b177f3Argyrios Kyrtzidis /// "source location address space". 72b73377eeb3eff76be134203aebb6068244b177f3Argyrios Kyrtzidis unsigned MacroStartSLocOffset; 73b73377eeb3eff76be134203aebb6068244b177f3Argyrios Kyrtzidis 74b73377eeb3eff76be134203aebb6068244b177f3Argyrios Kyrtzidis /// \brief FileID/offset of the start of the macro definition. 75b73377eeb3eff76be134203aebb6068244b177f3Argyrios Kyrtzidis std::pair<FileID, unsigned> MacroDefStartInfo; 76b73377eeb3eff76be134203aebb6068244b177f3Argyrios Kyrtzidis 775f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer /// Lexical information about the expansion point of the macro: the identifier 785f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer /// that the macro expanded from had these properties. 799c683062752a26014197df1c8792a19efa9e93cfChris Lattner bool AtStartOfLine : 1; 809c683062752a26014197df1c8792a19efa9e93cfChris Lattner bool HasLeadingSpace : 1; 811eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 821543e9c69202b8e128c7fe34784ae7aa90964889Chris Lattner /// OwnsTokens - This is true if this TokenLexer allocated the Tokens 838d896438031f5990cb42eb95a7bae233db2eb333Chris Lattner /// array, and thus needs to free it when destroyed. For simple object-like 848d896438031f5990cb42eb95a7bae233db2eb333Chris Lattner /// macros (for example) we just point into the token buffer of the macro 858d896438031f5990cb42eb95a7bae233db2eb333Chris Lattner /// definition, we don't make a copy of it. 868d896438031f5990cb42eb95a7bae233db2eb333Chris Lattner bool OwnsTokens : 1; 871eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 886b884508c3bc97cc9df9516adb92fbf88dd0a2e4Chris Lattner /// DisableMacroExpansion - This is true when tokens lexed from the TokenLexer 896b884508c3bc97cc9df9516adb92fbf88dd0a2e4Chris Lattner /// should not be subject to further macro expansion. 906b884508c3bc97cc9df9516adb92fbf88dd0a2e4Chris Lattner bool DisableMacroExpansion : 1; 911eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 921543e9c69202b8e128c7fe34784ae7aa90964889Chris Lattner TokenLexer(const TokenLexer&); // DO NOT IMPLEMENT 931543e9c69202b8e128c7fe34784ae7aa90964889Chris Lattner void operator=(const TokenLexer&); // DO NOT IMPLEMENT 945f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencerpublic: 951543e9c69202b8e128c7fe34784ae7aa90964889Chris Lattner /// Create a TokenLexer for the specified macro with the specified actual 965f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer /// arguments. Note that this ctor takes ownership of the ActualArgs pointer. 97e7fb48466afcbf2c4ccdfa658824282fdc3c512cChris Lattner /// ILEnd specifies the location of the ')' for a function-like macro or the 98e7fb48466afcbf2c4ccdfa658824282fdc3c512cChris Lattner /// identifier for an object-like macro. 99e7fb48466afcbf2c4ccdfa658824282fdc3c512cChris Lattner TokenLexer(Token &Tok, SourceLocation ILEnd, MacroArgs *ActualArgs, 100e7fb48466afcbf2c4ccdfa658824282fdc3c512cChris Lattner Preprocessor &pp) 1018d896438031f5990cb42eb95a7bae233db2eb333Chris Lattner : Macro(0), ActualArgs(0), PP(pp), OwnsTokens(false) { 102e7fb48466afcbf2c4ccdfa658824282fdc3c512cChris Lattner Init(Tok, ILEnd, ActualArgs); 1039594acf32de2939b15eafa8fe818607bfc56bf66Chris Lattner } 1041eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 1051543e9c69202b8e128c7fe34784ae7aa90964889Chris Lattner /// Init - Initialize this TokenLexer to expand from the specified macro 1069594acf32de2939b15eafa8fe818607bfc56bf66Chris Lattner /// with the specified argument information. Note that this ctor takes 107e7fb48466afcbf2c4ccdfa658824282fdc3c512cChris Lattner /// ownership of the ActualArgs pointer. ILEnd specifies the location of the 108e7fb48466afcbf2c4ccdfa658824282fdc3c512cChris Lattner /// ')' for a function-like macro or the identifier for an object-like macro. 109e7fb48466afcbf2c4ccdfa658824282fdc3c512cChris Lattner void Init(Token &Tok, SourceLocation ILEnd, MacroArgs *ActualArgs); 1101eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 11121ec0e4c5db85e45b59cafabca66d640e0ee2dccChris Lattner /// Create a TokenLexer for the specified token stream. If 'OwnsTokens' is 11221ec0e4c5db85e45b59cafabca66d640e0ee2dccChris Lattner /// specified, this takes ownership of the tokens and delete[]'s them when 11321ec0e4c5db85e45b59cafabca66d640e0ee2dccChris Lattner /// the token lexer is empty. 1146b884508c3bc97cc9df9516adb92fbf88dd0a2e4Chris Lattner TokenLexer(const Token *TokArray, unsigned NumToks, bool DisableExpansion, 11521ec0e4c5db85e45b59cafabca66d640e0ee2dccChris Lattner bool ownsTokens, Preprocessor &pp) 1168d896438031f5990cb42eb95a7bae233db2eb333Chris Lattner : Macro(0), ActualArgs(0), PP(pp), OwnsTokens(false) { 11721ec0e4c5db85e45b59cafabca66d640e0ee2dccChris Lattner Init(TokArray, NumToks, DisableExpansion, ownsTokens); 1189594acf32de2939b15eafa8fe818607bfc56bf66Chris Lattner } 1191eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 1201543e9c69202b8e128c7fe34784ae7aa90964889Chris Lattner /// Init - Initialize this TokenLexer with the specified token stream. 1219594acf32de2939b15eafa8fe818607bfc56bf66Chris Lattner /// This does not take ownership of the specified token vector. 1226b884508c3bc97cc9df9516adb92fbf88dd0a2e4Chris Lattner /// 1231eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump /// DisableExpansion is true when macro expansion of tokens lexed from this 1246b884508c3bc97cc9df9516adb92fbf88dd0a2e4Chris Lattner /// stream should be disabled. 1256b884508c3bc97cc9df9516adb92fbf88dd0a2e4Chris Lattner void Init(const Token *TokArray, unsigned NumToks, 1266b884508c3bc97cc9df9516adb92fbf88dd0a2e4Chris Lattner bool DisableMacroExpansion, bool OwnsTokens); 1271eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 1281543e9c69202b8e128c7fe34784ae7aa90964889Chris Lattner ~TokenLexer() { destroy(); } 1291eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 1305f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer /// isNextTokenLParen - If the next token lexed will pop this macro off the 1315f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer /// expansion stack, return 2. If the next unexpanded token is a '(', return 1325f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer /// 1, otherwise return 0. 1335f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer unsigned isNextTokenLParen() const; 1341eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 1355f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer /// Lex - Lex and return a token from this macro stream. 136d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner void Lex(Token &Tok); 1371eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 138b2eb53d9fd973a1a02e05e67a3307b3efd12eff2Peter Collingbourne /// isParsingPreprocessorDirective - Return true if we are in the middle of a 139b2eb53d9fd973a1a02e05e67a3307b3efd12eff2Peter Collingbourne /// preprocessor directive. 140b2eb53d9fd973a1a02e05e67a3307b3efd12eff2Peter Collingbourne bool isParsingPreprocessorDirective() const; 141b2eb53d9fd973a1a02e05e67a3307b3efd12eff2Peter Collingbourne 1425f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencerprivate: 1439594acf32de2939b15eafa8fe818607bfc56bf66Chris Lattner void destroy(); 1441eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 1455f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer /// isAtEnd - Return true if the next lex call will pop this macro off the 1465f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer /// include stack. 1475f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer bool isAtEnd() const { 1488d896438031f5990cb42eb95a7bae233db2eb333Chris Lattner return CurToken == NumTokens; 1495f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 1501eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 1515f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer /// PasteTokens - Tok is the LHS of a ## operator, and CurToken is the ## 1525f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer /// operator. Read the ## and RHS, and paste the LHS/RHS together. If there 1535f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer /// are is another ## after it, chomp it iteratively. Return the result as 1543f1cc838f9caf469990f23fccd0940263c0c61ffChris Lattner /// Tok. If this returns true, the caller should immediately return the 1553f1cc838f9caf469990f23fccd0940263c0c61ffChris Lattner /// token. 1563f1cc838f9caf469990f23fccd0940263c0c61ffChris Lattner bool PasteTokens(Token &Tok); 1571eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 1585f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer /// Expand the arguments of a function-like macro so that we can quickly 1598d896438031f5990cb42eb95a7bae233db2eb333Chris Lattner /// return preexpanded tokens from Tokens. 1605f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer void ExpandFunctionArguments(); 1611eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 1623f1cc838f9caf469990f23fccd0940263c0c61ffChris Lattner /// HandleMicrosoftCommentPaste - In microsoft compatibility mode, /##/ pastes 1633f1cc838f9caf469990f23fccd0940263c0c61ffChris Lattner /// together to form a comment that comments out everything in the current 1643f1cc838f9caf469990f23fccd0940263c0c61ffChris Lattner /// macro, other active macros, and anything left on the current physical 165d0f5e443ceed37e1a72015bd109db8e5fc6206cfChandler Carruth /// source line of the expanded buffer. Handle this by returning the 1663f1cc838f9caf469990f23fccd0940263c0c61ffChris Lattner /// first token on the next line. 1673f1cc838f9caf469990f23fccd0940263c0c61ffChris Lattner void HandleMicrosoftCommentPaste(Token &Tok); 168b73377eeb3eff76be134203aebb6068244b177f3Argyrios Kyrtzidis 169b73377eeb3eff76be134203aebb6068244b177f3Argyrios Kyrtzidis /// \brief If \arg loc is a FileID and points inside the current macro 170b73377eeb3eff76be134203aebb6068244b177f3Argyrios Kyrtzidis /// definition, returns the appropriate source location pointing at the 171b73377eeb3eff76be134203aebb6068244b177f3Argyrios Kyrtzidis /// macro expansion source location entry. 172ec3b5389a5032433116ec2f2ee266b9666ad2eb4Argyrios Kyrtzidis SourceLocation getExpansionLocForMacroDefLoc(SourceLocation loc) const; 173d60a34a4e514ec0dfddd05ef2744be104e111f45Argyrios Kyrtzidis 174d60a34a4e514ec0dfddd05ef2744be104e111f45Argyrios Kyrtzidis /// \brief Creates SLocEntries and updates the locations of macro argument 175d60a34a4e514ec0dfddd05ef2744be104e111f45Argyrios Kyrtzidis /// tokens to their new expanded locations. 176d60a34a4e514ec0dfddd05ef2744be104e111f45Argyrios Kyrtzidis /// 177d60a34a4e514ec0dfddd05ef2744be104e111f45Argyrios Kyrtzidis /// \param ArgIdSpellLoc the location of the macro argument id inside the 178d60a34a4e514ec0dfddd05ef2744be104e111f45Argyrios Kyrtzidis /// macro definition. 179d60a34a4e514ec0dfddd05ef2744be104e111f45Argyrios Kyrtzidis void updateLocForMacroArgTokens(SourceLocation ArgIdSpellLoc, 180d60a34a4e514ec0dfddd05ef2744be104e111f45Argyrios Kyrtzidis Token *begin_tokens, Token *end_tokens); 1815f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer}; 1825f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 1835f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} // end namespace clang 1845f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 1855f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer#endif 186