11543e9c69202b8e128c7fe34784ae7aa90964889Chris Lattner//===--- TokenLexer.cpp - Lex from a token stream -------------------------===// 25f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// 35f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// The LLVM Compiler Infrastructure 45f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// 50bc735ffcfb223c0186419547abaa5c84482663eChris Lattner// This file is distributed under the University of Illinois Open Source 60bc735ffcfb223c0186419547abaa5c84482663eChris Lattner// License. See LICENSE.TXT for details. 75f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// 85f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer//===----------------------------------------------------------------------===// 95f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// 101543e9c69202b8e128c7fe34784ae7aa90964889Chris Lattner// This file implements the TokenLexer interface. 115f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// 125f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer//===----------------------------------------------------------------------===// 135f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 145d75de0f821023f4ed4815825bf3aea8a0b5e40dChris Lattner#include "clang/Lex/TokenLexer.h" 155f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer#include "clang/Basic/SourceManager.h" 16500d3297d2a21edeac4d46cbcbe21bc2352c2a28Chris Lattner#include "clang/Lex/LexDiagnostic.h" 17651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines#include "clang/Lex/MacroArgs.h" 1855fc873017f10f6f566b182b70f6fc22aefa3464Chandler Carruth#include "clang/Lex/MacroInfo.h" 1955fc873017f10f6f566b182b70f6fc22aefa3464Chandler Carruth#include "clang/Lex/Preprocessor.h" 208fe83e1df954d72c0f4ffc15d20a5222ec151c21Benjamin Kramer#include "llvm/ADT/SmallString.h" 215f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencerusing namespace clang; 225f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 235f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 241543e9c69202b8e128c7fe34784ae7aa90964889Chris Lattner/// Create a TokenLexer for the specified macro with the specified actual 255f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// arguments. Note that this ctor takes ownership of the ActualArgs pointer. 26c30981a563a8947cb26b1e308d122fa2ef90fcebRichard Smithvoid TokenLexer::Init(Token &Tok, SourceLocation ELEnd, MacroInfo *MI, 27c30981a563a8947cb26b1e308d122fa2ef90fcebRichard Smith MacroArgs *Actuals) { 281543e9c69202b8e128c7fe34784ae7aa90964889Chris Lattner // If the client is reusing a TokenLexer, make sure to free any memory 299594acf32de2939b15eafa8fe818607bfc56bf66Chris Lattner // associated with it. 309594acf32de2939b15eafa8fe818607bfc56bf66Chris Lattner destroy(); 311eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 32c30981a563a8947cb26b1e308d122fa2ef90fcebRichard Smith Macro = MI; 339594acf32de2939b15eafa8fe818607bfc56bf66Chris Lattner ActualArgs = Actuals; 349594acf32de2939b15eafa8fe818607bfc56bf66Chris Lattner CurToken = 0; 351eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 36d0f5e443ceed37e1a72015bd109db8e5fc6206cfChandler Carruth ExpandLocStart = Tok.getLocation(); 37d0f5e443ceed37e1a72015bd109db8e5fc6206cfChandler Carruth ExpandLocEnd = ELEnd; 389594acf32de2939b15eafa8fe818607bfc56bf66Chris Lattner AtStartOfLine = Tok.isAtStartOfLine(); 399594acf32de2939b15eafa8fe818607bfc56bf66Chris Lattner HasLeadingSpace = Tok.hasLeadingSpace(); 40651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NextTokGetsSpace = false; 418d896438031f5990cb42eb95a7bae233db2eb333Chris Lattner Tokens = &*Macro->tokens_begin(); 428d896438031f5990cb42eb95a7bae233db2eb333Chris Lattner OwnsTokens = false; 436b884508c3bc97cc9df9516adb92fbf88dd0a2e4Chris Lattner DisableMacroExpansion = false; 448d896438031f5990cb42eb95a7bae233db2eb333Chris Lattner NumTokens = Macro->tokens_end()-Macro->tokens_begin(); 45b73377eeb3eff76be134203aebb6068244b177f3Argyrios Kyrtzidis MacroExpansionStart = SourceLocation(); 46b73377eeb3eff76be134203aebb6068244b177f3Argyrios Kyrtzidis 47b73377eeb3eff76be134203aebb6068244b177f3Argyrios Kyrtzidis SourceManager &SM = PP.getSourceManager(); 48f62d43d2afe1960755a1b5813cae1e5983bcac1bDouglas Gregor MacroStartSLocOffset = SM.getNextLocalOffset(); 49b73377eeb3eff76be134203aebb6068244b177f3Argyrios Kyrtzidis 50b73377eeb3eff76be134203aebb6068244b177f3Argyrios Kyrtzidis if (NumTokens > 0) { 51b73377eeb3eff76be134203aebb6068244b177f3Argyrios Kyrtzidis assert(Tokens[0].getLocation().isValid()); 52b73377eeb3eff76be134203aebb6068244b177f3Argyrios Kyrtzidis assert((Tokens[0].getLocation().isFileID() || Tokens[0].is(tok::comment)) && 53b73377eeb3eff76be134203aebb6068244b177f3Argyrios Kyrtzidis "Macro defined in macro?"); 54d0f5e443ceed37e1a72015bd109db8e5fc6206cfChandler Carruth assert(ExpandLocStart.isValid()); 55b73377eeb3eff76be134203aebb6068244b177f3Argyrios Kyrtzidis 56b73377eeb3eff76be134203aebb6068244b177f3Argyrios Kyrtzidis // Reserve a source location entry chunk for the length of the macro 57b73377eeb3eff76be134203aebb6068244b177f3Argyrios Kyrtzidis // definition. Tokens that get lexed directly from the definition will 58b73377eeb3eff76be134203aebb6068244b177f3Argyrios Kyrtzidis // have their locations pointing inside this chunk. This is to avoid 59b73377eeb3eff76be134203aebb6068244b177f3Argyrios Kyrtzidis // creating separate source location entries for each token. 60499ea5550d6e2fc5cfbd33b47f06d92ce25d7a13Argyrios Kyrtzidis MacroDefStart = SM.getExpansionLoc(Tokens[0].getLocation()); 61499ea5550d6e2fc5cfbd33b47f06d92ce25d7a13Argyrios Kyrtzidis MacroDefLength = Macro->getDefinitionLength(SM); 62499ea5550d6e2fc5cfbd33b47f06d92ce25d7a13Argyrios Kyrtzidis MacroExpansionStart = SM.createExpansionLoc(MacroDefStart, 63bf340e452339e374ea6eef78c1f0a2abdd16c5a3Chandler Carruth ExpandLocStart, 64bf340e452339e374ea6eef78c1f0a2abdd16c5a3Chandler Carruth ExpandLocEnd, 65499ea5550d6e2fc5cfbd33b47f06d92ce25d7a13Argyrios Kyrtzidis MacroDefLength); 66b73377eeb3eff76be134203aebb6068244b177f3Argyrios Kyrtzidis } 675f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 685f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If this is a function-like macro, expand the arguments and change 698d896438031f5990cb42eb95a7bae233db2eb333Chris Lattner // Tokens to point to the expanded tokens. 705f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (Macro->isFunctionLike() && Macro->getNumArgs()) 715f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer ExpandFunctionArguments(); 721eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 735f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Mark the macro as currently disabled, so that it is not recursively 745f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // expanded. The macro must be disabled only after argument pre-expansion of 755f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // function-like macro arguments occurs. 765f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Macro->DisableMacro(); 775f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 785f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 799594acf32de2939b15eafa8fe818607bfc56bf66Chris Lattner 809594acf32de2939b15eafa8fe818607bfc56bf66Chris Lattner 811543e9c69202b8e128c7fe34784ae7aa90964889Chris Lattner/// Create a TokenLexer for the specified token stream. This does not 825f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// take ownership of the specified token vector. 836b884508c3bc97cc9df9516adb92fbf88dd0a2e4Chris Lattnervoid TokenLexer::Init(const Token *TokArray, unsigned NumToks, 846b884508c3bc97cc9df9516adb92fbf88dd0a2e4Chris Lattner bool disableMacroExpansion, bool ownsTokens) { 851543e9c69202b8e128c7fe34784ae7aa90964889Chris Lattner // If the client is reusing a TokenLexer, make sure to free any memory 869594acf32de2939b15eafa8fe818607bfc56bf66Chris Lattner // associated with it. 879594acf32de2939b15eafa8fe818607bfc56bf66Chris Lattner destroy(); 881eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 896bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Macro = nullptr; 906bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines ActualArgs = nullptr; 918d896438031f5990cb42eb95a7bae233db2eb333Chris Lattner Tokens = TokArray; 926b884508c3bc97cc9df9516adb92fbf88dd0a2e4Chris Lattner OwnsTokens = ownsTokens; 936b884508c3bc97cc9df9516adb92fbf88dd0a2e4Chris Lattner DisableMacroExpansion = disableMacroExpansion; 948d896438031f5990cb42eb95a7bae233db2eb333Chris Lattner NumTokens = NumToks; 959594acf32de2939b15eafa8fe818607bfc56bf66Chris Lattner CurToken = 0; 96d0f5e443ceed37e1a72015bd109db8e5fc6206cfChandler Carruth ExpandLocStart = ExpandLocEnd = SourceLocation(); 979594acf32de2939b15eafa8fe818607bfc56bf66Chris Lattner AtStartOfLine = false; 989594acf32de2939b15eafa8fe818607bfc56bf66Chris Lattner HasLeadingSpace = false; 99651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NextTokGetsSpace = false; 100b73377eeb3eff76be134203aebb6068244b177f3Argyrios Kyrtzidis MacroExpansionStart = SourceLocation(); 1011eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 1025f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Set HasLeadingSpace/AtStartOfLine so that the first token will be 1035f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // returned unmodified. 1045f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (NumToks != 0) { 1055f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer AtStartOfLine = TokArray[0].isAtStartOfLine(); 1065f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer HasLeadingSpace = TokArray[0].hasLeadingSpace(); 1075f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 1085f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 1095f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 1105f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 1111543e9c69202b8e128c7fe34784ae7aa90964889Chris Lattnervoid TokenLexer::destroy() { 1125f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If this was a function-like macro that actually uses its arguments, delete 1135f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // the expanded tokens. 1148d896438031f5990cb42eb95a7bae233db2eb333Chris Lattner if (OwnsTokens) { 1158d896438031f5990cb42eb95a7bae233db2eb333Chris Lattner delete [] Tokens; 1166bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines Tokens = nullptr; 11732fca722dd974b8202d0fb9c71b6c185c0767da6Chris Lattner OwnsTokens = false; 1189c683062752a26014197df1c8792a19efa9e93cfChris Lattner } 1191eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 1201543e9c69202b8e128c7fe34784ae7aa90964889Chris Lattner // TokenLexer owns its formal arguments. 121561395bead16b9dc9b0d5bfb6b257a9ed4545db6Chris Lattner if (ActualArgs) ActualArgs->destroy(PP); 1225f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 1235f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 1246bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hinesbool TokenLexer::MaybeRemoveCommaBeforeVaArgs( 1256bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines SmallVectorImpl<Token> &ResultToks, bool HasPasteOperator, MacroInfo *Macro, 1266bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines unsigned MacroArgNo, Preprocessor &PP) { 12702f0022dfced0d082d979ebfd9185211f4dd0496Andy Gibbs // Is the macro argument __VA_ARGS__? 12802f0022dfced0d082d979ebfd9185211f4dd0496Andy Gibbs if (!Macro->isVariadic() || MacroArgNo != Macro->getNumArgs()-1) 12902f0022dfced0d082d979ebfd9185211f4dd0496Andy Gibbs return false; 13002f0022dfced0d082d979ebfd9185211f4dd0496Andy Gibbs 13102f0022dfced0d082d979ebfd9185211f4dd0496Andy Gibbs // In Microsoft-compatibility mode, a comma is removed in the expansion 13202f0022dfced0d082d979ebfd9185211f4dd0496Andy Gibbs // of " ... , __VA_ARGS__ " if __VA_ARGS__ is empty. This extension is 13302f0022dfced0d082d979ebfd9185211f4dd0496Andy Gibbs // not supported by gcc. 134651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (!HasPasteOperator && !PP.getLangOpts().MSVCCompat) 13502f0022dfced0d082d979ebfd9185211f4dd0496Andy Gibbs return false; 13602f0022dfced0d082d979ebfd9185211f4dd0496Andy Gibbs 13702f0022dfced0d082d979ebfd9185211f4dd0496Andy Gibbs // GCC removes the comma in the expansion of " ... , ## __VA_ARGS__ " if 13802f0022dfced0d082d979ebfd9185211f4dd0496Andy Gibbs // __VA_ARGS__ is empty, but not in strict C99 mode where there are no 13902f0022dfced0d082d979ebfd9185211f4dd0496Andy Gibbs // named arguments, where it remains. In all other modes, including C99 14002f0022dfced0d082d979ebfd9185211f4dd0496Andy Gibbs // with GNU extensions, it is removed regardless of named arguments. 14102f0022dfced0d082d979ebfd9185211f4dd0496Andy Gibbs // Microsoft also appears to support this extension, unofficially. 14202f0022dfced0d082d979ebfd9185211f4dd0496Andy Gibbs if (PP.getLangOpts().C99 && !PP.getLangOpts().GNUMode 14302f0022dfced0d082d979ebfd9185211f4dd0496Andy Gibbs && Macro->getNumArgs() < 2) 14402f0022dfced0d082d979ebfd9185211f4dd0496Andy Gibbs return false; 14502f0022dfced0d082d979ebfd9185211f4dd0496Andy Gibbs 14602f0022dfced0d082d979ebfd9185211f4dd0496Andy Gibbs // Is a comma available to be removed? 14702f0022dfced0d082d979ebfd9185211f4dd0496Andy Gibbs if (ResultToks.empty() || !ResultToks.back().is(tok::comma)) 14802f0022dfced0d082d979ebfd9185211f4dd0496Andy Gibbs return false; 14902f0022dfced0d082d979ebfd9185211f4dd0496Andy Gibbs 15002f0022dfced0d082d979ebfd9185211f4dd0496Andy Gibbs // Issue an extension diagnostic for the paste operator. 15102f0022dfced0d082d979ebfd9185211f4dd0496Andy Gibbs if (HasPasteOperator) 15202f0022dfced0d082d979ebfd9185211f4dd0496Andy Gibbs PP.Diag(ResultToks.back().getLocation(), diag::ext_paste_comma); 15302f0022dfced0d082d979ebfd9185211f4dd0496Andy Gibbs 15402f0022dfced0d082d979ebfd9185211f4dd0496Andy Gibbs // Remove the comma. 15502f0022dfced0d082d979ebfd9185211f4dd0496Andy Gibbs ResultToks.pop_back(); 15602f0022dfced0d082d979ebfd9185211f4dd0496Andy Gibbs 15702f0022dfced0d082d979ebfd9185211f4dd0496Andy Gibbs // If the comma was right after another paste (e.g. "X##,##__VA_ARGS__"), 15802f0022dfced0d082d979ebfd9185211f4dd0496Andy Gibbs // then removal of the comma should produce a placemarker token (in C99 15902f0022dfced0d082d979ebfd9185211f4dd0496Andy Gibbs // terms) which we model by popping off the previous ##, giving us a plain 16002f0022dfced0d082d979ebfd9185211f4dd0496Andy Gibbs // "X" when __VA_ARGS__ is empty. 16102f0022dfced0d082d979ebfd9185211f4dd0496Andy Gibbs if (!ResultToks.empty() && ResultToks.back().is(tok::hashhash)) 16202f0022dfced0d082d979ebfd9185211f4dd0496Andy Gibbs ResultToks.pop_back(); 16302f0022dfced0d082d979ebfd9185211f4dd0496Andy Gibbs 16402f0022dfced0d082d979ebfd9185211f4dd0496Andy Gibbs // Never add a space, even if the comma, ##, or arg had a space. 16502f0022dfced0d082d979ebfd9185211f4dd0496Andy Gibbs NextTokGetsSpace = false; 16602f0022dfced0d082d979ebfd9185211f4dd0496Andy Gibbs return true; 16702f0022dfced0d082d979ebfd9185211f4dd0496Andy Gibbs} 16802f0022dfced0d082d979ebfd9185211f4dd0496Andy Gibbs 1695f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// Expand the arguments of a function-like macro so that we can quickly 1708d896438031f5990cb42eb95a7bae233db2eb333Chris Lattner/// return preexpanded tokens from Tokens. 1711543e9c69202b8e128c7fe34784ae7aa90964889Chris Lattnervoid TokenLexer::ExpandFunctionArguments() { 1720fd9c48dfc6a324d8399b0b43ab4b943a1b1b843Argyrios Kyrtzidis 1735f9e272e632e951b1efe824cd16acb4d96077930Chris Lattner SmallVector<Token, 128> ResultToks; 1741eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 1758d896438031f5990cb42eb95a7bae233db2eb333Chris Lattner // Loop through 'Tokens', expanding them into ResultToks. Keep 1765f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // track of whether we change anything. If not, no need to keep them. If so, 1778d896438031f5990cb42eb95a7bae233db2eb333Chris Lattner // we install the newly expanded sequence as the new 'Tokens' list. 1785f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer bool MadeChange = false; 1791eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 1808d896438031f5990cb42eb95a7bae233db2eb333Chris Lattner for (unsigned i = 0, e = NumTokens; i != e; ++i) { 1815f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If we found the stringify operator, get the argument stringified. The 1825f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // preprocessor already verified that the following token is a macro name 1835f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // when the #define was parsed. 1848d896438031f5990cb42eb95a7bae233db2eb333Chris Lattner const Token &CurTok = Tokens[i]; 185651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (i != 0 && !Tokens[i-1].is(tok::hashhash) && CurTok.hasLeadingSpace()) 186651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines NextTokGetsSpace = true; 187651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 18822f6bbcafa8871f4f20c4402d9cbc5c024fee99aChris Lattner if (CurTok.is(tok::hash) || CurTok.is(tok::hashat)) { 1898d896438031f5990cb42eb95a7bae233db2eb333Chris Lattner int ArgNo = Macro->getArgumentNum(Tokens[i+1].getIdentifierInfo()); 1905f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer assert(ArgNo != -1 && "Token following # is not an argument?"); 1911eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 192a08529cc3f00e0b47a3c028823634129ac46847bAbramo Bagnara SourceLocation ExpansionLocStart = 193c0069d850efe74bac36c526dfec86b7dad1ede2fArgyrios Kyrtzidis getExpansionLocForMacroDefLoc(CurTok.getLocation()); 194a08529cc3f00e0b47a3c028823634129ac46847bAbramo Bagnara SourceLocation ExpansionLocEnd = 195a08529cc3f00e0b47a3c028823634129ac46847bAbramo Bagnara getExpansionLocForMacroDefLoc(Tokens[i+1].getLocation()); 196b73377eeb3eff76be134203aebb6068244b177f3Argyrios Kyrtzidis 197d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner Token Res; 19822f6bbcafa8871f4f20c4402d9cbc5c024fee99aChris Lattner if (CurTok.is(tok::hash)) // Stringify 199a08529cc3f00e0b47a3c028823634129ac46847bAbramo Bagnara Res = ActualArgs->getStringifiedArgument(ArgNo, PP, 200a08529cc3f00e0b47a3c028823634129ac46847bAbramo Bagnara ExpansionLocStart, 201a08529cc3f00e0b47a3c028823634129ac46847bAbramo Bagnara ExpansionLocEnd); 2025f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer else { 2035f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // 'charify': don't bother caching these. 204e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner Res = MacroArgs::StringifyArgument(ActualArgs->getUnexpArgument(ArgNo), 205a08529cc3f00e0b47a3c028823634129ac46847bAbramo Bagnara PP, true, 206a08529cc3f00e0b47a3c028823634129ac46847bAbramo Bagnara ExpansionLocStart, 207a08529cc3f00e0b47a3c028823634129ac46847bAbramo Bagnara ExpansionLocEnd); 2085f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 2091eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 2105f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // The stringified/charified string leading space flag gets set to match 2115f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // the #/#@ operator. 212651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (NextTokGetsSpace) 213d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner Res.setFlag(Token::LeadingSpace); 2141eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 2155f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer ResultToks.push_back(Res); 2165f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer MadeChange = true; 2175f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer ++i; // Skip arg name. 2185f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer NextTokGetsSpace = false; 2195f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer continue; 2205f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 2211eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 222651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // Find out if there is a paste (##) operator before or after the token. 223651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines bool NonEmptyPasteBefore = 224651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines !ResultToks.empty() && ResultToks.back().is(tok::hashhash); 225651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines bool PasteBefore = i != 0 && Tokens[i-1].is(tok::hashhash); 226651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines bool PasteAfter = i+1 != e && Tokens[i+1].is(tok::hashhash); 227651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines assert(!NonEmptyPasteBefore || PasteBefore); 228651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 2295f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Otherwise, if this is not an argument token, just add the token to the 2305f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // output buffer. 2315f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer IdentifierInfo *II = CurTok.getIdentifierInfo(); 2325f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer int ArgNo = II ? Macro->getArgumentNum(II) : -1; 2335f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (ArgNo == -1) { 2345f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // This isn't an argument, just add it. 2355f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer ResultToks.push_back(CurTok); 2365f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 2375f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (NextTokGetsSpace) { 238d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner ResultToks.back().setFlag(Token::LeadingSpace); 2395f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer NextTokGetsSpace = false; 240651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } else if (PasteBefore && !NonEmptyPasteBefore) 241651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines ResultToks.back().clearFlag(Token::LeadingSpace); 242651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 2435f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer continue; 2445f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 2451eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 2465f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // An argument is expanded somehow, the result is different than the 2475f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // input. 2485f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer MadeChange = true; 2495f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 250651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // Otherwise, this is a use of the argument. 2511eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 25202f0022dfced0d082d979ebfd9185211f4dd0496Andy Gibbs // In Microsoft mode, remove the comma before __VA_ARGS__ to ensure there 25302f0022dfced0d082d979ebfd9185211f4dd0496Andy Gibbs // are no trailing commas if __VA_ARGS__ is empty. 25402f0022dfced0d082d979ebfd9185211f4dd0496Andy Gibbs if (!PasteBefore && ActualArgs->isVarargsElidedUse() && 255651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines MaybeRemoveCommaBeforeVaArgs(ResultToks, 25602f0022dfced0d082d979ebfd9185211f4dd0496Andy Gibbs /*HasPasteOperator=*/false, 25702f0022dfced0d082d979ebfd9185211f4dd0496Andy Gibbs Macro, ArgNo, PP)) 25802f0022dfced0d082d979ebfd9185211f4dd0496Andy Gibbs continue; 25902f0022dfced0d082d979ebfd9185211f4dd0496Andy Gibbs 2605f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If it is not the LHS/RHS of a ## operator, we must pre-expand the 2615f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // argument and substitute the expanded tokens into the result. This is 2625f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // C99 6.10.3.1p1. 2635f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (!PasteBefore && !PasteAfter) { 264d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner const Token *ResultArgToks; 2655f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 2665f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Only preexpand the argument if it could possibly need it. This 2675f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // avoids some work in common cases. 268d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner const Token *ArgTok = ActualArgs->getUnexpArgument(ArgNo); 269cc1a875f94630e58d24a55577ffbf0e89b7da8c7Chris Lattner if (ActualArgs->ArgNeedsPreexpansion(ArgTok, PP)) 270f5809a7b76fd5a86ae737d9b525a1eddb9339ee7Chris Lattner ResultArgToks = &ActualArgs->getPreExpArgument(ArgNo, Macro, PP)[0]; 2715f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer else 2725f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer ResultArgToks = ArgTok; // Use non-preexpanded tokens. 2731eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 2745f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If the arg token expanded into anything, append it. 27522f6bbcafa8871f4f20c4402d9cbc5c024fee99aChris Lattner if (ResultArgToks->isNot(tok::eof)) { 2765f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer unsigned FirstResult = ResultToks.size(); 2775f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer unsigned NumToks = MacroArgs::getArgLength(ResultArgToks); 2785f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer ResultToks.append(ResultArgToks, ResultArgToks+NumToks); 2791eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 28011be06464a338b7c6e007898a7709498dc456db2Reid Kleckner // In Microsoft-compatibility mode, we follow MSVC's preprocessing 28111be06464a338b7c6e007898a7709498dc456db2Reid Kleckner // behavior by not considering single commas from nested macro 28211be06464a338b7c6e007898a7709498dc456db2Reid Kleckner // expansions as argument separators. Set a flag on the token so we can 28311be06464a338b7c6e007898a7709498dc456db2Reid Kleckner // test for this later when the macro expansion is processed. 284651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (PP.getLangOpts().MSVCCompat && NumToks == 1 && 28511be06464a338b7c6e007898a7709498dc456db2Reid Kleckner ResultToks.back().is(tok::comma)) 28611be06464a338b7c6e007898a7709498dc456db2Reid Kleckner ResultToks.back().setFlag(Token::IgnoredComma); 28711be06464a338b7c6e007898a7709498dc456db2Reid Kleckner 2883582a6c0a35f202249b721e100229fde9de0d6a6Argyrios Kyrtzidis // If the '##' came from expanding an argument, turn it into 'unknown' 2893582a6c0a35f202249b721e100229fde9de0d6a6Argyrios Kyrtzidis // to avoid pasting. 2903582a6c0a35f202249b721e100229fde9de0d6a6Argyrios Kyrtzidis for (unsigned i = FirstResult, e = ResultToks.size(); i != e; ++i) { 2913582a6c0a35f202249b721e100229fde9de0d6a6Argyrios Kyrtzidis Token &Tok = ResultToks[i]; 2923582a6c0a35f202249b721e100229fde9de0d6a6Argyrios Kyrtzidis if (Tok.is(tok::hashhash)) 2933582a6c0a35f202249b721e100229fde9de0d6a6Argyrios Kyrtzidis Tok.setKind(tok::unknown); 2943582a6c0a35f202249b721e100229fde9de0d6a6Argyrios Kyrtzidis } 2953582a6c0a35f202249b721e100229fde9de0d6a6Argyrios Kyrtzidis 296d0f5e443ceed37e1a72015bd109db8e5fc6206cfChandler Carruth if(ExpandLocStart.isValid()) { 297d60a34a4e514ec0dfddd05ef2744be104e111f45Argyrios Kyrtzidis updateLocForMacroArgTokens(CurTok.getLocation(), 298d60a34a4e514ec0dfddd05ef2744be104e111f45Argyrios Kyrtzidis ResultToks.begin()+FirstResult, 299d60a34a4e514ec0dfddd05ef2744be104e111f45Argyrios Kyrtzidis ResultToks.end()); 300b73377eeb3eff76be134203aebb6068244b177f3Argyrios Kyrtzidis } 301b73377eeb3eff76be134203aebb6068244b177f3Argyrios Kyrtzidis 3025f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If any tokens were substituted from the argument, the whitespace 3035f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // before the first token should match the whitespace of the arg 3045f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // identifier. 305d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner ResultToks[FirstResult].setFlagValue(Token::LeadingSpace, 3065f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer NextTokGetsSpace); 3075f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer NextTokGetsSpace = false; 3085f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 3095f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer continue; 3105f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 3111eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 3125f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Okay, we have a token that is either the LHS or RHS of a paste (##) 3135f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // argument. It gets substituted as its non-pre-expanded tokens. 314d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner const Token *ArgToks = ActualArgs->getUnexpArgument(ArgNo); 3155f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer unsigned NumToks = MacroArgs::getArgLength(ArgToks); 3165f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (NumToks) { // Not an empty argument? 3179f728cd37476c6588b06d241fa778d2df6e277daRichard Smith // If this is the GNU ", ## __VA_ARGS__" extension, and we just learned 3189f728cd37476c6588b06d241fa778d2df6e277daRichard Smith // that __VA_ARGS__ expands to multiple tokens, avoid a pasting error when 3199f728cd37476c6588b06d241fa778d2df6e277daRichard Smith // the expander trys to paste ',' with the first token of the __VA_ARGS__ 32071a3a8d50643c450823d3fbcf3307a3d7db1a265Chris Lattner // expansion. 321461f2d820ffb628ebdf6ef4a4474a43f8ff914e5Argyrios Kyrtzidis if (NonEmptyPasteBefore && ResultToks.size() >= 2 && 32271a3a8d50643c450823d3fbcf3307a3d7db1a265Chris Lattner ResultToks[ResultToks.size()-2].is(tok::comma) && 32371a3a8d50643c450823d3fbcf3307a3d7db1a265Chris Lattner (unsigned)ArgNo == Macro->getNumArgs()-1 && 32471a3a8d50643c450823d3fbcf3307a3d7db1a265Chris Lattner Macro->isVariadic()) { 32571a3a8d50643c450823d3fbcf3307a3d7db1a265Chris Lattner // Remove the paste operator, report use of the extension. 326344472ebeded2fca2ed5013b9e87f81d09bfa908Robert Wilhelm PP.Diag(ResultToks.pop_back_val().getLocation(), diag::ext_paste_comma); 32771a3a8d50643c450823d3fbcf3307a3d7db1a265Chris Lattner } 3281eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 3295f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer ResultToks.append(ArgToks, ArgToks+NumToks); 3301eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 3313582a6c0a35f202249b721e100229fde9de0d6a6Argyrios Kyrtzidis // If the '##' came from expanding an argument, turn it into 'unknown' 3323582a6c0a35f202249b721e100229fde9de0d6a6Argyrios Kyrtzidis // to avoid pasting. 3333582a6c0a35f202249b721e100229fde9de0d6a6Argyrios Kyrtzidis for (unsigned i = ResultToks.size() - NumToks, e = ResultToks.size(); 3343582a6c0a35f202249b721e100229fde9de0d6a6Argyrios Kyrtzidis i != e; ++i) { 3353582a6c0a35f202249b721e100229fde9de0d6a6Argyrios Kyrtzidis Token &Tok = ResultToks[i]; 3363582a6c0a35f202249b721e100229fde9de0d6a6Argyrios Kyrtzidis if (Tok.is(tok::hashhash)) 3373582a6c0a35f202249b721e100229fde9de0d6a6Argyrios Kyrtzidis Tok.setKind(tok::unknown); 3383582a6c0a35f202249b721e100229fde9de0d6a6Argyrios Kyrtzidis } 3393582a6c0a35f202249b721e100229fde9de0d6a6Argyrios Kyrtzidis 340d0f5e443ceed37e1a72015bd109db8e5fc6206cfChandler Carruth if (ExpandLocStart.isValid()) { 341d60a34a4e514ec0dfddd05ef2744be104e111f45Argyrios Kyrtzidis updateLocForMacroArgTokens(CurTok.getLocation(), 342d60a34a4e514ec0dfddd05ef2744be104e111f45Argyrios Kyrtzidis ResultToks.end()-NumToks, ResultToks.end()); 343b73377eeb3eff76be134203aebb6068244b177f3Argyrios Kyrtzidis } 344b73377eeb3eff76be134203aebb6068244b177f3Argyrios Kyrtzidis 345442a661a2b44e78e3c66e35bf842c3dc119dfe92Chris Lattner // If this token (the macro argument) was supposed to get leading 346442a661a2b44e78e3c66e35bf842c3dc119dfe92Chris Lattner // whitespace, transfer this information onto the first token of the 347442a661a2b44e78e3c66e35bf842c3dc119dfe92Chris Lattner // expansion. 348442a661a2b44e78e3c66e35bf842c3dc119dfe92Chris Lattner // 349442a661a2b44e78e3c66e35bf842c3dc119dfe92Chris Lattner // Do not do this if the paste operator occurs before the macro argument, 350442a661a2b44e78e3c66e35bf842c3dc119dfe92Chris Lattner // as in "A ## MACROARG". In valid code, the first token will get 351442a661a2b44e78e3c66e35bf842c3dc119dfe92Chris Lattner // smooshed onto the preceding one anyway (forming AMACROARG). In 352442a661a2b44e78e3c66e35bf842c3dc119dfe92Chris Lattner // assembler-with-cpp mode, invalid pastes are allowed through: in this 353442a661a2b44e78e3c66e35bf842c3dc119dfe92Chris Lattner // case, we do not want the extra whitespace to be added. For example, 354442a661a2b44e78e3c66e35bf842c3dc119dfe92Chris Lattner // we want ". ## foo" -> ".foo" not ". foo". 355651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (NextTokGetsSpace) 356442a661a2b44e78e3c66e35bf842c3dc119dfe92Chris Lattner ResultToks[ResultToks.size()-NumToks].setFlag(Token::LeadingSpace); 3571eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 358442a661a2b44e78e3c66e35bf842c3dc119dfe92Chris Lattner NextTokGetsSpace = false; 3595f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer continue; 3605f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 3611eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 3625f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If an empty argument is on the LHS or RHS of a paste, the standard (C99 3635f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // 6.10.3.3p2,3) calls for a bunch of placemarker stuff to occur. We 3645f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // implement this by eating ## operators when a LHS or RHS expands to 3655f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // empty. 3665f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (PasteAfter) { 3675f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Discard the argument token and skip (don't copy to the expansion 3685f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // buffer) the paste operator after it. 3695f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer ++i; 3705f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer continue; 3715f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 3721eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 3735f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If this is on the RHS of a paste operator, we've already copied the 374461f2d820ffb628ebdf6ef4a4474a43f8ff914e5Argyrios Kyrtzidis // paste operator to the ResultToks list, unless the LHS was empty too. 375461f2d820ffb628ebdf6ef4a4474a43f8ff914e5Argyrios Kyrtzidis // Remove it. 376461f2d820ffb628ebdf6ef4a4474a43f8ff914e5Argyrios Kyrtzidis assert(PasteBefore); 377461f2d820ffb628ebdf6ef4a4474a43f8ff914e5Argyrios Kyrtzidis if (NonEmptyPasteBefore) { 378461f2d820ffb628ebdf6ef4a4474a43f8ff914e5Argyrios Kyrtzidis assert(ResultToks.back().is(tok::hashhash)); 379651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines ResultToks.pop_back(); 380461f2d820ffb628ebdf6ef4a4474a43f8ff914e5Argyrios Kyrtzidis } 3811eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 3825f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If this is the __VA_ARGS__ token, and if the argument wasn't provided, 3835f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // and if the macro had at least one real argument, and if the token before 38402f0022dfced0d082d979ebfd9185211f4dd0496Andy Gibbs // the ## was a comma, remove the comma. This is a GCC extension which is 38502f0022dfced0d082d979ebfd9185211f4dd0496Andy Gibbs // disabled when using -std=c99. 38602f0022dfced0d082d979ebfd9185211f4dd0496Andy Gibbs if (ActualArgs->isVarargsElidedUse()) 387651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines MaybeRemoveCommaBeforeVaArgs(ResultToks, 38802f0022dfced0d082d979ebfd9185211f4dd0496Andy Gibbs /*HasPasteOperator=*/true, 38902f0022dfced0d082d979ebfd9185211f4dd0496Andy Gibbs Macro, ArgNo, PP); 39002f0022dfced0d082d979ebfd9185211f4dd0496Andy Gibbs 3915f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer continue; 3925f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 3931eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 3948d896438031f5990cb42eb95a7bae233db2eb333Chris Lattner // If anything changed, install this as the new Tokens list. 3955f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (MadeChange) { 39632fca722dd974b8202d0fb9c71b6c185c0767da6Chris Lattner assert(!OwnsTokens && "This would leak if we already own the token list"); 3975f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // This is deleted in the dtor. 3988d896438031f5990cb42eb95a7bae233db2eb333Chris Lattner NumTokens = ResultToks.size(); 3995b3284a9275a27f5c4410e25eb8933be540601d6Argyrios Kyrtzidis // The tokens will be added to Preprocessor's cache and will be removed 4005b3284a9275a27f5c4410e25eb8933be540601d6Argyrios Kyrtzidis // when this TokenLexer finishes lexing them. 4015b3284a9275a27f5c4410e25eb8933be540601d6Argyrios Kyrtzidis Tokens = PP.cacheMacroExpandedTokens(this, ResultToks); 4025b3284a9275a27f5c4410e25eb8933be540601d6Argyrios Kyrtzidis 4035b3284a9275a27f5c4410e25eb8933be540601d6Argyrios Kyrtzidis // The preprocessor cache of macro expanded tokens owns these tokens,not us. 40432fca722dd974b8202d0fb9c71b6c185c0767da6Chris Lattner OwnsTokens = false; 4055f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 4065f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 4075f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 4085f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// Lex - Lex and return a token from this macro stream. 4095f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// 410d2f9308220af22bfc1bcd3bc2cad118dbd8be013Eli Friedmanbool TokenLexer::Lex(Token &Tok) { 4115f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Lexing off the end of the macro, pop this macro off the expansion stack. 4125f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (isAtEnd()) { 4135f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If this is a macro (not a token stream), mark the macro enabled now 4145f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // that it is no longer being expanded. 4155f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (Macro) Macro->EnableMacro(); 4165f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 417d2f9308220af22bfc1bcd3bc2cad118dbd8be013Eli Friedman Tok.startToken(); 418d2f9308220af22bfc1bcd3bc2cad118dbd8be013Eli Friedman Tok.setFlagValue(Token::StartOfLine , AtStartOfLine); 419651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines Tok.setFlagValue(Token::LeadingSpace, HasLeadingSpace || NextTokGetsSpace); 420d2f9308220af22bfc1bcd3bc2cad118dbd8be013Eli Friedman if (CurToken == 0) 421d2f9308220af22bfc1bcd3bc2cad118dbd8be013Eli Friedman Tok.setFlag(Token::LeadingEmptyMacro); 422d2f9308220af22bfc1bcd3bc2cad118dbd8be013Eli Friedman return PP.HandleEndOfTokenLexer(Tok); 4235f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 4241eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 4250fd9c48dfc6a324d8399b0b43ab4b943a1b1b843Argyrios Kyrtzidis SourceManager &SM = PP.getSourceManager(); 4260fd9c48dfc6a324d8399b0b43ab4b943a1b1b843Argyrios Kyrtzidis 4275f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If this is the first token of the expanded result, we inherit spacing 4285f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // properties later. 4295f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer bool isFirstToken = CurToken == 0; 4301eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 4315f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Get the next token to return. 4328d896438031f5990cb42eb95a7bae233db2eb333Chris Lattner Tok = Tokens[CurToken++]; 4331eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 434332ee08e2fe78fde281e0868ec039b546cb6e0f4Chris Lattner bool TokenIsFromPaste = false; 4351eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 4365f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If this token is followed by a token paste (##) operator, paste the tokens! 437b11e43c31dc5d395a7ec6a07259c078dadd4f47bChris Lattner // Note that ## is a normal token when not expanding a macro. 4383582a6c0a35f202249b721e100229fde9de0d6a6Argyrios Kyrtzidis if (!isAtEnd() && Tokens[CurToken].is(tok::hashhash) && Macro) { 43904f2bd859ba6a23faf98eb3cc7d40e9adf6d5eaaChris Lattner // When handling the microsoft /##/ extension, the final token is 44004f2bd859ba6a23faf98eb3cc7d40e9adf6d5eaaChris Lattner // returned by PasteTokens, not the pasted token. 44104f2bd859ba6a23faf98eb3cc7d40e9adf6d5eaaChris Lattner if (PasteTokens(Tok)) 442d2f9308220af22bfc1bcd3bc2cad118dbd8be013Eli Friedman return true; 44365cc1e889c0c2c617a81a0ae032fa4fb24760f38Kovarththanan Rajaratnam 44404f2bd859ba6a23faf98eb3cc7d40e9adf6d5eaaChris Lattner TokenIsFromPaste = true; 4451eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump } 4465f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 4475f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // The token's current location indicate where the token was lexed from. We 4485f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // need this information to compute the spelling of the token, but any 4495f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // diagnostics for the expanded token should appear as if they came from 450d0f5e443ceed37e1a72015bd109db8e5fc6206cfChandler Carruth // ExpansionLoc. Pull this information together into a new SourceLocation 4515f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // that captures all of this. 452d0f5e443ceed37e1a72015bd109db8e5fc6206cfChandler Carruth if (ExpandLocStart.isValid() && // Don't do this for token streams. 453b73377eeb3eff76be134203aebb6068244b177f3Argyrios Kyrtzidis // Check that the token's location was not already set properly. 4545d579e7b0463a2e65328df85e4fed8e07799fd9eArgyrios Kyrtzidis SM.isBeforeInSLocAddrSpace(Tok.getLocation(), MacroStartSLocOffset)) { 455b73377eeb3eff76be134203aebb6068244b177f3Argyrios Kyrtzidis SourceLocation instLoc; 456b73377eeb3eff76be134203aebb6068244b177f3Argyrios Kyrtzidis if (Tok.is(tok::comment)) { 457bf340e452339e374ea6eef78c1f0a2abdd16c5a3Chandler Carruth instLoc = SM.createExpansionLoc(Tok.getLocation(), 458bf340e452339e374ea6eef78c1f0a2abdd16c5a3Chandler Carruth ExpandLocStart, 459bf340e452339e374ea6eef78c1f0a2abdd16c5a3Chandler Carruth ExpandLocEnd, 460bf340e452339e374ea6eef78c1f0a2abdd16c5a3Chandler Carruth Tok.getLength()); 461b73377eeb3eff76be134203aebb6068244b177f3Argyrios Kyrtzidis } else { 462ec3b5389a5032433116ec2f2ee266b9666ad2eb4Argyrios Kyrtzidis instLoc = getExpansionLocForMacroDefLoc(Tok.getLocation()); 463b73377eeb3eff76be134203aebb6068244b177f3Argyrios Kyrtzidis } 464b73377eeb3eff76be134203aebb6068244b177f3Argyrios Kyrtzidis 465b73377eeb3eff76be134203aebb6068244b177f3Argyrios Kyrtzidis Tok.setLocation(instLoc); 4665f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 4671eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 4685f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If this is the first token, set the lexical properties of the token to 4695f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // match the lexical properties of the macro identifier. 4705f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (isFirstToken) { 471d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner Tok.setFlagValue(Token::StartOfLine , AtStartOfLine); 472d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner Tok.setFlagValue(Token::LeadingSpace, HasLeadingSpace); 473651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } else { 474651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // If this is not the first token, we may still need to pass through 475651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // leading whitespace if we've expanded a macro. 476651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (AtStartOfLine) Tok.setFlag(Token::StartOfLine); 477651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (HasLeadingSpace) Tok.setFlag(Token::LeadingSpace); 4785f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 479651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines AtStartOfLine = false; 480651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines HasLeadingSpace = false; 4811eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 4825f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Handle recursive expansion! 4836bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines if (!Tok.isAnnotation() && Tok.getIdentifierInfo() != nullptr) { 484863c486fcb6162495a94fddf7ac8409de2638995Chris Lattner // Change the kind of this identifier to the appropriate token kind, e.g. 485863c486fcb6162495a94fddf7ac8409de2638995Chris Lattner // turning "for" into a keyword. 4866f421f91849418883186592051fce05339ee62e9Argyrios Kyrtzidis IdentifierInfo *II = Tok.getIdentifierInfo(); 487863c486fcb6162495a94fddf7ac8409de2638995Chris Lattner Tok.setKind(II->getTokenID()); 4881eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 489332ee08e2fe78fde281e0868ec039b546cb6e0f4Chris Lattner // If this identifier was poisoned and from a paste, emit an error. This 490332ee08e2fe78fde281e0868ec039b546cb6e0f4Chris Lattner // won't be handled by Preprocessor::HandleIdentifier because this is coming 491332ee08e2fe78fde281e0868ec039b546cb6e0f4Chris Lattner // from a macro expansion. 492332ee08e2fe78fde281e0868ec039b546cb6e0f4Chris Lattner if (II->isPoisoned() && TokenIsFromPaste) { 49328bbe4b8acc338476fe0825769b41fb32b423c72John Wiegley PP.HandlePoisonedIdentifier(Tok); 494332ee08e2fe78fde281e0868ec039b546cb6e0f4Chris Lattner } 4951eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 496863c486fcb6162495a94fddf7ac8409de2638995Chris Lattner if (!DisableMacroExpansion && II->isHandleIdentifierCase()) 497d2f9308220af22bfc1bcd3bc2cad118dbd8be013Eli Friedman return PP.HandleIdentifier(Tok); 498863c486fcb6162495a94fddf7ac8409de2638995Chris Lattner } 4995f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 5005f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Otherwise, return a normal token. 501d2f9308220af22bfc1bcd3bc2cad118dbd8be013Eli Friedman return true; 5025f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 5035f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 5045f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// PasteTokens - Tok is the LHS of a ## operator, and CurToken is the ## 5055f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// operator. Read the ## and RHS, and paste the LHS/RHS together. If there 50648002c89578eaeb008a0bb5a10d8216e61906459Nico Weber/// are more ## after it, chomp them iteratively. Return the result as Tok. 5073f1cc838f9caf469990f23fccd0940263c0c61ffChris Lattner/// If this returns true, the caller should immediately return the token. 5081543e9c69202b8e128c7fe34784ae7aa90964889Chris Lattnerbool TokenLexer::PasteTokens(Token &Tok) { 509f7ccbad5d9949e7ddd1cbef43d482553b811e026Dylan Noblesmith SmallString<128> Buffer; 5106bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines const char *ResultTokStrPtr = nullptr; 511a08529cc3f00e0b47a3c028823634129ac46847bAbramo Bagnara SourceLocation StartLoc = Tok.getLocation(); 512b73377eeb3eff76be134203aebb6068244b177f3Argyrios Kyrtzidis SourceLocation PasteOpLoc; 5135f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer do { 5145f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Consume the ## operator. 515b73377eeb3eff76be134203aebb6068244b177f3Argyrios Kyrtzidis PasteOpLoc = Tokens[CurToken].getLocation(); 5165f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer ++CurToken; 5175f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer assert(!isAtEnd() && "No token on the RHS of a paste operator!"); 5181eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 5195f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Get the RHS token. 5208d896438031f5990cb42eb95a7bae233db2eb333Chris Lattner const Token &RHS = Tokens[CurToken]; 5211eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 5225f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Allocate space for the result token. This is guaranteed to be enough for 5231fa495304c81e03f07f278a47b5efe9317104aabChris Lattner // the two tokens. 5241fa495304c81e03f07f278a47b5efe9317104aabChris Lattner Buffer.resize(Tok.getLength() + RHS.getLength()); 5251eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 5265f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Get the spelling of the LHS token in Buffer. 5275f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer const char *BufPtr = &Buffer[0]; 528453091cc2082e207ea2c2dda645a9bc01b37fb0cDouglas Gregor bool Invalid = false; 529453091cc2082e207ea2c2dda645a9bc01b37fb0cDouglas Gregor unsigned LHSLen = PP.getSpelling(Tok, BufPtr, &Invalid); 5305f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (BufPtr != &Buffer[0]) // Really, we want the chars in Buffer! 5315f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer memcpy(&Buffer[0], BufPtr, LHSLen); 532453091cc2082e207ea2c2dda645a9bc01b37fb0cDouglas Gregor if (Invalid) 533453091cc2082e207ea2c2dda645a9bc01b37fb0cDouglas Gregor return true; 534453091cc2082e207ea2c2dda645a9bc01b37fb0cDouglas Gregor 5355f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer BufPtr = &Buffer[LHSLen]; 536453091cc2082e207ea2c2dda645a9bc01b37fb0cDouglas Gregor unsigned RHSLen = PP.getSpelling(RHS, BufPtr, &Invalid); 537453091cc2082e207ea2c2dda645a9bc01b37fb0cDouglas Gregor if (Invalid) 538453091cc2082e207ea2c2dda645a9bc01b37fb0cDouglas Gregor return true; 5395f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (BufPtr != &Buffer[LHSLen]) // Really, we want the chars in Buffer! 5405f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer memcpy(&Buffer[LHSLen], BufPtr, RHSLen); 5411eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 5425f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Trim excess space. 5431fa495304c81e03f07f278a47b5efe9317104aabChris Lattner Buffer.resize(LHSLen+RHSLen); 5441eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 5455f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Plop the pasted result (including the trailing newline and null) into a 5465f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // scratch buffer where we can lex it. 54747246be8ac5b0ddde6c402b8fc6946b6135487b5Chris Lattner Token ResultTokTmp; 54847246be8ac5b0ddde6c402b8fc6946b6135487b5Chris Lattner ResultTokTmp.startToken(); 5491eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 55047246be8ac5b0ddde6c402b8fc6946b6135487b5Chris Lattner // Claim that the tmp token is a string_literal so that we can get the 551917f14387470cc98b8ff90342f5307bc921234fbChris Lattner // character pointer back from CreateString in getLiteralData(). 55247246be8ac5b0ddde6c402b8fc6946b6135487b5Chris Lattner ResultTokTmp.setKind(tok::string_literal); 553374b3837d676133fcc1eb70a25c8baf8ec4a5c4aDmitri Gribenko PP.CreateString(Buffer, ResultTokTmp); 55447246be8ac5b0ddde6c402b8fc6946b6135487b5Chris Lattner SourceLocation ResultTokLoc = ResultTokTmp.getLocation(); 55547246be8ac5b0ddde6c402b8fc6946b6135487b5Chris Lattner ResultTokStrPtr = ResultTokTmp.getLiteralData(); 55647246be8ac5b0ddde6c402b8fc6946b6135487b5Chris Lattner 5575f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Lex the resultant pasted token into Result. 558d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner Token Result; 5591eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 560c4bf2b9afb7d47445a9dc6bc848657098a4e3851Abramo Bagnara if (Tok.isAnyIdentifier() && RHS.isAnyIdentifier()) { 5615f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Common paste case: identifier+identifier = identifier. Avoid creating 5625f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // a lexer and other overhead. 5635f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer PP.IncrementPasteCounter(true); 5645f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Result.startToken(); 565c4bf2b9afb7d47445a9dc6bc848657098a4e3851Abramo Bagnara Result.setKind(tok::raw_identifier); 566c4bf2b9afb7d47445a9dc6bc848657098a4e3851Abramo Bagnara Result.setRawIdentifierData(ResultTokStrPtr); 5675f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Result.setLocation(ResultTokLoc); 5685f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Result.setLength(LHSLen+RHSLen); 5695f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } else { 5705f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer PP.IncrementPasteCounter(false); 5711eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 572bcc2a67e5180612417727cbdd8afd0f79fdf726dChris Lattner assert(ResultTokLoc.isFileID() && 573bcc2a67e5180612417727cbdd8afd0f79fdf726dChris Lattner "Should be a raw location into scratch buffer"); 5745f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer SourceManager &SourceMgr = PP.getSourceManager(); 57547246be8ac5b0ddde6c402b8fc6946b6135487b5Chris Lattner FileID LocFileID = SourceMgr.getFileID(ResultTokLoc); 5761eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 577f715ca12bfc9fddfde75f98a197424434428b821Douglas Gregor bool Invalid = false; 578aea67dbd653a2dd6dd5cc2159279e81e855b2482Douglas Gregor const char *ScratchBufStart 579f6ac97b101c8840efa92bf29166077ce4049e293Benjamin Kramer = SourceMgr.getBufferData(LocFileID, &Invalid).data(); 580f715ca12bfc9fddfde75f98a197424434428b821Douglas Gregor if (Invalid) 581aea67dbd653a2dd6dd5cc2159279e81e855b2482Douglas Gregor return false; 5821eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 583bcc2a67e5180612417727cbdd8afd0f79fdf726dChris Lattner // Make a lexer to lex this string from. Lex just this one token. 5845f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Make a lexer object so that we lex and expand the paste result. 58547246be8ac5b0ddde6c402b8fc6946b6135487b5Chris Lattner Lexer TL(SourceMgr.getLocForStartOfFile(LocFileID), 5864e4d08403ca5cfd4d558fa2936215d3a4e5a528dDavid Blaikie PP.getLangOpts(), ScratchBufStart, 5871fa495304c81e03f07f278a47b5efe9317104aabChris Lattner ResultTokStrPtr, ResultTokStrPtr+LHSLen+RHSLen); 5881eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 5895f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Lex a token in raw mode. This way it won't look up identifiers 5905f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // automatically, lexing off the end will return an eof token, and 5915f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // warnings are disabled. This returns true if the result token is the 5925f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // entire buffer. 5931fa495304c81e03f07f278a47b5efe9317104aabChris Lattner bool isInvalid = !TL.LexFromRawLexer(Result); 5941eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 5955f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If we got an EOF token, we didn't form even ONE token. For example, we 5965f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // did "/ ## /" to get "//". 5971fa495304c81e03f07f278a47b5efe9317104aabChris Lattner isInvalid |= Result.is(tok::eof); 5981eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 5991fa495304c81e03f07f278a47b5efe9317104aabChris Lattner // If pasting the two tokens didn't form a full new token, this is an 6001fa495304c81e03f07f278a47b5efe9317104aabChris Lattner // error. This occurs with "x ## +" and other stuff. Return with Tok 6011fa495304c81e03f07f278a47b5efe9317104aabChris Lattner // unmodified and with RHS as the next token to lex. 6021fa495304c81e03f07f278a47b5efe9317104aabChris Lattner if (isInvalid) { 6031fa495304c81e03f07f278a47b5efe9317104aabChris Lattner // Test for the Microsoft extension of /##/ turning into // here on the 6041fa495304c81e03f07f278a47b5efe9317104aabChris Lattner // error path. 6054e4d08403ca5cfd4d558fa2936215d3a4e5a528dDavid Blaikie if (PP.getLangOpts().MicrosoftExt && Tok.is(tok::slash) && 6061fa495304c81e03f07f278a47b5efe9317104aabChris Lattner RHS.is(tok::slash)) { 6071fa495304c81e03f07f278a47b5efe9317104aabChris Lattner HandleMicrosoftCommentPaste(Tok); 6081fa495304c81e03f07f278a47b5efe9317104aabChris Lattner return true; 6091fa495304c81e03f07f278a47b5efe9317104aabChris Lattner } 6101eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 611920bf45bd739334c25633ec9a29d903cab50be23Chris Lattner // Do not emit the error when preprocessing assembler code. 6124e4d08403ca5cfd4d558fa2936215d3a4e5a528dDavid Blaikie if (!PP.getLangOpts().AsmPreprocessor) { 613d0f5e443ceed37e1a72015bd109db8e5fc6206cfChandler Carruth // Explicitly convert the token location to have proper expansion 6143c17b8a4306b84c05454168128bc705b33a79ca3Chris Lattner // information so that the user knows where it came from. 6153c17b8a4306b84c05454168128bc705b33a79ca3Chris Lattner SourceManager &SM = PP.getSourceManager(); 616b11e43c31dc5d395a7ec6a07259c078dadd4f47bChris Lattner SourceLocation Loc = 617bf340e452339e374ea6eef78c1f0a2abdd16c5a3Chandler Carruth SM.createExpansionLoc(PasteOpLoc, ExpandLocStart, ExpandLocEnd, 2); 618920bf45bd739334c25633ec9a29d903cab50be23Chris Lattner // If we're in microsoft extensions mode, downgrade this from a hard 619651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // error to an extension that defaults to an error. This allows 620920bf45bd739334c25633ec9a29d903cab50be23Chris Lattner // disabling it. 621651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines PP.Diag(Loc, PP.getLangOpts().MicrosoftExt ? diag::ext_pp_bad_paste_ms 622651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines : diag::err_pp_bad_paste) 623651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines << Buffer.str(); 6243c17b8a4306b84c05454168128bc705b33a79ca3Chris Lattner } 6251eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 626c5f7459b2d60ad55ed2ab3cfd281d9c718f5a8dfRichard Smith // An error has occurred so exit loop. 627c5f7459b2d60ad55ed2ab3cfd281d9c718f5a8dfRichard Smith break; 6281fa495304c81e03f07f278a47b5efe9317104aabChris Lattner } 6291eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 6301fa495304c81e03f07f278a47b5efe9317104aabChris Lattner // Turn ## into 'unknown' to avoid # ## # from looking like a paste 6311fa495304c81e03f07f278a47b5efe9317104aabChris Lattner // operator. 6321fa495304c81e03f07f278a47b5efe9317104aabChris Lattner if (Result.is(tok::hashhash)) 6331fa495304c81e03f07f278a47b5efe9317104aabChris Lattner Result.setKind(tok::unknown); 6341fa495304c81e03f07f278a47b5efe9317104aabChris Lattner } 6351eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 636bed28ac1d1463adca3ecf24fca5c30646fa9dbb2Sylvestre Ledru // Transfer properties of the LHS over the Result. 637d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner Result.setFlagValue(Token::StartOfLine , Tok.isAtStartOfLine()); 638d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner Result.setFlagValue(Token::LeadingSpace, Tok.hasLeadingSpace()); 639b73377eeb3eff76be134203aebb6068244b177f3Argyrios Kyrtzidis 6405f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Finally, replace LHS with the result, consume the RHS, and iterate. 6415f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer ++CurToken; 6425f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Tok = Result; 6438d896438031f5990cb42eb95a7bae233db2eb333Chris Lattner } while (!isAtEnd() && Tokens[CurToken].is(tok::hashhash)); 6441eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 645a08529cc3f00e0b47a3c028823634129ac46847bAbramo Bagnara SourceLocation EndLoc = Tokens[CurToken - 1].getLocation(); 646a08529cc3f00e0b47a3c028823634129ac46847bAbramo Bagnara 647b73377eeb3eff76be134203aebb6068244b177f3Argyrios Kyrtzidis // The token's current location indicate where the token was lexed from. We 648b73377eeb3eff76be134203aebb6068244b177f3Argyrios Kyrtzidis // need this information to compute the spelling of the token, but any 649b73377eeb3eff76be134203aebb6068244b177f3Argyrios Kyrtzidis // diagnostics for the expanded token should appear as if the token was 650a08529cc3f00e0b47a3c028823634129ac46847bAbramo Bagnara // expanded from the full ## expression. Pull this information together into 651b73377eeb3eff76be134203aebb6068244b177f3Argyrios Kyrtzidis // a new SourceLocation that captures all of this. 652c0069d850efe74bac36c526dfec86b7dad1ede2fArgyrios Kyrtzidis SourceManager &SM = PP.getSourceManager(); 653a08529cc3f00e0b47a3c028823634129ac46847bAbramo Bagnara if (StartLoc.isFileID()) 654a08529cc3f00e0b47a3c028823634129ac46847bAbramo Bagnara StartLoc = getExpansionLocForMacroDefLoc(StartLoc); 655a08529cc3f00e0b47a3c028823634129ac46847bAbramo Bagnara if (EndLoc.isFileID()) 656a08529cc3f00e0b47a3c028823634129ac46847bAbramo Bagnara EndLoc = getExpansionLocForMacroDefLoc(EndLoc); 65719c8ce0cf534caa909955ba9a17b7841f574a3beEli Friedman FileID MacroFID = SM.getFileID(MacroExpansionStart); 65819c8ce0cf534caa909955ba9a17b7841f574a3beEli Friedman while (SM.getFileID(StartLoc) != MacroFID) 65919c8ce0cf534caa909955ba9a17b7841f574a3beEli Friedman StartLoc = SM.getImmediateExpansionRange(StartLoc).first; 66019c8ce0cf534caa909955ba9a17b7841f574a3beEli Friedman while (SM.getFileID(EndLoc) != MacroFID) 66119c8ce0cf534caa909955ba9a17b7841f574a3beEli Friedman EndLoc = SM.getImmediateExpansionRange(EndLoc).second; 66219c8ce0cf534caa909955ba9a17b7841f574a3beEli Friedman 663a08529cc3f00e0b47a3c028823634129ac46847bAbramo Bagnara Tok.setLocation(SM.createExpansionLoc(Tok.getLocation(), StartLoc, EndLoc, 664c0069d850efe74bac36c526dfec86b7dad1ede2fArgyrios Kyrtzidis Tok.getLength())); 665b73377eeb3eff76be134203aebb6068244b177f3Argyrios Kyrtzidis 6665f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Now that we got the result token, it will be subject to expansion. Since 6675f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // token pasting re-lexes the result token in raw mode, identifier information 6685f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // isn't looked up. As such, if the result is an identifier, look up id info. 669c4bf2b9afb7d47445a9dc6bc848657098a4e3851Abramo Bagnara if (Tok.is(tok::raw_identifier)) { 6705f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Look up the identifier info for the token. We disabled identifier lookup 6715f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // by saying we're skipping contents, so we need to do this manually. 672c4bf2b9afb7d47445a9dc6bc848657098a4e3851Abramo Bagnara PP.LookUpIdentifierInfo(Tok); 6735f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 6743f1cc838f9caf469990f23fccd0940263c0c61ffChris Lattner return false; 6755f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 6765f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 6775f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// isNextTokenLParen - If the next token lexed will pop this macro off the 6785f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// expansion stack, return 2. If the next unexpanded token is a '(', return 6795f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// 1, otherwise return 0. 6801543e9c69202b8e128c7fe34784ae7aa90964889Chris Lattnerunsigned TokenLexer::isNextTokenLParen() const { 6815f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Out of tokens? 6825f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (isAtEnd()) 6835f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer return 2; 6848d896438031f5990cb42eb95a7bae233db2eb333Chris Lattner return Tokens[CurToken].is(tok::l_paren); 6855f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 6863f1cc838f9caf469990f23fccd0940263c0c61ffChris Lattner 687b2eb53d9fd973a1a02e05e67a3307b3efd12eff2Peter Collingbourne/// isParsingPreprocessorDirective - Return true if we are in the middle of a 688b2eb53d9fd973a1a02e05e67a3307b3efd12eff2Peter Collingbourne/// preprocessor directive. 689b2eb53d9fd973a1a02e05e67a3307b3efd12eff2Peter Collingbournebool TokenLexer::isParsingPreprocessorDirective() const { 69084021556baceb76eedf7d44be8ba71d9b8cfaccePeter Collingbourne return Tokens[NumTokens-1].is(tok::eod) && !isAtEnd(); 691b2eb53d9fd973a1a02e05e67a3307b3efd12eff2Peter Collingbourne} 6923f1cc838f9caf469990f23fccd0940263c0c61ffChris Lattner 6933f1cc838f9caf469990f23fccd0940263c0c61ffChris Lattner/// HandleMicrosoftCommentPaste - In microsoft compatibility mode, /##/ pastes 6943f1cc838f9caf469990f23fccd0940263c0c61ffChris Lattner/// together to form a comment that comments out everything in the current 6953f1cc838f9caf469990f23fccd0940263c0c61ffChris Lattner/// macro, other active macros, and anything left on the current physical 696d0f5e443ceed37e1a72015bd109db8e5fc6206cfChandler Carruth/// source line of the expanded buffer. Handle this by returning the 6973f1cc838f9caf469990f23fccd0940263c0c61ffChris Lattner/// first token on the next line. 6981543e9c69202b8e128c7fe34784ae7aa90964889Chris Lattnervoid TokenLexer::HandleMicrosoftCommentPaste(Token &Tok) { 6993f1cc838f9caf469990f23fccd0940263c0c61ffChris Lattner // We 'comment out' the rest of this macro by just ignoring the rest of the 7003f1cc838f9caf469990f23fccd0940263c0c61ffChris Lattner // tokens that have not been lexed yet, if any. 7011eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 7023f1cc838f9caf469990f23fccd0940263c0c61ffChris Lattner // Since this must be a macro, mark the macro enabled now that it is no longer 7033f1cc838f9caf469990f23fccd0940263c0c61ffChris Lattner // being expanded. 7043f1cc838f9caf469990f23fccd0940263c0c61ffChris Lattner assert(Macro && "Token streams can't paste comments"); 7053f1cc838f9caf469990f23fccd0940263c0c61ffChris Lattner Macro->EnableMacro(); 7061eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 7073f1cc838f9caf469990f23fccd0940263c0c61ffChris Lattner PP.HandleMicrosoftCommentPaste(Tok); 7083f1cc838f9caf469990f23fccd0940263c0c61ffChris Lattner} 709b73377eeb3eff76be134203aebb6068244b177f3Argyrios Kyrtzidis 710ec3b5389a5032433116ec2f2ee266b9666ad2eb4Argyrios Kyrtzidis/// \brief If \arg loc is a file ID and points inside the current macro 711b73377eeb3eff76be134203aebb6068244b177f3Argyrios Kyrtzidis/// definition, returns the appropriate source location pointing at the 712ec3b5389a5032433116ec2f2ee266b9666ad2eb4Argyrios Kyrtzidis/// macro expansion source location entry, otherwise it returns an invalid 713ec3b5389a5032433116ec2f2ee266b9666ad2eb4Argyrios Kyrtzidis/// SourceLocation. 714ec3b5389a5032433116ec2f2ee266b9666ad2eb4Argyrios KyrtzidisSourceLocation 715ec3b5389a5032433116ec2f2ee266b9666ad2eb4Argyrios KyrtzidisTokenLexer::getExpansionLocForMacroDefLoc(SourceLocation loc) const { 716d0f5e443ceed37e1a72015bd109db8e5fc6206cfChandler Carruth assert(ExpandLocStart.isValid() && MacroExpansionStart.isValid() && 717b73377eeb3eff76be134203aebb6068244b177f3Argyrios Kyrtzidis "Not appropriate for token streams"); 718c0069d850efe74bac36c526dfec86b7dad1ede2fArgyrios Kyrtzidis assert(loc.isValid() && loc.isFileID()); 719b73377eeb3eff76be134203aebb6068244b177f3Argyrios Kyrtzidis 720b73377eeb3eff76be134203aebb6068244b177f3Argyrios Kyrtzidis SourceManager &SM = PP.getSourceManager(); 721499ea5550d6e2fc5cfbd33b47f06d92ce25d7a13Argyrios Kyrtzidis assert(SM.isInSLocAddrSpace(loc, MacroDefStart, MacroDefLength) && 722499ea5550d6e2fc5cfbd33b47f06d92ce25d7a13Argyrios Kyrtzidis "Expected loc to come from the macro definition"); 723499ea5550d6e2fc5cfbd33b47f06d92ce25d7a13Argyrios Kyrtzidis 724499ea5550d6e2fc5cfbd33b47f06d92ce25d7a13Argyrios Kyrtzidis unsigned relativeOffset = 0; 725499ea5550d6e2fc5cfbd33b47f06d92ce25d7a13Argyrios Kyrtzidis SM.isInSLocAddrSpace(loc, MacroDefStart, MacroDefLength, &relativeOffset); 726a64ccefdf0ea4e03ec88805d71b0af74950c7472Argyrios Kyrtzidis return MacroExpansionStart.getLocWithOffset(relativeOffset); 727b73377eeb3eff76be134203aebb6068244b177f3Argyrios Kyrtzidis} 728d60a34a4e514ec0dfddd05ef2744be104e111f45Argyrios Kyrtzidis 729d60a34a4e514ec0dfddd05ef2744be104e111f45Argyrios Kyrtzidis/// \brief Finds the tokens that are consecutive (from the same FileID) 730d60a34a4e514ec0dfddd05ef2744be104e111f45Argyrios Kyrtzidis/// creates a single SLocEntry, and assigns SourceLocations to each token that 731d60a34a4e514ec0dfddd05ef2744be104e111f45Argyrios Kyrtzidis/// point to that SLocEntry. e.g for 732d60a34a4e514ec0dfddd05ef2744be104e111f45Argyrios Kyrtzidis/// assert(foo == bar); 733d60a34a4e514ec0dfddd05ef2744be104e111f45Argyrios Kyrtzidis/// There will be a single SLocEntry for the "foo == bar" chunk and locations 734d60a34a4e514ec0dfddd05ef2744be104e111f45Argyrios Kyrtzidis/// for the 'foo', '==', 'bar' tokens will point inside that chunk. 735d60a34a4e514ec0dfddd05ef2744be104e111f45Argyrios Kyrtzidis/// 736d60a34a4e514ec0dfddd05ef2744be104e111f45Argyrios Kyrtzidis/// \arg begin_tokens will be updated to a position past all the found 737d60a34a4e514ec0dfddd05ef2744be104e111f45Argyrios Kyrtzidis/// consecutive tokens. 738d60a34a4e514ec0dfddd05ef2744be104e111f45Argyrios Kyrtzidisstatic void updateConsecutiveMacroArgTokens(SourceManager &SM, 739d60a34a4e514ec0dfddd05ef2744be104e111f45Argyrios Kyrtzidis SourceLocation InstLoc, 740d60a34a4e514ec0dfddd05ef2744be104e111f45Argyrios Kyrtzidis Token *&begin_tokens, 741d60a34a4e514ec0dfddd05ef2744be104e111f45Argyrios Kyrtzidis Token * end_tokens) { 742d60a34a4e514ec0dfddd05ef2744be104e111f45Argyrios Kyrtzidis assert(begin_tokens < end_tokens); 743d60a34a4e514ec0dfddd05ef2744be104e111f45Argyrios Kyrtzidis 744b6c465e17ec37390667223a18a340e8652c212ffArgyrios Kyrtzidis SourceLocation FirstLoc = begin_tokens->getLocation(); 745b6c465e17ec37390667223a18a340e8652c212ffArgyrios Kyrtzidis SourceLocation CurLoc = FirstLoc; 746b6c465e17ec37390667223a18a340e8652c212ffArgyrios Kyrtzidis 747b6c465e17ec37390667223a18a340e8652c212ffArgyrios Kyrtzidis // Compare the source location offset of tokens and group together tokens that 748b6c465e17ec37390667223a18a340e8652c212ffArgyrios Kyrtzidis // are close, even if their locations point to different FileIDs. e.g. 749b6c465e17ec37390667223a18a340e8652c212ffArgyrios Kyrtzidis // 750b6c465e17ec37390667223a18a340e8652c212ffArgyrios Kyrtzidis // |bar | foo | cake | (3 tokens from 3 consecutive FileIDs) 751b6c465e17ec37390667223a18a340e8652c212ffArgyrios Kyrtzidis // ^ ^ 752b6c465e17ec37390667223a18a340e8652c212ffArgyrios Kyrtzidis // |bar foo cake| (one SLocEntry chunk for all tokens) 753b6c465e17ec37390667223a18a340e8652c212ffArgyrios Kyrtzidis // 754b6c465e17ec37390667223a18a340e8652c212ffArgyrios Kyrtzidis // we can perform this "merge" since the token's spelling location depends 755b6c465e17ec37390667223a18a340e8652c212ffArgyrios Kyrtzidis // on the relative offset. 756b6c465e17ec37390667223a18a340e8652c212ffArgyrios Kyrtzidis 757b6c465e17ec37390667223a18a340e8652c212ffArgyrios Kyrtzidis Token *NextTok = begin_tokens + 1; 758b6c465e17ec37390667223a18a340e8652c212ffArgyrios Kyrtzidis for (; NextTok < end_tokens; ++NextTok) { 759b29740ae158f9421096a28bcc7ad6af7171b1874Argyrios Kyrtzidis SourceLocation NextLoc = NextTok->getLocation(); 760b29740ae158f9421096a28bcc7ad6af7171b1874Argyrios Kyrtzidis if (CurLoc.isFileID() != NextLoc.isFileID()) 761b29740ae158f9421096a28bcc7ad6af7171b1874Argyrios Kyrtzidis break; // Token from different kind of FileID. 762b29740ae158f9421096a28bcc7ad6af7171b1874Argyrios Kyrtzidis 763b6c465e17ec37390667223a18a340e8652c212ffArgyrios Kyrtzidis int RelOffs; 764b29740ae158f9421096a28bcc7ad6af7171b1874Argyrios Kyrtzidis if (!SM.isInSameSLocAddrSpace(CurLoc, NextLoc, &RelOffs)) 765b6c465e17ec37390667223a18a340e8652c212ffArgyrios Kyrtzidis break; // Token from different local/loaded location. 766b6c465e17ec37390667223a18a340e8652c212ffArgyrios Kyrtzidis // Check that token is not before the previous token or more than 50 767b6c465e17ec37390667223a18a340e8652c212ffArgyrios Kyrtzidis // "characters" away. 768b6c465e17ec37390667223a18a340e8652c212ffArgyrios Kyrtzidis if (RelOffs < 0 || RelOffs > 50) 769d60a34a4e514ec0dfddd05ef2744be104e111f45Argyrios Kyrtzidis break; 770b29740ae158f9421096a28bcc7ad6af7171b1874Argyrios Kyrtzidis CurLoc = NextLoc; 771b6c465e17ec37390667223a18a340e8652c212ffArgyrios Kyrtzidis } 772d60a34a4e514ec0dfddd05ef2744be104e111f45Argyrios Kyrtzidis 773d60a34a4e514ec0dfddd05ef2744be104e111f45Argyrios Kyrtzidis // For the consecutive tokens, find the length of the SLocEntry to contain 774d60a34a4e514ec0dfddd05ef2744be104e111f45Argyrios Kyrtzidis // all of them. 775b6c465e17ec37390667223a18a340e8652c212ffArgyrios Kyrtzidis Token &LastConsecutiveTok = *(NextTok-1); 776c4b5582908e743cc05782e5220546f27286be19bArgyrios Kyrtzidis int LastRelOffs = 0; 777b6c465e17ec37390667223a18a340e8652c212ffArgyrios Kyrtzidis SM.isInSameSLocAddrSpace(FirstLoc, LastConsecutiveTok.getLocation(), 778b6c465e17ec37390667223a18a340e8652c212ffArgyrios Kyrtzidis &LastRelOffs); 779b6c465e17ec37390667223a18a340e8652c212ffArgyrios Kyrtzidis unsigned FullLength = LastRelOffs + LastConsecutiveTok.getLength(); 780d60a34a4e514ec0dfddd05ef2744be104e111f45Argyrios Kyrtzidis 781d60a34a4e514ec0dfddd05ef2744be104e111f45Argyrios Kyrtzidis // Create a macro expansion SLocEntry that will "contain" all of the tokens. 782d60a34a4e514ec0dfddd05ef2744be104e111f45Argyrios Kyrtzidis SourceLocation Expansion = 783b6c465e17ec37390667223a18a340e8652c212ffArgyrios Kyrtzidis SM.createMacroArgExpansionLoc(FirstLoc, InstLoc,FullLength); 784d60a34a4e514ec0dfddd05ef2744be104e111f45Argyrios Kyrtzidis 785d60a34a4e514ec0dfddd05ef2744be104e111f45Argyrios Kyrtzidis // Change the location of the tokens from the spelling location to the new 786d60a34a4e514ec0dfddd05ef2744be104e111f45Argyrios Kyrtzidis // expanded location. 787b6c465e17ec37390667223a18a340e8652c212ffArgyrios Kyrtzidis for (; begin_tokens < NextTok; ++begin_tokens) { 788d60a34a4e514ec0dfddd05ef2744be104e111f45Argyrios Kyrtzidis Token &Tok = *begin_tokens; 789c4b5582908e743cc05782e5220546f27286be19bArgyrios Kyrtzidis int RelOffs = 0; 790b6c465e17ec37390667223a18a340e8652c212ffArgyrios Kyrtzidis SM.isInSameSLocAddrSpace(FirstLoc, Tok.getLocation(), &RelOffs); 791a64ccefdf0ea4e03ec88805d71b0af74950c7472Argyrios Kyrtzidis Tok.setLocation(Expansion.getLocWithOffset(RelOffs)); 792d60a34a4e514ec0dfddd05ef2744be104e111f45Argyrios Kyrtzidis } 793d60a34a4e514ec0dfddd05ef2744be104e111f45Argyrios Kyrtzidis} 794d60a34a4e514ec0dfddd05ef2744be104e111f45Argyrios Kyrtzidis 795d60a34a4e514ec0dfddd05ef2744be104e111f45Argyrios Kyrtzidis/// \brief Creates SLocEntries and updates the locations of macro argument 796d60a34a4e514ec0dfddd05ef2744be104e111f45Argyrios Kyrtzidis/// tokens to their new expanded locations. 797d60a34a4e514ec0dfddd05ef2744be104e111f45Argyrios Kyrtzidis/// 798d60a34a4e514ec0dfddd05ef2744be104e111f45Argyrios Kyrtzidis/// \param ArgIdDefLoc the location of the macro argument id inside the macro 799d60a34a4e514ec0dfddd05ef2744be104e111f45Argyrios Kyrtzidis/// definition. 800d60a34a4e514ec0dfddd05ef2744be104e111f45Argyrios Kyrtzidis/// \param Tokens the macro argument tokens to update. 801d60a34a4e514ec0dfddd05ef2744be104e111f45Argyrios Kyrtzidisvoid TokenLexer::updateLocForMacroArgTokens(SourceLocation ArgIdSpellLoc, 802d60a34a4e514ec0dfddd05ef2744be104e111f45Argyrios Kyrtzidis Token *begin_tokens, 803d60a34a4e514ec0dfddd05ef2744be104e111f45Argyrios Kyrtzidis Token *end_tokens) { 804d60a34a4e514ec0dfddd05ef2744be104e111f45Argyrios Kyrtzidis SourceManager &SM = PP.getSourceManager(); 805d60a34a4e514ec0dfddd05ef2744be104e111f45Argyrios Kyrtzidis 806b6c465e17ec37390667223a18a340e8652c212ffArgyrios Kyrtzidis SourceLocation InstLoc = 807d60a34a4e514ec0dfddd05ef2744be104e111f45Argyrios Kyrtzidis getExpansionLocForMacroDefLoc(ArgIdSpellLoc); 808d60a34a4e514ec0dfddd05ef2744be104e111f45Argyrios Kyrtzidis 809b6c465e17ec37390667223a18a340e8652c212ffArgyrios Kyrtzidis while (begin_tokens < end_tokens) { 810b6c465e17ec37390667223a18a340e8652c212ffArgyrios Kyrtzidis // If there's only one token just create a SLocEntry for it. 811b6c465e17ec37390667223a18a340e8652c212ffArgyrios Kyrtzidis if (end_tokens - begin_tokens == 1) { 812b6c465e17ec37390667223a18a340e8652c212ffArgyrios Kyrtzidis Token &Tok = *begin_tokens; 813b6c465e17ec37390667223a18a340e8652c212ffArgyrios Kyrtzidis Tok.setLocation(SM.createMacroArgExpansionLoc(Tok.getLocation(), 814b6c465e17ec37390667223a18a340e8652c212ffArgyrios Kyrtzidis InstLoc, 815b6c465e17ec37390667223a18a340e8652c212ffArgyrios Kyrtzidis Tok.getLength())); 816b6c465e17ec37390667223a18a340e8652c212ffArgyrios Kyrtzidis return; 817b6c465e17ec37390667223a18a340e8652c212ffArgyrios Kyrtzidis } 818b6c465e17ec37390667223a18a340e8652c212ffArgyrios Kyrtzidis 819b6c465e17ec37390667223a18a340e8652c212ffArgyrios Kyrtzidis updateConsecutiveMacroArgTokens(SM, InstLoc, begin_tokens, end_tokens); 820b6c465e17ec37390667223a18a340e8652c212ffArgyrios Kyrtzidis } 821d60a34a4e514ec0dfddd05ef2744be104e111f45Argyrios Kyrtzidis} 822d2f9308220af22bfc1bcd3bc2cad118dbd8be013Eli Friedman 823d2f9308220af22bfc1bcd3bc2cad118dbd8be013Eli Friedmanvoid TokenLexer::PropagateLineStartLeadingSpaceInfo(Token &Result) { 824d2f9308220af22bfc1bcd3bc2cad118dbd8be013Eli Friedman AtStartOfLine = Result.isAtStartOfLine(); 825d2f9308220af22bfc1bcd3bc2cad118dbd8be013Eli Friedman HasLeadingSpace = Result.hasLeadingSpace(); 826d2f9308220af22bfc1bcd3bc2cad118dbd8be013Eli Friedman} 827