TokenLexer.cpp revision 0af574270d3be2b0e73a3379dfaa633746f8fc6f
11543e9c69202b8e128c7fe34784ae7aa90964889Chris Lattner//===--- TokenLexer.cpp - Lex from a token stream -------------------------===// 25f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// 35f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// The LLVM Compiler Infrastructure 45f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// 50bc735ffcfb223c0186419547abaa5c84482663eChris Lattner// This file is distributed under the University of Illinois Open Source 60bc735ffcfb223c0186419547abaa5c84482663eChris Lattner// License. See LICENSE.TXT for details. 75f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// 85f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer//===----------------------------------------------------------------------===// 95f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// 101543e9c69202b8e128c7fe34784ae7aa90964889Chris Lattner// This file implements the TokenLexer interface. 115f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// 125f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer//===----------------------------------------------------------------------===// 135f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 145d75de0f821023f4ed4815825bf3aea8a0b5e40dChris Lattner#include "clang/Lex/TokenLexer.h" 15e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner#include "MacroArgs.h" 165f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer#include "clang/Lex/MacroInfo.h" 175f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer#include "clang/Lex/Preprocessor.h" 185f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer#include "clang/Basic/SourceManager.h" 195f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer#include "clang/Basic/Diagnostic.h" 205f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer#include "llvm/ADT/SmallVector.h" 215f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencerusing namespace clang; 225f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 235f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 241543e9c69202b8e128c7fe34784ae7aa90964889Chris Lattner/// Create a TokenLexer for the specified macro with the specified actual 255f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// arguments. Note that this ctor takes ownership of the ActualArgs pointer. 261543e9c69202b8e128c7fe34784ae7aa90964889Chris Lattnervoid TokenLexer::Init(Token &Tok, MacroArgs *Actuals) { 271543e9c69202b8e128c7fe34784ae7aa90964889Chris Lattner // If the client is reusing a TokenLexer, make sure to free any memory 289594acf32de2939b15eafa8fe818607bfc56bf66Chris Lattner // associated with it. 299594acf32de2939b15eafa8fe818607bfc56bf66Chris Lattner destroy(); 309594acf32de2939b15eafa8fe818607bfc56bf66Chris Lattner 31cc1a875f94630e58d24a55577ffbf0e89b7da8c7Chris Lattner Macro = PP.getMacroInfo(Tok.getIdentifierInfo()); 329594acf32de2939b15eafa8fe818607bfc56bf66Chris Lattner ActualArgs = Actuals; 339594acf32de2939b15eafa8fe818607bfc56bf66Chris Lattner CurToken = 0; 349594acf32de2939b15eafa8fe818607bfc56bf66Chris Lattner InstantiateLoc = Tok.getLocation(); 359594acf32de2939b15eafa8fe818607bfc56bf66Chris Lattner AtStartOfLine = Tok.isAtStartOfLine(); 369594acf32de2939b15eafa8fe818607bfc56bf66Chris Lattner HasLeadingSpace = Tok.hasLeadingSpace(); 378d896438031f5990cb42eb95a7bae233db2eb333Chris Lattner Tokens = &*Macro->tokens_begin(); 388d896438031f5990cb42eb95a7bae233db2eb333Chris Lattner OwnsTokens = false; 396b884508c3bc97cc9df9516adb92fbf88dd0a2e4Chris Lattner DisableMacroExpansion = false; 408d896438031f5990cb42eb95a7bae233db2eb333Chris Lattner NumTokens = Macro->tokens_end()-Macro->tokens_begin(); 415f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 425f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If this is a function-like macro, expand the arguments and change 438d896438031f5990cb42eb95a7bae233db2eb333Chris Lattner // Tokens to point to the expanded tokens. 445f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (Macro->isFunctionLike() && Macro->getNumArgs()) 455f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer ExpandFunctionArguments(); 465f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 475f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Mark the macro as currently disabled, so that it is not recursively 485f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // expanded. The macro must be disabled only after argument pre-expansion of 495f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // function-like macro arguments occurs. 505f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Macro->DisableMacro(); 515f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 525f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 539594acf32de2939b15eafa8fe818607bfc56bf66Chris Lattner 549594acf32de2939b15eafa8fe818607bfc56bf66Chris Lattner 551543e9c69202b8e128c7fe34784ae7aa90964889Chris Lattner/// Create a TokenLexer for the specified token stream. This does not 565f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// take ownership of the specified token vector. 576b884508c3bc97cc9df9516adb92fbf88dd0a2e4Chris Lattnervoid TokenLexer::Init(const Token *TokArray, unsigned NumToks, 586b884508c3bc97cc9df9516adb92fbf88dd0a2e4Chris Lattner bool disableMacroExpansion, bool ownsTokens) { 591543e9c69202b8e128c7fe34784ae7aa90964889Chris Lattner // If the client is reusing a TokenLexer, make sure to free any memory 609594acf32de2939b15eafa8fe818607bfc56bf66Chris Lattner // associated with it. 619594acf32de2939b15eafa8fe818607bfc56bf66Chris Lattner destroy(); 629594acf32de2939b15eafa8fe818607bfc56bf66Chris Lattner 639594acf32de2939b15eafa8fe818607bfc56bf66Chris Lattner Macro = 0; 649594acf32de2939b15eafa8fe818607bfc56bf66Chris Lattner ActualArgs = 0; 658d896438031f5990cb42eb95a7bae233db2eb333Chris Lattner Tokens = TokArray; 666b884508c3bc97cc9df9516adb92fbf88dd0a2e4Chris Lattner OwnsTokens = ownsTokens; 676b884508c3bc97cc9df9516adb92fbf88dd0a2e4Chris Lattner DisableMacroExpansion = disableMacroExpansion; 688d896438031f5990cb42eb95a7bae233db2eb333Chris Lattner NumTokens = NumToks; 699594acf32de2939b15eafa8fe818607bfc56bf66Chris Lattner CurToken = 0; 709594acf32de2939b15eafa8fe818607bfc56bf66Chris Lattner InstantiateLoc = SourceLocation(); 719594acf32de2939b15eafa8fe818607bfc56bf66Chris Lattner AtStartOfLine = false; 729594acf32de2939b15eafa8fe818607bfc56bf66Chris Lattner HasLeadingSpace = false; 735f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 745f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Set HasLeadingSpace/AtStartOfLine so that the first token will be 755f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // returned unmodified. 765f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (NumToks != 0) { 775f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer AtStartOfLine = TokArray[0].isAtStartOfLine(); 785f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer HasLeadingSpace = TokArray[0].hasLeadingSpace(); 795f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 805f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 815f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 825f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 831543e9c69202b8e128c7fe34784ae7aa90964889Chris Lattnervoid TokenLexer::destroy() { 845f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If this was a function-like macro that actually uses its arguments, delete 855f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // the expanded tokens. 868d896438031f5990cb42eb95a7bae233db2eb333Chris Lattner if (OwnsTokens) { 878d896438031f5990cb42eb95a7bae233db2eb333Chris Lattner delete [] Tokens; 888d896438031f5990cb42eb95a7bae233db2eb333Chris Lattner Tokens = 0; 899c683062752a26014197df1c8792a19efa9e93cfChris Lattner } 905f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 911543e9c69202b8e128c7fe34784ae7aa90964889Chris Lattner // TokenLexer owns its formal arguments. 925f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (ActualArgs) ActualArgs->destroy(); 935f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 945f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 955f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// Expand the arguments of a function-like macro so that we can quickly 968d896438031f5990cb42eb95a7bae233db2eb333Chris Lattner/// return preexpanded tokens from Tokens. 971543e9c69202b8e128c7fe34784ae7aa90964889Chris Lattnervoid TokenLexer::ExpandFunctionArguments() { 98d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner llvm::SmallVector<Token, 128> ResultToks; 995f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 1008d896438031f5990cb42eb95a7bae233db2eb333Chris Lattner // Loop through 'Tokens', expanding them into ResultToks. Keep 1015f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // track of whether we change anything. If not, no need to keep them. If so, 1028d896438031f5990cb42eb95a7bae233db2eb333Chris Lattner // we install the newly expanded sequence as the new 'Tokens' list. 1035f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer bool MadeChange = false; 1045f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 1055f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // NextTokGetsSpace - When this is true, the next token appended to the 1065f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // output list will get a leading space, regardless of whether it had one to 1075f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // begin with or not. This is used for placemarker support. 1085f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer bool NextTokGetsSpace = false; 1095f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 1108d896438031f5990cb42eb95a7bae233db2eb333Chris Lattner for (unsigned i = 0, e = NumTokens; i != e; ++i) { 1115f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If we found the stringify operator, get the argument stringified. The 1125f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // preprocessor already verified that the following token is a macro name 1135f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // when the #define was parsed. 1148d896438031f5990cb42eb95a7bae233db2eb333Chris Lattner const Token &CurTok = Tokens[i]; 11522f6bbcafa8871f4f20c4402d9cbc5c024fee99aChris Lattner if (CurTok.is(tok::hash) || CurTok.is(tok::hashat)) { 1168d896438031f5990cb42eb95a7bae233db2eb333Chris Lattner int ArgNo = Macro->getArgumentNum(Tokens[i+1].getIdentifierInfo()); 1175f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer assert(ArgNo != -1 && "Token following # is not an argument?"); 1185f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 119d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner Token Res; 12022f6bbcafa8871f4f20c4402d9cbc5c024fee99aChris Lattner if (CurTok.is(tok::hash)) // Stringify 1215f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Res = ActualArgs->getStringifiedArgument(ArgNo, PP); 1225f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer else { 1235f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // 'charify': don't bother caching these. 124e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner Res = MacroArgs::StringifyArgument(ActualArgs->getUnexpArgument(ArgNo), 125e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner PP, true); 1265f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 1275f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 1285f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // The stringified/charified string leading space flag gets set to match 1295f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // the #/#@ operator. 1305f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (CurTok.hasLeadingSpace() || NextTokGetsSpace) 131d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner Res.setFlag(Token::LeadingSpace); 1325f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 1335f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer ResultToks.push_back(Res); 1345f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer MadeChange = true; 1355f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer ++i; // Skip arg name. 1365f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer NextTokGetsSpace = false; 1375f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer continue; 1385f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 1395f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 1405f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Otherwise, if this is not an argument token, just add the token to the 1415f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // output buffer. 1425f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer IdentifierInfo *II = CurTok.getIdentifierInfo(); 1435f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer int ArgNo = II ? Macro->getArgumentNum(II) : -1; 1445f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (ArgNo == -1) { 1455f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // This isn't an argument, just add it. 1465f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer ResultToks.push_back(CurTok); 1475f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 1485f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (NextTokGetsSpace) { 149d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner ResultToks.back().setFlag(Token::LeadingSpace); 1505f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer NextTokGetsSpace = false; 1515f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 1525f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer continue; 1535f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 1545f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 1555f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // An argument is expanded somehow, the result is different than the 1565f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // input. 1575f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer MadeChange = true; 1585f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 1595f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Otherwise, this is a use of the argument. Find out if there is a paste 1605f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // (##) operator before or after the argument. 1615f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer bool PasteBefore = 16222f6bbcafa8871f4f20c4402d9cbc5c024fee99aChris Lattner !ResultToks.empty() && ResultToks.back().is(tok::hashhash); 1638d896438031f5990cb42eb95a7bae233db2eb333Chris Lattner bool PasteAfter = i+1 != e && Tokens[i+1].is(tok::hashhash); 1645f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 1655f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If it is not the LHS/RHS of a ## operator, we must pre-expand the 1665f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // argument and substitute the expanded tokens into the result. This is 1675f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // C99 6.10.3.1p1. 1685f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (!PasteBefore && !PasteAfter) { 169d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner const Token *ResultArgToks; 1705f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 1715f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Only preexpand the argument if it could possibly need it. This 1725f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // avoids some work in common cases. 173d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner const Token *ArgTok = ActualArgs->getUnexpArgument(ArgNo); 174cc1a875f94630e58d24a55577ffbf0e89b7da8c7Chris Lattner if (ActualArgs->ArgNeedsPreexpansion(ArgTok, PP)) 1755f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer ResultArgToks = &ActualArgs->getPreExpArgument(ArgNo, PP)[0]; 1765f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer else 1775f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer ResultArgToks = ArgTok; // Use non-preexpanded tokens. 1785f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 1795f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If the arg token expanded into anything, append it. 18022f6bbcafa8871f4f20c4402d9cbc5c024fee99aChris Lattner if (ResultArgToks->isNot(tok::eof)) { 1815f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer unsigned FirstResult = ResultToks.size(); 1825f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer unsigned NumToks = MacroArgs::getArgLength(ResultArgToks); 1835f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer ResultToks.append(ResultArgToks, ResultArgToks+NumToks); 1845f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 1855f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If any tokens were substituted from the argument, the whitespace 1865f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // before the first token should match the whitespace of the arg 1875f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // identifier. 188d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner ResultToks[FirstResult].setFlagValue(Token::LeadingSpace, 1895f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurTok.hasLeadingSpace() || 1905f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer NextTokGetsSpace); 1915f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer NextTokGetsSpace = false; 1925f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } else { 1935f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If this is an empty argument, and if there was whitespace before the 1945f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // formal token, make sure the next token gets whitespace before it. 1955f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer NextTokGetsSpace = CurTok.hasLeadingSpace(); 1965f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 1975f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer continue; 1985f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 1995f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 2005f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Okay, we have a token that is either the LHS or RHS of a paste (##) 2015f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // argument. It gets substituted as its non-pre-expanded tokens. 202d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner const Token *ArgToks = ActualArgs->getUnexpArgument(ArgNo); 2035f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer unsigned NumToks = MacroArgs::getArgLength(ArgToks); 2045f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (NumToks) { // Not an empty argument? 20571a3a8d50643c450823d3fbcf3307a3d7db1a265Chris Lattner // If this is the GNU ", ## __VA_ARG__" extension, and we just learned 20671a3a8d50643c450823d3fbcf3307a3d7db1a265Chris Lattner // that __VA_ARG__ expands to multiple tokens, avoid a pasting error when 20771a3a8d50643c450823d3fbcf3307a3d7db1a265Chris Lattner // the expander trys to paste ',' with the first token of the __VA_ARG__ 20871a3a8d50643c450823d3fbcf3307a3d7db1a265Chris Lattner // expansion. 20971a3a8d50643c450823d3fbcf3307a3d7db1a265Chris Lattner if (PasteBefore && ResultToks.size() >= 2 && 21071a3a8d50643c450823d3fbcf3307a3d7db1a265Chris Lattner ResultToks[ResultToks.size()-2].is(tok::comma) && 21171a3a8d50643c450823d3fbcf3307a3d7db1a265Chris Lattner (unsigned)ArgNo == Macro->getNumArgs()-1 && 21271a3a8d50643c450823d3fbcf3307a3d7db1a265Chris Lattner Macro->isVariadic()) { 21371a3a8d50643c450823d3fbcf3307a3d7db1a265Chris Lattner // Remove the paste operator, report use of the extension. 21471a3a8d50643c450823d3fbcf3307a3d7db1a265Chris Lattner PP.Diag(ResultToks.back().getLocation(), diag::ext_paste_comma); 21571a3a8d50643c450823d3fbcf3307a3d7db1a265Chris Lattner ResultToks.pop_back(); 21671a3a8d50643c450823d3fbcf3307a3d7db1a265Chris Lattner } 21771a3a8d50643c450823d3fbcf3307a3d7db1a265Chris Lattner 2185f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer ResultToks.append(ArgToks, ArgToks+NumToks); 2195f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 2205f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If the next token was supposed to get leading whitespace, ensure it has 2215f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // it now. 2225f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (NextTokGetsSpace) { 223d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner ResultToks[ResultToks.size()-NumToks].setFlag(Token::LeadingSpace); 2245f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer NextTokGetsSpace = false; 2255f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 2265f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer continue; 2275f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 2285f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 2295f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If an empty argument is on the LHS or RHS of a paste, the standard (C99 2305f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // 6.10.3.3p2,3) calls for a bunch of placemarker stuff to occur. We 2315f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // implement this by eating ## operators when a LHS or RHS expands to 2325f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // empty. 2335f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer NextTokGetsSpace |= CurTok.hasLeadingSpace(); 2345f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (PasteAfter) { 2355f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Discard the argument token and skip (don't copy to the expansion 2365f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // buffer) the paste operator after it. 2378d896438031f5990cb42eb95a7bae233db2eb333Chris Lattner NextTokGetsSpace |= Tokens[i+1].hasLeadingSpace(); 2385f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer ++i; 2395f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer continue; 2405f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 2415f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 2425f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If this is on the RHS of a paste operator, we've already copied the 2435f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // paste operator to the ResultToks list. Remove it. 24422f6bbcafa8871f4f20c4402d9cbc5c024fee99aChris Lattner assert(PasteBefore && ResultToks.back().is(tok::hashhash)); 2455f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer NextTokGetsSpace |= ResultToks.back().hasLeadingSpace(); 2465f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer ResultToks.pop_back(); 2475f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 2485f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If this is the __VA_ARGS__ token, and if the argument wasn't provided, 2495f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // and if the macro had at least one real argument, and if the token before 2505f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // the ## was a comma, remove the comma. 2515f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if ((unsigned)ArgNo == Macro->getNumArgs()-1 && // is __VA_ARGS__ 2525f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer ActualArgs->isVarargsElidedUse() && // Argument elided. 25322f6bbcafa8871f4f20c4402d9cbc5c024fee99aChris Lattner !ResultToks.empty() && ResultToks.back().is(tok::comma)) { 2545f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Never add a space, even if the comma, ##, or arg had a space. 2555f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer NextTokGetsSpace = false; 25671a3a8d50643c450823d3fbcf3307a3d7db1a265Chris Lattner // Remove the paste operator, report use of the extension. 25771a3a8d50643c450823d3fbcf3307a3d7db1a265Chris Lattner PP.Diag(ResultToks.back().getLocation(), diag::ext_paste_comma); 2585f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer ResultToks.pop_back(); 2595f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 2605f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer continue; 2615f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 2625f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 2638d896438031f5990cb42eb95a7bae233db2eb333Chris Lattner // If anything changed, install this as the new Tokens list. 2645f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (MadeChange) { 2655f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // This is deleted in the dtor. 2668d896438031f5990cb42eb95a7bae233db2eb333Chris Lattner NumTokens = ResultToks.size(); 267d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner Token *Res = new Token[ResultToks.size()]; 2688d896438031f5990cb42eb95a7bae233db2eb333Chris Lattner if (NumTokens) 2698d896438031f5990cb42eb95a7bae233db2eb333Chris Lattner memcpy(Res, &ResultToks[0], NumTokens*sizeof(Token)); 2708d896438031f5990cb42eb95a7bae233db2eb333Chris Lattner Tokens = Res; 2718d896438031f5990cb42eb95a7bae233db2eb333Chris Lattner OwnsTokens = true; 2725f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 2735f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 2745f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 2755f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// Lex - Lex and return a token from this macro stream. 2765f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// 2771543e9c69202b8e128c7fe34784ae7aa90964889Chris Lattnervoid TokenLexer::Lex(Token &Tok) { 2785f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Lexing off the end of the macro, pop this macro off the expansion stack. 2795f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (isAtEnd()) { 2805f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If this is a macro (not a token stream), mark the macro enabled now 2815f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // that it is no longer being expanded. 2825f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (Macro) Macro->EnableMacro(); 2835f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 2845f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Pop this context off the preprocessors lexer stack and get the next 2855f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // token. This will delete "this" so remember the PP instance var. 2865f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Preprocessor &PPCache = PP; 287fde2bf9befede63e3f01f84519784c17b4c81480Chris Lattner if (PP.HandleEndOfTokenLexer(Tok)) 2885f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer return; 2895f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 290fde2bf9befede63e3f01f84519784c17b4c81480Chris Lattner // HandleEndOfTokenLexer may not return a token. If it doesn't, lex 291fde2bf9befede63e3f01f84519784c17b4c81480Chris Lattner // whatever is next. 2925f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer return PPCache.Lex(Tok); 2935f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 2945f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 2955f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If this is the first token of the expanded result, we inherit spacing 2965f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // properties later. 2975f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer bool isFirstToken = CurToken == 0; 2985f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 2995f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Get the next token to return. 3008d896438031f5990cb42eb95a7bae233db2eb333Chris Lattner Tok = Tokens[CurToken++]; 3015f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 3025f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If this token is followed by a token paste (##) operator, paste the tokens! 3038d896438031f5990cb42eb95a7bae233db2eb333Chris Lattner if (!isAtEnd() && Tokens[CurToken].is(tok::hashhash)) 3043f1cc838f9caf469990f23fccd0940263c0c61ffChris Lattner if (PasteTokens(Tok)) { 3053f1cc838f9caf469990f23fccd0940263c0c61ffChris Lattner // When handling the microsoft /##/ extension, the final token is 3063f1cc838f9caf469990f23fccd0940263c0c61ffChris Lattner // returned by PasteTokens, not the pasted token. 3073f1cc838f9caf469990f23fccd0940263c0c61ffChris Lattner return; 3083f1cc838f9caf469990f23fccd0940263c0c61ffChris Lattner } 3095f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 3105f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // The token's current location indicate where the token was lexed from. We 3115f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // need this information to compute the spelling of the token, but any 3125f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // diagnostics for the expanded token should appear as if they came from 3135f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // InstantiationLoc. Pull this information together into a new SourceLocation 3145f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // that captures all of this. 3155f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (InstantiateLoc.isValid()) { // Don't do this for token streams. 3165f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer SourceManager &SrcMgr = PP.getSourceManager(); 317abca2bba54197256f2830f757218cfbea2ed9cd1Chris Lattner Tok.setLocation(SrcMgr.getInstantiationLoc(Tok.getLocation(), 318abca2bba54197256f2830f757218cfbea2ed9cd1Chris Lattner InstantiateLoc)); 3195f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 3205f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 3215f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If this is the first token, set the lexical properties of the token to 3225f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // match the lexical properties of the macro identifier. 3235f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (isFirstToken) { 324d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner Tok.setFlagValue(Token::StartOfLine , AtStartOfLine); 325d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner Tok.setFlagValue(Token::LeadingSpace, HasLeadingSpace); 3265f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 3275f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 3285f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Handle recursive expansion! 3296b884508c3bc97cc9df9516adb92fbf88dd0a2e4Chris Lattner if (Tok.getIdentifierInfo() && !DisableMacroExpansion) 3305f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer return PP.HandleIdentifier(Tok); 3315f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 3325f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Otherwise, return a normal token. 3335f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 3345f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 3355f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// PasteTokens - Tok is the LHS of a ## operator, and CurToken is the ## 3365f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// operator. Read the ## and RHS, and paste the LHS/RHS together. If there 33748002c89578eaeb008a0bb5a10d8216e61906459Nico Weber/// are more ## after it, chomp them iteratively. Return the result as Tok. 3383f1cc838f9caf469990f23fccd0940263c0c61ffChris Lattner/// If this returns true, the caller should immediately return the token. 3391543e9c69202b8e128c7fe34784ae7aa90964889Chris Lattnerbool TokenLexer::PasteTokens(Token &Tok) { 3405f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer llvm::SmallVector<char, 128> Buffer; 3415f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer do { 3425f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Consume the ## operator. 3438d896438031f5990cb42eb95a7bae233db2eb333Chris Lattner SourceLocation PasteOpLoc = Tokens[CurToken].getLocation(); 3445f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer ++CurToken; 3455f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer assert(!isAtEnd() && "No token on the RHS of a paste operator!"); 3465f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 3475f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Get the RHS token. 3488d896438031f5990cb42eb95a7bae233db2eb333Chris Lattner const Token &RHS = Tokens[CurToken]; 3495f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 3505f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer bool isInvalid = false; 3515f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 3525f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Allocate space for the result token. This is guaranteed to be enough for 3535f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // the two tokens and a null terminator. 3545f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Buffer.resize(Tok.getLength() + RHS.getLength() + 1); 3555f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 3565f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Get the spelling of the LHS token in Buffer. 3575f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer const char *BufPtr = &Buffer[0]; 3585f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer unsigned LHSLen = PP.getSpelling(Tok, BufPtr); 3595f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (BufPtr != &Buffer[0]) // Really, we want the chars in Buffer! 3605f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer memcpy(&Buffer[0], BufPtr, LHSLen); 3615f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 3625f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer BufPtr = &Buffer[LHSLen]; 3635f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer unsigned RHSLen = PP.getSpelling(RHS, BufPtr); 3645f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (BufPtr != &Buffer[LHSLen]) // Really, we want the chars in Buffer! 3655f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer memcpy(&Buffer[LHSLen], BufPtr, RHSLen); 3665f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 3675f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Add null terminator. 3685f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Buffer[LHSLen+RHSLen] = '\0'; 3695f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 3705f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Trim excess space. 3715f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Buffer.resize(LHSLen+RHSLen+1); 3725f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 3735f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Plop the pasted result (including the trailing newline and null) into a 3745f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // scratch buffer where we can lex it. 3755f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer SourceLocation ResultTokLoc = PP.CreateString(&Buffer[0], Buffer.size()); 3765f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 3775f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Lex the resultant pasted token into Result. 378d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner Token Result; 3795f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 3800af574270d3be2b0e73a3379dfaa633746f8fc6fChris Lattner if (Tok.is(tok::identifier) && RHS.is(tok::identifier)) { 3815f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Common paste case: identifier+identifier = identifier. Avoid creating 3825f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // a lexer and other overhead. 3835f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer PP.IncrementPasteCounter(true); 3845f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Result.startToken(); 3855f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Result.setKind(tok::identifier); 3865f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Result.setLocation(ResultTokLoc); 3875f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Result.setLength(LHSLen+RHSLen); 3885f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } else { 3895f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer PP.IncrementPasteCounter(false); 3905f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 3915f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Make a lexer to lex this string from. 3925f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer SourceManager &SourceMgr = PP.getSourceManager(); 3935f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer const char *ResultStrData = SourceMgr.getCharacterData(ResultTokLoc); 3945f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 395590f0cc643274267d4d41125b62557e1d87886c3Chris Lattner const llvm::MemoryBuffer *Buffer = 396590f0cc643274267d4d41125b62557e1d87886c3Chris Lattner SourceMgr.getBuffer(ResultTokLoc.getFileID()); 397590f0cc643274267d4d41125b62557e1d87886c3Chris Lattner 3985f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Make a lexer object so that we lex and expand the paste result. 399590f0cc643274267d4d41125b62557e1d87886c3Chris Lattner Lexer TL(ResultTokLoc, PP.getLangOptions(), ResultStrData, 400590f0cc643274267d4d41125b62557e1d87886c3Chris Lattner ResultStrData+LHSLen+RHSLen /*don't include null*/, Buffer); 4015f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 4025f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Lex a token in raw mode. This way it won't look up identifiers 4035f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // automatically, lexing off the end will return an eof token, and 4045f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // warnings are disabled. This returns true if the result token is the 4055f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // entire buffer. 406590f0cc643274267d4d41125b62557e1d87886c3Chris Lattner bool IsComplete = TL.LexFromRawLexer(Result); 4075f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 4085f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If we got an EOF token, we didn't form even ONE token. For example, we 4095f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // did "/ ## /" to get "//". 41022f6bbcafa8871f4f20c4402d9cbc5c024fee99aChris Lattner IsComplete &= Result.isNot(tok::eof); 4115f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer isInvalid = !IsComplete; 4125f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 4135f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 4145f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If pasting the two tokens didn't form a full new token, this is an error. 4155f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // This occurs with "x ## +" and other stuff. Return with Tok unmodified 4165f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // and with RHS as the next token to lex. 4175f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (isInvalid) { 4183f1cc838f9caf469990f23fccd0940263c0c61ffChris Lattner // Test for the Microsoft extension of /##/ turning into // here on the 4193f1cc838f9caf469990f23fccd0940263c0c61ffChris Lattner // error path. 4203f1cc838f9caf469990f23fccd0940263c0c61ffChris Lattner if (PP.getLangOptions().Microsoft && Tok.is(tok::slash) && 4213f1cc838f9caf469990f23fccd0940263c0c61ffChris Lattner RHS.is(tok::slash)) { 4223f1cc838f9caf469990f23fccd0940263c0c61ffChris Lattner HandleMicrosoftCommentPaste(Tok); 4233f1cc838f9caf469990f23fccd0940263c0c61ffChris Lattner return true; 4243f1cc838f9caf469990f23fccd0940263c0c61ffChris Lattner } else { 4253f1cc838f9caf469990f23fccd0940263c0c61ffChris Lattner // TODO: If not in assembler language mode. 4263f1cc838f9caf469990f23fccd0940263c0c61ffChris Lattner PP.Diag(PasteOpLoc, diag::err_pp_bad_paste, 4273f1cc838f9caf469990f23fccd0940263c0c61ffChris Lattner std::string(Buffer.begin(), Buffer.end()-1)); 4283f1cc838f9caf469990f23fccd0940263c0c61ffChris Lattner return false; 4293f1cc838f9caf469990f23fccd0940263c0c61ffChris Lattner } 4305f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 4315f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 4321543e9c69202b8e128c7fe34784ae7aa90964889Chris Lattner // Turn ## into 'unknown' to avoid # ## # from looking like a paste 4331543e9c69202b8e128c7fe34784ae7aa90964889Chris Lattner // operator. 43422f6bbcafa8871f4f20c4402d9cbc5c024fee99aChris Lattner if (Result.is(tok::hashhash)) 4355f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Result.setKind(tok::unknown); 43671a3a8d50643c450823d3fbcf3307a3d7db1a265Chris Lattner // FIXME: Turn __VA_ARGS__ into "not a token"? 4375f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 4385f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Transfer properties of the LHS over the the Result. 439d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner Result.setFlagValue(Token::StartOfLine , Tok.isAtStartOfLine()); 440d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner Result.setFlagValue(Token::LeadingSpace, Tok.hasLeadingSpace()); 4415f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 4425f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Finally, replace LHS with the result, consume the RHS, and iterate. 4435f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer ++CurToken; 4445f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Tok = Result; 4458d896438031f5990cb42eb95a7bae233db2eb333Chris Lattner } while (!isAtEnd() && Tokens[CurToken].is(tok::hashhash)); 4465f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 4475f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Now that we got the result token, it will be subject to expansion. Since 4485f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // token pasting re-lexes the result token in raw mode, identifier information 4495f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // isn't looked up. As such, if the result is an identifier, look up id info. 45022f6bbcafa8871f4f20c4402d9cbc5c024fee99aChris Lattner if (Tok.is(tok::identifier)) { 4515f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Look up the identifier info for the token. We disabled identifier lookup 4525f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // by saying we're skipping contents, so we need to do this manually. 4535f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Tok.setIdentifierInfo(PP.LookUpIdentifierInfo(Tok)); 4545f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 4553f1cc838f9caf469990f23fccd0940263c0c61ffChris Lattner return false; 4565f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 4575f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 4585f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// isNextTokenLParen - If the next token lexed will pop this macro off the 4595f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// expansion stack, return 2. If the next unexpanded token is a '(', return 4605f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// 1, otherwise return 0. 4611543e9c69202b8e128c7fe34784ae7aa90964889Chris Lattnerunsigned TokenLexer::isNextTokenLParen() const { 4625f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Out of tokens? 4635f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (isAtEnd()) 4645f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer return 2; 4658d896438031f5990cb42eb95a7bae233db2eb333Chris Lattner return Tokens[CurToken].is(tok::l_paren); 4665f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 4673f1cc838f9caf469990f23fccd0940263c0c61ffChris Lattner 4683f1cc838f9caf469990f23fccd0940263c0c61ffChris Lattner 4693f1cc838f9caf469990f23fccd0940263c0c61ffChris Lattner/// HandleMicrosoftCommentPaste - In microsoft compatibility mode, /##/ pastes 4703f1cc838f9caf469990f23fccd0940263c0c61ffChris Lattner/// together to form a comment that comments out everything in the current 4713f1cc838f9caf469990f23fccd0940263c0c61ffChris Lattner/// macro, other active macros, and anything left on the current physical 4723f1cc838f9caf469990f23fccd0940263c0c61ffChris Lattner/// source line of the instantiated buffer. Handle this by returning the 4733f1cc838f9caf469990f23fccd0940263c0c61ffChris Lattner/// first token on the next line. 4741543e9c69202b8e128c7fe34784ae7aa90964889Chris Lattnervoid TokenLexer::HandleMicrosoftCommentPaste(Token &Tok) { 4753f1cc838f9caf469990f23fccd0940263c0c61ffChris Lattner // We 'comment out' the rest of this macro by just ignoring the rest of the 4763f1cc838f9caf469990f23fccd0940263c0c61ffChris Lattner // tokens that have not been lexed yet, if any. 4773f1cc838f9caf469990f23fccd0940263c0c61ffChris Lattner 4783f1cc838f9caf469990f23fccd0940263c0c61ffChris Lattner // Since this must be a macro, mark the macro enabled now that it is no longer 4793f1cc838f9caf469990f23fccd0940263c0c61ffChris Lattner // being expanded. 4803f1cc838f9caf469990f23fccd0940263c0c61ffChris Lattner assert(Macro && "Token streams can't paste comments"); 4813f1cc838f9caf469990f23fccd0940263c0c61ffChris Lattner Macro->EnableMacro(); 4823f1cc838f9caf469990f23fccd0940263c0c61ffChris Lattner 4833f1cc838f9caf469990f23fccd0940263c0c61ffChris Lattner PP.HandleMicrosoftCommentPaste(Tok); 4843f1cc838f9caf469990f23fccd0940263c0c61ffChris Lattner} 485