TokenLexer.cpp revision 461f2d820ffb628ebdf6ef4a4474a43f8ff914e5
1//===--- TokenLexer.cpp - Lex from a token stream -------------------------===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file implements the TokenLexer interface. 11// 12//===----------------------------------------------------------------------===// 13 14#include "clang/Lex/TokenLexer.h" 15#include "clang/Lex/MacroArgs.h" 16#include "clang/Basic/SourceManager.h" 17#include "clang/Lex/LexDiagnostic.h" 18#include "clang/Lex/MacroInfo.h" 19#include "clang/Lex/Preprocessor.h" 20#include "llvm/ADT/SmallString.h" 21using namespace clang; 22 23 24/// Create a TokenLexer for the specified macro with the specified actual 25/// arguments. Note that this ctor takes ownership of the ActualArgs pointer. 26void TokenLexer::Init(Token &Tok, SourceLocation ELEnd, MacroInfo *MI, 27 MacroArgs *Actuals) { 28 // If the client is reusing a TokenLexer, make sure to free any memory 29 // associated with it. 30 destroy(); 31 32 Macro = MI; 33 ActualArgs = Actuals; 34 CurToken = 0; 35 36 ExpandLocStart = Tok.getLocation(); 37 ExpandLocEnd = ELEnd; 38 AtStartOfLine = Tok.isAtStartOfLine(); 39 HasLeadingSpace = Tok.hasLeadingSpace(); 40 Tokens = &*Macro->tokens_begin(); 41 OwnsTokens = false; 42 DisableMacroExpansion = false; 43 NumTokens = Macro->tokens_end()-Macro->tokens_begin(); 44 MacroExpansionStart = SourceLocation(); 45 46 SourceManager &SM = PP.getSourceManager(); 47 MacroStartSLocOffset = SM.getNextLocalOffset(); 48 49 if (NumTokens > 0) { 50 assert(Tokens[0].getLocation().isValid()); 51 assert((Tokens[0].getLocation().isFileID() || Tokens[0].is(tok::comment)) && 52 "Macro defined in macro?"); 53 assert(ExpandLocStart.isValid()); 54 55 // Reserve a source location entry chunk for the length of the macro 56 // definition. Tokens that get lexed directly from the definition will 57 // have their locations pointing inside this chunk. This is to avoid 58 // creating separate source location entries for each token. 59 MacroDefStart = SM.getExpansionLoc(Tokens[0].getLocation()); 60 MacroDefLength = Macro->getDefinitionLength(SM); 61 MacroExpansionStart = SM.createExpansionLoc(MacroDefStart, 62 ExpandLocStart, 63 ExpandLocEnd, 64 MacroDefLength); 65 } 66 67 // If this is a function-like macro, expand the arguments and change 68 // Tokens to point to the expanded tokens. 69 if (Macro->isFunctionLike() && Macro->getNumArgs()) 70 ExpandFunctionArguments(); 71 72 // Mark the macro as currently disabled, so that it is not recursively 73 // expanded. The macro must be disabled only after argument pre-expansion of 74 // function-like macro arguments occurs. 75 Macro->DisableMacro(); 76} 77 78 79 80/// Create a TokenLexer for the specified token stream. This does not 81/// take ownership of the specified token vector. 82void TokenLexer::Init(const Token *TokArray, unsigned NumToks, 83 bool disableMacroExpansion, bool ownsTokens) { 84 // If the client is reusing a TokenLexer, make sure to free any memory 85 // associated with it. 86 destroy(); 87 88 Macro = 0; 89 ActualArgs = 0; 90 Tokens = TokArray; 91 OwnsTokens = ownsTokens; 92 DisableMacroExpansion = disableMacroExpansion; 93 NumTokens = NumToks; 94 CurToken = 0; 95 ExpandLocStart = ExpandLocEnd = SourceLocation(); 96 AtStartOfLine = false; 97 HasLeadingSpace = false; 98 MacroExpansionStart = SourceLocation(); 99 100 // Set HasLeadingSpace/AtStartOfLine so that the first token will be 101 // returned unmodified. 102 if (NumToks != 0) { 103 AtStartOfLine = TokArray[0].isAtStartOfLine(); 104 HasLeadingSpace = TokArray[0].hasLeadingSpace(); 105 } 106} 107 108 109void TokenLexer::destroy() { 110 // If this was a function-like macro that actually uses its arguments, delete 111 // the expanded tokens. 112 if (OwnsTokens) { 113 delete [] Tokens; 114 Tokens = 0; 115 OwnsTokens = false; 116 } 117 118 // TokenLexer owns its formal arguments. 119 if (ActualArgs) ActualArgs->destroy(PP); 120} 121 122/// Remove comma ahead of __VA_ARGS__, if present, according to compiler dialect 123/// settings. Returns true if the comma is removed. 124static bool MaybeRemoveCommaBeforeVaArgs(SmallVector<Token, 128> &ResultToks, 125 bool &NextTokGetsSpace, 126 bool HasPasteOperator, 127 MacroInfo *Macro, unsigned MacroArgNo, 128 Preprocessor &PP) { 129 // Is the macro argument __VA_ARGS__? 130 if (!Macro->isVariadic() || MacroArgNo != Macro->getNumArgs()-1) 131 return false; 132 133 // In Microsoft-compatibility mode, a comma is removed in the expansion 134 // of " ... , __VA_ARGS__ " if __VA_ARGS__ is empty. This extension is 135 // not supported by gcc. 136 if (!HasPasteOperator && !PP.getLangOpts().MicrosoftMode) 137 return false; 138 139 // GCC removes the comma in the expansion of " ... , ## __VA_ARGS__ " if 140 // __VA_ARGS__ is empty, but not in strict C99 mode where there are no 141 // named arguments, where it remains. In all other modes, including C99 142 // with GNU extensions, it is removed regardless of named arguments. 143 // Microsoft also appears to support this extension, unofficially. 144 if (PP.getLangOpts().C99 && !PP.getLangOpts().GNUMode 145 && Macro->getNumArgs() < 2) 146 return false; 147 148 // Is a comma available to be removed? 149 if (ResultToks.empty() || !ResultToks.back().is(tok::comma)) 150 return false; 151 152 // Issue an extension diagnostic for the paste operator. 153 if (HasPasteOperator) 154 PP.Diag(ResultToks.back().getLocation(), diag::ext_paste_comma); 155 156 // Remove the comma. 157 ResultToks.pop_back(); 158 159 // If the comma was right after another paste (e.g. "X##,##__VA_ARGS__"), 160 // then removal of the comma should produce a placemarker token (in C99 161 // terms) which we model by popping off the previous ##, giving us a plain 162 // "X" when __VA_ARGS__ is empty. 163 if (!ResultToks.empty() && ResultToks.back().is(tok::hashhash)) 164 ResultToks.pop_back(); 165 166 // Never add a space, even if the comma, ##, or arg had a space. 167 NextTokGetsSpace = false; 168 return true; 169} 170 171/// Expand the arguments of a function-like macro so that we can quickly 172/// return preexpanded tokens from Tokens. 173void TokenLexer::ExpandFunctionArguments() { 174 175 SmallVector<Token, 128> ResultToks; 176 177 // Loop through 'Tokens', expanding them into ResultToks. Keep 178 // track of whether we change anything. If not, no need to keep them. If so, 179 // we install the newly expanded sequence as the new 'Tokens' list. 180 bool MadeChange = false; 181 182 // NextTokGetsSpace - When this is true, the next token appended to the 183 // output list will get a leading space, regardless of whether it had one to 184 // begin with or not. This is used for placemarker support. 185 bool NextTokGetsSpace = false; 186 187 for (unsigned i = 0, e = NumTokens; i != e; ++i) { 188 // If we found the stringify operator, get the argument stringified. The 189 // preprocessor already verified that the following token is a macro name 190 // when the #define was parsed. 191 const Token &CurTok = Tokens[i]; 192 if (CurTok.is(tok::hash) || CurTok.is(tok::hashat)) { 193 int ArgNo = Macro->getArgumentNum(Tokens[i+1].getIdentifierInfo()); 194 assert(ArgNo != -1 && "Token following # is not an argument?"); 195 196 SourceLocation ExpansionLocStart = 197 getExpansionLocForMacroDefLoc(CurTok.getLocation()); 198 SourceLocation ExpansionLocEnd = 199 getExpansionLocForMacroDefLoc(Tokens[i+1].getLocation()); 200 201 Token Res; 202 if (CurTok.is(tok::hash)) // Stringify 203 Res = ActualArgs->getStringifiedArgument(ArgNo, PP, 204 ExpansionLocStart, 205 ExpansionLocEnd); 206 else { 207 // 'charify': don't bother caching these. 208 Res = MacroArgs::StringifyArgument(ActualArgs->getUnexpArgument(ArgNo), 209 PP, true, 210 ExpansionLocStart, 211 ExpansionLocEnd); 212 } 213 214 // The stringified/charified string leading space flag gets set to match 215 // the #/#@ operator. 216 if (CurTok.hasLeadingSpace() || NextTokGetsSpace) 217 Res.setFlag(Token::LeadingSpace); 218 219 ResultToks.push_back(Res); 220 MadeChange = true; 221 ++i; // Skip arg name. 222 NextTokGetsSpace = false; 223 continue; 224 } 225 226 // Otherwise, if this is not an argument token, just add the token to the 227 // output buffer. 228 IdentifierInfo *II = CurTok.getIdentifierInfo(); 229 int ArgNo = II ? Macro->getArgumentNum(II) : -1; 230 if (ArgNo == -1) { 231 // This isn't an argument, just add it. 232 ResultToks.push_back(CurTok); 233 234 if (NextTokGetsSpace) { 235 ResultToks.back().setFlag(Token::LeadingSpace); 236 NextTokGetsSpace = false; 237 } 238 continue; 239 } 240 241 // An argument is expanded somehow, the result is different than the 242 // input. 243 MadeChange = true; 244 245 // Otherwise, this is a use of the argument. Find out if there is a paste 246 // (##) operator before or after the argument. 247 bool NonEmptyPasteBefore = 248 !ResultToks.empty() && ResultToks.back().is(tok::hashhash); 249 bool PasteBefore = i != 0 && Tokens[i-1].is(tok::hashhash); 250 bool PasteAfter = i+1 != e && Tokens[i+1].is(tok::hashhash); 251 assert(!NonEmptyPasteBefore || PasteBefore); 252 253 // In Microsoft mode, remove the comma before __VA_ARGS__ to ensure there 254 // are no trailing commas if __VA_ARGS__ is empty. 255 if (!PasteBefore && ActualArgs->isVarargsElidedUse() && 256 MaybeRemoveCommaBeforeVaArgs(ResultToks, NextTokGetsSpace, 257 /*HasPasteOperator=*/false, 258 Macro, ArgNo, PP)) 259 continue; 260 261 // If it is not the LHS/RHS of a ## operator, we must pre-expand the 262 // argument and substitute the expanded tokens into the result. This is 263 // C99 6.10.3.1p1. 264 if (!PasteBefore && !PasteAfter) { 265 const Token *ResultArgToks; 266 267 // Only preexpand the argument if it could possibly need it. This 268 // avoids some work in common cases. 269 const Token *ArgTok = ActualArgs->getUnexpArgument(ArgNo); 270 if (ActualArgs->ArgNeedsPreexpansion(ArgTok, PP)) 271 ResultArgToks = &ActualArgs->getPreExpArgument(ArgNo, Macro, PP)[0]; 272 else 273 ResultArgToks = ArgTok; // Use non-preexpanded tokens. 274 275 // If the arg token expanded into anything, append it. 276 if (ResultArgToks->isNot(tok::eof)) { 277 unsigned FirstResult = ResultToks.size(); 278 unsigned NumToks = MacroArgs::getArgLength(ResultArgToks); 279 ResultToks.append(ResultArgToks, ResultArgToks+NumToks); 280 281 // If the '##' came from expanding an argument, turn it into 'unknown' 282 // to avoid pasting. 283 for (unsigned i = FirstResult, e = ResultToks.size(); i != e; ++i) { 284 Token &Tok = ResultToks[i]; 285 if (Tok.is(tok::hashhash)) 286 Tok.setKind(tok::unknown); 287 } 288 289 if(ExpandLocStart.isValid()) { 290 updateLocForMacroArgTokens(CurTok.getLocation(), 291 ResultToks.begin()+FirstResult, 292 ResultToks.end()); 293 } 294 295 // If any tokens were substituted from the argument, the whitespace 296 // before the first token should match the whitespace of the arg 297 // identifier. 298 ResultToks[FirstResult].setFlagValue(Token::LeadingSpace, 299 CurTok.hasLeadingSpace() || 300 NextTokGetsSpace); 301 NextTokGetsSpace = false; 302 } else { 303 // If this is an empty argument, and if there was whitespace before the 304 // formal token, make sure the next token gets whitespace before it. 305 NextTokGetsSpace = CurTok.hasLeadingSpace(); 306 } 307 continue; 308 } 309 310 // Okay, we have a token that is either the LHS or RHS of a paste (##) 311 // argument. It gets substituted as its non-pre-expanded tokens. 312 const Token *ArgToks = ActualArgs->getUnexpArgument(ArgNo); 313 unsigned NumToks = MacroArgs::getArgLength(ArgToks); 314 if (NumToks) { // Not an empty argument? 315 // If this is the GNU ", ## __VA_ARGS__" extension, and we just learned 316 // that __VA_ARGS__ expands to multiple tokens, avoid a pasting error when 317 // the expander trys to paste ',' with the first token of the __VA_ARGS__ 318 // expansion. 319 if (NonEmptyPasteBefore && ResultToks.size() >= 2 && 320 ResultToks[ResultToks.size()-2].is(tok::comma) && 321 (unsigned)ArgNo == Macro->getNumArgs()-1 && 322 Macro->isVariadic()) { 323 // Remove the paste operator, report use of the extension. 324 PP.Diag(ResultToks.back().getLocation(), diag::ext_paste_comma); 325 ResultToks.pop_back(); 326 } 327 328 ResultToks.append(ArgToks, ArgToks+NumToks); 329 330 // If the '##' came from expanding an argument, turn it into 'unknown' 331 // to avoid pasting. 332 for (unsigned i = ResultToks.size() - NumToks, e = ResultToks.size(); 333 i != e; ++i) { 334 Token &Tok = ResultToks[i]; 335 if (Tok.is(tok::hashhash)) 336 Tok.setKind(tok::unknown); 337 } 338 339 if (ExpandLocStart.isValid()) { 340 updateLocForMacroArgTokens(CurTok.getLocation(), 341 ResultToks.end()-NumToks, ResultToks.end()); 342 } 343 344 // If this token (the macro argument) was supposed to get leading 345 // whitespace, transfer this information onto the first token of the 346 // expansion. 347 // 348 // Do not do this if the paste operator occurs before the macro argument, 349 // as in "A ## MACROARG". In valid code, the first token will get 350 // smooshed onto the preceding one anyway (forming AMACROARG). In 351 // assembler-with-cpp mode, invalid pastes are allowed through: in this 352 // case, we do not want the extra whitespace to be added. For example, 353 // we want ". ## foo" -> ".foo" not ". foo". 354 if ((CurTok.hasLeadingSpace() || NextTokGetsSpace) && 355 !NonEmptyPasteBefore) 356 ResultToks[ResultToks.size()-NumToks].setFlag(Token::LeadingSpace); 357 358 NextTokGetsSpace = false; 359 continue; 360 } 361 362 // If an empty argument is on the LHS or RHS of a paste, the standard (C99 363 // 6.10.3.3p2,3) calls for a bunch of placemarker stuff to occur. We 364 // implement this by eating ## operators when a LHS or RHS expands to 365 // empty. 366 NextTokGetsSpace |= CurTok.hasLeadingSpace(); 367 if (PasteAfter) { 368 // Discard the argument token and skip (don't copy to the expansion 369 // buffer) the paste operator after it. 370 NextTokGetsSpace |= Tokens[i+1].hasLeadingSpace(); 371 ++i; 372 continue; 373 } 374 375 // If this is on the RHS of a paste operator, we've already copied the 376 // paste operator to the ResultToks list, unless the LHS was empty too. 377 // Remove it. 378 assert(PasteBefore); 379 if (NonEmptyPasteBefore) { 380 assert(ResultToks.back().is(tok::hashhash)); 381 NextTokGetsSpace |= ResultToks.back().hasLeadingSpace(); 382 ResultToks.pop_back(); 383 } 384 385 // If this is the __VA_ARGS__ token, and if the argument wasn't provided, 386 // and if the macro had at least one real argument, and if the token before 387 // the ## was a comma, remove the comma. This is a GCC extension which is 388 // disabled when using -std=c99. 389 if (ActualArgs->isVarargsElidedUse()) 390 MaybeRemoveCommaBeforeVaArgs(ResultToks, NextTokGetsSpace, 391 /*HasPasteOperator=*/true, 392 Macro, ArgNo, PP); 393 394 continue; 395 } 396 397 // If anything changed, install this as the new Tokens list. 398 if (MadeChange) { 399 assert(!OwnsTokens && "This would leak if we already own the token list"); 400 // This is deleted in the dtor. 401 NumTokens = ResultToks.size(); 402 // The tokens will be added to Preprocessor's cache and will be removed 403 // when this TokenLexer finishes lexing them. 404 Tokens = PP.cacheMacroExpandedTokens(this, ResultToks); 405 406 // The preprocessor cache of macro expanded tokens owns these tokens,not us. 407 OwnsTokens = false; 408 } 409} 410 411/// Lex - Lex and return a token from this macro stream. 412/// 413void TokenLexer::Lex(Token &Tok) { 414 // Lexing off the end of the macro, pop this macro off the expansion stack. 415 if (isAtEnd()) { 416 // If this is a macro (not a token stream), mark the macro enabled now 417 // that it is no longer being expanded. 418 if (Macro) Macro->EnableMacro(); 419 420 // Pop this context off the preprocessors lexer stack and get the next 421 // token. This will delete "this" so remember the PP instance var. 422 Preprocessor &PPCache = PP; 423 if (PP.HandleEndOfTokenLexer(Tok)) 424 return; 425 426 // HandleEndOfTokenLexer may not return a token. If it doesn't, lex 427 // whatever is next. 428 return PPCache.Lex(Tok); 429 } 430 431 SourceManager &SM = PP.getSourceManager(); 432 433 // If this is the first token of the expanded result, we inherit spacing 434 // properties later. 435 bool isFirstToken = CurToken == 0; 436 437 // Get the next token to return. 438 Tok = Tokens[CurToken++]; 439 440 bool TokenIsFromPaste = false; 441 442 // If this token is followed by a token paste (##) operator, paste the tokens! 443 // Note that ## is a normal token when not expanding a macro. 444 if (!isAtEnd() && Tokens[CurToken].is(tok::hashhash) && Macro) { 445 // When handling the microsoft /##/ extension, the final token is 446 // returned by PasteTokens, not the pasted token. 447 if (PasteTokens(Tok)) 448 return; 449 450 TokenIsFromPaste = true; 451 } 452 453 // The token's current location indicate where the token was lexed from. We 454 // need this information to compute the spelling of the token, but any 455 // diagnostics for the expanded token should appear as if they came from 456 // ExpansionLoc. Pull this information together into a new SourceLocation 457 // that captures all of this. 458 if (ExpandLocStart.isValid() && // Don't do this for token streams. 459 // Check that the token's location was not already set properly. 460 SM.isBeforeInSLocAddrSpace(Tok.getLocation(), MacroStartSLocOffset)) { 461 SourceLocation instLoc; 462 if (Tok.is(tok::comment)) { 463 instLoc = SM.createExpansionLoc(Tok.getLocation(), 464 ExpandLocStart, 465 ExpandLocEnd, 466 Tok.getLength()); 467 } else { 468 instLoc = getExpansionLocForMacroDefLoc(Tok.getLocation()); 469 } 470 471 Tok.setLocation(instLoc); 472 } 473 474 // If this is the first token, set the lexical properties of the token to 475 // match the lexical properties of the macro identifier. 476 if (isFirstToken) { 477 Tok.setFlagValue(Token::StartOfLine , AtStartOfLine); 478 Tok.setFlagValue(Token::LeadingSpace, HasLeadingSpace); 479 } 480 481 // Handle recursive expansion! 482 if (!Tok.isAnnotation() && Tok.getIdentifierInfo() != 0) { 483 // Change the kind of this identifier to the appropriate token kind, e.g. 484 // turning "for" into a keyword. 485 IdentifierInfo *II = Tok.getIdentifierInfo(); 486 Tok.setKind(II->getTokenID()); 487 488 // If this identifier was poisoned and from a paste, emit an error. This 489 // won't be handled by Preprocessor::HandleIdentifier because this is coming 490 // from a macro expansion. 491 if (II->isPoisoned() && TokenIsFromPaste) { 492 PP.HandlePoisonedIdentifier(Tok); 493 } 494 495 if (!DisableMacroExpansion && II->isHandleIdentifierCase()) 496 PP.HandleIdentifier(Tok); 497 } 498 499 // Otherwise, return a normal token. 500} 501 502/// PasteTokens - Tok is the LHS of a ## operator, and CurToken is the ## 503/// operator. Read the ## and RHS, and paste the LHS/RHS together. If there 504/// are more ## after it, chomp them iteratively. Return the result as Tok. 505/// If this returns true, the caller should immediately return the token. 506bool TokenLexer::PasteTokens(Token &Tok) { 507 SmallString<128> Buffer; 508 const char *ResultTokStrPtr = 0; 509 SourceLocation StartLoc = Tok.getLocation(); 510 SourceLocation PasteOpLoc; 511 do { 512 // Consume the ## operator. 513 PasteOpLoc = Tokens[CurToken].getLocation(); 514 ++CurToken; 515 assert(!isAtEnd() && "No token on the RHS of a paste operator!"); 516 517 // Get the RHS token. 518 const Token &RHS = Tokens[CurToken]; 519 520 // Allocate space for the result token. This is guaranteed to be enough for 521 // the two tokens. 522 Buffer.resize(Tok.getLength() + RHS.getLength()); 523 524 // Get the spelling of the LHS token in Buffer. 525 const char *BufPtr = &Buffer[0]; 526 bool Invalid = false; 527 unsigned LHSLen = PP.getSpelling(Tok, BufPtr, &Invalid); 528 if (BufPtr != &Buffer[0]) // Really, we want the chars in Buffer! 529 memcpy(&Buffer[0], BufPtr, LHSLen); 530 if (Invalid) 531 return true; 532 533 BufPtr = &Buffer[LHSLen]; 534 unsigned RHSLen = PP.getSpelling(RHS, BufPtr, &Invalid); 535 if (Invalid) 536 return true; 537 if (BufPtr != &Buffer[LHSLen]) // Really, we want the chars in Buffer! 538 memcpy(&Buffer[LHSLen], BufPtr, RHSLen); 539 540 // Trim excess space. 541 Buffer.resize(LHSLen+RHSLen); 542 543 // Plop the pasted result (including the trailing newline and null) into a 544 // scratch buffer where we can lex it. 545 Token ResultTokTmp; 546 ResultTokTmp.startToken(); 547 548 // Claim that the tmp token is a string_literal so that we can get the 549 // character pointer back from CreateString in getLiteralData(). 550 ResultTokTmp.setKind(tok::string_literal); 551 PP.CreateString(Buffer, ResultTokTmp); 552 SourceLocation ResultTokLoc = ResultTokTmp.getLocation(); 553 ResultTokStrPtr = ResultTokTmp.getLiteralData(); 554 555 // Lex the resultant pasted token into Result. 556 Token Result; 557 558 if (Tok.isAnyIdentifier() && RHS.isAnyIdentifier()) { 559 // Common paste case: identifier+identifier = identifier. Avoid creating 560 // a lexer and other overhead. 561 PP.IncrementPasteCounter(true); 562 Result.startToken(); 563 Result.setKind(tok::raw_identifier); 564 Result.setRawIdentifierData(ResultTokStrPtr); 565 Result.setLocation(ResultTokLoc); 566 Result.setLength(LHSLen+RHSLen); 567 } else { 568 PP.IncrementPasteCounter(false); 569 570 assert(ResultTokLoc.isFileID() && 571 "Should be a raw location into scratch buffer"); 572 SourceManager &SourceMgr = PP.getSourceManager(); 573 FileID LocFileID = SourceMgr.getFileID(ResultTokLoc); 574 575 bool Invalid = false; 576 const char *ScratchBufStart 577 = SourceMgr.getBufferData(LocFileID, &Invalid).data(); 578 if (Invalid) 579 return false; 580 581 // Make a lexer to lex this string from. Lex just this one token. 582 // Make a lexer object so that we lex and expand the paste result. 583 Lexer TL(SourceMgr.getLocForStartOfFile(LocFileID), 584 PP.getLangOpts(), ScratchBufStart, 585 ResultTokStrPtr, ResultTokStrPtr+LHSLen+RHSLen); 586 587 // Lex a token in raw mode. This way it won't look up identifiers 588 // automatically, lexing off the end will return an eof token, and 589 // warnings are disabled. This returns true if the result token is the 590 // entire buffer. 591 bool isInvalid = !TL.LexFromRawLexer(Result); 592 593 // If we got an EOF token, we didn't form even ONE token. For example, we 594 // did "/ ## /" to get "//". 595 isInvalid |= Result.is(tok::eof); 596 597 // If pasting the two tokens didn't form a full new token, this is an 598 // error. This occurs with "x ## +" and other stuff. Return with Tok 599 // unmodified and with RHS as the next token to lex. 600 if (isInvalid) { 601 // Test for the Microsoft extension of /##/ turning into // here on the 602 // error path. 603 if (PP.getLangOpts().MicrosoftExt && Tok.is(tok::slash) && 604 RHS.is(tok::slash)) { 605 HandleMicrosoftCommentPaste(Tok); 606 return true; 607 } 608 609 // Do not emit the error when preprocessing assembler code. 610 if (!PP.getLangOpts().AsmPreprocessor) { 611 // Explicitly convert the token location to have proper expansion 612 // information so that the user knows where it came from. 613 SourceManager &SM = PP.getSourceManager(); 614 SourceLocation Loc = 615 SM.createExpansionLoc(PasteOpLoc, ExpandLocStart, ExpandLocEnd, 2); 616 // If we're in microsoft extensions mode, downgrade this from a hard 617 // error to a warning that defaults to an error. This allows 618 // disabling it. 619 PP.Diag(Loc, 620 PP.getLangOpts().MicrosoftExt ? diag::err_pp_bad_paste_ms 621 : diag::err_pp_bad_paste) 622 << Buffer.str(); 623 } 624 625 // An error has occurred so exit loop. 626 break; 627 } 628 629 // Turn ## into 'unknown' to avoid # ## # from looking like a paste 630 // operator. 631 if (Result.is(tok::hashhash)) 632 Result.setKind(tok::unknown); 633 } 634 635 // Transfer properties of the LHS over the Result. 636 Result.setFlagValue(Token::StartOfLine , Tok.isAtStartOfLine()); 637 Result.setFlagValue(Token::LeadingSpace, Tok.hasLeadingSpace()); 638 639 // Finally, replace LHS with the result, consume the RHS, and iterate. 640 ++CurToken; 641 Tok = Result; 642 } while (!isAtEnd() && Tokens[CurToken].is(tok::hashhash)); 643 644 SourceLocation EndLoc = Tokens[CurToken - 1].getLocation(); 645 646 // The token's current location indicate where the token was lexed from. We 647 // need this information to compute the spelling of the token, but any 648 // diagnostics for the expanded token should appear as if the token was 649 // expanded from the full ## expression. Pull this information together into 650 // a new SourceLocation that captures all of this. 651 SourceManager &SM = PP.getSourceManager(); 652 if (StartLoc.isFileID()) 653 StartLoc = getExpansionLocForMacroDefLoc(StartLoc); 654 if (EndLoc.isFileID()) 655 EndLoc = getExpansionLocForMacroDefLoc(EndLoc); 656 FileID MacroFID = SM.getFileID(MacroExpansionStart); 657 while (SM.getFileID(StartLoc) != MacroFID) 658 StartLoc = SM.getImmediateExpansionRange(StartLoc).first; 659 while (SM.getFileID(EndLoc) != MacroFID) 660 EndLoc = SM.getImmediateExpansionRange(EndLoc).second; 661 662 Tok.setLocation(SM.createExpansionLoc(Tok.getLocation(), StartLoc, EndLoc, 663 Tok.getLength())); 664 665 // Now that we got the result token, it will be subject to expansion. Since 666 // token pasting re-lexes the result token in raw mode, identifier information 667 // isn't looked up. As such, if the result is an identifier, look up id info. 668 if (Tok.is(tok::raw_identifier)) { 669 // Look up the identifier info for the token. We disabled identifier lookup 670 // by saying we're skipping contents, so we need to do this manually. 671 PP.LookUpIdentifierInfo(Tok); 672 } 673 return false; 674} 675 676/// isNextTokenLParen - If the next token lexed will pop this macro off the 677/// expansion stack, return 2. If the next unexpanded token is a '(', return 678/// 1, otherwise return 0. 679unsigned TokenLexer::isNextTokenLParen() const { 680 // Out of tokens? 681 if (isAtEnd()) 682 return 2; 683 return Tokens[CurToken].is(tok::l_paren); 684} 685 686/// isParsingPreprocessorDirective - Return true if we are in the middle of a 687/// preprocessor directive. 688bool TokenLexer::isParsingPreprocessorDirective() const { 689 return Tokens[NumTokens-1].is(tok::eod) && !isAtEnd(); 690} 691 692/// HandleMicrosoftCommentPaste - In microsoft compatibility mode, /##/ pastes 693/// together to form a comment that comments out everything in the current 694/// macro, other active macros, and anything left on the current physical 695/// source line of the expanded buffer. Handle this by returning the 696/// first token on the next line. 697void TokenLexer::HandleMicrosoftCommentPaste(Token &Tok) { 698 // We 'comment out' the rest of this macro by just ignoring the rest of the 699 // tokens that have not been lexed yet, if any. 700 701 // Since this must be a macro, mark the macro enabled now that it is no longer 702 // being expanded. 703 assert(Macro && "Token streams can't paste comments"); 704 Macro->EnableMacro(); 705 706 PP.HandleMicrosoftCommentPaste(Tok); 707} 708 709/// \brief If \arg loc is a file ID and points inside the current macro 710/// definition, returns the appropriate source location pointing at the 711/// macro expansion source location entry, otherwise it returns an invalid 712/// SourceLocation. 713SourceLocation 714TokenLexer::getExpansionLocForMacroDefLoc(SourceLocation loc) const { 715 assert(ExpandLocStart.isValid() && MacroExpansionStart.isValid() && 716 "Not appropriate for token streams"); 717 assert(loc.isValid() && loc.isFileID()); 718 719 SourceManager &SM = PP.getSourceManager(); 720 assert(SM.isInSLocAddrSpace(loc, MacroDefStart, MacroDefLength) && 721 "Expected loc to come from the macro definition"); 722 723 unsigned relativeOffset = 0; 724 SM.isInSLocAddrSpace(loc, MacroDefStart, MacroDefLength, &relativeOffset); 725 return MacroExpansionStart.getLocWithOffset(relativeOffset); 726} 727 728/// \brief Finds the tokens that are consecutive (from the same FileID) 729/// creates a single SLocEntry, and assigns SourceLocations to each token that 730/// point to that SLocEntry. e.g for 731/// assert(foo == bar); 732/// There will be a single SLocEntry for the "foo == bar" chunk and locations 733/// for the 'foo', '==', 'bar' tokens will point inside that chunk. 734/// 735/// \arg begin_tokens will be updated to a position past all the found 736/// consecutive tokens. 737static void updateConsecutiveMacroArgTokens(SourceManager &SM, 738 SourceLocation InstLoc, 739 Token *&begin_tokens, 740 Token * end_tokens) { 741 assert(begin_tokens < end_tokens); 742 743 SourceLocation FirstLoc = begin_tokens->getLocation(); 744 SourceLocation CurLoc = FirstLoc; 745 746 // Compare the source location offset of tokens and group together tokens that 747 // are close, even if their locations point to different FileIDs. e.g. 748 // 749 // |bar | foo | cake | (3 tokens from 3 consecutive FileIDs) 750 // ^ ^ 751 // |bar foo cake| (one SLocEntry chunk for all tokens) 752 // 753 // we can perform this "merge" since the token's spelling location depends 754 // on the relative offset. 755 756 Token *NextTok = begin_tokens + 1; 757 for (; NextTok < end_tokens; ++NextTok) { 758 SourceLocation NextLoc = NextTok->getLocation(); 759 if (CurLoc.isFileID() != NextLoc.isFileID()) 760 break; // Token from different kind of FileID. 761 762 int RelOffs; 763 if (!SM.isInSameSLocAddrSpace(CurLoc, NextLoc, &RelOffs)) 764 break; // Token from different local/loaded location. 765 // Check that token is not before the previous token or more than 50 766 // "characters" away. 767 if (RelOffs < 0 || RelOffs > 50) 768 break; 769 CurLoc = NextLoc; 770 } 771 772 // For the consecutive tokens, find the length of the SLocEntry to contain 773 // all of them. 774 Token &LastConsecutiveTok = *(NextTok-1); 775 int LastRelOffs = 0; 776 SM.isInSameSLocAddrSpace(FirstLoc, LastConsecutiveTok.getLocation(), 777 &LastRelOffs); 778 unsigned FullLength = LastRelOffs + LastConsecutiveTok.getLength(); 779 780 // Create a macro expansion SLocEntry that will "contain" all of the tokens. 781 SourceLocation Expansion = 782 SM.createMacroArgExpansionLoc(FirstLoc, InstLoc,FullLength); 783 784 // Change the location of the tokens from the spelling location to the new 785 // expanded location. 786 for (; begin_tokens < NextTok; ++begin_tokens) { 787 Token &Tok = *begin_tokens; 788 int RelOffs = 0; 789 SM.isInSameSLocAddrSpace(FirstLoc, Tok.getLocation(), &RelOffs); 790 Tok.setLocation(Expansion.getLocWithOffset(RelOffs)); 791 } 792} 793 794/// \brief Creates SLocEntries and updates the locations of macro argument 795/// tokens to their new expanded locations. 796/// 797/// \param ArgIdDefLoc the location of the macro argument id inside the macro 798/// definition. 799/// \param Tokens the macro argument tokens to update. 800void TokenLexer::updateLocForMacroArgTokens(SourceLocation ArgIdSpellLoc, 801 Token *begin_tokens, 802 Token *end_tokens) { 803 SourceManager &SM = PP.getSourceManager(); 804 805 SourceLocation InstLoc = 806 getExpansionLocForMacroDefLoc(ArgIdSpellLoc); 807 808 while (begin_tokens < end_tokens) { 809 // If there's only one token just create a SLocEntry for it. 810 if (end_tokens - begin_tokens == 1) { 811 Token &Tok = *begin_tokens; 812 Tok.setLocation(SM.createMacroArgExpansionLoc(Tok.getLocation(), 813 InstLoc, 814 Tok.getLength())); 815 return; 816 } 817 818 updateConsecutiveMacroArgTokens(SM, InstLoc, begin_tokens, end_tokens); 819 } 820} 821