Lexer.cpp revision 8f1900376906c51aabf5fd18a39524e2318276ba
1//===--- Lexer.cpp - C Language Family Lexer ------------------------------===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file implements the Lexer and Token interfaces. 11// 12//===----------------------------------------------------------------------===// 13// 14// TODO: GCC Diagnostics emitted by the lexer: 15// PEDWARN: (form feed|vertical tab) in preprocessing directive 16// 17// Universal characters, unicode, char mapping: 18// WARNING: `%.*s' is not in NFKC 19// WARNING: `%.*s' is not in NFC 20// 21// Other: 22// TODO: Options to support: 23// -fexec-charset,-fwide-exec-charset 24// 25//===----------------------------------------------------------------------===// 26 27#include "clang/Lex/Lexer.h" 28#include "clang/Basic/CharInfo.h" 29#include "clang/Basic/SourceManager.h" 30#include "clang/Lex/CodeCompletionHandler.h" 31#include "clang/Lex/LexDiagnostic.h" 32#include "clang/Lex/Preprocessor.h" 33#include "llvm/ADT/STLExtras.h" 34#include "llvm/ADT/StringExtras.h" 35#include "llvm/ADT/StringSwitch.h" 36#include "llvm/Support/Compiler.h" 37#include "llvm/Support/ConvertUTF.h" 38#include "llvm/Support/MemoryBuffer.h" 39#include "UnicodeCharSets.h" 40#include <cstring> 41using namespace clang; 42 43//===----------------------------------------------------------------------===// 44// Token Class Implementation 45//===----------------------------------------------------------------------===// 46 47/// isObjCAtKeyword - Return true if we have an ObjC keyword identifier. 48bool Token::isObjCAtKeyword(tok::ObjCKeywordKind objcKey) const { 49 if (IdentifierInfo *II = getIdentifierInfo()) 50 return II->getObjCKeywordID() == objcKey; 51 return false; 52} 53 54/// getObjCKeywordID - Return the ObjC keyword kind. 55tok::ObjCKeywordKind Token::getObjCKeywordID() const { 56 IdentifierInfo *specId = getIdentifierInfo(); 57 return specId ? specId->getObjCKeywordID() : tok::objc_not_keyword; 58} 59 60 61//===----------------------------------------------------------------------===// 62// Lexer Class Implementation 63//===----------------------------------------------------------------------===// 64 65void Lexer::anchor() { } 66 67void Lexer::InitLexer(const char *BufStart, const char *BufPtr, 68 const char *BufEnd) { 69 BufferStart = BufStart; 70 BufferPtr = BufPtr; 71 BufferEnd = BufEnd; 72 73 assert(BufEnd[0] == 0 && 74 "We assume that the input buffer has a null character at the end" 75 " to simplify lexing!"); 76 77 // Check whether we have a BOM in the beginning of the buffer. If yes - act 78 // accordingly. Right now we support only UTF-8 with and without BOM, so, just 79 // skip the UTF-8 BOM if it's present. 80 if (BufferStart == BufferPtr) { 81 // Determine the size of the BOM. 82 StringRef Buf(BufferStart, BufferEnd - BufferStart); 83 size_t BOMLength = llvm::StringSwitch<size_t>(Buf) 84 .StartsWith("\xEF\xBB\xBF", 3) // UTF-8 BOM 85 .Default(0); 86 87 // Skip the BOM. 88 BufferPtr += BOMLength; 89 } 90 91 Is_PragmaLexer = false; 92 CurrentConflictMarkerState = CMK_None; 93 94 // Start of the file is a start of line. 95 IsAtStartOfLine = true; 96 97 // We are not after parsing a #. 98 ParsingPreprocessorDirective = false; 99 100 // We are not after parsing #include. 101 ParsingFilename = false; 102 103 // We are not in raw mode. Raw mode disables diagnostics and interpretation 104 // of tokens (e.g. identifiers, thus disabling macro expansion). It is used 105 // to quickly lex the tokens of the buffer, e.g. when handling a "#if 0" block 106 // or otherwise skipping over tokens. 107 LexingRawMode = false; 108 109 // Default to not keeping comments. 110 ExtendedTokenMode = 0; 111} 112 113/// Lexer constructor - Create a new lexer object for the specified buffer 114/// with the specified preprocessor managing the lexing process. This lexer 115/// assumes that the associated file buffer and Preprocessor objects will 116/// outlive it, so it doesn't take ownership of either of them. 117Lexer::Lexer(FileID FID, const llvm::MemoryBuffer *InputFile, Preprocessor &PP) 118 : PreprocessorLexer(&PP, FID), 119 FileLoc(PP.getSourceManager().getLocForStartOfFile(FID)), 120 LangOpts(PP.getLangOpts()) { 121 122 InitLexer(InputFile->getBufferStart(), InputFile->getBufferStart(), 123 InputFile->getBufferEnd()); 124 125 resetExtendedTokenMode(); 126} 127 128void Lexer::resetExtendedTokenMode() { 129 assert(PP && "Cannot reset token mode without a preprocessor"); 130 if (LangOpts.TraditionalCPP) 131 SetKeepWhitespaceMode(true); 132 else 133 SetCommentRetentionState(PP->getCommentRetentionState()); 134} 135 136/// Lexer constructor - Create a new raw lexer object. This object is only 137/// suitable for calls to 'LexFromRawLexer'. This lexer assumes that the text 138/// range will outlive it, so it doesn't take ownership of it. 139Lexer::Lexer(SourceLocation fileloc, const LangOptions &langOpts, 140 const char *BufStart, const char *BufPtr, const char *BufEnd) 141 : FileLoc(fileloc), LangOpts(langOpts) { 142 143 InitLexer(BufStart, BufPtr, BufEnd); 144 145 // We *are* in raw mode. 146 LexingRawMode = true; 147} 148 149/// Lexer constructor - Create a new raw lexer object. This object is only 150/// suitable for calls to 'LexFromRawLexer'. This lexer assumes that the text 151/// range will outlive it, so it doesn't take ownership of it. 152Lexer::Lexer(FileID FID, const llvm::MemoryBuffer *FromFile, 153 const SourceManager &SM, const LangOptions &langOpts) 154 : FileLoc(SM.getLocForStartOfFile(FID)), LangOpts(langOpts) { 155 156 InitLexer(FromFile->getBufferStart(), FromFile->getBufferStart(), 157 FromFile->getBufferEnd()); 158 159 // We *are* in raw mode. 160 LexingRawMode = true; 161} 162 163/// Create_PragmaLexer: Lexer constructor - Create a new lexer object for 164/// _Pragma expansion. This has a variety of magic semantics that this method 165/// sets up. It returns a new'd Lexer that must be delete'd when done. 166/// 167/// On entrance to this routine, TokStartLoc is a macro location which has a 168/// spelling loc that indicates the bytes to be lexed for the token and an 169/// expansion location that indicates where all lexed tokens should be 170/// "expanded from". 171/// 172/// FIXME: It would really be nice to make _Pragma just be a wrapper around a 173/// normal lexer that remaps tokens as they fly by. This would require making 174/// Preprocessor::Lex virtual. Given that, we could just dump in a magic lexer 175/// interface that could handle this stuff. This would pull GetMappedTokenLoc 176/// out of the critical path of the lexer! 177/// 178Lexer *Lexer::Create_PragmaLexer(SourceLocation SpellingLoc, 179 SourceLocation ExpansionLocStart, 180 SourceLocation ExpansionLocEnd, 181 unsigned TokLen, Preprocessor &PP) { 182 SourceManager &SM = PP.getSourceManager(); 183 184 // Create the lexer as if we were going to lex the file normally. 185 FileID SpellingFID = SM.getFileID(SpellingLoc); 186 const llvm::MemoryBuffer *InputFile = SM.getBuffer(SpellingFID); 187 Lexer *L = new Lexer(SpellingFID, InputFile, PP); 188 189 // Now that the lexer is created, change the start/end locations so that we 190 // just lex the subsection of the file that we want. This is lexing from a 191 // scratch buffer. 192 const char *StrData = SM.getCharacterData(SpellingLoc); 193 194 L->BufferPtr = StrData; 195 L->BufferEnd = StrData+TokLen; 196 assert(L->BufferEnd[0] == 0 && "Buffer is not nul terminated!"); 197 198 // Set the SourceLocation with the remapping information. This ensures that 199 // GetMappedTokenLoc will remap the tokens as they are lexed. 200 L->FileLoc = SM.createExpansionLoc(SM.getLocForStartOfFile(SpellingFID), 201 ExpansionLocStart, 202 ExpansionLocEnd, TokLen); 203 204 // Ensure that the lexer thinks it is inside a directive, so that end \n will 205 // return an EOD token. 206 L->ParsingPreprocessorDirective = true; 207 208 // This lexer really is for _Pragma. 209 L->Is_PragmaLexer = true; 210 return L; 211} 212 213 214/// Stringify - Convert the specified string into a C string, with surrounding 215/// ""'s, and with escaped \ and " characters. 216std::string Lexer::Stringify(const std::string &Str, bool Charify) { 217 std::string Result = Str; 218 char Quote = Charify ? '\'' : '"'; 219 for (unsigned i = 0, e = Result.size(); i != e; ++i) { 220 if (Result[i] == '\\' || Result[i] == Quote) { 221 Result.insert(Result.begin()+i, '\\'); 222 ++i; ++e; 223 } 224 } 225 return Result; 226} 227 228/// Stringify - Convert the specified string into a C string by escaping '\' 229/// and " characters. This does not add surrounding ""'s to the string. 230void Lexer::Stringify(SmallVectorImpl<char> &Str) { 231 for (unsigned i = 0, e = Str.size(); i != e; ++i) { 232 if (Str[i] == '\\' || Str[i] == '"') { 233 Str.insert(Str.begin()+i, '\\'); 234 ++i; ++e; 235 } 236 } 237} 238 239//===----------------------------------------------------------------------===// 240// Token Spelling 241//===----------------------------------------------------------------------===// 242 243/// \brief Slow case of getSpelling. Extract the characters comprising the 244/// spelling of this token from the provided input buffer. 245static size_t getSpellingSlow(const Token &Tok, const char *BufPtr, 246 const LangOptions &LangOpts, char *Spelling) { 247 assert(Tok.needsCleaning() && "getSpellingSlow called on simple token"); 248 249 size_t Length = 0; 250 const char *BufEnd = BufPtr + Tok.getLength(); 251 252 if (Tok.is(tok::string_literal)) { 253 // Munch the encoding-prefix and opening double-quote. 254 while (BufPtr < BufEnd) { 255 unsigned Size; 256 Spelling[Length++] = Lexer::getCharAndSizeNoWarn(BufPtr, Size, LangOpts); 257 BufPtr += Size; 258 259 if (Spelling[Length - 1] == '"') 260 break; 261 } 262 263 // Raw string literals need special handling; trigraph expansion and line 264 // splicing do not occur within their d-char-sequence nor within their 265 // r-char-sequence. 266 if (Length >= 2 && 267 Spelling[Length - 2] == 'R' && Spelling[Length - 1] == '"') { 268 // Search backwards from the end of the token to find the matching closing 269 // quote. 270 const char *RawEnd = BufEnd; 271 do --RawEnd; while (*RawEnd != '"'); 272 size_t RawLength = RawEnd - BufPtr + 1; 273 274 // Everything between the quotes is included verbatim in the spelling. 275 memcpy(Spelling + Length, BufPtr, RawLength); 276 Length += RawLength; 277 BufPtr += RawLength; 278 279 // The rest of the token is lexed normally. 280 } 281 } 282 283 while (BufPtr < BufEnd) { 284 unsigned Size; 285 Spelling[Length++] = Lexer::getCharAndSizeNoWarn(BufPtr, Size, LangOpts); 286 BufPtr += Size; 287 } 288 289 assert(Length < Tok.getLength() && 290 "NeedsCleaning flag set on token that didn't need cleaning!"); 291 return Length; 292} 293 294/// getSpelling() - Return the 'spelling' of this token. The spelling of a 295/// token are the characters used to represent the token in the source file 296/// after trigraph expansion and escaped-newline folding. In particular, this 297/// wants to get the true, uncanonicalized, spelling of things like digraphs 298/// UCNs, etc. 299StringRef Lexer::getSpelling(SourceLocation loc, 300 SmallVectorImpl<char> &buffer, 301 const SourceManager &SM, 302 const LangOptions &options, 303 bool *invalid) { 304 // Break down the source location. 305 std::pair<FileID, unsigned> locInfo = SM.getDecomposedLoc(loc); 306 307 // Try to the load the file buffer. 308 bool invalidTemp = false; 309 StringRef file = SM.getBufferData(locInfo.first, &invalidTemp); 310 if (invalidTemp) { 311 if (invalid) *invalid = true; 312 return StringRef(); 313 } 314 315 const char *tokenBegin = file.data() + locInfo.second; 316 317 // Lex from the start of the given location. 318 Lexer lexer(SM.getLocForStartOfFile(locInfo.first), options, 319 file.begin(), tokenBegin, file.end()); 320 Token token; 321 lexer.LexFromRawLexer(token); 322 323 unsigned length = token.getLength(); 324 325 // Common case: no need for cleaning. 326 if (!token.needsCleaning()) 327 return StringRef(tokenBegin, length); 328 329 // Hard case, we need to relex the characters into the string. 330 buffer.resize(length); 331 buffer.resize(getSpellingSlow(token, tokenBegin, options, buffer.data())); 332 return StringRef(buffer.data(), buffer.size()); 333} 334 335/// getSpelling() - Return the 'spelling' of this token. The spelling of a 336/// token are the characters used to represent the token in the source file 337/// after trigraph expansion and escaped-newline folding. In particular, this 338/// wants to get the true, uncanonicalized, spelling of things like digraphs 339/// UCNs, etc. 340std::string Lexer::getSpelling(const Token &Tok, const SourceManager &SourceMgr, 341 const LangOptions &LangOpts, bool *Invalid) { 342 assert((int)Tok.getLength() >= 0 && "Token character range is bogus!"); 343 344 bool CharDataInvalid = false; 345 const char *TokStart = SourceMgr.getCharacterData(Tok.getLocation(), 346 &CharDataInvalid); 347 if (Invalid) 348 *Invalid = CharDataInvalid; 349 if (CharDataInvalid) 350 return std::string(); 351 352 // If this token contains nothing interesting, return it directly. 353 if (!Tok.needsCleaning()) 354 return std::string(TokStart, TokStart + Tok.getLength()); 355 356 std::string Result; 357 Result.resize(Tok.getLength()); 358 Result.resize(getSpellingSlow(Tok, TokStart, LangOpts, &*Result.begin())); 359 return Result; 360} 361 362/// getSpelling - This method is used to get the spelling of a token into a 363/// preallocated buffer, instead of as an std::string. The caller is required 364/// to allocate enough space for the token, which is guaranteed to be at least 365/// Tok.getLength() bytes long. The actual length of the token is returned. 366/// 367/// Note that this method may do two possible things: it may either fill in 368/// the buffer specified with characters, or it may *change the input pointer* 369/// to point to a constant buffer with the data already in it (avoiding a 370/// copy). The caller is not allowed to modify the returned buffer pointer 371/// if an internal buffer is returned. 372unsigned Lexer::getSpelling(const Token &Tok, const char *&Buffer, 373 const SourceManager &SourceMgr, 374 const LangOptions &LangOpts, bool *Invalid) { 375 assert((int)Tok.getLength() >= 0 && "Token character range is bogus!"); 376 377 const char *TokStart = 0; 378 // NOTE: this has to be checked *before* testing for an IdentifierInfo. 379 if (Tok.is(tok::raw_identifier)) 380 TokStart = Tok.getRawIdentifierData(); 381 else if (!Tok.hasUCN()) { 382 if (const IdentifierInfo *II = Tok.getIdentifierInfo()) { 383 // Just return the string from the identifier table, which is very quick. 384 Buffer = II->getNameStart(); 385 return II->getLength(); 386 } 387 } 388 389 // NOTE: this can be checked even after testing for an IdentifierInfo. 390 if (Tok.isLiteral()) 391 TokStart = Tok.getLiteralData(); 392 393 if (TokStart == 0) { 394 // Compute the start of the token in the input lexer buffer. 395 bool CharDataInvalid = false; 396 TokStart = SourceMgr.getCharacterData(Tok.getLocation(), &CharDataInvalid); 397 if (Invalid) 398 *Invalid = CharDataInvalid; 399 if (CharDataInvalid) { 400 Buffer = ""; 401 return 0; 402 } 403 } 404 405 // If this token contains nothing interesting, return it directly. 406 if (!Tok.needsCleaning()) { 407 Buffer = TokStart; 408 return Tok.getLength(); 409 } 410 411 // Otherwise, hard case, relex the characters into the string. 412 return getSpellingSlow(Tok, TokStart, LangOpts, const_cast<char*>(Buffer)); 413} 414 415 416/// MeasureTokenLength - Relex the token at the specified location and return 417/// its length in bytes in the input file. If the token needs cleaning (e.g. 418/// includes a trigraph or an escaped newline) then this count includes bytes 419/// that are part of that. 420unsigned Lexer::MeasureTokenLength(SourceLocation Loc, 421 const SourceManager &SM, 422 const LangOptions &LangOpts) { 423 Token TheTok; 424 if (getRawToken(Loc, TheTok, SM, LangOpts)) 425 return 0; 426 return TheTok.getLength(); 427} 428 429/// \brief Relex the token at the specified location. 430/// \returns true if there was a failure, false on success. 431bool Lexer::getRawToken(SourceLocation Loc, Token &Result, 432 const SourceManager &SM, 433 const LangOptions &LangOpts) { 434 // TODO: this could be special cased for common tokens like identifiers, ')', 435 // etc to make this faster, if it mattered. Just look at StrData[0] to handle 436 // all obviously single-char tokens. This could use 437 // Lexer::isObviouslySimpleCharacter for example to handle identifiers or 438 // something. 439 440 // If this comes from a macro expansion, we really do want the macro name, not 441 // the token this macro expanded to. 442 Loc = SM.getExpansionLoc(Loc); 443 std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(Loc); 444 bool Invalid = false; 445 StringRef Buffer = SM.getBufferData(LocInfo.first, &Invalid); 446 if (Invalid) 447 return true; 448 449 const char *StrData = Buffer.data()+LocInfo.second; 450 451 if (isWhitespace(StrData[0])) 452 return true; 453 454 // Create a lexer starting at the beginning of this token. 455 Lexer TheLexer(SM.getLocForStartOfFile(LocInfo.first), LangOpts, 456 Buffer.begin(), StrData, Buffer.end()); 457 TheLexer.SetCommentRetentionState(true); 458 TheLexer.LexFromRawLexer(Result); 459 return false; 460} 461 462static SourceLocation getBeginningOfFileToken(SourceLocation Loc, 463 const SourceManager &SM, 464 const LangOptions &LangOpts) { 465 assert(Loc.isFileID()); 466 std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(Loc); 467 if (LocInfo.first.isInvalid()) 468 return Loc; 469 470 bool Invalid = false; 471 StringRef Buffer = SM.getBufferData(LocInfo.first, &Invalid); 472 if (Invalid) 473 return Loc; 474 475 // Back up from the current location until we hit the beginning of a line 476 // (or the buffer). We'll relex from that point. 477 const char *BufStart = Buffer.data(); 478 if (LocInfo.second >= Buffer.size()) 479 return Loc; 480 481 const char *StrData = BufStart+LocInfo.second; 482 if (StrData[0] == '\n' || StrData[0] == '\r') 483 return Loc; 484 485 const char *LexStart = StrData; 486 while (LexStart != BufStart) { 487 if (LexStart[0] == '\n' || LexStart[0] == '\r') { 488 ++LexStart; 489 break; 490 } 491 492 --LexStart; 493 } 494 495 // Create a lexer starting at the beginning of this token. 496 SourceLocation LexerStartLoc = Loc.getLocWithOffset(-LocInfo.second); 497 Lexer TheLexer(LexerStartLoc, LangOpts, BufStart, LexStart, Buffer.end()); 498 TheLexer.SetCommentRetentionState(true); 499 500 // Lex tokens until we find the token that contains the source location. 501 Token TheTok; 502 do { 503 TheLexer.LexFromRawLexer(TheTok); 504 505 if (TheLexer.getBufferLocation() > StrData) { 506 // Lexing this token has taken the lexer past the source location we're 507 // looking for. If the current token encompasses our source location, 508 // return the beginning of that token. 509 if (TheLexer.getBufferLocation() - TheTok.getLength() <= StrData) 510 return TheTok.getLocation(); 511 512 // We ended up skipping over the source location entirely, which means 513 // that it points into whitespace. We're done here. 514 break; 515 } 516 } while (TheTok.getKind() != tok::eof); 517 518 // We've passed our source location; just return the original source location. 519 return Loc; 520} 521 522SourceLocation Lexer::GetBeginningOfToken(SourceLocation Loc, 523 const SourceManager &SM, 524 const LangOptions &LangOpts) { 525 if (Loc.isFileID()) 526 return getBeginningOfFileToken(Loc, SM, LangOpts); 527 528 if (!SM.isMacroArgExpansion(Loc)) 529 return Loc; 530 531 SourceLocation FileLoc = SM.getSpellingLoc(Loc); 532 SourceLocation BeginFileLoc = getBeginningOfFileToken(FileLoc, SM, LangOpts); 533 std::pair<FileID, unsigned> FileLocInfo = SM.getDecomposedLoc(FileLoc); 534 std::pair<FileID, unsigned> BeginFileLocInfo 535 = SM.getDecomposedLoc(BeginFileLoc); 536 assert(FileLocInfo.first == BeginFileLocInfo.first && 537 FileLocInfo.second >= BeginFileLocInfo.second); 538 return Loc.getLocWithOffset(BeginFileLocInfo.second - FileLocInfo.second); 539} 540 541namespace { 542 enum PreambleDirectiveKind { 543 PDK_Skipped, 544 PDK_StartIf, 545 PDK_EndIf, 546 PDK_Unknown 547 }; 548} 549 550std::pair<unsigned, bool> 551Lexer::ComputePreamble(const llvm::MemoryBuffer *Buffer, 552 const LangOptions &LangOpts, unsigned MaxLines) { 553 // Create a lexer starting at the beginning of the file. Note that we use a 554 // "fake" file source location at offset 1 so that the lexer will track our 555 // position within the file. 556 const unsigned StartOffset = 1; 557 SourceLocation FileLoc = SourceLocation::getFromRawEncoding(StartOffset); 558 Lexer TheLexer(FileLoc, LangOpts, Buffer->getBufferStart(), 559 Buffer->getBufferStart(), Buffer->getBufferEnd()); 560 TheLexer.SetCommentRetentionState(true); 561 562 // StartLoc will differ from FileLoc if there is a BOM that was skipped. 563 SourceLocation StartLoc = TheLexer.getSourceLocation(); 564 565 bool InPreprocessorDirective = false; 566 Token TheTok; 567 Token IfStartTok; 568 unsigned IfCount = 0; 569 SourceLocation ActiveCommentLoc; 570 571 unsigned MaxLineOffset = 0; 572 if (MaxLines) { 573 const char *CurPtr = Buffer->getBufferStart(); 574 unsigned CurLine = 0; 575 while (CurPtr != Buffer->getBufferEnd()) { 576 char ch = *CurPtr++; 577 if (ch == '\n') { 578 ++CurLine; 579 if (CurLine == MaxLines) 580 break; 581 } 582 } 583 if (CurPtr != Buffer->getBufferEnd()) 584 MaxLineOffset = CurPtr - Buffer->getBufferStart(); 585 } 586 587 do { 588 TheLexer.LexFromRawLexer(TheTok); 589 590 if (InPreprocessorDirective) { 591 // If we've hit the end of the file, we're done. 592 if (TheTok.getKind() == tok::eof) { 593 break; 594 } 595 596 // If we haven't hit the end of the preprocessor directive, skip this 597 // token. 598 if (!TheTok.isAtStartOfLine()) 599 continue; 600 601 // We've passed the end of the preprocessor directive, and will look 602 // at this token again below. 603 InPreprocessorDirective = false; 604 } 605 606 // Keep track of the # of lines in the preamble. 607 if (TheTok.isAtStartOfLine()) { 608 unsigned TokOffset = TheTok.getLocation().getRawEncoding() - StartOffset; 609 610 // If we were asked to limit the number of lines in the preamble, 611 // and we're about to exceed that limit, we're done. 612 if (MaxLineOffset && TokOffset >= MaxLineOffset) 613 break; 614 } 615 616 // Comments are okay; skip over them. 617 if (TheTok.getKind() == tok::comment) { 618 if (ActiveCommentLoc.isInvalid()) 619 ActiveCommentLoc = TheTok.getLocation(); 620 continue; 621 } 622 623 if (TheTok.isAtStartOfLine() && TheTok.getKind() == tok::hash) { 624 // This is the start of a preprocessor directive. 625 Token HashTok = TheTok; 626 InPreprocessorDirective = true; 627 ActiveCommentLoc = SourceLocation(); 628 629 // Figure out which directive this is. Since we're lexing raw tokens, 630 // we don't have an identifier table available. Instead, just look at 631 // the raw identifier to recognize and categorize preprocessor directives. 632 TheLexer.LexFromRawLexer(TheTok); 633 if (TheTok.getKind() == tok::raw_identifier && !TheTok.needsCleaning()) { 634 StringRef Keyword(TheTok.getRawIdentifierData(), 635 TheTok.getLength()); 636 PreambleDirectiveKind PDK 637 = llvm::StringSwitch<PreambleDirectiveKind>(Keyword) 638 .Case("include", PDK_Skipped) 639 .Case("__include_macros", PDK_Skipped) 640 .Case("define", PDK_Skipped) 641 .Case("undef", PDK_Skipped) 642 .Case("line", PDK_Skipped) 643 .Case("error", PDK_Skipped) 644 .Case("pragma", PDK_Skipped) 645 .Case("import", PDK_Skipped) 646 .Case("include_next", PDK_Skipped) 647 .Case("warning", PDK_Skipped) 648 .Case("ident", PDK_Skipped) 649 .Case("sccs", PDK_Skipped) 650 .Case("assert", PDK_Skipped) 651 .Case("unassert", PDK_Skipped) 652 .Case("if", PDK_StartIf) 653 .Case("ifdef", PDK_StartIf) 654 .Case("ifndef", PDK_StartIf) 655 .Case("elif", PDK_Skipped) 656 .Case("else", PDK_Skipped) 657 .Case("endif", PDK_EndIf) 658 .Default(PDK_Unknown); 659 660 switch (PDK) { 661 case PDK_Skipped: 662 continue; 663 664 case PDK_StartIf: 665 if (IfCount == 0) 666 IfStartTok = HashTok; 667 668 ++IfCount; 669 continue; 670 671 case PDK_EndIf: 672 // Mismatched #endif. The preamble ends here. 673 if (IfCount == 0) 674 break; 675 676 --IfCount; 677 continue; 678 679 case PDK_Unknown: 680 // We don't know what this directive is; stop at the '#'. 681 break; 682 } 683 } 684 685 // We only end up here if we didn't recognize the preprocessor 686 // directive or it was one that can't occur in the preamble at this 687 // point. Roll back the current token to the location of the '#'. 688 InPreprocessorDirective = false; 689 TheTok = HashTok; 690 } 691 692 // We hit a token that we don't recognize as being in the 693 // "preprocessing only" part of the file, so we're no longer in 694 // the preamble. 695 break; 696 } while (true); 697 698 SourceLocation End; 699 if (IfCount) 700 End = IfStartTok.getLocation(); 701 else if (ActiveCommentLoc.isValid()) 702 End = ActiveCommentLoc; // don't truncate a decl comment. 703 else 704 End = TheTok.getLocation(); 705 706 return std::make_pair(End.getRawEncoding() - StartLoc.getRawEncoding(), 707 IfCount? IfStartTok.isAtStartOfLine() 708 : TheTok.isAtStartOfLine()); 709} 710 711 712/// AdvanceToTokenCharacter - Given a location that specifies the start of a 713/// token, return a new location that specifies a character within the token. 714SourceLocation Lexer::AdvanceToTokenCharacter(SourceLocation TokStart, 715 unsigned CharNo, 716 const SourceManager &SM, 717 const LangOptions &LangOpts) { 718 // Figure out how many physical characters away the specified expansion 719 // character is. This needs to take into consideration newlines and 720 // trigraphs. 721 bool Invalid = false; 722 const char *TokPtr = SM.getCharacterData(TokStart, &Invalid); 723 724 // If they request the first char of the token, we're trivially done. 725 if (Invalid || (CharNo == 0 && Lexer::isObviouslySimpleCharacter(*TokPtr))) 726 return TokStart; 727 728 unsigned PhysOffset = 0; 729 730 // The usual case is that tokens don't contain anything interesting. Skip 731 // over the uninteresting characters. If a token only consists of simple 732 // chars, this method is extremely fast. 733 while (Lexer::isObviouslySimpleCharacter(*TokPtr)) { 734 if (CharNo == 0) 735 return TokStart.getLocWithOffset(PhysOffset); 736 ++TokPtr, --CharNo, ++PhysOffset; 737 } 738 739 // If we have a character that may be a trigraph or escaped newline, use a 740 // lexer to parse it correctly. 741 for (; CharNo; --CharNo) { 742 unsigned Size; 743 Lexer::getCharAndSizeNoWarn(TokPtr, Size, LangOpts); 744 TokPtr += Size; 745 PhysOffset += Size; 746 } 747 748 // Final detail: if we end up on an escaped newline, we want to return the 749 // location of the actual byte of the token. For example foo\<newline>bar 750 // advanced by 3 should return the location of b, not of \\. One compounding 751 // detail of this is that the escape may be made by a trigraph. 752 if (!Lexer::isObviouslySimpleCharacter(*TokPtr)) 753 PhysOffset += Lexer::SkipEscapedNewLines(TokPtr)-TokPtr; 754 755 return TokStart.getLocWithOffset(PhysOffset); 756} 757 758/// \brief Computes the source location just past the end of the 759/// token at this source location. 760/// 761/// This routine can be used to produce a source location that 762/// points just past the end of the token referenced by \p Loc, and 763/// is generally used when a diagnostic needs to point just after a 764/// token where it expected something different that it received. If 765/// the returned source location would not be meaningful (e.g., if 766/// it points into a macro), this routine returns an invalid 767/// source location. 768/// 769/// \param Offset an offset from the end of the token, where the source 770/// location should refer to. The default offset (0) produces a source 771/// location pointing just past the end of the token; an offset of 1 produces 772/// a source location pointing to the last character in the token, etc. 773SourceLocation Lexer::getLocForEndOfToken(SourceLocation Loc, unsigned Offset, 774 const SourceManager &SM, 775 const LangOptions &LangOpts) { 776 if (Loc.isInvalid()) 777 return SourceLocation(); 778 779 if (Loc.isMacroID()) { 780 if (Offset > 0 || !isAtEndOfMacroExpansion(Loc, SM, LangOpts, &Loc)) 781 return SourceLocation(); // Points inside the macro expansion. 782 } 783 784 unsigned Len = Lexer::MeasureTokenLength(Loc, SM, LangOpts); 785 if (Len > Offset) 786 Len = Len - Offset; 787 else 788 return Loc; 789 790 return Loc.getLocWithOffset(Len); 791} 792 793/// \brief Returns true if the given MacroID location points at the first 794/// token of the macro expansion. 795bool Lexer::isAtStartOfMacroExpansion(SourceLocation loc, 796 const SourceManager &SM, 797 const LangOptions &LangOpts, 798 SourceLocation *MacroBegin) { 799 assert(loc.isValid() && loc.isMacroID() && "Expected a valid macro loc"); 800 801 std::pair<FileID, unsigned> infoLoc = SM.getDecomposedLoc(loc); 802 // FIXME: If the token comes from the macro token paste operator ('##') 803 // this function will always return false; 804 if (infoLoc.second > 0) 805 return false; // Does not point at the start of token. 806 807 SourceLocation expansionLoc = 808 SM.getSLocEntry(infoLoc.first).getExpansion().getExpansionLocStart(); 809 if (expansionLoc.isFileID()) { 810 // No other macro expansions, this is the first. 811 if (MacroBegin) 812 *MacroBegin = expansionLoc; 813 return true; 814 } 815 816 return isAtStartOfMacroExpansion(expansionLoc, SM, LangOpts, MacroBegin); 817} 818 819/// \brief Returns true if the given MacroID location points at the last 820/// token of the macro expansion. 821bool Lexer::isAtEndOfMacroExpansion(SourceLocation loc, 822 const SourceManager &SM, 823 const LangOptions &LangOpts, 824 SourceLocation *MacroEnd) { 825 assert(loc.isValid() && loc.isMacroID() && "Expected a valid macro loc"); 826 827 SourceLocation spellLoc = SM.getSpellingLoc(loc); 828 unsigned tokLen = MeasureTokenLength(spellLoc, SM, LangOpts); 829 if (tokLen == 0) 830 return false; 831 832 FileID FID = SM.getFileID(loc); 833 SourceLocation afterLoc = loc.getLocWithOffset(tokLen+1); 834 if (SM.isInFileID(afterLoc, FID)) 835 return false; // Still in the same FileID, does not point to the last token. 836 837 // FIXME: If the token comes from the macro token paste operator ('##') 838 // or the stringify operator ('#') this function will always return false; 839 840 SourceLocation expansionLoc = 841 SM.getSLocEntry(FID).getExpansion().getExpansionLocEnd(); 842 if (expansionLoc.isFileID()) { 843 // No other macro expansions. 844 if (MacroEnd) 845 *MacroEnd = expansionLoc; 846 return true; 847 } 848 849 return isAtEndOfMacroExpansion(expansionLoc, SM, LangOpts, MacroEnd); 850} 851 852static CharSourceRange makeRangeFromFileLocs(CharSourceRange Range, 853 const SourceManager &SM, 854 const LangOptions &LangOpts) { 855 SourceLocation Begin = Range.getBegin(); 856 SourceLocation End = Range.getEnd(); 857 assert(Begin.isFileID() && End.isFileID()); 858 if (Range.isTokenRange()) { 859 End = Lexer::getLocForEndOfToken(End, 0, SM,LangOpts); 860 if (End.isInvalid()) 861 return CharSourceRange(); 862 } 863 864 // Break down the source locations. 865 FileID FID; 866 unsigned BeginOffs; 867 llvm::tie(FID, BeginOffs) = SM.getDecomposedLoc(Begin); 868 if (FID.isInvalid()) 869 return CharSourceRange(); 870 871 unsigned EndOffs; 872 if (!SM.isInFileID(End, FID, &EndOffs) || 873 BeginOffs > EndOffs) 874 return CharSourceRange(); 875 876 return CharSourceRange::getCharRange(Begin, End); 877} 878 879CharSourceRange Lexer::makeFileCharRange(CharSourceRange Range, 880 const SourceManager &SM, 881 const LangOptions &LangOpts) { 882 SourceLocation Begin = Range.getBegin(); 883 SourceLocation End = Range.getEnd(); 884 if (Begin.isInvalid() || End.isInvalid()) 885 return CharSourceRange(); 886 887 if (Begin.isFileID() && End.isFileID()) 888 return makeRangeFromFileLocs(Range, SM, LangOpts); 889 890 if (Begin.isMacroID() && End.isFileID()) { 891 if (!isAtStartOfMacroExpansion(Begin, SM, LangOpts, &Begin)) 892 return CharSourceRange(); 893 Range.setBegin(Begin); 894 return makeRangeFromFileLocs(Range, SM, LangOpts); 895 } 896 897 if (Begin.isFileID() && End.isMacroID()) { 898 if ((Range.isTokenRange() && !isAtEndOfMacroExpansion(End, SM, LangOpts, 899 &End)) || 900 (Range.isCharRange() && !isAtStartOfMacroExpansion(End, SM, LangOpts, 901 &End))) 902 return CharSourceRange(); 903 Range.setEnd(End); 904 return makeRangeFromFileLocs(Range, SM, LangOpts); 905 } 906 907 assert(Begin.isMacroID() && End.isMacroID()); 908 SourceLocation MacroBegin, MacroEnd; 909 if (isAtStartOfMacroExpansion(Begin, SM, LangOpts, &MacroBegin) && 910 ((Range.isTokenRange() && isAtEndOfMacroExpansion(End, SM, LangOpts, 911 &MacroEnd)) || 912 (Range.isCharRange() && isAtStartOfMacroExpansion(End, SM, LangOpts, 913 &MacroEnd)))) { 914 Range.setBegin(MacroBegin); 915 Range.setEnd(MacroEnd); 916 return makeRangeFromFileLocs(Range, SM, LangOpts); 917 } 918 919 FileID FID; 920 unsigned BeginOffs; 921 llvm::tie(FID, BeginOffs) = SM.getDecomposedLoc(Begin); 922 if (FID.isInvalid()) 923 return CharSourceRange(); 924 925 unsigned EndOffs; 926 if (!SM.isInFileID(End, FID, &EndOffs) || 927 BeginOffs > EndOffs) 928 return CharSourceRange(); 929 930 const SrcMgr::SLocEntry *E = &SM.getSLocEntry(FID); 931 const SrcMgr::ExpansionInfo &Expansion = E->getExpansion(); 932 if (Expansion.isMacroArgExpansion() && 933 Expansion.getSpellingLoc().isFileID()) { 934 SourceLocation SpellLoc = Expansion.getSpellingLoc(); 935 Range.setBegin(SpellLoc.getLocWithOffset(BeginOffs)); 936 Range.setEnd(SpellLoc.getLocWithOffset(EndOffs)); 937 return makeRangeFromFileLocs(Range, SM, LangOpts); 938 } 939 940 return CharSourceRange(); 941} 942 943StringRef Lexer::getSourceText(CharSourceRange Range, 944 const SourceManager &SM, 945 const LangOptions &LangOpts, 946 bool *Invalid) { 947 Range = makeFileCharRange(Range, SM, LangOpts); 948 if (Range.isInvalid()) { 949 if (Invalid) *Invalid = true; 950 return StringRef(); 951 } 952 953 // Break down the source location. 954 std::pair<FileID, unsigned> beginInfo = SM.getDecomposedLoc(Range.getBegin()); 955 if (beginInfo.first.isInvalid()) { 956 if (Invalid) *Invalid = true; 957 return StringRef(); 958 } 959 960 unsigned EndOffs; 961 if (!SM.isInFileID(Range.getEnd(), beginInfo.first, &EndOffs) || 962 beginInfo.second > EndOffs) { 963 if (Invalid) *Invalid = true; 964 return StringRef(); 965 } 966 967 // Try to the load the file buffer. 968 bool invalidTemp = false; 969 StringRef file = SM.getBufferData(beginInfo.first, &invalidTemp); 970 if (invalidTemp) { 971 if (Invalid) *Invalid = true; 972 return StringRef(); 973 } 974 975 if (Invalid) *Invalid = false; 976 return file.substr(beginInfo.second, EndOffs - beginInfo.second); 977} 978 979StringRef Lexer::getImmediateMacroName(SourceLocation Loc, 980 const SourceManager &SM, 981 const LangOptions &LangOpts) { 982 assert(Loc.isMacroID() && "Only reasonble to call this on macros"); 983 984 // Find the location of the immediate macro expansion. 985 while (1) { 986 FileID FID = SM.getFileID(Loc); 987 const SrcMgr::SLocEntry *E = &SM.getSLocEntry(FID); 988 const SrcMgr::ExpansionInfo &Expansion = E->getExpansion(); 989 Loc = Expansion.getExpansionLocStart(); 990 if (!Expansion.isMacroArgExpansion()) 991 break; 992 993 // For macro arguments we need to check that the argument did not come 994 // from an inner macro, e.g: "MAC1( MAC2(foo) )" 995 996 // Loc points to the argument id of the macro definition, move to the 997 // macro expansion. 998 Loc = SM.getImmediateExpansionRange(Loc).first; 999 SourceLocation SpellLoc = Expansion.getSpellingLoc(); 1000 if (SpellLoc.isFileID()) 1001 break; // No inner macro. 1002 1003 // If spelling location resides in the same FileID as macro expansion 1004 // location, it means there is no inner macro. 1005 FileID MacroFID = SM.getFileID(Loc); 1006 if (SM.isInFileID(SpellLoc, MacroFID)) 1007 break; 1008 1009 // Argument came from inner macro. 1010 Loc = SpellLoc; 1011 } 1012 1013 // Find the spelling location of the start of the non-argument expansion 1014 // range. This is where the macro name was spelled in order to begin 1015 // expanding this macro. 1016 Loc = SM.getSpellingLoc(Loc); 1017 1018 // Dig out the buffer where the macro name was spelled and the extents of the 1019 // name so that we can render it into the expansion note. 1020 std::pair<FileID, unsigned> ExpansionInfo = SM.getDecomposedLoc(Loc); 1021 unsigned MacroTokenLength = Lexer::MeasureTokenLength(Loc, SM, LangOpts); 1022 StringRef ExpansionBuffer = SM.getBufferData(ExpansionInfo.first); 1023 return ExpansionBuffer.substr(ExpansionInfo.second, MacroTokenLength); 1024} 1025 1026bool Lexer::isIdentifierBodyChar(char c, const LangOptions &LangOpts) { 1027 return isIdentifierBody(c, LangOpts.DollarIdents); 1028} 1029 1030 1031//===----------------------------------------------------------------------===// 1032// Diagnostics forwarding code. 1033//===----------------------------------------------------------------------===// 1034 1035/// GetMappedTokenLoc - If lexing out of a 'mapped buffer', where we pretend the 1036/// lexer buffer was all expanded at a single point, perform the mapping. 1037/// This is currently only used for _Pragma implementation, so it is the slow 1038/// path of the hot getSourceLocation method. Do not allow it to be inlined. 1039static LLVM_ATTRIBUTE_NOINLINE SourceLocation GetMappedTokenLoc( 1040 Preprocessor &PP, SourceLocation FileLoc, unsigned CharNo, unsigned TokLen); 1041static SourceLocation GetMappedTokenLoc(Preprocessor &PP, 1042 SourceLocation FileLoc, 1043 unsigned CharNo, unsigned TokLen) { 1044 assert(FileLoc.isMacroID() && "Must be a macro expansion"); 1045 1046 // Otherwise, we're lexing "mapped tokens". This is used for things like 1047 // _Pragma handling. Combine the expansion location of FileLoc with the 1048 // spelling location. 1049 SourceManager &SM = PP.getSourceManager(); 1050 1051 // Create a new SLoc which is expanded from Expansion(FileLoc) but whose 1052 // characters come from spelling(FileLoc)+Offset. 1053 SourceLocation SpellingLoc = SM.getSpellingLoc(FileLoc); 1054 SpellingLoc = SpellingLoc.getLocWithOffset(CharNo); 1055 1056 // Figure out the expansion loc range, which is the range covered by the 1057 // original _Pragma(...) sequence. 1058 std::pair<SourceLocation,SourceLocation> II = 1059 SM.getImmediateExpansionRange(FileLoc); 1060 1061 return SM.createExpansionLoc(SpellingLoc, II.first, II.second, TokLen); 1062} 1063 1064/// getSourceLocation - Return a source location identifier for the specified 1065/// offset in the current file. 1066SourceLocation Lexer::getSourceLocation(const char *Loc, 1067 unsigned TokLen) const { 1068 assert(Loc >= BufferStart && Loc <= BufferEnd && 1069 "Location out of range for this buffer!"); 1070 1071 // In the normal case, we're just lexing from a simple file buffer, return 1072 // the file id from FileLoc with the offset specified. 1073 unsigned CharNo = Loc-BufferStart; 1074 if (FileLoc.isFileID()) 1075 return FileLoc.getLocWithOffset(CharNo); 1076 1077 // Otherwise, this is the _Pragma lexer case, which pretends that all of the 1078 // tokens are lexed from where the _Pragma was defined. 1079 assert(PP && "This doesn't work on raw lexers"); 1080 return GetMappedTokenLoc(*PP, FileLoc, CharNo, TokLen); 1081} 1082 1083/// Diag - Forwarding function for diagnostics. This translate a source 1084/// position in the current buffer into a SourceLocation object for rendering. 1085DiagnosticBuilder Lexer::Diag(const char *Loc, unsigned DiagID) const { 1086 return PP->Diag(getSourceLocation(Loc), DiagID); 1087} 1088 1089//===----------------------------------------------------------------------===// 1090// Trigraph and Escaped Newline Handling Code. 1091//===----------------------------------------------------------------------===// 1092 1093/// GetTrigraphCharForLetter - Given a character that occurs after a ?? pair, 1094/// return the decoded trigraph letter it corresponds to, or '\0' if nothing. 1095static char GetTrigraphCharForLetter(char Letter) { 1096 switch (Letter) { 1097 default: return 0; 1098 case '=': return '#'; 1099 case ')': return ']'; 1100 case '(': return '['; 1101 case '!': return '|'; 1102 case '\'': return '^'; 1103 case '>': return '}'; 1104 case '/': return '\\'; 1105 case '<': return '{'; 1106 case '-': return '~'; 1107 } 1108} 1109 1110/// DecodeTrigraphChar - If the specified character is a legal trigraph when 1111/// prefixed with ??, emit a trigraph warning. If trigraphs are enabled, 1112/// return the result character. Finally, emit a warning about trigraph use 1113/// whether trigraphs are enabled or not. 1114static char DecodeTrigraphChar(const char *CP, Lexer *L) { 1115 char Res = GetTrigraphCharForLetter(*CP); 1116 if (!Res || !L) return Res; 1117 1118 if (!L->getLangOpts().Trigraphs) { 1119 if (!L->isLexingRawMode()) 1120 L->Diag(CP-2, diag::trigraph_ignored); 1121 return 0; 1122 } 1123 1124 if (!L->isLexingRawMode()) 1125 L->Diag(CP-2, diag::trigraph_converted) << StringRef(&Res, 1); 1126 return Res; 1127} 1128 1129/// getEscapedNewLineSize - Return the size of the specified escaped newline, 1130/// or 0 if it is not an escaped newline. P[-1] is known to be a "\" or a 1131/// trigraph equivalent on entry to this function. 1132unsigned Lexer::getEscapedNewLineSize(const char *Ptr) { 1133 unsigned Size = 0; 1134 while (isWhitespace(Ptr[Size])) { 1135 ++Size; 1136 1137 if (Ptr[Size-1] != '\n' && Ptr[Size-1] != '\r') 1138 continue; 1139 1140 // If this is a \r\n or \n\r, skip the other half. 1141 if ((Ptr[Size] == '\r' || Ptr[Size] == '\n') && 1142 Ptr[Size-1] != Ptr[Size]) 1143 ++Size; 1144 1145 return Size; 1146 } 1147 1148 // Not an escaped newline, must be a \t or something else. 1149 return 0; 1150} 1151 1152/// SkipEscapedNewLines - If P points to an escaped newline (or a series of 1153/// them), skip over them and return the first non-escaped-newline found, 1154/// otherwise return P. 1155const char *Lexer::SkipEscapedNewLines(const char *P) { 1156 while (1) { 1157 const char *AfterEscape; 1158 if (*P == '\\') { 1159 AfterEscape = P+1; 1160 } else if (*P == '?') { 1161 // If not a trigraph for escape, bail out. 1162 if (P[1] != '?' || P[2] != '/') 1163 return P; 1164 AfterEscape = P+3; 1165 } else { 1166 return P; 1167 } 1168 1169 unsigned NewLineSize = Lexer::getEscapedNewLineSize(AfterEscape); 1170 if (NewLineSize == 0) return P; 1171 P = AfterEscape+NewLineSize; 1172 } 1173} 1174 1175/// \brief Checks that the given token is the first token that occurs after the 1176/// given location (this excludes comments and whitespace). Returns the location 1177/// immediately after the specified token. If the token is not found or the 1178/// location is inside a macro, the returned source location will be invalid. 1179SourceLocation Lexer::findLocationAfterToken(SourceLocation Loc, 1180 tok::TokenKind TKind, 1181 const SourceManager &SM, 1182 const LangOptions &LangOpts, 1183 bool SkipTrailingWhitespaceAndNewLine) { 1184 if (Loc.isMacroID()) { 1185 if (!Lexer::isAtEndOfMacroExpansion(Loc, SM, LangOpts, &Loc)) 1186 return SourceLocation(); 1187 } 1188 Loc = Lexer::getLocForEndOfToken(Loc, 0, SM, LangOpts); 1189 1190 // Break down the source location. 1191 std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(Loc); 1192 1193 // Try to load the file buffer. 1194 bool InvalidTemp = false; 1195 StringRef File = SM.getBufferData(LocInfo.first, &InvalidTemp); 1196 if (InvalidTemp) 1197 return SourceLocation(); 1198 1199 const char *TokenBegin = File.data() + LocInfo.second; 1200 1201 // Lex from the start of the given location. 1202 Lexer lexer(SM.getLocForStartOfFile(LocInfo.first), LangOpts, File.begin(), 1203 TokenBegin, File.end()); 1204 // Find the token. 1205 Token Tok; 1206 lexer.LexFromRawLexer(Tok); 1207 if (Tok.isNot(TKind)) 1208 return SourceLocation(); 1209 SourceLocation TokenLoc = Tok.getLocation(); 1210 1211 // Calculate how much whitespace needs to be skipped if any. 1212 unsigned NumWhitespaceChars = 0; 1213 if (SkipTrailingWhitespaceAndNewLine) { 1214 const char *TokenEnd = SM.getCharacterData(TokenLoc) + 1215 Tok.getLength(); 1216 unsigned char C = *TokenEnd; 1217 while (isHorizontalWhitespace(C)) { 1218 C = *(++TokenEnd); 1219 NumWhitespaceChars++; 1220 } 1221 1222 // Skip \r, \n, \r\n, or \n\r 1223 if (C == '\n' || C == '\r') { 1224 char PrevC = C; 1225 C = *(++TokenEnd); 1226 NumWhitespaceChars++; 1227 if ((C == '\n' || C == '\r') && C != PrevC) 1228 NumWhitespaceChars++; 1229 } 1230 } 1231 1232 return TokenLoc.getLocWithOffset(Tok.getLength() + NumWhitespaceChars); 1233} 1234 1235/// getCharAndSizeSlow - Peek a single 'character' from the specified buffer, 1236/// get its size, and return it. This is tricky in several cases: 1237/// 1. If currently at the start of a trigraph, we warn about the trigraph, 1238/// then either return the trigraph (skipping 3 chars) or the '?', 1239/// depending on whether trigraphs are enabled or not. 1240/// 2. If this is an escaped newline (potentially with whitespace between 1241/// the backslash and newline), implicitly skip the newline and return 1242/// the char after it. 1243/// 1244/// This handles the slow/uncommon case of the getCharAndSize method. Here we 1245/// know that we can accumulate into Size, and that we have already incremented 1246/// Ptr by Size bytes. 1247/// 1248/// NOTE: When this method is updated, getCharAndSizeSlowNoWarn (below) should 1249/// be updated to match. 1250/// 1251char Lexer::getCharAndSizeSlow(const char *Ptr, unsigned &Size, 1252 Token *Tok) { 1253 // If we have a slash, look for an escaped newline. 1254 if (Ptr[0] == '\\') { 1255 ++Size; 1256 ++Ptr; 1257Slash: 1258 // Common case, backslash-char where the char is not whitespace. 1259 if (!isWhitespace(Ptr[0])) return '\\'; 1260 1261 // See if we have optional whitespace characters between the slash and 1262 // newline. 1263 if (unsigned EscapedNewLineSize = getEscapedNewLineSize(Ptr)) { 1264 // Remember that this token needs to be cleaned. 1265 if (Tok) Tok->setFlag(Token::NeedsCleaning); 1266 1267 // Warn if there was whitespace between the backslash and newline. 1268 if (Ptr[0] != '\n' && Ptr[0] != '\r' && Tok && !isLexingRawMode()) 1269 Diag(Ptr, diag::backslash_newline_space); 1270 1271 // Found backslash<whitespace><newline>. Parse the char after it. 1272 Size += EscapedNewLineSize; 1273 Ptr += EscapedNewLineSize; 1274 1275 // If the char that we finally got was a \n, then we must have had 1276 // something like \<newline><newline>. We don't want to consume the 1277 // second newline. 1278 if (*Ptr == '\n' || *Ptr == '\r' || *Ptr == '\0') 1279 return ' '; 1280 1281 // Use slow version to accumulate a correct size field. 1282 return getCharAndSizeSlow(Ptr, Size, Tok); 1283 } 1284 1285 // Otherwise, this is not an escaped newline, just return the slash. 1286 return '\\'; 1287 } 1288 1289 // If this is a trigraph, process it. 1290 if (Ptr[0] == '?' && Ptr[1] == '?') { 1291 // If this is actually a legal trigraph (not something like "??x"), emit 1292 // a trigraph warning. If so, and if trigraphs are enabled, return it. 1293 if (char C = DecodeTrigraphChar(Ptr+2, Tok ? this : 0)) { 1294 // Remember that this token needs to be cleaned. 1295 if (Tok) Tok->setFlag(Token::NeedsCleaning); 1296 1297 Ptr += 3; 1298 Size += 3; 1299 if (C == '\\') goto Slash; 1300 return C; 1301 } 1302 } 1303 1304 // If this is neither, return a single character. 1305 ++Size; 1306 return *Ptr; 1307} 1308 1309 1310/// getCharAndSizeSlowNoWarn - Handle the slow/uncommon case of the 1311/// getCharAndSizeNoWarn method. Here we know that we can accumulate into Size, 1312/// and that we have already incremented Ptr by Size bytes. 1313/// 1314/// NOTE: When this method is updated, getCharAndSizeSlow (above) should 1315/// be updated to match. 1316char Lexer::getCharAndSizeSlowNoWarn(const char *Ptr, unsigned &Size, 1317 const LangOptions &LangOpts) { 1318 // If we have a slash, look for an escaped newline. 1319 if (Ptr[0] == '\\') { 1320 ++Size; 1321 ++Ptr; 1322Slash: 1323 // Common case, backslash-char where the char is not whitespace. 1324 if (!isWhitespace(Ptr[0])) return '\\'; 1325 1326 // See if we have optional whitespace characters followed by a newline. 1327 if (unsigned EscapedNewLineSize = getEscapedNewLineSize(Ptr)) { 1328 // Found backslash<whitespace><newline>. Parse the char after it. 1329 Size += EscapedNewLineSize; 1330 Ptr += EscapedNewLineSize; 1331 1332 // If the char that we finally got was a \n, then we must have had 1333 // something like \<newline><newline>. We don't want to consume the 1334 // second newline. 1335 if (*Ptr == '\n' || *Ptr == '\r' || *Ptr == '\0') 1336 return ' '; 1337 1338 // Use slow version to accumulate a correct size field. 1339 return getCharAndSizeSlowNoWarn(Ptr, Size, LangOpts); 1340 } 1341 1342 // Otherwise, this is not an escaped newline, just return the slash. 1343 return '\\'; 1344 } 1345 1346 // If this is a trigraph, process it. 1347 if (LangOpts.Trigraphs && Ptr[0] == '?' && Ptr[1] == '?') { 1348 // If this is actually a legal trigraph (not something like "??x"), return 1349 // it. 1350 if (char C = GetTrigraphCharForLetter(Ptr[2])) { 1351 Ptr += 3; 1352 Size += 3; 1353 if (C == '\\') goto Slash; 1354 return C; 1355 } 1356 } 1357 1358 // If this is neither, return a single character. 1359 ++Size; 1360 return *Ptr; 1361} 1362 1363//===----------------------------------------------------------------------===// 1364// Helper methods for lexing. 1365//===----------------------------------------------------------------------===// 1366 1367/// \brief Routine that indiscriminately skips bytes in the source file. 1368void Lexer::SkipBytes(unsigned Bytes, bool StartOfLine) { 1369 BufferPtr += Bytes; 1370 if (BufferPtr > BufferEnd) 1371 BufferPtr = BufferEnd; 1372 IsAtStartOfLine = StartOfLine; 1373} 1374 1375static bool isAllowedIDChar(uint32_t C, const LangOptions &LangOpts) { 1376 if (LangOpts.CPlusPlus11 || LangOpts.C11) 1377 return isCharInSet(C, C11AllowedIDChars); 1378 else if (LangOpts.CPlusPlus) 1379 return isCharInSet(C, CXX03AllowedIDChars); 1380 else 1381 return isCharInSet(C, C99AllowedIDChars); 1382} 1383 1384static bool isAllowedInitiallyIDChar(uint32_t C, const LangOptions &LangOpts) { 1385 assert(isAllowedIDChar(C, LangOpts)); 1386 if (LangOpts.CPlusPlus11 || LangOpts.C11) 1387 return !isCharInSet(C, C11DisallowedInitialIDChars); 1388 else if (LangOpts.CPlusPlus) 1389 return true; 1390 else 1391 return !isCharInSet(C, C99DisallowedInitialIDChars); 1392} 1393 1394static inline CharSourceRange makeCharRange(Lexer &L, const char *Begin, 1395 const char *End) { 1396 return CharSourceRange::getCharRange(L.getSourceLocation(Begin), 1397 L.getSourceLocation(End)); 1398} 1399 1400static void maybeDiagnoseIDCharCompat(DiagnosticsEngine &Diags, uint32_t C, 1401 CharSourceRange Range, bool IsFirst) { 1402 // Check C99 compatibility. 1403 if (Diags.getDiagnosticLevel(diag::warn_c99_compat_unicode_id, 1404 Range.getBegin()) > DiagnosticsEngine::Ignored) { 1405 enum { 1406 CannotAppearInIdentifier = 0, 1407 CannotStartIdentifier 1408 }; 1409 1410 if (!isCharInSet(C, C99AllowedIDChars)) { 1411 Diags.Report(Range.getBegin(), diag::warn_c99_compat_unicode_id) 1412 << Range 1413 << CannotAppearInIdentifier; 1414 } else if (IsFirst && isCharInSet(C, C99DisallowedInitialIDChars)) { 1415 Diags.Report(Range.getBegin(), diag::warn_c99_compat_unicode_id) 1416 << Range 1417 << CannotStartIdentifier; 1418 } 1419 } 1420 1421 // Check C++98 compatibility. 1422 if (Diags.getDiagnosticLevel(diag::warn_cxx98_compat_unicode_id, 1423 Range.getBegin()) > DiagnosticsEngine::Ignored) { 1424 if (!isCharInSet(C, CXX03AllowedIDChars)) { 1425 Diags.Report(Range.getBegin(), diag::warn_cxx98_compat_unicode_id) 1426 << Range; 1427 } 1428 } 1429 } 1430 1431void Lexer::LexIdentifier(Token &Result, const char *CurPtr) { 1432 // Match [_A-Za-z0-9]*, we have already matched [_A-Za-z$] 1433 unsigned Size; 1434 unsigned char C = *CurPtr++; 1435 while (isIdentifierBody(C)) 1436 C = *CurPtr++; 1437 1438 --CurPtr; // Back up over the skipped character. 1439 1440 // Fast path, no $,\,? in identifier found. '\' might be an escaped newline 1441 // or UCN, and ? might be a trigraph for '\', an escaped newline or UCN. 1442 // 1443 // TODO: Could merge these checks into an InfoTable flag to make the 1444 // comparison cheaper 1445 if (isASCII(C) && C != '\\' && C != '?' && 1446 (C != '$' || !LangOpts.DollarIdents)) { 1447FinishIdentifier: 1448 const char *IdStart = BufferPtr; 1449 FormTokenWithChars(Result, CurPtr, tok::raw_identifier); 1450 Result.setRawIdentifierData(IdStart); 1451 1452 // If we are in raw mode, return this identifier raw. There is no need to 1453 // look up identifier information or attempt to macro expand it. 1454 if (LexingRawMode) 1455 return; 1456 1457 // Fill in Result.IdentifierInfo and update the token kind, 1458 // looking up the identifier in the identifier table. 1459 IdentifierInfo *II = PP->LookUpIdentifierInfo(Result); 1460 1461 // Finally, now that we know we have an identifier, pass this off to the 1462 // preprocessor, which may macro expand it or something. 1463 if (II->isHandleIdentifierCase()) 1464 PP->HandleIdentifier(Result); 1465 1466 return; 1467 } 1468 1469 // Otherwise, $,\,? in identifier found. Enter slower path. 1470 1471 C = getCharAndSize(CurPtr, Size); 1472 while (1) { 1473 if (C == '$') { 1474 // If we hit a $ and they are not supported in identifiers, we are done. 1475 if (!LangOpts.DollarIdents) goto FinishIdentifier; 1476 1477 // Otherwise, emit a diagnostic and continue. 1478 if (!isLexingRawMode()) 1479 Diag(CurPtr, diag::ext_dollar_in_identifier); 1480 CurPtr = ConsumeChar(CurPtr, Size, Result); 1481 C = getCharAndSize(CurPtr, Size); 1482 continue; 1483 1484 } else if (C == '\\') { 1485 const char *UCNPtr = CurPtr + Size; 1486 uint32_t CodePoint = tryReadUCN(UCNPtr, CurPtr, /*Token=*/0); 1487 if (CodePoint == 0 || !isAllowedIDChar(CodePoint, LangOpts)) 1488 goto FinishIdentifier; 1489 1490 if (!isLexingRawMode()) { 1491 maybeDiagnoseIDCharCompat(PP->getDiagnostics(), CodePoint, 1492 makeCharRange(*this, CurPtr, UCNPtr), 1493 /*IsFirst=*/false); 1494 } 1495 1496 Result.setFlag(Token::HasUCN); 1497 if ((UCNPtr - CurPtr == 6 && CurPtr[1] == 'u') || 1498 (UCNPtr - CurPtr == 10 && CurPtr[1] == 'U')) 1499 CurPtr = UCNPtr; 1500 else 1501 while (CurPtr != UCNPtr) 1502 (void)getAndAdvanceChar(CurPtr, Result); 1503 1504 C = getCharAndSize(CurPtr, Size); 1505 continue; 1506 } else if (!isASCII(C)) { 1507 const char *UnicodePtr = CurPtr; 1508 UTF32 CodePoint; 1509 ConversionResult Result = 1510 llvm::convertUTF8Sequence((const UTF8 **)&UnicodePtr, 1511 (const UTF8 *)BufferEnd, 1512 &CodePoint, 1513 strictConversion); 1514 if (Result != conversionOK || 1515 !isAllowedIDChar(static_cast<uint32_t>(CodePoint), LangOpts)) 1516 goto FinishIdentifier; 1517 1518 if (!isLexingRawMode()) { 1519 maybeDiagnoseIDCharCompat(PP->getDiagnostics(), CodePoint, 1520 makeCharRange(*this, CurPtr, UnicodePtr), 1521 /*IsFirst=*/false); 1522 } 1523 1524 CurPtr = UnicodePtr; 1525 C = getCharAndSize(CurPtr, Size); 1526 continue; 1527 } else if (!isIdentifierBody(C)) { 1528 goto FinishIdentifier; 1529 } 1530 1531 // Otherwise, this character is good, consume it. 1532 CurPtr = ConsumeChar(CurPtr, Size, Result); 1533 1534 C = getCharAndSize(CurPtr, Size); 1535 while (isIdentifierBody(C)) { 1536 CurPtr = ConsumeChar(CurPtr, Size, Result); 1537 C = getCharAndSize(CurPtr, Size); 1538 } 1539 } 1540} 1541 1542/// isHexaLiteral - Return true if Start points to a hex constant. 1543/// in microsoft mode (where this is supposed to be several different tokens). 1544bool Lexer::isHexaLiteral(const char *Start, const LangOptions &LangOpts) { 1545 unsigned Size; 1546 char C1 = Lexer::getCharAndSizeNoWarn(Start, Size, LangOpts); 1547 if (C1 != '0') 1548 return false; 1549 char C2 = Lexer::getCharAndSizeNoWarn(Start + Size, Size, LangOpts); 1550 return (C2 == 'x' || C2 == 'X'); 1551} 1552 1553/// LexNumericConstant - Lex the remainder of a integer or floating point 1554/// constant. From[-1] is the first character lexed. Return the end of the 1555/// constant. 1556void Lexer::LexNumericConstant(Token &Result, const char *CurPtr) { 1557 unsigned Size; 1558 char C = getCharAndSize(CurPtr, Size); 1559 char PrevCh = 0; 1560 while (isPreprocessingNumberBody(C)) { // FIXME: UCNs in ud-suffix. 1561 CurPtr = ConsumeChar(CurPtr, Size, Result); 1562 PrevCh = C; 1563 C = getCharAndSize(CurPtr, Size); 1564 } 1565 1566 // If we fell out, check for a sign, due to 1e+12. If we have one, continue. 1567 if ((C == '-' || C == '+') && (PrevCh == 'E' || PrevCh == 'e')) { 1568 // If we are in Microsoft mode, don't continue if the constant is hex. 1569 // For example, MSVC will accept the following as 3 tokens: 0x1234567e+1 1570 if (!LangOpts.MicrosoftExt || !isHexaLiteral(BufferPtr, LangOpts)) 1571 return LexNumericConstant(Result, ConsumeChar(CurPtr, Size, Result)); 1572 } 1573 1574 // If we have a hex FP constant, continue. 1575 if ((C == '-' || C == '+') && (PrevCh == 'P' || PrevCh == 'p')) { 1576 // Outside C99, we accept hexadecimal floating point numbers as a 1577 // not-quite-conforming extension. Only do so if this looks like it's 1578 // actually meant to be a hexfloat, and not if it has a ud-suffix. 1579 bool IsHexFloat = true; 1580 if (!LangOpts.C99) { 1581 if (!isHexaLiteral(BufferPtr, LangOpts)) 1582 IsHexFloat = false; 1583 else if (std::find(BufferPtr, CurPtr, '_') != CurPtr) 1584 IsHexFloat = false; 1585 } 1586 if (IsHexFloat) 1587 return LexNumericConstant(Result, ConsumeChar(CurPtr, Size, Result)); 1588 } 1589 1590 // Update the location of token as well as BufferPtr. 1591 const char *TokStart = BufferPtr; 1592 FormTokenWithChars(Result, CurPtr, tok::numeric_constant); 1593 Result.setLiteralData(TokStart); 1594} 1595 1596/// LexUDSuffix - Lex the ud-suffix production for user-defined literal suffixes 1597/// in C++11, or warn on a ud-suffix in C++98. 1598const char *Lexer::LexUDSuffix(Token &Result, const char *CurPtr) { 1599 assert(getLangOpts().CPlusPlus); 1600 1601 // Maximally munch an identifier. FIXME: UCNs. 1602 unsigned Size; 1603 char C = getCharAndSize(CurPtr, Size); 1604 if (isIdentifierHead(C)) { 1605 if (!getLangOpts().CPlusPlus11) { 1606 if (!isLexingRawMode()) 1607 Diag(CurPtr, 1608 C == '_' ? diag::warn_cxx11_compat_user_defined_literal 1609 : diag::warn_cxx11_compat_reserved_user_defined_literal) 1610 << FixItHint::CreateInsertion(getSourceLocation(CurPtr), " "); 1611 return CurPtr; 1612 } 1613 1614 // C++11 [lex.ext]p10, [usrlit.suffix]p1: A program containing a ud-suffix 1615 // that does not start with an underscore is ill-formed. As a conforming 1616 // extension, we treat all such suffixes as if they had whitespace before 1617 // them. 1618 if (C != '_') { 1619 if (!isLexingRawMode()) 1620 Diag(CurPtr, getLangOpts().MicrosoftMode ? 1621 diag::ext_ms_reserved_user_defined_literal : 1622 diag::ext_reserved_user_defined_literal) 1623 << FixItHint::CreateInsertion(getSourceLocation(CurPtr), " "); 1624 return CurPtr; 1625 } 1626 1627 Result.setFlag(Token::HasUDSuffix); 1628 do { 1629 CurPtr = ConsumeChar(CurPtr, Size, Result); 1630 C = getCharAndSize(CurPtr, Size); 1631 } while (isIdentifierBody(C)); 1632 } 1633 return CurPtr; 1634} 1635 1636/// LexStringLiteral - Lex the remainder of a string literal, after having lexed 1637/// either " or L" or u8" or u" or U". 1638void Lexer::LexStringLiteral(Token &Result, const char *CurPtr, 1639 tok::TokenKind Kind) { 1640 const char *NulCharacter = 0; // Does this string contain the \0 character? 1641 1642 if (!isLexingRawMode() && 1643 (Kind == tok::utf8_string_literal || 1644 Kind == tok::utf16_string_literal || 1645 Kind == tok::utf32_string_literal)) 1646 Diag(BufferPtr, getLangOpts().CPlusPlus 1647 ? diag::warn_cxx98_compat_unicode_literal 1648 : diag::warn_c99_compat_unicode_literal); 1649 1650 char C = getAndAdvanceChar(CurPtr, Result); 1651 while (C != '"') { 1652 // Skip escaped characters. Escaped newlines will already be processed by 1653 // getAndAdvanceChar. 1654 if (C == '\\') 1655 C = getAndAdvanceChar(CurPtr, Result); 1656 1657 if (C == '\n' || C == '\r' || // Newline. 1658 (C == 0 && CurPtr-1 == BufferEnd)) { // End of file. 1659 if (!isLexingRawMode() && !LangOpts.AsmPreprocessor) 1660 Diag(BufferPtr, diag::ext_unterminated_string); 1661 FormTokenWithChars(Result, CurPtr-1, tok::unknown); 1662 return; 1663 } 1664 1665 if (C == 0) { 1666 if (isCodeCompletionPoint(CurPtr-1)) { 1667 PP->CodeCompleteNaturalLanguage(); 1668 FormTokenWithChars(Result, CurPtr-1, tok::unknown); 1669 return cutOffLexing(); 1670 } 1671 1672 NulCharacter = CurPtr-1; 1673 } 1674 C = getAndAdvanceChar(CurPtr, Result); 1675 } 1676 1677 // If we are in C++11, lex the optional ud-suffix. 1678 if (getLangOpts().CPlusPlus) 1679 CurPtr = LexUDSuffix(Result, CurPtr); 1680 1681 // If a nul character existed in the string, warn about it. 1682 if (NulCharacter && !isLexingRawMode()) 1683 Diag(NulCharacter, diag::null_in_string); 1684 1685 // Update the location of the token as well as the BufferPtr instance var. 1686 const char *TokStart = BufferPtr; 1687 FormTokenWithChars(Result, CurPtr, Kind); 1688 Result.setLiteralData(TokStart); 1689} 1690 1691/// LexRawStringLiteral - Lex the remainder of a raw string literal, after 1692/// having lexed R", LR", u8R", uR", or UR". 1693void Lexer::LexRawStringLiteral(Token &Result, const char *CurPtr, 1694 tok::TokenKind Kind) { 1695 // This function doesn't use getAndAdvanceChar because C++0x [lex.pptoken]p3: 1696 // Between the initial and final double quote characters of the raw string, 1697 // any transformations performed in phases 1 and 2 (trigraphs, 1698 // universal-character-names, and line splicing) are reverted. 1699 1700 if (!isLexingRawMode()) 1701 Diag(BufferPtr, diag::warn_cxx98_compat_raw_string_literal); 1702 1703 unsigned PrefixLen = 0; 1704 1705 while (PrefixLen != 16 && isRawStringDelimBody(CurPtr[PrefixLen])) 1706 ++PrefixLen; 1707 1708 // If the last character was not a '(', then we didn't lex a valid delimiter. 1709 if (CurPtr[PrefixLen] != '(') { 1710 if (!isLexingRawMode()) { 1711 const char *PrefixEnd = &CurPtr[PrefixLen]; 1712 if (PrefixLen == 16) { 1713 Diag(PrefixEnd, diag::err_raw_delim_too_long); 1714 } else { 1715 Diag(PrefixEnd, diag::err_invalid_char_raw_delim) 1716 << StringRef(PrefixEnd, 1); 1717 } 1718 } 1719 1720 // Search for the next '"' in hopes of salvaging the lexer. Unfortunately, 1721 // it's possible the '"' was intended to be part of the raw string, but 1722 // there's not much we can do about that. 1723 while (1) { 1724 char C = *CurPtr++; 1725 1726 if (C == '"') 1727 break; 1728 if (C == 0 && CurPtr-1 == BufferEnd) { 1729 --CurPtr; 1730 break; 1731 } 1732 } 1733 1734 FormTokenWithChars(Result, CurPtr, tok::unknown); 1735 return; 1736 } 1737 1738 // Save prefix and move CurPtr past it 1739 const char *Prefix = CurPtr; 1740 CurPtr += PrefixLen + 1; // skip over prefix and '(' 1741 1742 while (1) { 1743 char C = *CurPtr++; 1744 1745 if (C == ')') { 1746 // Check for prefix match and closing quote. 1747 if (strncmp(CurPtr, Prefix, PrefixLen) == 0 && CurPtr[PrefixLen] == '"') { 1748 CurPtr += PrefixLen + 1; // skip over prefix and '"' 1749 break; 1750 } 1751 } else if (C == 0 && CurPtr-1 == BufferEnd) { // End of file. 1752 if (!isLexingRawMode()) 1753 Diag(BufferPtr, diag::err_unterminated_raw_string) 1754 << StringRef(Prefix, PrefixLen); 1755 FormTokenWithChars(Result, CurPtr-1, tok::unknown); 1756 return; 1757 } 1758 } 1759 1760 // If we are in C++11, lex the optional ud-suffix. 1761 if (getLangOpts().CPlusPlus) 1762 CurPtr = LexUDSuffix(Result, CurPtr); 1763 1764 // Update the location of token as well as BufferPtr. 1765 const char *TokStart = BufferPtr; 1766 FormTokenWithChars(Result, CurPtr, Kind); 1767 Result.setLiteralData(TokStart); 1768} 1769 1770/// LexAngledStringLiteral - Lex the remainder of an angled string literal, 1771/// after having lexed the '<' character. This is used for #include filenames. 1772void Lexer::LexAngledStringLiteral(Token &Result, const char *CurPtr) { 1773 const char *NulCharacter = 0; // Does this string contain the \0 character? 1774 const char *AfterLessPos = CurPtr; 1775 char C = getAndAdvanceChar(CurPtr, Result); 1776 while (C != '>') { 1777 // Skip escaped characters. 1778 if (C == '\\') { 1779 // Skip the escaped character. 1780 getAndAdvanceChar(CurPtr, Result); 1781 } else if (C == '\n' || C == '\r' || // Newline. 1782 (C == 0 && (CurPtr-1 == BufferEnd || // End of file. 1783 isCodeCompletionPoint(CurPtr-1)))) { 1784 // If the filename is unterminated, then it must just be a lone < 1785 // character. Return this as such. 1786 FormTokenWithChars(Result, AfterLessPos, tok::less); 1787 return; 1788 } else if (C == 0) { 1789 NulCharacter = CurPtr-1; 1790 } 1791 C = getAndAdvanceChar(CurPtr, Result); 1792 } 1793 1794 // If a nul character existed in the string, warn about it. 1795 if (NulCharacter && !isLexingRawMode()) 1796 Diag(NulCharacter, diag::null_in_string); 1797 1798 // Update the location of token as well as BufferPtr. 1799 const char *TokStart = BufferPtr; 1800 FormTokenWithChars(Result, CurPtr, tok::angle_string_literal); 1801 Result.setLiteralData(TokStart); 1802} 1803 1804 1805/// LexCharConstant - Lex the remainder of a character constant, after having 1806/// lexed either ' or L' or u' or U'. 1807void Lexer::LexCharConstant(Token &Result, const char *CurPtr, 1808 tok::TokenKind Kind) { 1809 const char *NulCharacter = 0; // Does this character contain the \0 character? 1810 1811 if (!isLexingRawMode() && 1812 (Kind == tok::utf16_char_constant || Kind == tok::utf32_char_constant)) 1813 Diag(BufferPtr, getLangOpts().CPlusPlus 1814 ? diag::warn_cxx98_compat_unicode_literal 1815 : diag::warn_c99_compat_unicode_literal); 1816 1817 char C = getAndAdvanceChar(CurPtr, Result); 1818 if (C == '\'') { 1819 if (!isLexingRawMode() && !LangOpts.AsmPreprocessor) 1820 Diag(BufferPtr, diag::ext_empty_character); 1821 FormTokenWithChars(Result, CurPtr, tok::unknown); 1822 return; 1823 } 1824 1825 while (C != '\'') { 1826 // Skip escaped characters. 1827 if (C == '\\') 1828 C = getAndAdvanceChar(CurPtr, Result); 1829 1830 if (C == '\n' || C == '\r' || // Newline. 1831 (C == 0 && CurPtr-1 == BufferEnd)) { // End of file. 1832 if (!isLexingRawMode() && !LangOpts.AsmPreprocessor) 1833 Diag(BufferPtr, diag::ext_unterminated_char); 1834 FormTokenWithChars(Result, CurPtr-1, tok::unknown); 1835 return; 1836 } 1837 1838 if (C == 0) { 1839 if (isCodeCompletionPoint(CurPtr-1)) { 1840 PP->CodeCompleteNaturalLanguage(); 1841 FormTokenWithChars(Result, CurPtr-1, tok::unknown); 1842 return cutOffLexing(); 1843 } 1844 1845 NulCharacter = CurPtr-1; 1846 } 1847 C = getAndAdvanceChar(CurPtr, Result); 1848 } 1849 1850 // If we are in C++11, lex the optional ud-suffix. 1851 if (getLangOpts().CPlusPlus) 1852 CurPtr = LexUDSuffix(Result, CurPtr); 1853 1854 // If a nul character existed in the character, warn about it. 1855 if (NulCharacter && !isLexingRawMode()) 1856 Diag(NulCharacter, diag::null_in_char); 1857 1858 // Update the location of token as well as BufferPtr. 1859 const char *TokStart = BufferPtr; 1860 FormTokenWithChars(Result, CurPtr, Kind); 1861 Result.setLiteralData(TokStart); 1862} 1863 1864/// SkipWhitespace - Efficiently skip over a series of whitespace characters. 1865/// Update BufferPtr to point to the next non-whitespace character and return. 1866/// 1867/// This method forms a token and returns true if KeepWhitespaceMode is enabled. 1868/// 1869bool Lexer::SkipWhitespace(Token &Result, const char *CurPtr) { 1870 // Whitespace - Skip it, then return the token after the whitespace. 1871 bool SawNewline = isVerticalWhitespace(CurPtr[-1]); 1872 1873 unsigned char Char = *CurPtr; 1874 1875 // Skip consecutive spaces efficiently. 1876 while (1) { 1877 // Skip horizontal whitespace very aggressively. 1878 while (isHorizontalWhitespace(Char)) 1879 Char = *++CurPtr; 1880 1881 // Otherwise if we have something other than whitespace, we're done. 1882 if (!isVerticalWhitespace(Char)) 1883 break; 1884 1885 if (ParsingPreprocessorDirective) { 1886 // End of preprocessor directive line, let LexTokenInternal handle this. 1887 BufferPtr = CurPtr; 1888 return false; 1889 } 1890 1891 // OK, but handle newline. 1892 SawNewline = true; 1893 Char = *++CurPtr; 1894 } 1895 1896 // If the client wants us to return whitespace, return it now. 1897 if (isKeepWhitespaceMode()) { 1898 FormTokenWithChars(Result, CurPtr, tok::unknown); 1899 if (SawNewline) 1900 IsAtStartOfLine = true; 1901 // FIXME: The next token will not have LeadingSpace set. 1902 return true; 1903 } 1904 1905 // If this isn't immediately after a newline, there is leading space. 1906 char PrevChar = CurPtr[-1]; 1907 bool HasLeadingSpace = !isVerticalWhitespace(PrevChar); 1908 1909 Result.setFlagValue(Token::LeadingSpace, HasLeadingSpace); 1910 if (SawNewline) 1911 Result.setFlag(Token::StartOfLine); 1912 1913 BufferPtr = CurPtr; 1914 return false; 1915} 1916 1917/// We have just read the // characters from input. Skip until we find the 1918/// newline character thats terminate the comment. Then update BufferPtr and 1919/// return. 1920/// 1921/// If we're in KeepCommentMode or any CommentHandler has inserted 1922/// some tokens, this will store the first token and return true. 1923bool Lexer::SkipLineComment(Token &Result, const char *CurPtr) { 1924 // If Line comments aren't explicitly enabled for this language, emit an 1925 // extension warning. 1926 if (!LangOpts.LineComment && !isLexingRawMode()) { 1927 Diag(BufferPtr, diag::ext_line_comment); 1928 1929 // Mark them enabled so we only emit one warning for this translation 1930 // unit. 1931 LangOpts.LineComment = true; 1932 } 1933 1934 // Scan over the body of the comment. The common case, when scanning, is that 1935 // the comment contains normal ascii characters with nothing interesting in 1936 // them. As such, optimize for this case with the inner loop. 1937 char C; 1938 do { 1939 C = *CurPtr; 1940 // Skip over characters in the fast loop. 1941 while (C != 0 && // Potentially EOF. 1942 C != '\n' && C != '\r') // Newline or DOS-style newline. 1943 C = *++CurPtr; 1944 1945 const char *NextLine = CurPtr; 1946 if (C != 0) { 1947 // We found a newline, see if it's escaped. 1948 const char *EscapePtr = CurPtr-1; 1949 while (isHorizontalWhitespace(*EscapePtr)) // Skip whitespace. 1950 --EscapePtr; 1951 1952 if (*EscapePtr == '\\') // Escaped newline. 1953 CurPtr = EscapePtr; 1954 else if (EscapePtr[0] == '/' && EscapePtr[-1] == '?' && 1955 EscapePtr[-2] == '?') // Trigraph-escaped newline. 1956 CurPtr = EscapePtr-2; 1957 else 1958 break; // This is a newline, we're done. 1959 } 1960 1961 // Otherwise, this is a hard case. Fall back on getAndAdvanceChar to 1962 // properly decode the character. Read it in raw mode to avoid emitting 1963 // diagnostics about things like trigraphs. If we see an escaped newline, 1964 // we'll handle it below. 1965 const char *OldPtr = CurPtr; 1966 bool OldRawMode = isLexingRawMode(); 1967 LexingRawMode = true; 1968 C = getAndAdvanceChar(CurPtr, Result); 1969 LexingRawMode = OldRawMode; 1970 1971 // If we only read only one character, then no special handling is needed. 1972 // We're done and can skip forward to the newline. 1973 if (C != 0 && CurPtr == OldPtr+1) { 1974 CurPtr = NextLine; 1975 break; 1976 } 1977 1978 // If we read multiple characters, and one of those characters was a \r or 1979 // \n, then we had an escaped newline within the comment. Emit diagnostic 1980 // unless the next line is also a // comment. 1981 if (CurPtr != OldPtr+1 && C != '/' && CurPtr[0] != '/') { 1982 for (; OldPtr != CurPtr; ++OldPtr) 1983 if (OldPtr[0] == '\n' || OldPtr[0] == '\r') { 1984 // Okay, we found a // comment that ends in a newline, if the next 1985 // line is also a // comment, but has spaces, don't emit a diagnostic. 1986 if (isWhitespace(C)) { 1987 const char *ForwardPtr = CurPtr; 1988 while (isWhitespace(*ForwardPtr)) // Skip whitespace. 1989 ++ForwardPtr; 1990 if (ForwardPtr[0] == '/' && ForwardPtr[1] == '/') 1991 break; 1992 } 1993 1994 if (!isLexingRawMode()) 1995 Diag(OldPtr-1, diag::ext_multi_line_line_comment); 1996 break; 1997 } 1998 } 1999 2000 if (CurPtr == BufferEnd+1) { 2001 --CurPtr; 2002 break; 2003 } 2004 2005 if (C == '\0' && isCodeCompletionPoint(CurPtr-1)) { 2006 PP->CodeCompleteNaturalLanguage(); 2007 cutOffLexing(); 2008 return false; 2009 } 2010 2011 } while (C != '\n' && C != '\r'); 2012 2013 // Found but did not consume the newline. Notify comment handlers about the 2014 // comment unless we're in a #if 0 block. 2015 if (PP && !isLexingRawMode() && 2016 PP->HandleComment(Result, SourceRange(getSourceLocation(BufferPtr), 2017 getSourceLocation(CurPtr)))) { 2018 BufferPtr = CurPtr; 2019 return true; // A token has to be returned. 2020 } 2021 2022 // If we are returning comments as tokens, return this comment as a token. 2023 if (inKeepCommentMode()) 2024 return SaveLineComment(Result, CurPtr); 2025 2026 // If we are inside a preprocessor directive and we see the end of line, 2027 // return immediately, so that the lexer can return this as an EOD token. 2028 if (ParsingPreprocessorDirective || CurPtr == BufferEnd) { 2029 BufferPtr = CurPtr; 2030 return false; 2031 } 2032 2033 // Otherwise, eat the \n character. We don't care if this is a \n\r or 2034 // \r\n sequence. This is an efficiency hack (because we know the \n can't 2035 // contribute to another token), it isn't needed for correctness. Note that 2036 // this is ok even in KeepWhitespaceMode, because we would have returned the 2037 /// comment above in that mode. 2038 ++CurPtr; 2039 2040 // The next returned token is at the start of the line. 2041 Result.setFlag(Token::StartOfLine); 2042 // No leading whitespace seen so far. 2043 Result.clearFlag(Token::LeadingSpace); 2044 BufferPtr = CurPtr; 2045 return false; 2046} 2047 2048/// If in save-comment mode, package up this Line comment in an appropriate 2049/// way and return it. 2050bool Lexer::SaveLineComment(Token &Result, const char *CurPtr) { 2051 // If we're not in a preprocessor directive, just return the // comment 2052 // directly. 2053 FormTokenWithChars(Result, CurPtr, tok::comment); 2054 2055 if (!ParsingPreprocessorDirective || LexingRawMode) 2056 return true; 2057 2058 // If this Line-style comment is in a macro definition, transmogrify it into 2059 // a C-style block comment. 2060 bool Invalid = false; 2061 std::string Spelling = PP->getSpelling(Result, &Invalid); 2062 if (Invalid) 2063 return true; 2064 2065 assert(Spelling[0] == '/' && Spelling[1] == '/' && "Not line comment?"); 2066 Spelling[1] = '*'; // Change prefix to "/*". 2067 Spelling += "*/"; // add suffix. 2068 2069 Result.setKind(tok::comment); 2070 PP->CreateString(Spelling, Result, 2071 Result.getLocation(), Result.getLocation()); 2072 return true; 2073} 2074 2075/// isBlockCommentEndOfEscapedNewLine - Return true if the specified newline 2076/// character (either \\n or \\r) is part of an escaped newline sequence. Issue 2077/// a diagnostic if so. We know that the newline is inside of a block comment. 2078static bool isEndOfBlockCommentWithEscapedNewLine(const char *CurPtr, 2079 Lexer *L) { 2080 assert(CurPtr[0] == '\n' || CurPtr[0] == '\r'); 2081 2082 // Back up off the newline. 2083 --CurPtr; 2084 2085 // If this is a two-character newline sequence, skip the other character. 2086 if (CurPtr[0] == '\n' || CurPtr[0] == '\r') { 2087 // \n\n or \r\r -> not escaped newline. 2088 if (CurPtr[0] == CurPtr[1]) 2089 return false; 2090 // \n\r or \r\n -> skip the newline. 2091 --CurPtr; 2092 } 2093 2094 // If we have horizontal whitespace, skip over it. We allow whitespace 2095 // between the slash and newline. 2096 bool HasSpace = false; 2097 while (isHorizontalWhitespace(*CurPtr) || *CurPtr == 0) { 2098 --CurPtr; 2099 HasSpace = true; 2100 } 2101 2102 // If we have a slash, we know this is an escaped newline. 2103 if (*CurPtr == '\\') { 2104 if (CurPtr[-1] != '*') return false; 2105 } else { 2106 // It isn't a slash, is it the ?? / trigraph? 2107 if (CurPtr[0] != '/' || CurPtr[-1] != '?' || CurPtr[-2] != '?' || 2108 CurPtr[-3] != '*') 2109 return false; 2110 2111 // This is the trigraph ending the comment. Emit a stern warning! 2112 CurPtr -= 2; 2113 2114 // If no trigraphs are enabled, warn that we ignored this trigraph and 2115 // ignore this * character. 2116 if (!L->getLangOpts().Trigraphs) { 2117 if (!L->isLexingRawMode()) 2118 L->Diag(CurPtr, diag::trigraph_ignored_block_comment); 2119 return false; 2120 } 2121 if (!L->isLexingRawMode()) 2122 L->Diag(CurPtr, diag::trigraph_ends_block_comment); 2123 } 2124 2125 // Warn about having an escaped newline between the */ characters. 2126 if (!L->isLexingRawMode()) 2127 L->Diag(CurPtr, diag::escaped_newline_block_comment_end); 2128 2129 // If there was space between the backslash and newline, warn about it. 2130 if (HasSpace && !L->isLexingRawMode()) 2131 L->Diag(CurPtr, diag::backslash_newline_space); 2132 2133 return true; 2134} 2135 2136#ifdef __SSE2__ 2137#include <emmintrin.h> 2138#elif __ALTIVEC__ 2139#include <altivec.h> 2140#undef bool 2141#endif 2142 2143/// We have just read from input the / and * characters that started a comment. 2144/// Read until we find the * and / characters that terminate the comment. 2145/// Note that we don't bother decoding trigraphs or escaped newlines in block 2146/// comments, because they cannot cause the comment to end. The only thing 2147/// that can happen is the comment could end with an escaped newline between 2148/// the terminating * and /. 2149/// 2150/// If we're in KeepCommentMode or any CommentHandler has inserted 2151/// some tokens, this will store the first token and return true. 2152bool Lexer::SkipBlockComment(Token &Result, const char *CurPtr) { 2153 // Scan one character past where we should, looking for a '/' character. Once 2154 // we find it, check to see if it was preceded by a *. This common 2155 // optimization helps people who like to put a lot of * characters in their 2156 // comments. 2157 2158 // The first character we get with newlines and trigraphs skipped to handle 2159 // the degenerate /*/ case below correctly if the * has an escaped newline 2160 // after it. 2161 unsigned CharSize; 2162 unsigned char C = getCharAndSize(CurPtr, CharSize); 2163 CurPtr += CharSize; 2164 if (C == 0 && CurPtr == BufferEnd+1) { 2165 if (!isLexingRawMode()) 2166 Diag(BufferPtr, diag::err_unterminated_block_comment); 2167 --CurPtr; 2168 2169 // KeepWhitespaceMode should return this broken comment as a token. Since 2170 // it isn't a well formed comment, just return it as an 'unknown' token. 2171 if (isKeepWhitespaceMode()) { 2172 FormTokenWithChars(Result, CurPtr, tok::unknown); 2173 return true; 2174 } 2175 2176 BufferPtr = CurPtr; 2177 return false; 2178 } 2179 2180 // Check to see if the first character after the '/*' is another /. If so, 2181 // then this slash does not end the block comment, it is part of it. 2182 if (C == '/') 2183 C = *CurPtr++; 2184 2185 while (1) { 2186 // Skip over all non-interesting characters until we find end of buffer or a 2187 // (probably ending) '/' character. 2188 if (CurPtr + 24 < BufferEnd && 2189 // If there is a code-completion point avoid the fast scan because it 2190 // doesn't check for '\0'. 2191 !(PP && PP->getCodeCompletionFileLoc() == FileLoc)) { 2192 // While not aligned to a 16-byte boundary. 2193 while (C != '/' && ((intptr_t)CurPtr & 0x0F) != 0) 2194 C = *CurPtr++; 2195 2196 if (C == '/') goto FoundSlash; 2197 2198#ifdef __SSE2__ 2199 __m128i Slashes = _mm_set1_epi8('/'); 2200 while (CurPtr+16 <= BufferEnd) { 2201 int cmp = _mm_movemask_epi8(_mm_cmpeq_epi8(*(const __m128i*)CurPtr, 2202 Slashes)); 2203 if (cmp != 0) { 2204 // Adjust the pointer to point directly after the first slash. It's 2205 // not necessary to set C here, it will be overwritten at the end of 2206 // the outer loop. 2207 CurPtr += llvm::CountTrailingZeros_32(cmp) + 1; 2208 goto FoundSlash; 2209 } 2210 CurPtr += 16; 2211 } 2212#elif __ALTIVEC__ 2213 __vector unsigned char Slashes = { 2214 '/', '/', '/', '/', '/', '/', '/', '/', 2215 '/', '/', '/', '/', '/', '/', '/', '/' 2216 }; 2217 while (CurPtr+16 <= BufferEnd && 2218 !vec_any_eq(*(vector unsigned char*)CurPtr, Slashes)) 2219 CurPtr += 16; 2220#else 2221 // Scan for '/' quickly. Many block comments are very large. 2222 while (CurPtr[0] != '/' && 2223 CurPtr[1] != '/' && 2224 CurPtr[2] != '/' && 2225 CurPtr[3] != '/' && 2226 CurPtr+4 < BufferEnd) { 2227 CurPtr += 4; 2228 } 2229#endif 2230 2231 // It has to be one of the bytes scanned, increment to it and read one. 2232 C = *CurPtr++; 2233 } 2234 2235 // Loop to scan the remainder. 2236 while (C != '/' && C != '\0') 2237 C = *CurPtr++; 2238 2239 if (C == '/') { 2240 FoundSlash: 2241 if (CurPtr[-2] == '*') // We found the final */. We're done! 2242 break; 2243 2244 if ((CurPtr[-2] == '\n' || CurPtr[-2] == '\r')) { 2245 if (isEndOfBlockCommentWithEscapedNewLine(CurPtr-2, this)) { 2246 // We found the final */, though it had an escaped newline between the 2247 // * and /. We're done! 2248 break; 2249 } 2250 } 2251 if (CurPtr[0] == '*' && CurPtr[1] != '/') { 2252 // If this is a /* inside of the comment, emit a warning. Don't do this 2253 // if this is a /*/, which will end the comment. This misses cases with 2254 // embedded escaped newlines, but oh well. 2255 if (!isLexingRawMode()) 2256 Diag(CurPtr-1, diag::warn_nested_block_comment); 2257 } 2258 } else if (C == 0 && CurPtr == BufferEnd+1) { 2259 if (!isLexingRawMode()) 2260 Diag(BufferPtr, diag::err_unterminated_block_comment); 2261 // Note: the user probably forgot a */. We could continue immediately 2262 // after the /*, but this would involve lexing a lot of what really is the 2263 // comment, which surely would confuse the parser. 2264 --CurPtr; 2265 2266 // KeepWhitespaceMode should return this broken comment as a token. Since 2267 // it isn't a well formed comment, just return it as an 'unknown' token. 2268 if (isKeepWhitespaceMode()) { 2269 FormTokenWithChars(Result, CurPtr, tok::unknown); 2270 return true; 2271 } 2272 2273 BufferPtr = CurPtr; 2274 return false; 2275 } else if (C == '\0' && isCodeCompletionPoint(CurPtr-1)) { 2276 PP->CodeCompleteNaturalLanguage(); 2277 cutOffLexing(); 2278 return false; 2279 } 2280 2281 C = *CurPtr++; 2282 } 2283 2284 // Notify comment handlers about the comment unless we're in a #if 0 block. 2285 if (PP && !isLexingRawMode() && 2286 PP->HandleComment(Result, SourceRange(getSourceLocation(BufferPtr), 2287 getSourceLocation(CurPtr)))) { 2288 BufferPtr = CurPtr; 2289 return true; // A token has to be returned. 2290 } 2291 2292 // If we are returning comments as tokens, return this comment as a token. 2293 if (inKeepCommentMode()) { 2294 FormTokenWithChars(Result, CurPtr, tok::comment); 2295 return true; 2296 } 2297 2298 // It is common for the tokens immediately after a /**/ comment to be 2299 // whitespace. Instead of going through the big switch, handle it 2300 // efficiently now. This is safe even in KeepWhitespaceMode because we would 2301 // have already returned above with the comment as a token. 2302 if (isHorizontalWhitespace(*CurPtr)) { 2303 SkipWhitespace(Result, CurPtr+1); 2304 return false; 2305 } 2306 2307 // Otherwise, just return so that the next character will be lexed as a token. 2308 BufferPtr = CurPtr; 2309 Result.setFlag(Token::LeadingSpace); 2310 return false; 2311} 2312 2313//===----------------------------------------------------------------------===// 2314// Primary Lexing Entry Points 2315//===----------------------------------------------------------------------===// 2316 2317/// ReadToEndOfLine - Read the rest of the current preprocessor line as an 2318/// uninterpreted string. This switches the lexer out of directive mode. 2319void Lexer::ReadToEndOfLine(SmallVectorImpl<char> *Result) { 2320 assert(ParsingPreprocessorDirective && ParsingFilename == false && 2321 "Must be in a preprocessing directive!"); 2322 Token Tmp; 2323 2324 // CurPtr - Cache BufferPtr in an automatic variable. 2325 const char *CurPtr = BufferPtr; 2326 while (1) { 2327 char Char = getAndAdvanceChar(CurPtr, Tmp); 2328 switch (Char) { 2329 default: 2330 if (Result) 2331 Result->push_back(Char); 2332 break; 2333 case 0: // Null. 2334 // Found end of file? 2335 if (CurPtr-1 != BufferEnd) { 2336 if (isCodeCompletionPoint(CurPtr-1)) { 2337 PP->CodeCompleteNaturalLanguage(); 2338 cutOffLexing(); 2339 return; 2340 } 2341 2342 // Nope, normal character, continue. 2343 if (Result) 2344 Result->push_back(Char); 2345 break; 2346 } 2347 // FALL THROUGH. 2348 case '\r': 2349 case '\n': 2350 // Okay, we found the end of the line. First, back up past the \0, \r, \n. 2351 assert(CurPtr[-1] == Char && "Trigraphs for newline?"); 2352 BufferPtr = CurPtr-1; 2353 2354 // Next, lex the character, which should handle the EOD transition. 2355 Lex(Tmp); 2356 if (Tmp.is(tok::code_completion)) { 2357 if (PP) 2358 PP->CodeCompleteNaturalLanguage(); 2359 Lex(Tmp); 2360 } 2361 assert(Tmp.is(tok::eod) && "Unexpected token!"); 2362 2363 // Finally, we're done; 2364 return; 2365 } 2366 } 2367} 2368 2369/// LexEndOfFile - CurPtr points to the end of this file. Handle this 2370/// condition, reporting diagnostics and handling other edge cases as required. 2371/// This returns true if Result contains a token, false if PP.Lex should be 2372/// called again. 2373bool Lexer::LexEndOfFile(Token &Result, const char *CurPtr) { 2374 // If we hit the end of the file while parsing a preprocessor directive, 2375 // end the preprocessor directive first. The next token returned will 2376 // then be the end of file. 2377 if (ParsingPreprocessorDirective) { 2378 // Done parsing the "line". 2379 ParsingPreprocessorDirective = false; 2380 // Update the location of token as well as BufferPtr. 2381 FormTokenWithChars(Result, CurPtr, tok::eod); 2382 2383 // Restore comment saving mode, in case it was disabled for directive. 2384 resetExtendedTokenMode(); 2385 return true; // Have a token. 2386 } 2387 2388 // If we are in raw mode, return this event as an EOF token. Let the caller 2389 // that put us in raw mode handle the event. 2390 if (isLexingRawMode()) { 2391 Result.startToken(); 2392 BufferPtr = BufferEnd; 2393 FormTokenWithChars(Result, BufferEnd, tok::eof); 2394 return true; 2395 } 2396 2397 // Issue diagnostics for unterminated #if and missing newline. 2398 2399 // If we are in a #if directive, emit an error. 2400 while (!ConditionalStack.empty()) { 2401 if (PP->getCodeCompletionFileLoc() != FileLoc) 2402 PP->Diag(ConditionalStack.back().IfLoc, 2403 diag::err_pp_unterminated_conditional); 2404 ConditionalStack.pop_back(); 2405 } 2406 2407 // C99 5.1.1.2p2: If the file is non-empty and didn't end in a newline, issue 2408 // a pedwarn. 2409 if (CurPtr != BufferStart && (CurPtr[-1] != '\n' && CurPtr[-1] != '\r')) 2410 Diag(BufferEnd, LangOpts.CPlusPlus11 ? // C++11 [lex.phases] 2.2 p2 2411 diag::warn_cxx98_compat_no_newline_eof : diag::ext_no_newline_eof) 2412 << FixItHint::CreateInsertion(getSourceLocation(BufferEnd), "\n"); 2413 2414 BufferPtr = CurPtr; 2415 2416 // Finally, let the preprocessor handle this. 2417 return PP->HandleEndOfFile(Result, isPragmaLexer()); 2418} 2419 2420/// isNextPPTokenLParen - Return 1 if the next unexpanded token lexed from 2421/// the specified lexer will return a tok::l_paren token, 0 if it is something 2422/// else and 2 if there are no more tokens in the buffer controlled by the 2423/// lexer. 2424unsigned Lexer::isNextPPTokenLParen() { 2425 assert(!LexingRawMode && "How can we expand a macro from a skipping buffer?"); 2426 2427 // Switch to 'skipping' mode. This will ensure that we can lex a token 2428 // without emitting diagnostics, disables macro expansion, and will cause EOF 2429 // to return an EOF token instead of popping the include stack. 2430 LexingRawMode = true; 2431 2432 // Save state that can be changed while lexing so that we can restore it. 2433 const char *TmpBufferPtr = BufferPtr; 2434 bool inPPDirectiveMode = ParsingPreprocessorDirective; 2435 2436 Token Tok; 2437 Tok.startToken(); 2438 LexTokenInternal(Tok); 2439 2440 // Restore state that may have changed. 2441 BufferPtr = TmpBufferPtr; 2442 ParsingPreprocessorDirective = inPPDirectiveMode; 2443 2444 // Restore the lexer back to non-skipping mode. 2445 LexingRawMode = false; 2446 2447 if (Tok.is(tok::eof)) 2448 return 2; 2449 return Tok.is(tok::l_paren); 2450} 2451 2452/// \brief Find the end of a version control conflict marker. 2453static const char *FindConflictEnd(const char *CurPtr, const char *BufferEnd, 2454 ConflictMarkerKind CMK) { 2455 const char *Terminator = CMK == CMK_Perforce ? "<<<<\n" : ">>>>>>>"; 2456 size_t TermLen = CMK == CMK_Perforce ? 5 : 7; 2457 StringRef RestOfBuffer(CurPtr+TermLen, BufferEnd-CurPtr-TermLen); 2458 size_t Pos = RestOfBuffer.find(Terminator); 2459 while (Pos != StringRef::npos) { 2460 // Must occur at start of line. 2461 if (RestOfBuffer[Pos-1] != '\r' && 2462 RestOfBuffer[Pos-1] != '\n') { 2463 RestOfBuffer = RestOfBuffer.substr(Pos+TermLen); 2464 Pos = RestOfBuffer.find(Terminator); 2465 continue; 2466 } 2467 return RestOfBuffer.data()+Pos; 2468 } 2469 return 0; 2470} 2471 2472/// IsStartOfConflictMarker - If the specified pointer is the start of a version 2473/// control conflict marker like '<<<<<<<', recognize it as such, emit an error 2474/// and recover nicely. This returns true if it is a conflict marker and false 2475/// if not. 2476bool Lexer::IsStartOfConflictMarker(const char *CurPtr) { 2477 // Only a conflict marker if it starts at the beginning of a line. 2478 if (CurPtr != BufferStart && 2479 CurPtr[-1] != '\n' && CurPtr[-1] != '\r') 2480 return false; 2481 2482 // Check to see if we have <<<<<<< or >>>>. 2483 if ((BufferEnd-CurPtr < 8 || StringRef(CurPtr, 7) != "<<<<<<<") && 2484 (BufferEnd-CurPtr < 6 || StringRef(CurPtr, 5) != ">>>> ")) 2485 return false; 2486 2487 // If we have a situation where we don't care about conflict markers, ignore 2488 // it. 2489 if (CurrentConflictMarkerState || isLexingRawMode()) 2490 return false; 2491 2492 ConflictMarkerKind Kind = *CurPtr == '<' ? CMK_Normal : CMK_Perforce; 2493 2494 // Check to see if there is an ending marker somewhere in the buffer at the 2495 // start of a line to terminate this conflict marker. 2496 if (FindConflictEnd(CurPtr, BufferEnd, Kind)) { 2497 // We found a match. We are really in a conflict marker. 2498 // Diagnose this, and ignore to the end of line. 2499 Diag(CurPtr, diag::err_conflict_marker); 2500 CurrentConflictMarkerState = Kind; 2501 2502 // Skip ahead to the end of line. We know this exists because the 2503 // end-of-conflict marker starts with \r or \n. 2504 while (*CurPtr != '\r' && *CurPtr != '\n') { 2505 assert(CurPtr != BufferEnd && "Didn't find end of line"); 2506 ++CurPtr; 2507 } 2508 BufferPtr = CurPtr; 2509 return true; 2510 } 2511 2512 // No end of conflict marker found. 2513 return false; 2514} 2515 2516 2517/// HandleEndOfConflictMarker - If this is a '====' or '||||' or '>>>>', or if 2518/// it is '<<<<' and the conflict marker started with a '>>>>' marker, then it 2519/// is the end of a conflict marker. Handle it by ignoring up until the end of 2520/// the line. This returns true if it is a conflict marker and false if not. 2521bool Lexer::HandleEndOfConflictMarker(const char *CurPtr) { 2522 // Only a conflict marker if it starts at the beginning of a line. 2523 if (CurPtr != BufferStart && 2524 CurPtr[-1] != '\n' && CurPtr[-1] != '\r') 2525 return false; 2526 2527 // If we have a situation where we don't care about conflict markers, ignore 2528 // it. 2529 if (!CurrentConflictMarkerState || isLexingRawMode()) 2530 return false; 2531 2532 // Check to see if we have the marker (4 characters in a row). 2533 for (unsigned i = 1; i != 4; ++i) 2534 if (CurPtr[i] != CurPtr[0]) 2535 return false; 2536 2537 // If we do have it, search for the end of the conflict marker. This could 2538 // fail if it got skipped with a '#if 0' or something. Note that CurPtr might 2539 // be the end of conflict marker. 2540 if (const char *End = FindConflictEnd(CurPtr, BufferEnd, 2541 CurrentConflictMarkerState)) { 2542 CurPtr = End; 2543 2544 // Skip ahead to the end of line. 2545 while (CurPtr != BufferEnd && *CurPtr != '\r' && *CurPtr != '\n') 2546 ++CurPtr; 2547 2548 BufferPtr = CurPtr; 2549 2550 // No longer in the conflict marker. 2551 CurrentConflictMarkerState = CMK_None; 2552 return true; 2553 } 2554 2555 return false; 2556} 2557 2558bool Lexer::isCodeCompletionPoint(const char *CurPtr) const { 2559 if (PP && PP->isCodeCompletionEnabled()) { 2560 SourceLocation Loc = FileLoc.getLocWithOffset(CurPtr-BufferStart); 2561 return Loc == PP->getCodeCompletionLoc(); 2562 } 2563 2564 return false; 2565} 2566 2567uint32_t Lexer::tryReadUCN(const char *&StartPtr, const char *SlashLoc, 2568 Token *Result) { 2569 unsigned CharSize; 2570 char Kind = getCharAndSize(StartPtr, CharSize); 2571 2572 unsigned NumHexDigits; 2573 if (Kind == 'u') 2574 NumHexDigits = 4; 2575 else if (Kind == 'U') 2576 NumHexDigits = 8; 2577 else 2578 return 0; 2579 2580 if (!LangOpts.CPlusPlus && !LangOpts.C99) { 2581 if (Result && !isLexingRawMode()) 2582 Diag(SlashLoc, diag::warn_ucn_not_valid_in_c89); 2583 return 0; 2584 } 2585 2586 const char *CurPtr = StartPtr + CharSize; 2587 const char *KindLoc = &CurPtr[-1]; 2588 2589 uint32_t CodePoint = 0; 2590 for (unsigned i = 0; i < NumHexDigits; ++i) { 2591 char C = getCharAndSize(CurPtr, CharSize); 2592 2593 unsigned Value = llvm::hexDigitValue(C); 2594 if (Value == -1U) { 2595 if (Result && !isLexingRawMode()) { 2596 if (i == 0) { 2597 Diag(BufferPtr, diag::warn_ucn_escape_no_digits) 2598 << StringRef(KindLoc, 1); 2599 } else { 2600 Diag(BufferPtr, diag::warn_ucn_escape_incomplete); 2601 2602 // If the user wrote \U1234, suggest a fixit to \u. 2603 if (i == 4 && NumHexDigits == 8) { 2604 CharSourceRange URange = makeCharRange(*this, KindLoc, KindLoc + 1); 2605 Diag(KindLoc, diag::note_ucn_four_not_eight) 2606 << FixItHint::CreateReplacement(URange, "u"); 2607 } 2608 } 2609 } 2610 2611 return 0; 2612 } 2613 2614 CodePoint <<= 4; 2615 CodePoint += Value; 2616 2617 CurPtr += CharSize; 2618 } 2619 2620 if (Result) { 2621 Result->setFlag(Token::HasUCN); 2622 if (CurPtr - StartPtr == (ptrdiff_t)NumHexDigits + 2) 2623 StartPtr = CurPtr; 2624 else 2625 while (StartPtr != CurPtr) 2626 (void)getAndAdvanceChar(StartPtr, *Result); 2627 } else { 2628 StartPtr = CurPtr; 2629 } 2630 2631 // C99 6.4.3p2: A universal character name shall not specify a character whose 2632 // short identifier is less than 00A0 other than 0024 ($), 0040 (@), or 2633 // 0060 (`), nor one in the range D800 through DFFF inclusive.) 2634 // C++11 [lex.charset]p2: If the hexadecimal value for a 2635 // universal-character-name corresponds to a surrogate code point (in the 2636 // range 0xD800-0xDFFF, inclusive), the program is ill-formed. Additionally, 2637 // if the hexadecimal value for a universal-character-name outside the 2638 // c-char-sequence, s-char-sequence, or r-char-sequence of a character or 2639 // string literal corresponds to a control character (in either of the 2640 // ranges 0x00-0x1F or 0x7F-0x9F, both inclusive) or to a character in the 2641 // basic source character set, the program is ill-formed. 2642 if (CodePoint < 0xA0) { 2643 if (CodePoint == 0x24 || CodePoint == 0x40 || CodePoint == 0x60) 2644 return CodePoint; 2645 2646 // We don't use isLexingRawMode() here because we need to warn about bad 2647 // UCNs even when skipping preprocessing tokens in a #if block. 2648 if (Result && PP) { 2649 if (CodePoint < 0x20 || CodePoint >= 0x7F) 2650 Diag(BufferPtr, diag::err_ucn_control_character); 2651 else { 2652 char C = static_cast<char>(CodePoint); 2653 Diag(BufferPtr, diag::err_ucn_escape_basic_scs) << StringRef(&C, 1); 2654 } 2655 } 2656 2657 return 0; 2658 2659 } else if (CodePoint >= 0xD800 && CodePoint <= 0xDFFF) { 2660 // C++03 allows UCNs representing surrogate characters. C99 and C++11 don't. 2661 // We don't use isLexingRawMode() here because we need to diagnose bad 2662 // UCNs even when skipping preprocessing tokens in a #if block. 2663 if (Result && PP) { 2664 if (LangOpts.CPlusPlus && !LangOpts.CPlusPlus11) 2665 Diag(BufferPtr, diag::warn_ucn_escape_surrogate); 2666 else 2667 Diag(BufferPtr, diag::err_ucn_escape_invalid); 2668 } 2669 return 0; 2670 } 2671 2672 return CodePoint; 2673} 2674 2675void Lexer::LexUnicode(Token &Result, uint32_t C, const char *CurPtr) { 2676 if (!isLexingRawMode() && !PP->isPreprocessedOutput() && 2677 isCharInSet(C, UnicodeWhitespaceChars)) { 2678 Diag(BufferPtr, diag::ext_unicode_whitespace) 2679 << makeCharRange(*this, BufferPtr, CurPtr); 2680 2681 Result.setFlag(Token::LeadingSpace); 2682 if (SkipWhitespace(Result, CurPtr)) 2683 return; // KeepWhitespaceMode 2684 2685 return LexTokenInternal(Result); 2686 } 2687 2688 if (isAllowedIDChar(C, LangOpts) && isAllowedInitiallyIDChar(C, LangOpts)) { 2689 if (!isLexingRawMode() && !ParsingPreprocessorDirective && 2690 !PP->isPreprocessedOutput()) { 2691 maybeDiagnoseIDCharCompat(PP->getDiagnostics(), C, 2692 makeCharRange(*this, BufferPtr, CurPtr), 2693 /*IsFirst=*/true); 2694 } 2695 2696 MIOpt.ReadToken(); 2697 return LexIdentifier(Result, CurPtr); 2698 } 2699 2700 if (!isLexingRawMode() && !ParsingPreprocessorDirective && 2701 !PP->isPreprocessedOutput() && 2702 !isASCII(*BufferPtr) && !isAllowedIDChar(C, LangOpts)) { 2703 // Non-ASCII characters tend to creep into source code unintentionally. 2704 // Instead of letting the parser complain about the unknown token, 2705 // just drop the character. 2706 // Note that we can /only/ do this when the non-ASCII character is actually 2707 // spelled as Unicode, not written as a UCN. The standard requires that 2708 // we not throw away any possible preprocessor tokens, but there's a 2709 // loophole in the mapping of Unicode characters to basic character set 2710 // characters that allows us to map these particular characters to, say, 2711 // whitespace. 2712 Diag(BufferPtr, diag::err_non_ascii) 2713 << FixItHint::CreateRemoval(makeCharRange(*this, BufferPtr, CurPtr)); 2714 2715 BufferPtr = CurPtr; 2716 return LexTokenInternal(Result); 2717 } 2718 2719 // Otherwise, we have an explicit UCN or a character that's unlikely to show 2720 // up by accident. 2721 MIOpt.ReadToken(); 2722 FormTokenWithChars(Result, CurPtr, tok::unknown); 2723} 2724 2725 2726/// LexTokenInternal - This implements a simple C family lexer. It is an 2727/// extremely performance critical piece of code. This assumes that the buffer 2728/// has a null character at the end of the file. This returns a preprocessing 2729/// token, not a normal token, as such, it is an internal interface. It assumes 2730/// that the Flags of result have been cleared before calling this. 2731void Lexer::LexTokenInternal(Token &Result) { 2732LexNextToken: 2733 // New token, can't need cleaning yet. 2734 Result.clearFlag(Token::NeedsCleaning); 2735 Result.setIdentifierInfo(0); 2736 2737 // CurPtr - Cache BufferPtr in an automatic variable. 2738 const char *CurPtr = BufferPtr; 2739 2740 // Small amounts of horizontal whitespace is very common between tokens. 2741 if ((*CurPtr == ' ') || (*CurPtr == '\t')) { 2742 ++CurPtr; 2743 while ((*CurPtr == ' ') || (*CurPtr == '\t')) 2744 ++CurPtr; 2745 2746 // If we are keeping whitespace and other tokens, just return what we just 2747 // skipped. The next lexer invocation will return the token after the 2748 // whitespace. 2749 if (isKeepWhitespaceMode()) { 2750 FormTokenWithChars(Result, CurPtr, tok::unknown); 2751 // FIXME: The next token will not have LeadingSpace set. 2752 return; 2753 } 2754 2755 BufferPtr = CurPtr; 2756 Result.setFlag(Token::LeadingSpace); 2757 } 2758 2759 unsigned SizeTmp, SizeTmp2; // Temporaries for use in cases below. 2760 2761 // Read a character, advancing over it. 2762 char Char = getAndAdvanceChar(CurPtr, Result); 2763 tok::TokenKind Kind; 2764 2765 switch (Char) { 2766 case 0: // Null. 2767 // Found end of file? 2768 if (CurPtr-1 == BufferEnd) { 2769 // Read the PP instance variable into an automatic variable, because 2770 // LexEndOfFile will often delete 'this'. 2771 Preprocessor *PPCache = PP; 2772 if (LexEndOfFile(Result, CurPtr-1)) // Retreat back into the file. 2773 return; // Got a token to return. 2774 assert(PPCache && "Raw buffer::LexEndOfFile should return a token"); 2775 return PPCache->Lex(Result); 2776 } 2777 2778 // Check if we are performing code completion. 2779 if (isCodeCompletionPoint(CurPtr-1)) { 2780 // Return the code-completion token. 2781 Result.startToken(); 2782 FormTokenWithChars(Result, CurPtr, tok::code_completion); 2783 return; 2784 } 2785 2786 if (!isLexingRawMode()) 2787 Diag(CurPtr-1, diag::null_in_file); 2788 Result.setFlag(Token::LeadingSpace); 2789 if (SkipWhitespace(Result, CurPtr)) 2790 return; // KeepWhitespaceMode 2791 2792 goto LexNextToken; // GCC isn't tail call eliminating. 2793 2794 case 26: // DOS & CP/M EOF: "^Z". 2795 // If we're in Microsoft extensions mode, treat this as end of file. 2796 if (LangOpts.MicrosoftExt) { 2797 // Read the PP instance variable into an automatic variable, because 2798 // LexEndOfFile will often delete 'this'. 2799 Preprocessor *PPCache = PP; 2800 if (LexEndOfFile(Result, CurPtr-1)) // Retreat back into the file. 2801 return; // Got a token to return. 2802 assert(PPCache && "Raw buffer::LexEndOfFile should return a token"); 2803 return PPCache->Lex(Result); 2804 } 2805 // If Microsoft extensions are disabled, this is just random garbage. 2806 Kind = tok::unknown; 2807 break; 2808 2809 case '\n': 2810 case '\r': 2811 // If we are inside a preprocessor directive and we see the end of line, 2812 // we know we are done with the directive, so return an EOD token. 2813 if (ParsingPreprocessorDirective) { 2814 // Done parsing the "line". 2815 ParsingPreprocessorDirective = false; 2816 2817 // Restore comment saving mode, in case it was disabled for directive. 2818 if (PP) 2819 resetExtendedTokenMode(); 2820 2821 // Since we consumed a newline, we are back at the start of a line. 2822 IsAtStartOfLine = true; 2823 2824 Kind = tok::eod; 2825 break; 2826 } 2827 2828 // No leading whitespace seen so far. 2829 Result.clearFlag(Token::LeadingSpace); 2830 2831 if (SkipWhitespace(Result, CurPtr)) 2832 return; // KeepWhitespaceMode 2833 goto LexNextToken; // GCC isn't tail call eliminating. 2834 case ' ': 2835 case '\t': 2836 case '\f': 2837 case '\v': 2838 SkipHorizontalWhitespace: 2839 Result.setFlag(Token::LeadingSpace); 2840 if (SkipWhitespace(Result, CurPtr)) 2841 return; // KeepWhitespaceMode 2842 2843 SkipIgnoredUnits: 2844 CurPtr = BufferPtr; 2845 2846 // If the next token is obviously a // or /* */ comment, skip it efficiently 2847 // too (without going through the big switch stmt). 2848 if (CurPtr[0] == '/' && CurPtr[1] == '/' && !inKeepCommentMode() && 2849 LangOpts.LineComment && !LangOpts.TraditionalCPP) { 2850 if (SkipLineComment(Result, CurPtr+2)) 2851 return; // There is a token to return. 2852 goto SkipIgnoredUnits; 2853 } else if (CurPtr[0] == '/' && CurPtr[1] == '*' && !inKeepCommentMode()) { 2854 if (SkipBlockComment(Result, CurPtr+2)) 2855 return; // There is a token to return. 2856 goto SkipIgnoredUnits; 2857 } else if (isHorizontalWhitespace(*CurPtr)) { 2858 goto SkipHorizontalWhitespace; 2859 } 2860 goto LexNextToken; // GCC isn't tail call eliminating. 2861 2862 // C99 6.4.4.1: Integer Constants. 2863 // C99 6.4.4.2: Floating Constants. 2864 case '0': case '1': case '2': case '3': case '4': 2865 case '5': case '6': case '7': case '8': case '9': 2866 // Notify MIOpt that we read a non-whitespace/non-comment token. 2867 MIOpt.ReadToken(); 2868 return LexNumericConstant(Result, CurPtr); 2869 2870 case 'u': // Identifier (uber) or C11/C++11 UTF-8 or UTF-16 string literal 2871 // Notify MIOpt that we read a non-whitespace/non-comment token. 2872 MIOpt.ReadToken(); 2873 2874 if (LangOpts.CPlusPlus11 || LangOpts.C11) { 2875 Char = getCharAndSize(CurPtr, SizeTmp); 2876 2877 // UTF-16 string literal 2878 if (Char == '"') 2879 return LexStringLiteral(Result, ConsumeChar(CurPtr, SizeTmp, Result), 2880 tok::utf16_string_literal); 2881 2882 // UTF-16 character constant 2883 if (Char == '\'') 2884 return LexCharConstant(Result, ConsumeChar(CurPtr, SizeTmp, Result), 2885 tok::utf16_char_constant); 2886 2887 // UTF-16 raw string literal 2888 if (Char == 'R' && LangOpts.CPlusPlus11 && 2889 getCharAndSize(CurPtr + SizeTmp, SizeTmp2) == '"') 2890 return LexRawStringLiteral(Result, 2891 ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 2892 SizeTmp2, Result), 2893 tok::utf16_string_literal); 2894 2895 if (Char == '8') { 2896 char Char2 = getCharAndSize(CurPtr + SizeTmp, SizeTmp2); 2897 2898 // UTF-8 string literal 2899 if (Char2 == '"') 2900 return LexStringLiteral(Result, 2901 ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 2902 SizeTmp2, Result), 2903 tok::utf8_string_literal); 2904 2905 if (Char2 == 'R' && LangOpts.CPlusPlus11) { 2906 unsigned SizeTmp3; 2907 char Char3 = getCharAndSize(CurPtr + SizeTmp + SizeTmp2, SizeTmp3); 2908 // UTF-8 raw string literal 2909 if (Char3 == '"') { 2910 return LexRawStringLiteral(Result, 2911 ConsumeChar(ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 2912 SizeTmp2, Result), 2913 SizeTmp3, Result), 2914 tok::utf8_string_literal); 2915 } 2916 } 2917 } 2918 } 2919 2920 // treat u like the start of an identifier. 2921 return LexIdentifier(Result, CurPtr); 2922 2923 case 'U': // Identifier (Uber) or C11/C++11 UTF-32 string literal 2924 // Notify MIOpt that we read a non-whitespace/non-comment token. 2925 MIOpt.ReadToken(); 2926 2927 if (LangOpts.CPlusPlus11 || LangOpts.C11) { 2928 Char = getCharAndSize(CurPtr, SizeTmp); 2929 2930 // UTF-32 string literal 2931 if (Char == '"') 2932 return LexStringLiteral(Result, ConsumeChar(CurPtr, SizeTmp, Result), 2933 tok::utf32_string_literal); 2934 2935 // UTF-32 character constant 2936 if (Char == '\'') 2937 return LexCharConstant(Result, ConsumeChar(CurPtr, SizeTmp, Result), 2938 tok::utf32_char_constant); 2939 2940 // UTF-32 raw string literal 2941 if (Char == 'R' && LangOpts.CPlusPlus11 && 2942 getCharAndSize(CurPtr + SizeTmp, SizeTmp2) == '"') 2943 return LexRawStringLiteral(Result, 2944 ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 2945 SizeTmp2, Result), 2946 tok::utf32_string_literal); 2947 } 2948 2949 // treat U like the start of an identifier. 2950 return LexIdentifier(Result, CurPtr); 2951 2952 case 'R': // Identifier or C++0x raw string literal 2953 // Notify MIOpt that we read a non-whitespace/non-comment token. 2954 MIOpt.ReadToken(); 2955 2956 if (LangOpts.CPlusPlus11) { 2957 Char = getCharAndSize(CurPtr, SizeTmp); 2958 2959 if (Char == '"') 2960 return LexRawStringLiteral(Result, 2961 ConsumeChar(CurPtr, SizeTmp, Result), 2962 tok::string_literal); 2963 } 2964 2965 // treat R like the start of an identifier. 2966 return LexIdentifier(Result, CurPtr); 2967 2968 case 'L': // Identifier (Loony) or wide literal (L'x' or L"xyz"). 2969 // Notify MIOpt that we read a non-whitespace/non-comment token. 2970 MIOpt.ReadToken(); 2971 Char = getCharAndSize(CurPtr, SizeTmp); 2972 2973 // Wide string literal. 2974 if (Char == '"') 2975 return LexStringLiteral(Result, ConsumeChar(CurPtr, SizeTmp, Result), 2976 tok::wide_string_literal); 2977 2978 // Wide raw string literal. 2979 if (LangOpts.CPlusPlus11 && Char == 'R' && 2980 getCharAndSize(CurPtr + SizeTmp, SizeTmp2) == '"') 2981 return LexRawStringLiteral(Result, 2982 ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 2983 SizeTmp2, Result), 2984 tok::wide_string_literal); 2985 2986 // Wide character constant. 2987 if (Char == '\'') 2988 return LexCharConstant(Result, ConsumeChar(CurPtr, SizeTmp, Result), 2989 tok::wide_char_constant); 2990 // FALL THROUGH, treating L like the start of an identifier. 2991 2992 // C99 6.4.2: Identifiers. 2993 case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': case 'G': 2994 case 'H': case 'I': case 'J': case 'K': /*'L'*/case 'M': case 'N': 2995 case 'O': case 'P': case 'Q': /*'R'*/case 'S': case 'T': /*'U'*/ 2996 case 'V': case 'W': case 'X': case 'Y': case 'Z': 2997 case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': case 'g': 2998 case 'h': case 'i': case 'j': case 'k': case 'l': case 'm': case 'n': 2999 case 'o': case 'p': case 'q': case 'r': case 's': case 't': /*'u'*/ 3000 case 'v': case 'w': case 'x': case 'y': case 'z': 3001 case '_': 3002 // Notify MIOpt that we read a non-whitespace/non-comment token. 3003 MIOpt.ReadToken(); 3004 return LexIdentifier(Result, CurPtr); 3005 3006 case '$': // $ in identifiers. 3007 if (LangOpts.DollarIdents) { 3008 if (!isLexingRawMode()) 3009 Diag(CurPtr-1, diag::ext_dollar_in_identifier); 3010 // Notify MIOpt that we read a non-whitespace/non-comment token. 3011 MIOpt.ReadToken(); 3012 return LexIdentifier(Result, CurPtr); 3013 } 3014 3015 Kind = tok::unknown; 3016 break; 3017 3018 // C99 6.4.4: Character Constants. 3019 case '\'': 3020 // Notify MIOpt that we read a non-whitespace/non-comment token. 3021 MIOpt.ReadToken(); 3022 return LexCharConstant(Result, CurPtr, tok::char_constant); 3023 3024 // C99 6.4.5: String Literals. 3025 case '"': 3026 // Notify MIOpt that we read a non-whitespace/non-comment token. 3027 MIOpt.ReadToken(); 3028 return LexStringLiteral(Result, CurPtr, tok::string_literal); 3029 3030 // C99 6.4.6: Punctuators. 3031 case '?': 3032 Kind = tok::question; 3033 break; 3034 case '[': 3035 Kind = tok::l_square; 3036 break; 3037 case ']': 3038 Kind = tok::r_square; 3039 break; 3040 case '(': 3041 Kind = tok::l_paren; 3042 break; 3043 case ')': 3044 Kind = tok::r_paren; 3045 break; 3046 case '{': 3047 Kind = tok::l_brace; 3048 break; 3049 case '}': 3050 Kind = tok::r_brace; 3051 break; 3052 case '.': 3053 Char = getCharAndSize(CurPtr, SizeTmp); 3054 if (Char >= '0' && Char <= '9') { 3055 // Notify MIOpt that we read a non-whitespace/non-comment token. 3056 MIOpt.ReadToken(); 3057 3058 return LexNumericConstant(Result, ConsumeChar(CurPtr, SizeTmp, Result)); 3059 } else if (LangOpts.CPlusPlus && Char == '*') { 3060 Kind = tok::periodstar; 3061 CurPtr += SizeTmp; 3062 } else if (Char == '.' && 3063 getCharAndSize(CurPtr+SizeTmp, SizeTmp2) == '.') { 3064 Kind = tok::ellipsis; 3065 CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 3066 SizeTmp2, Result); 3067 } else { 3068 Kind = tok::period; 3069 } 3070 break; 3071 case '&': 3072 Char = getCharAndSize(CurPtr, SizeTmp); 3073 if (Char == '&') { 3074 Kind = tok::ampamp; 3075 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3076 } else if (Char == '=') { 3077 Kind = tok::ampequal; 3078 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3079 } else { 3080 Kind = tok::amp; 3081 } 3082 break; 3083 case '*': 3084 if (getCharAndSize(CurPtr, SizeTmp) == '=') { 3085 Kind = tok::starequal; 3086 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3087 } else { 3088 Kind = tok::star; 3089 } 3090 break; 3091 case '+': 3092 Char = getCharAndSize(CurPtr, SizeTmp); 3093 if (Char == '+') { 3094 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3095 Kind = tok::plusplus; 3096 } else if (Char == '=') { 3097 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3098 Kind = tok::plusequal; 3099 } else { 3100 Kind = tok::plus; 3101 } 3102 break; 3103 case '-': 3104 Char = getCharAndSize(CurPtr, SizeTmp); 3105 if (Char == '-') { // -- 3106 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3107 Kind = tok::minusminus; 3108 } else if (Char == '>' && LangOpts.CPlusPlus && 3109 getCharAndSize(CurPtr+SizeTmp, SizeTmp2) == '*') { // C++ ->* 3110 CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 3111 SizeTmp2, Result); 3112 Kind = tok::arrowstar; 3113 } else if (Char == '>') { // -> 3114 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3115 Kind = tok::arrow; 3116 } else if (Char == '=') { // -= 3117 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3118 Kind = tok::minusequal; 3119 } else { 3120 Kind = tok::minus; 3121 } 3122 break; 3123 case '~': 3124 Kind = tok::tilde; 3125 break; 3126 case '!': 3127 if (getCharAndSize(CurPtr, SizeTmp) == '=') { 3128 Kind = tok::exclaimequal; 3129 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3130 } else { 3131 Kind = tok::exclaim; 3132 } 3133 break; 3134 case '/': 3135 // 6.4.9: Comments 3136 Char = getCharAndSize(CurPtr, SizeTmp); 3137 if (Char == '/') { // Line comment. 3138 // Even if Line comments are disabled (e.g. in C89 mode), we generally 3139 // want to lex this as a comment. There is one problem with this though, 3140 // that in one particular corner case, this can change the behavior of the 3141 // resultant program. For example, In "foo //**/ bar", C89 would lex 3142 // this as "foo / bar" and langauges with Line comments would lex it as 3143 // "foo". Check to see if the character after the second slash is a '*'. 3144 // If so, we will lex that as a "/" instead of the start of a comment. 3145 // However, we never do this if we are just preprocessing. 3146 bool TreatAsComment = LangOpts.LineComment && !LangOpts.TraditionalCPP; 3147 if (!TreatAsComment) 3148 if (!(PP && PP->isPreprocessedOutput())) 3149 TreatAsComment = getCharAndSize(CurPtr+SizeTmp, SizeTmp2) != '*'; 3150 3151 if (TreatAsComment) { 3152 if (SkipLineComment(Result, ConsumeChar(CurPtr, SizeTmp, Result))) 3153 return; // There is a token to return. 3154 3155 // It is common for the tokens immediately after a // comment to be 3156 // whitespace (indentation for the next line). Instead of going through 3157 // the big switch, handle it efficiently now. 3158 goto SkipIgnoredUnits; 3159 } 3160 } 3161 3162 if (Char == '*') { // /**/ comment. 3163 if (SkipBlockComment(Result, ConsumeChar(CurPtr, SizeTmp, Result))) 3164 return; // There is a token to return. 3165 goto LexNextToken; // GCC isn't tail call eliminating. 3166 } 3167 3168 if (Char == '=') { 3169 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3170 Kind = tok::slashequal; 3171 } else { 3172 Kind = tok::slash; 3173 } 3174 break; 3175 case '%': 3176 Char = getCharAndSize(CurPtr, SizeTmp); 3177 if (Char == '=') { 3178 Kind = tok::percentequal; 3179 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3180 } else if (LangOpts.Digraphs && Char == '>') { 3181 Kind = tok::r_brace; // '%>' -> '}' 3182 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3183 } else if (LangOpts.Digraphs && Char == ':') { 3184 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3185 Char = getCharAndSize(CurPtr, SizeTmp); 3186 if (Char == '%' && getCharAndSize(CurPtr+SizeTmp, SizeTmp2) == ':') { 3187 Kind = tok::hashhash; // '%:%:' -> '##' 3188 CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 3189 SizeTmp2, Result); 3190 } else if (Char == '@' && LangOpts.MicrosoftExt) {// %:@ -> #@ -> Charize 3191 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3192 if (!isLexingRawMode()) 3193 Diag(BufferPtr, diag::ext_charize_microsoft); 3194 Kind = tok::hashat; 3195 } else { // '%:' -> '#' 3196 // We parsed a # character. If this occurs at the start of the line, 3197 // it's actually the start of a preprocessing directive. Callback to 3198 // the preprocessor to handle it. 3199 // FIXME: -fpreprocessed mode?? 3200 if (Result.isAtStartOfLine() && !LexingRawMode && !Is_PragmaLexer) 3201 goto HandleDirective; 3202 3203 Kind = tok::hash; 3204 } 3205 } else { 3206 Kind = tok::percent; 3207 } 3208 break; 3209 case '<': 3210 Char = getCharAndSize(CurPtr, SizeTmp); 3211 if (ParsingFilename) { 3212 return LexAngledStringLiteral(Result, CurPtr); 3213 } else if (Char == '<') { 3214 char After = getCharAndSize(CurPtr+SizeTmp, SizeTmp2); 3215 if (After == '=') { 3216 Kind = tok::lesslessequal; 3217 CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 3218 SizeTmp2, Result); 3219 } else if (After == '<' && IsStartOfConflictMarker(CurPtr-1)) { 3220 // If this is actually a '<<<<<<<' version control conflict marker, 3221 // recognize it as such and recover nicely. 3222 goto LexNextToken; 3223 } else if (After == '<' && HandleEndOfConflictMarker(CurPtr-1)) { 3224 // If this is '<<<<' and we're in a Perforce-style conflict marker, 3225 // ignore it. 3226 goto LexNextToken; 3227 } else if (LangOpts.CUDA && After == '<') { 3228 Kind = tok::lesslessless; 3229 CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 3230 SizeTmp2, Result); 3231 } else { 3232 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3233 Kind = tok::lessless; 3234 } 3235 } else if (Char == '=') { 3236 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3237 Kind = tok::lessequal; 3238 } else if (LangOpts.Digraphs && Char == ':') { // '<:' -> '[' 3239 if (LangOpts.CPlusPlus11 && 3240 getCharAndSize(CurPtr + SizeTmp, SizeTmp2) == ':') { 3241 // C++0x [lex.pptoken]p3: 3242 // Otherwise, if the next three characters are <:: and the subsequent 3243 // character is neither : nor >, the < is treated as a preprocessor 3244 // token by itself and not as the first character of the alternative 3245 // token <:. 3246 unsigned SizeTmp3; 3247 char After = getCharAndSize(CurPtr + SizeTmp + SizeTmp2, SizeTmp3); 3248 if (After != ':' && After != '>') { 3249 Kind = tok::less; 3250 if (!isLexingRawMode()) 3251 Diag(BufferPtr, diag::warn_cxx98_compat_less_colon_colon); 3252 break; 3253 } 3254 } 3255 3256 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3257 Kind = tok::l_square; 3258 } else if (LangOpts.Digraphs && Char == '%') { // '<%' -> '{' 3259 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3260 Kind = tok::l_brace; 3261 } else { 3262 Kind = tok::less; 3263 } 3264 break; 3265 case '>': 3266 Char = getCharAndSize(CurPtr, SizeTmp); 3267 if (Char == '=') { 3268 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3269 Kind = tok::greaterequal; 3270 } else if (Char == '>') { 3271 char After = getCharAndSize(CurPtr+SizeTmp, SizeTmp2); 3272 if (After == '=') { 3273 CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 3274 SizeTmp2, Result); 3275 Kind = tok::greatergreaterequal; 3276 } else if (After == '>' && IsStartOfConflictMarker(CurPtr-1)) { 3277 // If this is actually a '>>>>' conflict marker, recognize it as such 3278 // and recover nicely. 3279 goto LexNextToken; 3280 } else if (After == '>' && HandleEndOfConflictMarker(CurPtr-1)) { 3281 // If this is '>>>>>>>' and we're in a conflict marker, ignore it. 3282 goto LexNextToken; 3283 } else if (LangOpts.CUDA && After == '>') { 3284 Kind = tok::greatergreatergreater; 3285 CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 3286 SizeTmp2, Result); 3287 } else { 3288 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3289 Kind = tok::greatergreater; 3290 } 3291 3292 } else { 3293 Kind = tok::greater; 3294 } 3295 break; 3296 case '^': 3297 Char = getCharAndSize(CurPtr, SizeTmp); 3298 if (Char == '=') { 3299 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3300 Kind = tok::caretequal; 3301 } else { 3302 Kind = tok::caret; 3303 } 3304 break; 3305 case '|': 3306 Char = getCharAndSize(CurPtr, SizeTmp); 3307 if (Char == '=') { 3308 Kind = tok::pipeequal; 3309 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3310 } else if (Char == '|') { 3311 // If this is '|||||||' and we're in a conflict marker, ignore it. 3312 if (CurPtr[1] == '|' && HandleEndOfConflictMarker(CurPtr-1)) 3313 goto LexNextToken; 3314 Kind = tok::pipepipe; 3315 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3316 } else { 3317 Kind = tok::pipe; 3318 } 3319 break; 3320 case ':': 3321 Char = getCharAndSize(CurPtr, SizeTmp); 3322 if (LangOpts.Digraphs && Char == '>') { 3323 Kind = tok::r_square; // ':>' -> ']' 3324 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3325 } else if (LangOpts.CPlusPlus && Char == ':') { 3326 Kind = tok::coloncolon; 3327 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3328 } else { 3329 Kind = tok::colon; 3330 } 3331 break; 3332 case ';': 3333 Kind = tok::semi; 3334 break; 3335 case '=': 3336 Char = getCharAndSize(CurPtr, SizeTmp); 3337 if (Char == '=') { 3338 // If this is '====' and we're in a conflict marker, ignore it. 3339 if (CurPtr[1] == '=' && HandleEndOfConflictMarker(CurPtr-1)) 3340 goto LexNextToken; 3341 3342 Kind = tok::equalequal; 3343 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3344 } else { 3345 Kind = tok::equal; 3346 } 3347 break; 3348 case ',': 3349 Kind = tok::comma; 3350 break; 3351 case '#': 3352 Char = getCharAndSize(CurPtr, SizeTmp); 3353 if (Char == '#') { 3354 Kind = tok::hashhash; 3355 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3356 } else if (Char == '@' && LangOpts.MicrosoftExt) { // #@ -> Charize 3357 Kind = tok::hashat; 3358 if (!isLexingRawMode()) 3359 Diag(BufferPtr, diag::ext_charize_microsoft); 3360 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3361 } else { 3362 // We parsed a # character. If this occurs at the start of the line, 3363 // it's actually the start of a preprocessing directive. Callback to 3364 // the preprocessor to handle it. 3365 // FIXME: -fpreprocessed mode?? 3366 if (Result.isAtStartOfLine() && !LexingRawMode && !Is_PragmaLexer) 3367 goto HandleDirective; 3368 3369 Kind = tok::hash; 3370 } 3371 break; 3372 3373 case '@': 3374 // Objective C support. 3375 if (CurPtr[-1] == '@' && LangOpts.ObjC1) 3376 Kind = tok::at; 3377 else 3378 Kind = tok::unknown; 3379 break; 3380 3381 // UCNs (C99 6.4.3, C++11 [lex.charset]p2) 3382 case '\\': 3383 if (uint32_t CodePoint = tryReadUCN(CurPtr, BufferPtr, &Result)) 3384 return LexUnicode(Result, CodePoint, CurPtr); 3385 3386 Kind = tok::unknown; 3387 break; 3388 3389 default: { 3390 if (isASCII(Char)) { 3391 Kind = tok::unknown; 3392 break; 3393 } 3394 3395 UTF32 CodePoint; 3396 3397 // We can't just reset CurPtr to BufferPtr because BufferPtr may point to 3398 // an escaped newline. 3399 --CurPtr; 3400 ConversionResult Status = 3401 llvm::convertUTF8Sequence((const UTF8 **)&CurPtr, 3402 (const UTF8 *)BufferEnd, 3403 &CodePoint, 3404 strictConversion); 3405 if (Status == conversionOK) 3406 return LexUnicode(Result, CodePoint, CurPtr); 3407 3408 if (isLexingRawMode() || ParsingPreprocessorDirective || 3409 PP->isPreprocessedOutput()) { 3410 ++CurPtr; 3411 Kind = tok::unknown; 3412 break; 3413 } 3414 3415 // Non-ASCII characters tend to creep into source code unintentionally. 3416 // Instead of letting the parser complain about the unknown token, 3417 // just diagnose the invalid UTF-8, then drop the character. 3418 Diag(CurPtr, diag::err_invalid_utf8); 3419 3420 BufferPtr = CurPtr+1; 3421 goto LexNextToken; 3422 } 3423 } 3424 3425 // Notify MIOpt that we read a non-whitespace/non-comment token. 3426 MIOpt.ReadToken(); 3427 3428 // Update the location of token as well as BufferPtr. 3429 FormTokenWithChars(Result, CurPtr, Kind); 3430 return; 3431 3432HandleDirective: 3433 // We parsed a # character and it's the start of a preprocessing directive. 3434 3435 FormTokenWithChars(Result, CurPtr, tok::hash); 3436 PP->HandleDirective(Result); 3437 3438 // As an optimization, if the preprocessor didn't switch lexers, tail 3439 // recurse. 3440 if (PP->isCurrentLexer(this)) { 3441 // Start a new token. If this is a #include or something, the PP may 3442 // want us starting at the beginning of the line again. If so, set 3443 // the StartOfLine flag and clear LeadingSpace. 3444 if (IsAtStartOfLine) { 3445 Result.setFlag(Token::StartOfLine); 3446 Result.clearFlag(Token::LeadingSpace); 3447 IsAtStartOfLine = false; 3448 } 3449 goto LexNextToken; // GCC isn't tail call eliminating. 3450 } 3451 return PP->Lex(Result); 3452} 3453