Lexer.cpp revision 263cea4485040bb590800ef3290448a81f0dbc4b
1//===--- Lexer.cpp - C Language Family Lexer ------------------------------===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file implements the Lexer and Token interfaces. 11// 12//===----------------------------------------------------------------------===// 13// 14// TODO: GCC Diagnostics emitted by the lexer: 15// PEDWARN: (form feed|vertical tab) in preprocessing directive 16// 17// Universal characters, unicode, char mapping: 18// WARNING: `%.*s' is not in NFKC 19// WARNING: `%.*s' is not in NFC 20// 21// Other: 22// TODO: Options to support: 23// -fexec-charset,-fwide-exec-charset 24// 25//===----------------------------------------------------------------------===// 26 27#include "clang/Lex/Lexer.h" 28#include "clang/Basic/CharInfo.h" 29#include "clang/Basic/SourceManager.h" 30#include "clang/Lex/CodeCompletionHandler.h" 31#include "clang/Lex/LexDiagnostic.h" 32#include "clang/Lex/Preprocessor.h" 33#include "llvm/ADT/STLExtras.h" 34#include "llvm/ADT/StringExtras.h" 35#include "llvm/ADT/StringSwitch.h" 36#include "llvm/Support/Compiler.h" 37#include "llvm/Support/ConvertUTF.h" 38#include "llvm/Support/MemoryBuffer.h" 39#include "UnicodeCharSets.h" 40#include <cstring> 41using namespace clang; 42 43//===----------------------------------------------------------------------===// 44// Token Class Implementation 45//===----------------------------------------------------------------------===// 46 47/// isObjCAtKeyword - Return true if we have an ObjC keyword identifier. 48bool Token::isObjCAtKeyword(tok::ObjCKeywordKind objcKey) const { 49 if (IdentifierInfo *II = getIdentifierInfo()) 50 return II->getObjCKeywordID() == objcKey; 51 return false; 52} 53 54/// getObjCKeywordID - Return the ObjC keyword kind. 55tok::ObjCKeywordKind Token::getObjCKeywordID() const { 56 IdentifierInfo *specId = getIdentifierInfo(); 57 return specId ? specId->getObjCKeywordID() : tok::objc_not_keyword; 58} 59 60 61//===----------------------------------------------------------------------===// 62// Lexer Class Implementation 63//===----------------------------------------------------------------------===// 64 65void Lexer::anchor() { } 66 67void Lexer::InitLexer(const char *BufStart, const char *BufPtr, 68 const char *BufEnd) { 69 BufferStart = BufStart; 70 BufferPtr = BufPtr; 71 BufferEnd = BufEnd; 72 73 assert(BufEnd[0] == 0 && 74 "We assume that the input buffer has a null character at the end" 75 " to simplify lexing!"); 76 77 // Check whether we have a BOM in the beginning of the buffer. If yes - act 78 // accordingly. Right now we support only UTF-8 with and without BOM, so, just 79 // skip the UTF-8 BOM if it's present. 80 if (BufferStart == BufferPtr) { 81 // Determine the size of the BOM. 82 StringRef Buf(BufferStart, BufferEnd - BufferStart); 83 size_t BOMLength = llvm::StringSwitch<size_t>(Buf) 84 .StartsWith("\xEF\xBB\xBF", 3) // UTF-8 BOM 85 .Default(0); 86 87 // Skip the BOM. 88 BufferPtr += BOMLength; 89 } 90 91 Is_PragmaLexer = false; 92 CurrentConflictMarkerState = CMK_None; 93 94 // Start of the file is a start of line. 95 IsAtStartOfLine = true; 96 97 // We are not after parsing a #. 98 ParsingPreprocessorDirective = false; 99 100 // We are not after parsing #include. 101 ParsingFilename = false; 102 103 // We are not in raw mode. Raw mode disables diagnostics and interpretation 104 // of tokens (e.g. identifiers, thus disabling macro expansion). It is used 105 // to quickly lex the tokens of the buffer, e.g. when handling a "#if 0" block 106 // or otherwise skipping over tokens. 107 LexingRawMode = false; 108 109 // Default to not keeping comments. 110 ExtendedTokenMode = 0; 111} 112 113/// Lexer constructor - Create a new lexer object for the specified buffer 114/// with the specified preprocessor managing the lexing process. This lexer 115/// assumes that the associated file buffer and Preprocessor objects will 116/// outlive it, so it doesn't take ownership of either of them. 117Lexer::Lexer(FileID FID, const llvm::MemoryBuffer *InputFile, Preprocessor &PP) 118 : PreprocessorLexer(&PP, FID), 119 FileLoc(PP.getSourceManager().getLocForStartOfFile(FID)), 120 LangOpts(PP.getLangOpts()) { 121 122 InitLexer(InputFile->getBufferStart(), InputFile->getBufferStart(), 123 InputFile->getBufferEnd()); 124 125 resetExtendedTokenMode(); 126} 127 128void Lexer::resetExtendedTokenMode() { 129 assert(PP && "Cannot reset token mode without a preprocessor"); 130 if (LangOpts.TraditionalCPP) 131 SetKeepWhitespaceMode(true); 132 else 133 SetCommentRetentionState(PP->getCommentRetentionState()); 134} 135 136/// Lexer constructor - Create a new raw lexer object. This object is only 137/// suitable for calls to 'LexFromRawLexer'. This lexer assumes that the text 138/// range will outlive it, so it doesn't take ownership of it. 139Lexer::Lexer(SourceLocation fileloc, const LangOptions &langOpts, 140 const char *BufStart, const char *BufPtr, const char *BufEnd) 141 : FileLoc(fileloc), LangOpts(langOpts) { 142 143 InitLexer(BufStart, BufPtr, BufEnd); 144 145 // We *are* in raw mode. 146 LexingRawMode = true; 147} 148 149/// Lexer constructor - Create a new raw lexer object. This object is only 150/// suitable for calls to 'LexFromRawLexer'. This lexer assumes that the text 151/// range will outlive it, so it doesn't take ownership of it. 152Lexer::Lexer(FileID FID, const llvm::MemoryBuffer *FromFile, 153 const SourceManager &SM, const LangOptions &langOpts) 154 : FileLoc(SM.getLocForStartOfFile(FID)), LangOpts(langOpts) { 155 156 InitLexer(FromFile->getBufferStart(), FromFile->getBufferStart(), 157 FromFile->getBufferEnd()); 158 159 // We *are* in raw mode. 160 LexingRawMode = true; 161} 162 163/// Create_PragmaLexer: Lexer constructor - Create a new lexer object for 164/// _Pragma expansion. This has a variety of magic semantics that this method 165/// sets up. It returns a new'd Lexer that must be delete'd when done. 166/// 167/// On entrance to this routine, TokStartLoc is a macro location which has a 168/// spelling loc that indicates the bytes to be lexed for the token and an 169/// expansion location that indicates where all lexed tokens should be 170/// "expanded from". 171/// 172/// FIXME: It would really be nice to make _Pragma just be a wrapper around a 173/// normal lexer that remaps tokens as they fly by. This would require making 174/// Preprocessor::Lex virtual. Given that, we could just dump in a magic lexer 175/// interface that could handle this stuff. This would pull GetMappedTokenLoc 176/// out of the critical path of the lexer! 177/// 178Lexer *Lexer::Create_PragmaLexer(SourceLocation SpellingLoc, 179 SourceLocation ExpansionLocStart, 180 SourceLocation ExpansionLocEnd, 181 unsigned TokLen, Preprocessor &PP) { 182 SourceManager &SM = PP.getSourceManager(); 183 184 // Create the lexer as if we were going to lex the file normally. 185 FileID SpellingFID = SM.getFileID(SpellingLoc); 186 const llvm::MemoryBuffer *InputFile = SM.getBuffer(SpellingFID); 187 Lexer *L = new Lexer(SpellingFID, InputFile, PP); 188 189 // Now that the lexer is created, change the start/end locations so that we 190 // just lex the subsection of the file that we want. This is lexing from a 191 // scratch buffer. 192 const char *StrData = SM.getCharacterData(SpellingLoc); 193 194 L->BufferPtr = StrData; 195 L->BufferEnd = StrData+TokLen; 196 assert(L->BufferEnd[0] == 0 && "Buffer is not nul terminated!"); 197 198 // Set the SourceLocation with the remapping information. This ensures that 199 // GetMappedTokenLoc will remap the tokens as they are lexed. 200 L->FileLoc = SM.createExpansionLoc(SM.getLocForStartOfFile(SpellingFID), 201 ExpansionLocStart, 202 ExpansionLocEnd, TokLen); 203 204 // Ensure that the lexer thinks it is inside a directive, so that end \n will 205 // return an EOD token. 206 L->ParsingPreprocessorDirective = true; 207 208 // This lexer really is for _Pragma. 209 L->Is_PragmaLexer = true; 210 return L; 211} 212 213 214/// Stringify - Convert the specified string into a C string, with surrounding 215/// ""'s, and with escaped \ and " characters. 216std::string Lexer::Stringify(const std::string &Str, bool Charify) { 217 std::string Result = Str; 218 char Quote = Charify ? '\'' : '"'; 219 for (unsigned i = 0, e = Result.size(); i != e; ++i) { 220 if (Result[i] == '\\' || Result[i] == Quote) { 221 Result.insert(Result.begin()+i, '\\'); 222 ++i; ++e; 223 } 224 } 225 return Result; 226} 227 228/// Stringify - Convert the specified string into a C string by escaping '\' 229/// and " characters. This does not add surrounding ""'s to the string. 230void Lexer::Stringify(SmallVectorImpl<char> &Str) { 231 for (unsigned i = 0, e = Str.size(); i != e; ++i) { 232 if (Str[i] == '\\' || Str[i] == '"') { 233 Str.insert(Str.begin()+i, '\\'); 234 ++i; ++e; 235 } 236 } 237} 238 239//===----------------------------------------------------------------------===// 240// Token Spelling 241//===----------------------------------------------------------------------===// 242 243/// \brief Slow case of getSpelling. Extract the characters comprising the 244/// spelling of this token from the provided input buffer. 245static size_t getSpellingSlow(const Token &Tok, const char *BufPtr, 246 const LangOptions &LangOpts, char *Spelling) { 247 assert(Tok.needsCleaning() && "getSpellingSlow called on simple token"); 248 249 size_t Length = 0; 250 const char *BufEnd = BufPtr + Tok.getLength(); 251 252 if (Tok.is(tok::string_literal)) { 253 // Munch the encoding-prefix and opening double-quote. 254 while (BufPtr < BufEnd) { 255 unsigned Size; 256 Spelling[Length++] = Lexer::getCharAndSizeNoWarn(BufPtr, Size, LangOpts); 257 BufPtr += Size; 258 259 if (Spelling[Length - 1] == '"') 260 break; 261 } 262 263 // Raw string literals need special handling; trigraph expansion and line 264 // splicing do not occur within their d-char-sequence nor within their 265 // r-char-sequence. 266 if (Length >= 2 && 267 Spelling[Length - 2] == 'R' && Spelling[Length - 1] == '"') { 268 // Search backwards from the end of the token to find the matching closing 269 // quote. 270 const char *RawEnd = BufEnd; 271 do --RawEnd; while (*RawEnd != '"'); 272 size_t RawLength = RawEnd - BufPtr + 1; 273 274 // Everything between the quotes is included verbatim in the spelling. 275 memcpy(Spelling + Length, BufPtr, RawLength); 276 Length += RawLength; 277 BufPtr += RawLength; 278 279 // The rest of the token is lexed normally. 280 } 281 } 282 283 while (BufPtr < BufEnd) { 284 unsigned Size; 285 Spelling[Length++] = Lexer::getCharAndSizeNoWarn(BufPtr, Size, LangOpts); 286 BufPtr += Size; 287 } 288 289 assert(Length < Tok.getLength() && 290 "NeedsCleaning flag set on token that didn't need cleaning!"); 291 return Length; 292} 293 294/// getSpelling() - Return the 'spelling' of this token. The spelling of a 295/// token are the characters used to represent the token in the source file 296/// after trigraph expansion and escaped-newline folding. In particular, this 297/// wants to get the true, uncanonicalized, spelling of things like digraphs 298/// UCNs, etc. 299StringRef Lexer::getSpelling(SourceLocation loc, 300 SmallVectorImpl<char> &buffer, 301 const SourceManager &SM, 302 const LangOptions &options, 303 bool *invalid) { 304 // Break down the source location. 305 std::pair<FileID, unsigned> locInfo = SM.getDecomposedLoc(loc); 306 307 // Try to the load the file buffer. 308 bool invalidTemp = false; 309 StringRef file = SM.getBufferData(locInfo.first, &invalidTemp); 310 if (invalidTemp) { 311 if (invalid) *invalid = true; 312 return StringRef(); 313 } 314 315 const char *tokenBegin = file.data() + locInfo.second; 316 317 // Lex from the start of the given location. 318 Lexer lexer(SM.getLocForStartOfFile(locInfo.first), options, 319 file.begin(), tokenBegin, file.end()); 320 Token token; 321 lexer.LexFromRawLexer(token); 322 323 unsigned length = token.getLength(); 324 325 // Common case: no need for cleaning. 326 if (!token.needsCleaning()) 327 return StringRef(tokenBegin, length); 328 329 // Hard case, we need to relex the characters into the string. 330 buffer.resize(length); 331 buffer.resize(getSpellingSlow(token, tokenBegin, options, buffer.data())); 332 return StringRef(buffer.data(), buffer.size()); 333} 334 335/// getSpelling() - Return the 'spelling' of this token. The spelling of a 336/// token are the characters used to represent the token in the source file 337/// after trigraph expansion and escaped-newline folding. In particular, this 338/// wants to get the true, uncanonicalized, spelling of things like digraphs 339/// UCNs, etc. 340std::string Lexer::getSpelling(const Token &Tok, const SourceManager &SourceMgr, 341 const LangOptions &LangOpts, bool *Invalid) { 342 assert((int)Tok.getLength() >= 0 && "Token character range is bogus!"); 343 344 bool CharDataInvalid = false; 345 const char *TokStart = SourceMgr.getCharacterData(Tok.getLocation(), 346 &CharDataInvalid); 347 if (Invalid) 348 *Invalid = CharDataInvalid; 349 if (CharDataInvalid) 350 return std::string(); 351 352 // If this token contains nothing interesting, return it directly. 353 if (!Tok.needsCleaning()) 354 return std::string(TokStart, TokStart + Tok.getLength()); 355 356 std::string Result; 357 Result.resize(Tok.getLength()); 358 Result.resize(getSpellingSlow(Tok, TokStart, LangOpts, &*Result.begin())); 359 return Result; 360} 361 362/// getSpelling - This method is used to get the spelling of a token into a 363/// preallocated buffer, instead of as an std::string. The caller is required 364/// to allocate enough space for the token, which is guaranteed to be at least 365/// Tok.getLength() bytes long. The actual length of the token is returned. 366/// 367/// Note that this method may do two possible things: it may either fill in 368/// the buffer specified with characters, or it may *change the input pointer* 369/// to point to a constant buffer with the data already in it (avoiding a 370/// copy). The caller is not allowed to modify the returned buffer pointer 371/// if an internal buffer is returned. 372unsigned Lexer::getSpelling(const Token &Tok, const char *&Buffer, 373 const SourceManager &SourceMgr, 374 const LangOptions &LangOpts, bool *Invalid) { 375 assert((int)Tok.getLength() >= 0 && "Token character range is bogus!"); 376 377 const char *TokStart = 0; 378 // NOTE: this has to be checked *before* testing for an IdentifierInfo. 379 if (Tok.is(tok::raw_identifier)) 380 TokStart = Tok.getRawIdentifierData(); 381 else if (!Tok.hasUCN()) { 382 if (const IdentifierInfo *II = Tok.getIdentifierInfo()) { 383 // Just return the string from the identifier table, which is very quick. 384 Buffer = II->getNameStart(); 385 return II->getLength(); 386 } 387 } 388 389 // NOTE: this can be checked even after testing for an IdentifierInfo. 390 if (Tok.isLiteral()) 391 TokStart = Tok.getLiteralData(); 392 393 if (TokStart == 0) { 394 // Compute the start of the token in the input lexer buffer. 395 bool CharDataInvalid = false; 396 TokStart = SourceMgr.getCharacterData(Tok.getLocation(), &CharDataInvalid); 397 if (Invalid) 398 *Invalid = CharDataInvalid; 399 if (CharDataInvalid) { 400 Buffer = ""; 401 return 0; 402 } 403 } 404 405 // If this token contains nothing interesting, return it directly. 406 if (!Tok.needsCleaning()) { 407 Buffer = TokStart; 408 return Tok.getLength(); 409 } 410 411 // Otherwise, hard case, relex the characters into the string. 412 return getSpellingSlow(Tok, TokStart, LangOpts, const_cast<char*>(Buffer)); 413} 414 415 416/// MeasureTokenLength - Relex the token at the specified location and return 417/// its length in bytes in the input file. If the token needs cleaning (e.g. 418/// includes a trigraph or an escaped newline) then this count includes bytes 419/// that are part of that. 420unsigned Lexer::MeasureTokenLength(SourceLocation Loc, 421 const SourceManager &SM, 422 const LangOptions &LangOpts) { 423 Token TheTok; 424 if (getRawToken(Loc, TheTok, SM, LangOpts)) 425 return 0; 426 return TheTok.getLength(); 427} 428 429/// \brief Relex the token at the specified location. 430/// \returns true if there was a failure, false on success. 431bool Lexer::getRawToken(SourceLocation Loc, Token &Result, 432 const SourceManager &SM, 433 const LangOptions &LangOpts, 434 bool IgnoreWhiteSpace) { 435 // TODO: this could be special cased for common tokens like identifiers, ')', 436 // etc to make this faster, if it mattered. Just look at StrData[0] to handle 437 // all obviously single-char tokens. This could use 438 // Lexer::isObviouslySimpleCharacter for example to handle identifiers or 439 // something. 440 441 // If this comes from a macro expansion, we really do want the macro name, not 442 // the token this macro expanded to. 443 Loc = SM.getExpansionLoc(Loc); 444 std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(Loc); 445 bool Invalid = false; 446 StringRef Buffer = SM.getBufferData(LocInfo.first, &Invalid); 447 if (Invalid) 448 return true; 449 450 const char *StrData = Buffer.data()+LocInfo.second; 451 452 if (!IgnoreWhiteSpace && isWhitespace(StrData[0])) 453 return true; 454 455 // Create a lexer starting at the beginning of this token. 456 Lexer TheLexer(SM.getLocForStartOfFile(LocInfo.first), LangOpts, 457 Buffer.begin(), StrData, Buffer.end()); 458 TheLexer.SetCommentRetentionState(true); 459 TheLexer.LexFromRawLexer(Result); 460 return false; 461} 462 463static SourceLocation getBeginningOfFileToken(SourceLocation Loc, 464 const SourceManager &SM, 465 const LangOptions &LangOpts) { 466 assert(Loc.isFileID()); 467 std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(Loc); 468 if (LocInfo.first.isInvalid()) 469 return Loc; 470 471 bool Invalid = false; 472 StringRef Buffer = SM.getBufferData(LocInfo.first, &Invalid); 473 if (Invalid) 474 return Loc; 475 476 // Back up from the current location until we hit the beginning of a line 477 // (or the buffer). We'll relex from that point. 478 const char *BufStart = Buffer.data(); 479 if (LocInfo.second >= Buffer.size()) 480 return Loc; 481 482 const char *StrData = BufStart+LocInfo.second; 483 if (StrData[0] == '\n' || StrData[0] == '\r') 484 return Loc; 485 486 const char *LexStart = StrData; 487 while (LexStart != BufStart) { 488 if (LexStart[0] == '\n' || LexStart[0] == '\r') { 489 ++LexStart; 490 break; 491 } 492 493 --LexStart; 494 } 495 496 // Create a lexer starting at the beginning of this token. 497 SourceLocation LexerStartLoc = Loc.getLocWithOffset(-LocInfo.second); 498 Lexer TheLexer(LexerStartLoc, LangOpts, BufStart, LexStart, Buffer.end()); 499 TheLexer.SetCommentRetentionState(true); 500 501 // Lex tokens until we find the token that contains the source location. 502 Token TheTok; 503 do { 504 TheLexer.LexFromRawLexer(TheTok); 505 506 if (TheLexer.getBufferLocation() > StrData) { 507 // Lexing this token has taken the lexer past the source location we're 508 // looking for. If the current token encompasses our source location, 509 // return the beginning of that token. 510 if (TheLexer.getBufferLocation() - TheTok.getLength() <= StrData) 511 return TheTok.getLocation(); 512 513 // We ended up skipping over the source location entirely, which means 514 // that it points into whitespace. We're done here. 515 break; 516 } 517 } while (TheTok.getKind() != tok::eof); 518 519 // We've passed our source location; just return the original source location. 520 return Loc; 521} 522 523SourceLocation Lexer::GetBeginningOfToken(SourceLocation Loc, 524 const SourceManager &SM, 525 const LangOptions &LangOpts) { 526 if (Loc.isFileID()) 527 return getBeginningOfFileToken(Loc, SM, LangOpts); 528 529 if (!SM.isMacroArgExpansion(Loc)) 530 return Loc; 531 532 SourceLocation FileLoc = SM.getSpellingLoc(Loc); 533 SourceLocation BeginFileLoc = getBeginningOfFileToken(FileLoc, SM, LangOpts); 534 std::pair<FileID, unsigned> FileLocInfo = SM.getDecomposedLoc(FileLoc); 535 std::pair<FileID, unsigned> BeginFileLocInfo 536 = SM.getDecomposedLoc(BeginFileLoc); 537 assert(FileLocInfo.first == BeginFileLocInfo.first && 538 FileLocInfo.second >= BeginFileLocInfo.second); 539 return Loc.getLocWithOffset(BeginFileLocInfo.second - FileLocInfo.second); 540} 541 542namespace { 543 enum PreambleDirectiveKind { 544 PDK_Skipped, 545 PDK_StartIf, 546 PDK_EndIf, 547 PDK_Unknown 548 }; 549} 550 551std::pair<unsigned, bool> 552Lexer::ComputePreamble(const llvm::MemoryBuffer *Buffer, 553 const LangOptions &LangOpts, unsigned MaxLines) { 554 // Create a lexer starting at the beginning of the file. Note that we use a 555 // "fake" file source location at offset 1 so that the lexer will track our 556 // position within the file. 557 const unsigned StartOffset = 1; 558 SourceLocation FileLoc = SourceLocation::getFromRawEncoding(StartOffset); 559 Lexer TheLexer(FileLoc, LangOpts, Buffer->getBufferStart(), 560 Buffer->getBufferStart(), Buffer->getBufferEnd()); 561 TheLexer.SetCommentRetentionState(true); 562 563 // StartLoc will differ from FileLoc if there is a BOM that was skipped. 564 SourceLocation StartLoc = TheLexer.getSourceLocation(); 565 566 bool InPreprocessorDirective = false; 567 Token TheTok; 568 Token IfStartTok; 569 unsigned IfCount = 0; 570 SourceLocation ActiveCommentLoc; 571 572 unsigned MaxLineOffset = 0; 573 if (MaxLines) { 574 const char *CurPtr = Buffer->getBufferStart(); 575 unsigned CurLine = 0; 576 while (CurPtr != Buffer->getBufferEnd()) { 577 char ch = *CurPtr++; 578 if (ch == '\n') { 579 ++CurLine; 580 if (CurLine == MaxLines) 581 break; 582 } 583 } 584 if (CurPtr != Buffer->getBufferEnd()) 585 MaxLineOffset = CurPtr - Buffer->getBufferStart(); 586 } 587 588 do { 589 TheLexer.LexFromRawLexer(TheTok); 590 591 if (InPreprocessorDirective) { 592 // If we've hit the end of the file, we're done. 593 if (TheTok.getKind() == tok::eof) { 594 break; 595 } 596 597 // If we haven't hit the end of the preprocessor directive, skip this 598 // token. 599 if (!TheTok.isAtStartOfLine()) 600 continue; 601 602 // We've passed the end of the preprocessor directive, and will look 603 // at this token again below. 604 InPreprocessorDirective = false; 605 } 606 607 // Keep track of the # of lines in the preamble. 608 if (TheTok.isAtStartOfLine()) { 609 unsigned TokOffset = TheTok.getLocation().getRawEncoding() - StartOffset; 610 611 // If we were asked to limit the number of lines in the preamble, 612 // and we're about to exceed that limit, we're done. 613 if (MaxLineOffset && TokOffset >= MaxLineOffset) 614 break; 615 } 616 617 // Comments are okay; skip over them. 618 if (TheTok.getKind() == tok::comment) { 619 if (ActiveCommentLoc.isInvalid()) 620 ActiveCommentLoc = TheTok.getLocation(); 621 continue; 622 } 623 624 if (TheTok.isAtStartOfLine() && TheTok.getKind() == tok::hash) { 625 // This is the start of a preprocessor directive. 626 Token HashTok = TheTok; 627 InPreprocessorDirective = true; 628 ActiveCommentLoc = SourceLocation(); 629 630 // Figure out which directive this is. Since we're lexing raw tokens, 631 // we don't have an identifier table available. Instead, just look at 632 // the raw identifier to recognize and categorize preprocessor directives. 633 TheLexer.LexFromRawLexer(TheTok); 634 if (TheTok.getKind() == tok::raw_identifier && !TheTok.needsCleaning()) { 635 StringRef Keyword(TheTok.getRawIdentifierData(), 636 TheTok.getLength()); 637 PreambleDirectiveKind PDK 638 = llvm::StringSwitch<PreambleDirectiveKind>(Keyword) 639 .Case("include", PDK_Skipped) 640 .Case("__include_macros", PDK_Skipped) 641 .Case("define", PDK_Skipped) 642 .Case("undef", PDK_Skipped) 643 .Case("line", PDK_Skipped) 644 .Case("error", PDK_Skipped) 645 .Case("pragma", PDK_Skipped) 646 .Case("import", PDK_Skipped) 647 .Case("include_next", PDK_Skipped) 648 .Case("warning", PDK_Skipped) 649 .Case("ident", PDK_Skipped) 650 .Case("sccs", PDK_Skipped) 651 .Case("assert", PDK_Skipped) 652 .Case("unassert", PDK_Skipped) 653 .Case("if", PDK_StartIf) 654 .Case("ifdef", PDK_StartIf) 655 .Case("ifndef", PDK_StartIf) 656 .Case("elif", PDK_Skipped) 657 .Case("else", PDK_Skipped) 658 .Case("endif", PDK_EndIf) 659 .Default(PDK_Unknown); 660 661 switch (PDK) { 662 case PDK_Skipped: 663 continue; 664 665 case PDK_StartIf: 666 if (IfCount == 0) 667 IfStartTok = HashTok; 668 669 ++IfCount; 670 continue; 671 672 case PDK_EndIf: 673 // Mismatched #endif. The preamble ends here. 674 if (IfCount == 0) 675 break; 676 677 --IfCount; 678 continue; 679 680 case PDK_Unknown: 681 // We don't know what this directive is; stop at the '#'. 682 break; 683 } 684 } 685 686 // We only end up here if we didn't recognize the preprocessor 687 // directive or it was one that can't occur in the preamble at this 688 // point. Roll back the current token to the location of the '#'. 689 InPreprocessorDirective = false; 690 TheTok = HashTok; 691 } 692 693 // We hit a token that we don't recognize as being in the 694 // "preprocessing only" part of the file, so we're no longer in 695 // the preamble. 696 break; 697 } while (true); 698 699 SourceLocation End; 700 if (IfCount) 701 End = IfStartTok.getLocation(); 702 else if (ActiveCommentLoc.isValid()) 703 End = ActiveCommentLoc; // don't truncate a decl comment. 704 else 705 End = TheTok.getLocation(); 706 707 return std::make_pair(End.getRawEncoding() - StartLoc.getRawEncoding(), 708 IfCount? IfStartTok.isAtStartOfLine() 709 : TheTok.isAtStartOfLine()); 710} 711 712 713/// AdvanceToTokenCharacter - Given a location that specifies the start of a 714/// token, return a new location that specifies a character within the token. 715SourceLocation Lexer::AdvanceToTokenCharacter(SourceLocation TokStart, 716 unsigned CharNo, 717 const SourceManager &SM, 718 const LangOptions &LangOpts) { 719 // Figure out how many physical characters away the specified expansion 720 // character is. This needs to take into consideration newlines and 721 // trigraphs. 722 bool Invalid = false; 723 const char *TokPtr = SM.getCharacterData(TokStart, &Invalid); 724 725 // If they request the first char of the token, we're trivially done. 726 if (Invalid || (CharNo == 0 && Lexer::isObviouslySimpleCharacter(*TokPtr))) 727 return TokStart; 728 729 unsigned PhysOffset = 0; 730 731 // The usual case is that tokens don't contain anything interesting. Skip 732 // over the uninteresting characters. If a token only consists of simple 733 // chars, this method is extremely fast. 734 while (Lexer::isObviouslySimpleCharacter(*TokPtr)) { 735 if (CharNo == 0) 736 return TokStart.getLocWithOffset(PhysOffset); 737 ++TokPtr, --CharNo, ++PhysOffset; 738 } 739 740 // If we have a character that may be a trigraph or escaped newline, use a 741 // lexer to parse it correctly. 742 for (; CharNo; --CharNo) { 743 unsigned Size; 744 Lexer::getCharAndSizeNoWarn(TokPtr, Size, LangOpts); 745 TokPtr += Size; 746 PhysOffset += Size; 747 } 748 749 // Final detail: if we end up on an escaped newline, we want to return the 750 // location of the actual byte of the token. For example foo\<newline>bar 751 // advanced by 3 should return the location of b, not of \\. One compounding 752 // detail of this is that the escape may be made by a trigraph. 753 if (!Lexer::isObviouslySimpleCharacter(*TokPtr)) 754 PhysOffset += Lexer::SkipEscapedNewLines(TokPtr)-TokPtr; 755 756 return TokStart.getLocWithOffset(PhysOffset); 757} 758 759/// \brief Computes the source location just past the end of the 760/// token at this source location. 761/// 762/// This routine can be used to produce a source location that 763/// points just past the end of the token referenced by \p Loc, and 764/// is generally used when a diagnostic needs to point just after a 765/// token where it expected something different that it received. If 766/// the returned source location would not be meaningful (e.g., if 767/// it points into a macro), this routine returns an invalid 768/// source location. 769/// 770/// \param Offset an offset from the end of the token, where the source 771/// location should refer to. The default offset (0) produces a source 772/// location pointing just past the end of the token; an offset of 1 produces 773/// a source location pointing to the last character in the token, etc. 774SourceLocation Lexer::getLocForEndOfToken(SourceLocation Loc, unsigned Offset, 775 const SourceManager &SM, 776 const LangOptions &LangOpts) { 777 if (Loc.isInvalid()) 778 return SourceLocation(); 779 780 if (Loc.isMacroID()) { 781 if (Offset > 0 || !isAtEndOfMacroExpansion(Loc, SM, LangOpts, &Loc)) 782 return SourceLocation(); // Points inside the macro expansion. 783 } 784 785 unsigned Len = Lexer::MeasureTokenLength(Loc, SM, LangOpts); 786 if (Len > Offset) 787 Len = Len - Offset; 788 else 789 return Loc; 790 791 return Loc.getLocWithOffset(Len); 792} 793 794/// \brief Returns true if the given MacroID location points at the first 795/// token of the macro expansion. 796bool Lexer::isAtStartOfMacroExpansion(SourceLocation loc, 797 const SourceManager &SM, 798 const LangOptions &LangOpts, 799 SourceLocation *MacroBegin) { 800 assert(loc.isValid() && loc.isMacroID() && "Expected a valid macro loc"); 801 802 SourceLocation expansionLoc; 803 if (!SM.isAtStartOfImmediateMacroExpansion(loc, &expansionLoc)) 804 return false; 805 806 if (expansionLoc.isFileID()) { 807 // No other macro expansions, this is the first. 808 if (MacroBegin) 809 *MacroBegin = expansionLoc; 810 return true; 811 } 812 813 return isAtStartOfMacroExpansion(expansionLoc, SM, LangOpts, MacroBegin); 814} 815 816/// \brief Returns true if the given MacroID location points at the last 817/// token of the macro expansion. 818bool Lexer::isAtEndOfMacroExpansion(SourceLocation loc, 819 const SourceManager &SM, 820 const LangOptions &LangOpts, 821 SourceLocation *MacroEnd) { 822 assert(loc.isValid() && loc.isMacroID() && "Expected a valid macro loc"); 823 824 SourceLocation spellLoc = SM.getSpellingLoc(loc); 825 unsigned tokLen = MeasureTokenLength(spellLoc, SM, LangOpts); 826 if (tokLen == 0) 827 return false; 828 829 SourceLocation afterLoc = loc.getLocWithOffset(tokLen); 830 SourceLocation expansionLoc; 831 if (!SM.isAtEndOfImmediateMacroExpansion(afterLoc, &expansionLoc)) 832 return false; 833 834 if (expansionLoc.isFileID()) { 835 // No other macro expansions. 836 if (MacroEnd) 837 *MacroEnd = expansionLoc; 838 return true; 839 } 840 841 return isAtEndOfMacroExpansion(expansionLoc, SM, LangOpts, MacroEnd); 842} 843 844static CharSourceRange makeRangeFromFileLocs(CharSourceRange Range, 845 const SourceManager &SM, 846 const LangOptions &LangOpts) { 847 SourceLocation Begin = Range.getBegin(); 848 SourceLocation End = Range.getEnd(); 849 assert(Begin.isFileID() && End.isFileID()); 850 if (Range.isTokenRange()) { 851 End = Lexer::getLocForEndOfToken(End, 0, SM,LangOpts); 852 if (End.isInvalid()) 853 return CharSourceRange(); 854 } 855 856 // Break down the source locations. 857 FileID FID; 858 unsigned BeginOffs; 859 llvm::tie(FID, BeginOffs) = SM.getDecomposedLoc(Begin); 860 if (FID.isInvalid()) 861 return CharSourceRange(); 862 863 unsigned EndOffs; 864 if (!SM.isInFileID(End, FID, &EndOffs) || 865 BeginOffs > EndOffs) 866 return CharSourceRange(); 867 868 return CharSourceRange::getCharRange(Begin, End); 869} 870 871CharSourceRange Lexer::makeFileCharRange(CharSourceRange Range, 872 const SourceManager &SM, 873 const LangOptions &LangOpts) { 874 SourceLocation Begin = Range.getBegin(); 875 SourceLocation End = Range.getEnd(); 876 if (Begin.isInvalid() || End.isInvalid()) 877 return CharSourceRange(); 878 879 if (Begin.isFileID() && End.isFileID()) 880 return makeRangeFromFileLocs(Range, SM, LangOpts); 881 882 if (Begin.isMacroID() && End.isFileID()) { 883 if (!isAtStartOfMacroExpansion(Begin, SM, LangOpts, &Begin)) 884 return CharSourceRange(); 885 Range.setBegin(Begin); 886 return makeRangeFromFileLocs(Range, SM, LangOpts); 887 } 888 889 if (Begin.isFileID() && End.isMacroID()) { 890 if ((Range.isTokenRange() && !isAtEndOfMacroExpansion(End, SM, LangOpts, 891 &End)) || 892 (Range.isCharRange() && !isAtStartOfMacroExpansion(End, SM, LangOpts, 893 &End))) 894 return CharSourceRange(); 895 Range.setEnd(End); 896 return makeRangeFromFileLocs(Range, SM, LangOpts); 897 } 898 899 assert(Begin.isMacroID() && End.isMacroID()); 900 SourceLocation MacroBegin, MacroEnd; 901 if (isAtStartOfMacroExpansion(Begin, SM, LangOpts, &MacroBegin) && 902 ((Range.isTokenRange() && isAtEndOfMacroExpansion(End, SM, LangOpts, 903 &MacroEnd)) || 904 (Range.isCharRange() && isAtStartOfMacroExpansion(End, SM, LangOpts, 905 &MacroEnd)))) { 906 Range.setBegin(MacroBegin); 907 Range.setEnd(MacroEnd); 908 return makeRangeFromFileLocs(Range, SM, LangOpts); 909 } 910 911 bool Invalid = false; 912 const SrcMgr::SLocEntry &BeginEntry = SM.getSLocEntry(SM.getFileID(Begin), 913 &Invalid); 914 if (Invalid) 915 return CharSourceRange(); 916 917 if (BeginEntry.getExpansion().isMacroArgExpansion()) { 918 const SrcMgr::SLocEntry &EndEntry = SM.getSLocEntry(SM.getFileID(End), 919 &Invalid); 920 if (Invalid) 921 return CharSourceRange(); 922 923 if (EndEntry.getExpansion().isMacroArgExpansion() && 924 BeginEntry.getExpansion().getExpansionLocStart() == 925 EndEntry.getExpansion().getExpansionLocStart()) { 926 Range.setBegin(SM.getImmediateSpellingLoc(Begin)); 927 Range.setEnd(SM.getImmediateSpellingLoc(End)); 928 return makeFileCharRange(Range, SM, LangOpts); 929 } 930 } 931 932 return CharSourceRange(); 933} 934 935StringRef Lexer::getSourceText(CharSourceRange Range, 936 const SourceManager &SM, 937 const LangOptions &LangOpts, 938 bool *Invalid) { 939 Range = makeFileCharRange(Range, SM, LangOpts); 940 if (Range.isInvalid()) { 941 if (Invalid) *Invalid = true; 942 return StringRef(); 943 } 944 945 // Break down the source location. 946 std::pair<FileID, unsigned> beginInfo = SM.getDecomposedLoc(Range.getBegin()); 947 if (beginInfo.first.isInvalid()) { 948 if (Invalid) *Invalid = true; 949 return StringRef(); 950 } 951 952 unsigned EndOffs; 953 if (!SM.isInFileID(Range.getEnd(), beginInfo.first, &EndOffs) || 954 beginInfo.second > EndOffs) { 955 if (Invalid) *Invalid = true; 956 return StringRef(); 957 } 958 959 // Try to the load the file buffer. 960 bool invalidTemp = false; 961 StringRef file = SM.getBufferData(beginInfo.first, &invalidTemp); 962 if (invalidTemp) { 963 if (Invalid) *Invalid = true; 964 return StringRef(); 965 } 966 967 if (Invalid) *Invalid = false; 968 return file.substr(beginInfo.second, EndOffs - beginInfo.second); 969} 970 971StringRef Lexer::getImmediateMacroName(SourceLocation Loc, 972 const SourceManager &SM, 973 const LangOptions &LangOpts) { 974 assert(Loc.isMacroID() && "Only reasonble to call this on macros"); 975 976 // Find the location of the immediate macro expansion. 977 while (1) { 978 FileID FID = SM.getFileID(Loc); 979 const SrcMgr::SLocEntry *E = &SM.getSLocEntry(FID); 980 const SrcMgr::ExpansionInfo &Expansion = E->getExpansion(); 981 Loc = Expansion.getExpansionLocStart(); 982 if (!Expansion.isMacroArgExpansion()) 983 break; 984 985 // For macro arguments we need to check that the argument did not come 986 // from an inner macro, e.g: "MAC1( MAC2(foo) )" 987 988 // Loc points to the argument id of the macro definition, move to the 989 // macro expansion. 990 Loc = SM.getImmediateExpansionRange(Loc).first; 991 SourceLocation SpellLoc = Expansion.getSpellingLoc(); 992 if (SpellLoc.isFileID()) 993 break; // No inner macro. 994 995 // If spelling location resides in the same FileID as macro expansion 996 // location, it means there is no inner macro. 997 FileID MacroFID = SM.getFileID(Loc); 998 if (SM.isInFileID(SpellLoc, MacroFID)) 999 break; 1000 1001 // Argument came from inner macro. 1002 Loc = SpellLoc; 1003 } 1004 1005 // Find the spelling location of the start of the non-argument expansion 1006 // range. This is where the macro name was spelled in order to begin 1007 // expanding this macro. 1008 Loc = SM.getSpellingLoc(Loc); 1009 1010 // Dig out the buffer where the macro name was spelled and the extents of the 1011 // name so that we can render it into the expansion note. 1012 std::pair<FileID, unsigned> ExpansionInfo = SM.getDecomposedLoc(Loc); 1013 unsigned MacroTokenLength = Lexer::MeasureTokenLength(Loc, SM, LangOpts); 1014 StringRef ExpansionBuffer = SM.getBufferData(ExpansionInfo.first); 1015 return ExpansionBuffer.substr(ExpansionInfo.second, MacroTokenLength); 1016} 1017 1018bool Lexer::isIdentifierBodyChar(char c, const LangOptions &LangOpts) { 1019 return isIdentifierBody(c, LangOpts.DollarIdents); 1020} 1021 1022 1023//===----------------------------------------------------------------------===// 1024// Diagnostics forwarding code. 1025//===----------------------------------------------------------------------===// 1026 1027/// GetMappedTokenLoc - If lexing out of a 'mapped buffer', where we pretend the 1028/// lexer buffer was all expanded at a single point, perform the mapping. 1029/// This is currently only used for _Pragma implementation, so it is the slow 1030/// path of the hot getSourceLocation method. Do not allow it to be inlined. 1031static LLVM_ATTRIBUTE_NOINLINE SourceLocation GetMappedTokenLoc( 1032 Preprocessor &PP, SourceLocation FileLoc, unsigned CharNo, unsigned TokLen); 1033static SourceLocation GetMappedTokenLoc(Preprocessor &PP, 1034 SourceLocation FileLoc, 1035 unsigned CharNo, unsigned TokLen) { 1036 assert(FileLoc.isMacroID() && "Must be a macro expansion"); 1037 1038 // Otherwise, we're lexing "mapped tokens". This is used for things like 1039 // _Pragma handling. Combine the expansion location of FileLoc with the 1040 // spelling location. 1041 SourceManager &SM = PP.getSourceManager(); 1042 1043 // Create a new SLoc which is expanded from Expansion(FileLoc) but whose 1044 // characters come from spelling(FileLoc)+Offset. 1045 SourceLocation SpellingLoc = SM.getSpellingLoc(FileLoc); 1046 SpellingLoc = SpellingLoc.getLocWithOffset(CharNo); 1047 1048 // Figure out the expansion loc range, which is the range covered by the 1049 // original _Pragma(...) sequence. 1050 std::pair<SourceLocation,SourceLocation> II = 1051 SM.getImmediateExpansionRange(FileLoc); 1052 1053 return SM.createExpansionLoc(SpellingLoc, II.first, II.second, TokLen); 1054} 1055 1056/// getSourceLocation - Return a source location identifier for the specified 1057/// offset in the current file. 1058SourceLocation Lexer::getSourceLocation(const char *Loc, 1059 unsigned TokLen) const { 1060 assert(Loc >= BufferStart && Loc <= BufferEnd && 1061 "Location out of range for this buffer!"); 1062 1063 // In the normal case, we're just lexing from a simple file buffer, return 1064 // the file id from FileLoc with the offset specified. 1065 unsigned CharNo = Loc-BufferStart; 1066 if (FileLoc.isFileID()) 1067 return FileLoc.getLocWithOffset(CharNo); 1068 1069 // Otherwise, this is the _Pragma lexer case, which pretends that all of the 1070 // tokens are lexed from where the _Pragma was defined. 1071 assert(PP && "This doesn't work on raw lexers"); 1072 return GetMappedTokenLoc(*PP, FileLoc, CharNo, TokLen); 1073} 1074 1075/// Diag - Forwarding function for diagnostics. This translate a source 1076/// position in the current buffer into a SourceLocation object for rendering. 1077DiagnosticBuilder Lexer::Diag(const char *Loc, unsigned DiagID) const { 1078 return PP->Diag(getSourceLocation(Loc), DiagID); 1079} 1080 1081//===----------------------------------------------------------------------===// 1082// Trigraph and Escaped Newline Handling Code. 1083//===----------------------------------------------------------------------===// 1084 1085/// GetTrigraphCharForLetter - Given a character that occurs after a ?? pair, 1086/// return the decoded trigraph letter it corresponds to, or '\0' if nothing. 1087static char GetTrigraphCharForLetter(char Letter) { 1088 switch (Letter) { 1089 default: return 0; 1090 case '=': return '#'; 1091 case ')': return ']'; 1092 case '(': return '['; 1093 case '!': return '|'; 1094 case '\'': return '^'; 1095 case '>': return '}'; 1096 case '/': return '\\'; 1097 case '<': return '{'; 1098 case '-': return '~'; 1099 } 1100} 1101 1102/// DecodeTrigraphChar - If the specified character is a legal trigraph when 1103/// prefixed with ??, emit a trigraph warning. If trigraphs are enabled, 1104/// return the result character. Finally, emit a warning about trigraph use 1105/// whether trigraphs are enabled or not. 1106static char DecodeTrigraphChar(const char *CP, Lexer *L) { 1107 char Res = GetTrigraphCharForLetter(*CP); 1108 if (!Res || !L) return Res; 1109 1110 if (!L->getLangOpts().Trigraphs) { 1111 if (!L->isLexingRawMode()) 1112 L->Diag(CP-2, diag::trigraph_ignored); 1113 return 0; 1114 } 1115 1116 if (!L->isLexingRawMode()) 1117 L->Diag(CP-2, diag::trigraph_converted) << StringRef(&Res, 1); 1118 return Res; 1119} 1120 1121/// getEscapedNewLineSize - Return the size of the specified escaped newline, 1122/// or 0 if it is not an escaped newline. P[-1] is known to be a "\" or a 1123/// trigraph equivalent on entry to this function. 1124unsigned Lexer::getEscapedNewLineSize(const char *Ptr) { 1125 unsigned Size = 0; 1126 while (isWhitespace(Ptr[Size])) { 1127 ++Size; 1128 1129 if (Ptr[Size-1] != '\n' && Ptr[Size-1] != '\r') 1130 continue; 1131 1132 // If this is a \r\n or \n\r, skip the other half. 1133 if ((Ptr[Size] == '\r' || Ptr[Size] == '\n') && 1134 Ptr[Size-1] != Ptr[Size]) 1135 ++Size; 1136 1137 return Size; 1138 } 1139 1140 // Not an escaped newline, must be a \t or something else. 1141 return 0; 1142} 1143 1144/// SkipEscapedNewLines - If P points to an escaped newline (or a series of 1145/// them), skip over them and return the first non-escaped-newline found, 1146/// otherwise return P. 1147const char *Lexer::SkipEscapedNewLines(const char *P) { 1148 while (1) { 1149 const char *AfterEscape; 1150 if (*P == '\\') { 1151 AfterEscape = P+1; 1152 } else if (*P == '?') { 1153 // If not a trigraph for escape, bail out. 1154 if (P[1] != '?' || P[2] != '/') 1155 return P; 1156 AfterEscape = P+3; 1157 } else { 1158 return P; 1159 } 1160 1161 unsigned NewLineSize = Lexer::getEscapedNewLineSize(AfterEscape); 1162 if (NewLineSize == 0) return P; 1163 P = AfterEscape+NewLineSize; 1164 } 1165} 1166 1167/// \brief Checks that the given token is the first token that occurs after the 1168/// given location (this excludes comments and whitespace). Returns the location 1169/// immediately after the specified token. If the token is not found or the 1170/// location is inside a macro, the returned source location will be invalid. 1171SourceLocation Lexer::findLocationAfterToken(SourceLocation Loc, 1172 tok::TokenKind TKind, 1173 const SourceManager &SM, 1174 const LangOptions &LangOpts, 1175 bool SkipTrailingWhitespaceAndNewLine) { 1176 if (Loc.isMacroID()) { 1177 if (!Lexer::isAtEndOfMacroExpansion(Loc, SM, LangOpts, &Loc)) 1178 return SourceLocation(); 1179 } 1180 Loc = Lexer::getLocForEndOfToken(Loc, 0, SM, LangOpts); 1181 1182 // Break down the source location. 1183 std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(Loc); 1184 1185 // Try to load the file buffer. 1186 bool InvalidTemp = false; 1187 StringRef File = SM.getBufferData(LocInfo.first, &InvalidTemp); 1188 if (InvalidTemp) 1189 return SourceLocation(); 1190 1191 const char *TokenBegin = File.data() + LocInfo.second; 1192 1193 // Lex from the start of the given location. 1194 Lexer lexer(SM.getLocForStartOfFile(LocInfo.first), LangOpts, File.begin(), 1195 TokenBegin, File.end()); 1196 // Find the token. 1197 Token Tok; 1198 lexer.LexFromRawLexer(Tok); 1199 if (Tok.isNot(TKind)) 1200 return SourceLocation(); 1201 SourceLocation TokenLoc = Tok.getLocation(); 1202 1203 // Calculate how much whitespace needs to be skipped if any. 1204 unsigned NumWhitespaceChars = 0; 1205 if (SkipTrailingWhitespaceAndNewLine) { 1206 const char *TokenEnd = SM.getCharacterData(TokenLoc) + 1207 Tok.getLength(); 1208 unsigned char C = *TokenEnd; 1209 while (isHorizontalWhitespace(C)) { 1210 C = *(++TokenEnd); 1211 NumWhitespaceChars++; 1212 } 1213 1214 // Skip \r, \n, \r\n, or \n\r 1215 if (C == '\n' || C == '\r') { 1216 char PrevC = C; 1217 C = *(++TokenEnd); 1218 NumWhitespaceChars++; 1219 if ((C == '\n' || C == '\r') && C != PrevC) 1220 NumWhitespaceChars++; 1221 } 1222 } 1223 1224 return TokenLoc.getLocWithOffset(Tok.getLength() + NumWhitespaceChars); 1225} 1226 1227/// getCharAndSizeSlow - Peek a single 'character' from the specified buffer, 1228/// get its size, and return it. This is tricky in several cases: 1229/// 1. If currently at the start of a trigraph, we warn about the trigraph, 1230/// then either return the trigraph (skipping 3 chars) or the '?', 1231/// depending on whether trigraphs are enabled or not. 1232/// 2. If this is an escaped newline (potentially with whitespace between 1233/// the backslash and newline), implicitly skip the newline and return 1234/// the char after it. 1235/// 1236/// This handles the slow/uncommon case of the getCharAndSize method. Here we 1237/// know that we can accumulate into Size, and that we have already incremented 1238/// Ptr by Size bytes. 1239/// 1240/// NOTE: When this method is updated, getCharAndSizeSlowNoWarn (below) should 1241/// be updated to match. 1242/// 1243char Lexer::getCharAndSizeSlow(const char *Ptr, unsigned &Size, 1244 Token *Tok) { 1245 // If we have a slash, look for an escaped newline. 1246 if (Ptr[0] == '\\') { 1247 ++Size; 1248 ++Ptr; 1249Slash: 1250 // Common case, backslash-char where the char is not whitespace. 1251 if (!isWhitespace(Ptr[0])) return '\\'; 1252 1253 // See if we have optional whitespace characters between the slash and 1254 // newline. 1255 if (unsigned EscapedNewLineSize = getEscapedNewLineSize(Ptr)) { 1256 // Remember that this token needs to be cleaned. 1257 if (Tok) Tok->setFlag(Token::NeedsCleaning); 1258 1259 // Warn if there was whitespace between the backslash and newline. 1260 if (Ptr[0] != '\n' && Ptr[0] != '\r' && Tok && !isLexingRawMode()) 1261 Diag(Ptr, diag::backslash_newline_space); 1262 1263 // Found backslash<whitespace><newline>. Parse the char after it. 1264 Size += EscapedNewLineSize; 1265 Ptr += EscapedNewLineSize; 1266 1267 // If the char that we finally got was a \n, then we must have had 1268 // something like \<newline><newline>. We don't want to consume the 1269 // second newline. 1270 if (*Ptr == '\n' || *Ptr == '\r' || *Ptr == '\0') 1271 return ' '; 1272 1273 // Use slow version to accumulate a correct size field. 1274 return getCharAndSizeSlow(Ptr, Size, Tok); 1275 } 1276 1277 // Otherwise, this is not an escaped newline, just return the slash. 1278 return '\\'; 1279 } 1280 1281 // If this is a trigraph, process it. 1282 if (Ptr[0] == '?' && Ptr[1] == '?') { 1283 // If this is actually a legal trigraph (not something like "??x"), emit 1284 // a trigraph warning. If so, and if trigraphs are enabled, return it. 1285 if (char C = DecodeTrigraphChar(Ptr+2, Tok ? this : 0)) { 1286 // Remember that this token needs to be cleaned. 1287 if (Tok) Tok->setFlag(Token::NeedsCleaning); 1288 1289 Ptr += 3; 1290 Size += 3; 1291 if (C == '\\') goto Slash; 1292 return C; 1293 } 1294 } 1295 1296 // If this is neither, return a single character. 1297 ++Size; 1298 return *Ptr; 1299} 1300 1301 1302/// getCharAndSizeSlowNoWarn - Handle the slow/uncommon case of the 1303/// getCharAndSizeNoWarn method. Here we know that we can accumulate into Size, 1304/// and that we have already incremented Ptr by Size bytes. 1305/// 1306/// NOTE: When this method is updated, getCharAndSizeSlow (above) should 1307/// be updated to match. 1308char Lexer::getCharAndSizeSlowNoWarn(const char *Ptr, unsigned &Size, 1309 const LangOptions &LangOpts) { 1310 // If we have a slash, look for an escaped newline. 1311 if (Ptr[0] == '\\') { 1312 ++Size; 1313 ++Ptr; 1314Slash: 1315 // Common case, backslash-char where the char is not whitespace. 1316 if (!isWhitespace(Ptr[0])) return '\\'; 1317 1318 // See if we have optional whitespace characters followed by a newline. 1319 if (unsigned EscapedNewLineSize = getEscapedNewLineSize(Ptr)) { 1320 // Found backslash<whitespace><newline>. Parse the char after it. 1321 Size += EscapedNewLineSize; 1322 Ptr += EscapedNewLineSize; 1323 1324 // If the char that we finally got was a \n, then we must have had 1325 // something like \<newline><newline>. We don't want to consume the 1326 // second newline. 1327 if (*Ptr == '\n' || *Ptr == '\r' || *Ptr == '\0') 1328 return ' '; 1329 1330 // Use slow version to accumulate a correct size field. 1331 return getCharAndSizeSlowNoWarn(Ptr, Size, LangOpts); 1332 } 1333 1334 // Otherwise, this is not an escaped newline, just return the slash. 1335 return '\\'; 1336 } 1337 1338 // If this is a trigraph, process it. 1339 if (LangOpts.Trigraphs && Ptr[0] == '?' && Ptr[1] == '?') { 1340 // If this is actually a legal trigraph (not something like "??x"), return 1341 // it. 1342 if (char C = GetTrigraphCharForLetter(Ptr[2])) { 1343 Ptr += 3; 1344 Size += 3; 1345 if (C == '\\') goto Slash; 1346 return C; 1347 } 1348 } 1349 1350 // If this is neither, return a single character. 1351 ++Size; 1352 return *Ptr; 1353} 1354 1355//===----------------------------------------------------------------------===// 1356// Helper methods for lexing. 1357//===----------------------------------------------------------------------===// 1358 1359/// \brief Routine that indiscriminately skips bytes in the source file. 1360void Lexer::SkipBytes(unsigned Bytes, bool StartOfLine) { 1361 BufferPtr += Bytes; 1362 if (BufferPtr > BufferEnd) 1363 BufferPtr = BufferEnd; 1364 IsAtStartOfLine = StartOfLine; 1365} 1366 1367static bool isAllowedIDChar(uint32_t C, const LangOptions &LangOpts) { 1368 if (LangOpts.CPlusPlus11 || LangOpts.C11) { 1369 static const llvm::sys::UnicodeCharSet C11AllowedIDChars( 1370 C11AllowedIDCharRanges); 1371 return C11AllowedIDChars.contains(C); 1372 } else if (LangOpts.CPlusPlus) { 1373 static const llvm::sys::UnicodeCharSet CXX03AllowedIDChars( 1374 CXX03AllowedIDCharRanges); 1375 return CXX03AllowedIDChars.contains(C); 1376 } else { 1377 static const llvm::sys::UnicodeCharSet C99AllowedIDChars( 1378 C99AllowedIDCharRanges); 1379 return C99AllowedIDChars.contains(C); 1380 } 1381} 1382 1383static bool isAllowedInitiallyIDChar(uint32_t C, const LangOptions &LangOpts) { 1384 assert(isAllowedIDChar(C, LangOpts)); 1385 if (LangOpts.CPlusPlus11 || LangOpts.C11) { 1386 static const llvm::sys::UnicodeCharSet C11DisallowedInitialIDChars( 1387 C11DisallowedInitialIDCharRanges); 1388 return !C11DisallowedInitialIDChars.contains(C); 1389 } else if (LangOpts.CPlusPlus) { 1390 return true; 1391 } else { 1392 static const llvm::sys::UnicodeCharSet C99DisallowedInitialIDChars( 1393 C99DisallowedInitialIDCharRanges); 1394 return !C99DisallowedInitialIDChars.contains(C); 1395 } 1396} 1397 1398static inline CharSourceRange makeCharRange(Lexer &L, const char *Begin, 1399 const char *End) { 1400 return CharSourceRange::getCharRange(L.getSourceLocation(Begin), 1401 L.getSourceLocation(End)); 1402} 1403 1404static void maybeDiagnoseIDCharCompat(DiagnosticsEngine &Diags, uint32_t C, 1405 CharSourceRange Range, bool IsFirst) { 1406 // Check C99 compatibility. 1407 if (Diags.getDiagnosticLevel(diag::warn_c99_compat_unicode_id, 1408 Range.getBegin()) > DiagnosticsEngine::Ignored) { 1409 enum { 1410 CannotAppearInIdentifier = 0, 1411 CannotStartIdentifier 1412 }; 1413 1414 static const llvm::sys::UnicodeCharSet C99AllowedIDChars( 1415 C99AllowedIDCharRanges); 1416 static const llvm::sys::UnicodeCharSet C99DisallowedInitialIDChars( 1417 C99DisallowedInitialIDCharRanges); 1418 if (!C99AllowedIDChars.contains(C)) { 1419 Diags.Report(Range.getBegin(), diag::warn_c99_compat_unicode_id) 1420 << Range 1421 << CannotAppearInIdentifier; 1422 } else if (IsFirst && C99DisallowedInitialIDChars.contains(C)) { 1423 Diags.Report(Range.getBegin(), diag::warn_c99_compat_unicode_id) 1424 << Range 1425 << CannotStartIdentifier; 1426 } 1427 } 1428 1429 // Check C++98 compatibility. 1430 if (Diags.getDiagnosticLevel(diag::warn_cxx98_compat_unicode_id, 1431 Range.getBegin()) > DiagnosticsEngine::Ignored) { 1432 static const llvm::sys::UnicodeCharSet CXX03AllowedIDChars( 1433 CXX03AllowedIDCharRanges); 1434 if (!CXX03AllowedIDChars.contains(C)) { 1435 Diags.Report(Range.getBegin(), diag::warn_cxx98_compat_unicode_id) 1436 << Range; 1437 } 1438 } 1439 } 1440 1441void Lexer::LexIdentifier(Token &Result, const char *CurPtr) { 1442 // Match [_A-Za-z0-9]*, we have already matched [_A-Za-z$] 1443 unsigned Size; 1444 unsigned char C = *CurPtr++; 1445 while (isIdentifierBody(C)) 1446 C = *CurPtr++; 1447 1448 --CurPtr; // Back up over the skipped character. 1449 1450 // Fast path, no $,\,? in identifier found. '\' might be an escaped newline 1451 // or UCN, and ? might be a trigraph for '\', an escaped newline or UCN. 1452 // 1453 // TODO: Could merge these checks into an InfoTable flag to make the 1454 // comparison cheaper 1455 if (isASCII(C) && C != '\\' && C != '?' && 1456 (C != '$' || !LangOpts.DollarIdents)) { 1457FinishIdentifier: 1458 const char *IdStart = BufferPtr; 1459 FormTokenWithChars(Result, CurPtr, tok::raw_identifier); 1460 Result.setRawIdentifierData(IdStart); 1461 1462 // If we are in raw mode, return this identifier raw. There is no need to 1463 // look up identifier information or attempt to macro expand it. 1464 if (LexingRawMode) 1465 return; 1466 1467 // Fill in Result.IdentifierInfo and update the token kind, 1468 // looking up the identifier in the identifier table. 1469 IdentifierInfo *II = PP->LookUpIdentifierInfo(Result); 1470 1471 // Finally, now that we know we have an identifier, pass this off to the 1472 // preprocessor, which may macro expand it or something. 1473 if (II->isHandleIdentifierCase()) 1474 PP->HandleIdentifier(Result); 1475 1476 return; 1477 } 1478 1479 // Otherwise, $,\,? in identifier found. Enter slower path. 1480 1481 C = getCharAndSize(CurPtr, Size); 1482 while (1) { 1483 if (C == '$') { 1484 // If we hit a $ and they are not supported in identifiers, we are done. 1485 if (!LangOpts.DollarIdents) goto FinishIdentifier; 1486 1487 // Otherwise, emit a diagnostic and continue. 1488 if (!isLexingRawMode()) 1489 Diag(CurPtr, diag::ext_dollar_in_identifier); 1490 CurPtr = ConsumeChar(CurPtr, Size, Result); 1491 C = getCharAndSize(CurPtr, Size); 1492 continue; 1493 1494 } else if (C == '\\') { 1495 const char *UCNPtr = CurPtr + Size; 1496 uint32_t CodePoint = tryReadUCN(UCNPtr, CurPtr, /*Token=*/0); 1497 if (CodePoint == 0 || !isAllowedIDChar(CodePoint, LangOpts)) 1498 goto FinishIdentifier; 1499 1500 if (!isLexingRawMode()) { 1501 maybeDiagnoseIDCharCompat(PP->getDiagnostics(), CodePoint, 1502 makeCharRange(*this, CurPtr, UCNPtr), 1503 /*IsFirst=*/false); 1504 } 1505 1506 Result.setFlag(Token::HasUCN); 1507 if ((UCNPtr - CurPtr == 6 && CurPtr[1] == 'u') || 1508 (UCNPtr - CurPtr == 10 && CurPtr[1] == 'U')) 1509 CurPtr = UCNPtr; 1510 else 1511 while (CurPtr != UCNPtr) 1512 (void)getAndAdvanceChar(CurPtr, Result); 1513 1514 C = getCharAndSize(CurPtr, Size); 1515 continue; 1516 } else if (!isASCII(C)) { 1517 const char *UnicodePtr = CurPtr; 1518 UTF32 CodePoint; 1519 ConversionResult Result = 1520 llvm::convertUTF8Sequence((const UTF8 **)&UnicodePtr, 1521 (const UTF8 *)BufferEnd, 1522 &CodePoint, 1523 strictConversion); 1524 if (Result != conversionOK || 1525 !isAllowedIDChar(static_cast<uint32_t>(CodePoint), LangOpts)) 1526 goto FinishIdentifier; 1527 1528 if (!isLexingRawMode()) { 1529 maybeDiagnoseIDCharCompat(PP->getDiagnostics(), CodePoint, 1530 makeCharRange(*this, CurPtr, UnicodePtr), 1531 /*IsFirst=*/false); 1532 } 1533 1534 CurPtr = UnicodePtr; 1535 C = getCharAndSize(CurPtr, Size); 1536 continue; 1537 } else if (!isIdentifierBody(C)) { 1538 goto FinishIdentifier; 1539 } 1540 1541 // Otherwise, this character is good, consume it. 1542 CurPtr = ConsumeChar(CurPtr, Size, Result); 1543 1544 C = getCharAndSize(CurPtr, Size); 1545 while (isIdentifierBody(C)) { 1546 CurPtr = ConsumeChar(CurPtr, Size, Result); 1547 C = getCharAndSize(CurPtr, Size); 1548 } 1549 } 1550} 1551 1552/// isHexaLiteral - Return true if Start points to a hex constant. 1553/// in microsoft mode (where this is supposed to be several different tokens). 1554bool Lexer::isHexaLiteral(const char *Start, const LangOptions &LangOpts) { 1555 unsigned Size; 1556 char C1 = Lexer::getCharAndSizeNoWarn(Start, Size, LangOpts); 1557 if (C1 != '0') 1558 return false; 1559 char C2 = Lexer::getCharAndSizeNoWarn(Start + Size, Size, LangOpts); 1560 return (C2 == 'x' || C2 == 'X'); 1561} 1562 1563/// LexNumericConstant - Lex the remainder of a integer or floating point 1564/// constant. From[-1] is the first character lexed. Return the end of the 1565/// constant. 1566void Lexer::LexNumericConstant(Token &Result, const char *CurPtr) { 1567 unsigned Size; 1568 char C = getCharAndSize(CurPtr, Size); 1569 char PrevCh = 0; 1570 while (isPreprocessingNumberBody(C)) { // FIXME: UCNs in ud-suffix. 1571 CurPtr = ConsumeChar(CurPtr, Size, Result); 1572 PrevCh = C; 1573 C = getCharAndSize(CurPtr, Size); 1574 } 1575 1576 // If we fell out, check for a sign, due to 1e+12. If we have one, continue. 1577 if ((C == '-' || C == '+') && (PrevCh == 'E' || PrevCh == 'e')) { 1578 // If we are in Microsoft mode, don't continue if the constant is hex. 1579 // For example, MSVC will accept the following as 3 tokens: 0x1234567e+1 1580 if (!LangOpts.MicrosoftExt || !isHexaLiteral(BufferPtr, LangOpts)) 1581 return LexNumericConstant(Result, ConsumeChar(CurPtr, Size, Result)); 1582 } 1583 1584 // If we have a hex FP constant, continue. 1585 if ((C == '-' || C == '+') && (PrevCh == 'P' || PrevCh == 'p')) { 1586 // Outside C99, we accept hexadecimal floating point numbers as a 1587 // not-quite-conforming extension. Only do so if this looks like it's 1588 // actually meant to be a hexfloat, and not if it has a ud-suffix. 1589 bool IsHexFloat = true; 1590 if (!LangOpts.C99) { 1591 if (!isHexaLiteral(BufferPtr, LangOpts)) 1592 IsHexFloat = false; 1593 else if (std::find(BufferPtr, CurPtr, '_') != CurPtr) 1594 IsHexFloat = false; 1595 } 1596 if (IsHexFloat) 1597 return LexNumericConstant(Result, ConsumeChar(CurPtr, Size, Result)); 1598 } 1599 1600 // Update the location of token as well as BufferPtr. 1601 const char *TokStart = BufferPtr; 1602 FormTokenWithChars(Result, CurPtr, tok::numeric_constant); 1603 Result.setLiteralData(TokStart); 1604} 1605 1606/// LexUDSuffix - Lex the ud-suffix production for user-defined literal suffixes 1607/// in C++11, or warn on a ud-suffix in C++98. 1608const char *Lexer::LexUDSuffix(Token &Result, const char *CurPtr, 1609 bool IsStringLiteral) { 1610 assert(getLangOpts().CPlusPlus); 1611 1612 // Maximally munch an identifier. FIXME: UCNs. 1613 unsigned Size; 1614 char C = getCharAndSize(CurPtr, Size); 1615 if (isIdentifierHead(C)) { 1616 if (!getLangOpts().CPlusPlus11) { 1617 if (!isLexingRawMode()) 1618 Diag(CurPtr, 1619 C == '_' ? diag::warn_cxx11_compat_user_defined_literal 1620 : diag::warn_cxx11_compat_reserved_user_defined_literal) 1621 << FixItHint::CreateInsertion(getSourceLocation(CurPtr), " "); 1622 return CurPtr; 1623 } 1624 1625 // C++11 [lex.ext]p10, [usrlit.suffix]p1: A program containing a ud-suffix 1626 // that does not start with an underscore is ill-formed. As a conforming 1627 // extension, we treat all such suffixes as if they had whitespace before 1628 // them. 1629 bool IsUDSuffix = false; 1630 if (C == '_') 1631 IsUDSuffix = true; 1632 else if (IsStringLiteral && C == 's' && getLangOpts().CPlusPlus1y) { 1633 // In C++1y, "s" is a valid ud-suffix for a string literal. 1634 unsigned NextSize; 1635 if (!isIdentifierBody(getCharAndSizeNoWarn(CurPtr + Size, NextSize, 1636 getLangOpts()))) 1637 IsUDSuffix = true; 1638 } 1639 1640 if (!IsUDSuffix) { 1641 if (!isLexingRawMode()) 1642 Diag(CurPtr, getLangOpts().MicrosoftMode ? 1643 diag::ext_ms_reserved_user_defined_literal : 1644 diag::ext_reserved_user_defined_literal) 1645 << FixItHint::CreateInsertion(getSourceLocation(CurPtr), " "); 1646 return CurPtr; 1647 } 1648 1649 Result.setFlag(Token::HasUDSuffix); 1650 do { 1651 CurPtr = ConsumeChar(CurPtr, Size, Result); 1652 C = getCharAndSize(CurPtr, Size); 1653 } while (isIdentifierBody(C)); 1654 } 1655 return CurPtr; 1656} 1657 1658/// LexStringLiteral - Lex the remainder of a string literal, after having lexed 1659/// either " or L" or u8" or u" or U". 1660void Lexer::LexStringLiteral(Token &Result, const char *CurPtr, 1661 tok::TokenKind Kind) { 1662 const char *NulCharacter = 0; // Does this string contain the \0 character? 1663 1664 if (!isLexingRawMode() && 1665 (Kind == tok::utf8_string_literal || 1666 Kind == tok::utf16_string_literal || 1667 Kind == tok::utf32_string_literal)) 1668 Diag(BufferPtr, getLangOpts().CPlusPlus 1669 ? diag::warn_cxx98_compat_unicode_literal 1670 : diag::warn_c99_compat_unicode_literal); 1671 1672 char C = getAndAdvanceChar(CurPtr, Result); 1673 while (C != '"') { 1674 // Skip escaped characters. Escaped newlines will already be processed by 1675 // getAndAdvanceChar. 1676 if (C == '\\') 1677 C = getAndAdvanceChar(CurPtr, Result); 1678 1679 if (C == '\n' || C == '\r' || // Newline. 1680 (C == 0 && CurPtr-1 == BufferEnd)) { // End of file. 1681 if (!isLexingRawMode() && !LangOpts.AsmPreprocessor) 1682 Diag(BufferPtr, diag::ext_unterminated_string); 1683 FormTokenWithChars(Result, CurPtr-1, tok::unknown); 1684 return; 1685 } 1686 1687 if (C == 0) { 1688 if (isCodeCompletionPoint(CurPtr-1)) { 1689 PP->CodeCompleteNaturalLanguage(); 1690 FormTokenWithChars(Result, CurPtr-1, tok::unknown); 1691 return cutOffLexing(); 1692 } 1693 1694 NulCharacter = CurPtr-1; 1695 } 1696 C = getAndAdvanceChar(CurPtr, Result); 1697 } 1698 1699 // If we are in C++11, lex the optional ud-suffix. 1700 if (getLangOpts().CPlusPlus) 1701 CurPtr = LexUDSuffix(Result, CurPtr, true); 1702 1703 // If a nul character existed in the string, warn about it. 1704 if (NulCharacter && !isLexingRawMode()) 1705 Diag(NulCharacter, diag::null_in_string); 1706 1707 // Update the location of the token as well as the BufferPtr instance var. 1708 const char *TokStart = BufferPtr; 1709 FormTokenWithChars(Result, CurPtr, Kind); 1710 Result.setLiteralData(TokStart); 1711} 1712 1713/// LexRawStringLiteral - Lex the remainder of a raw string literal, after 1714/// having lexed R", LR", u8R", uR", or UR". 1715void Lexer::LexRawStringLiteral(Token &Result, const char *CurPtr, 1716 tok::TokenKind Kind) { 1717 // This function doesn't use getAndAdvanceChar because C++0x [lex.pptoken]p3: 1718 // Between the initial and final double quote characters of the raw string, 1719 // any transformations performed in phases 1 and 2 (trigraphs, 1720 // universal-character-names, and line splicing) are reverted. 1721 1722 if (!isLexingRawMode()) 1723 Diag(BufferPtr, diag::warn_cxx98_compat_raw_string_literal); 1724 1725 unsigned PrefixLen = 0; 1726 1727 while (PrefixLen != 16 && isRawStringDelimBody(CurPtr[PrefixLen])) 1728 ++PrefixLen; 1729 1730 // If the last character was not a '(', then we didn't lex a valid delimiter. 1731 if (CurPtr[PrefixLen] != '(') { 1732 if (!isLexingRawMode()) { 1733 const char *PrefixEnd = &CurPtr[PrefixLen]; 1734 if (PrefixLen == 16) { 1735 Diag(PrefixEnd, diag::err_raw_delim_too_long); 1736 } else { 1737 Diag(PrefixEnd, diag::err_invalid_char_raw_delim) 1738 << StringRef(PrefixEnd, 1); 1739 } 1740 } 1741 1742 // Search for the next '"' in hopes of salvaging the lexer. Unfortunately, 1743 // it's possible the '"' was intended to be part of the raw string, but 1744 // there's not much we can do about that. 1745 while (1) { 1746 char C = *CurPtr++; 1747 1748 if (C == '"') 1749 break; 1750 if (C == 0 && CurPtr-1 == BufferEnd) { 1751 --CurPtr; 1752 break; 1753 } 1754 } 1755 1756 FormTokenWithChars(Result, CurPtr, tok::unknown); 1757 return; 1758 } 1759 1760 // Save prefix and move CurPtr past it 1761 const char *Prefix = CurPtr; 1762 CurPtr += PrefixLen + 1; // skip over prefix and '(' 1763 1764 while (1) { 1765 char C = *CurPtr++; 1766 1767 if (C == ')') { 1768 // Check for prefix match and closing quote. 1769 if (strncmp(CurPtr, Prefix, PrefixLen) == 0 && CurPtr[PrefixLen] == '"') { 1770 CurPtr += PrefixLen + 1; // skip over prefix and '"' 1771 break; 1772 } 1773 } else if (C == 0 && CurPtr-1 == BufferEnd) { // End of file. 1774 if (!isLexingRawMode()) 1775 Diag(BufferPtr, diag::err_unterminated_raw_string) 1776 << StringRef(Prefix, PrefixLen); 1777 FormTokenWithChars(Result, CurPtr-1, tok::unknown); 1778 return; 1779 } 1780 } 1781 1782 // If we are in C++11, lex the optional ud-suffix. 1783 if (getLangOpts().CPlusPlus) 1784 CurPtr = LexUDSuffix(Result, CurPtr, true); 1785 1786 // Update the location of token as well as BufferPtr. 1787 const char *TokStart = BufferPtr; 1788 FormTokenWithChars(Result, CurPtr, Kind); 1789 Result.setLiteralData(TokStart); 1790} 1791 1792/// LexAngledStringLiteral - Lex the remainder of an angled string literal, 1793/// after having lexed the '<' character. This is used for #include filenames. 1794void Lexer::LexAngledStringLiteral(Token &Result, const char *CurPtr) { 1795 const char *NulCharacter = 0; // Does this string contain the \0 character? 1796 const char *AfterLessPos = CurPtr; 1797 char C = getAndAdvanceChar(CurPtr, Result); 1798 while (C != '>') { 1799 // Skip escaped characters. 1800 if (C == '\\') { 1801 // Skip the escaped character. 1802 getAndAdvanceChar(CurPtr, Result); 1803 } else if (C == '\n' || C == '\r' || // Newline. 1804 (C == 0 && (CurPtr-1 == BufferEnd || // End of file. 1805 isCodeCompletionPoint(CurPtr-1)))) { 1806 // If the filename is unterminated, then it must just be a lone < 1807 // character. Return this as such. 1808 FormTokenWithChars(Result, AfterLessPos, tok::less); 1809 return; 1810 } else if (C == 0) { 1811 NulCharacter = CurPtr-1; 1812 } 1813 C = getAndAdvanceChar(CurPtr, Result); 1814 } 1815 1816 // If a nul character existed in the string, warn about it. 1817 if (NulCharacter && !isLexingRawMode()) 1818 Diag(NulCharacter, diag::null_in_string); 1819 1820 // Update the location of token as well as BufferPtr. 1821 const char *TokStart = BufferPtr; 1822 FormTokenWithChars(Result, CurPtr, tok::angle_string_literal); 1823 Result.setLiteralData(TokStart); 1824} 1825 1826 1827/// LexCharConstant - Lex the remainder of a character constant, after having 1828/// lexed either ' or L' or u' or U'. 1829void Lexer::LexCharConstant(Token &Result, const char *CurPtr, 1830 tok::TokenKind Kind) { 1831 const char *NulCharacter = 0; // Does this character contain the \0 character? 1832 1833 if (!isLexingRawMode() && 1834 (Kind == tok::utf16_char_constant || Kind == tok::utf32_char_constant)) 1835 Diag(BufferPtr, getLangOpts().CPlusPlus 1836 ? diag::warn_cxx98_compat_unicode_literal 1837 : diag::warn_c99_compat_unicode_literal); 1838 1839 char C = getAndAdvanceChar(CurPtr, Result); 1840 if (C == '\'') { 1841 if (!isLexingRawMode() && !LangOpts.AsmPreprocessor) 1842 Diag(BufferPtr, diag::ext_empty_character); 1843 FormTokenWithChars(Result, CurPtr, tok::unknown); 1844 return; 1845 } 1846 1847 while (C != '\'') { 1848 // Skip escaped characters. 1849 if (C == '\\') 1850 C = getAndAdvanceChar(CurPtr, Result); 1851 1852 if (C == '\n' || C == '\r' || // Newline. 1853 (C == 0 && CurPtr-1 == BufferEnd)) { // End of file. 1854 if (!isLexingRawMode() && !LangOpts.AsmPreprocessor) 1855 Diag(BufferPtr, diag::ext_unterminated_char); 1856 FormTokenWithChars(Result, CurPtr-1, tok::unknown); 1857 return; 1858 } 1859 1860 if (C == 0) { 1861 if (isCodeCompletionPoint(CurPtr-1)) { 1862 PP->CodeCompleteNaturalLanguage(); 1863 FormTokenWithChars(Result, CurPtr-1, tok::unknown); 1864 return cutOffLexing(); 1865 } 1866 1867 NulCharacter = CurPtr-1; 1868 } 1869 C = getAndAdvanceChar(CurPtr, Result); 1870 } 1871 1872 // If we are in C++11, lex the optional ud-suffix. 1873 if (getLangOpts().CPlusPlus) 1874 CurPtr = LexUDSuffix(Result, CurPtr, false); 1875 1876 // If a nul character existed in the character, warn about it. 1877 if (NulCharacter && !isLexingRawMode()) 1878 Diag(NulCharacter, diag::null_in_char); 1879 1880 // Update the location of token as well as BufferPtr. 1881 const char *TokStart = BufferPtr; 1882 FormTokenWithChars(Result, CurPtr, Kind); 1883 Result.setLiteralData(TokStart); 1884} 1885 1886/// SkipWhitespace - Efficiently skip over a series of whitespace characters. 1887/// Update BufferPtr to point to the next non-whitespace character and return. 1888/// 1889/// This method forms a token and returns true if KeepWhitespaceMode is enabled. 1890/// 1891bool Lexer::SkipWhitespace(Token &Result, const char *CurPtr) { 1892 // Whitespace - Skip it, then return the token after the whitespace. 1893 bool SawNewline = isVerticalWhitespace(CurPtr[-1]); 1894 1895 unsigned char Char = *CurPtr; 1896 1897 // Skip consecutive spaces efficiently. 1898 while (1) { 1899 // Skip horizontal whitespace very aggressively. 1900 while (isHorizontalWhitespace(Char)) 1901 Char = *++CurPtr; 1902 1903 // Otherwise if we have something other than whitespace, we're done. 1904 if (!isVerticalWhitespace(Char)) 1905 break; 1906 1907 if (ParsingPreprocessorDirective) { 1908 // End of preprocessor directive line, let LexTokenInternal handle this. 1909 BufferPtr = CurPtr; 1910 return false; 1911 } 1912 1913 // OK, but handle newline. 1914 SawNewline = true; 1915 Char = *++CurPtr; 1916 } 1917 1918 // If the client wants us to return whitespace, return it now. 1919 if (isKeepWhitespaceMode()) { 1920 FormTokenWithChars(Result, CurPtr, tok::unknown); 1921 if (SawNewline) 1922 IsAtStartOfLine = true; 1923 // FIXME: The next token will not have LeadingSpace set. 1924 return true; 1925 } 1926 1927 // If this isn't immediately after a newline, there is leading space. 1928 char PrevChar = CurPtr[-1]; 1929 bool HasLeadingSpace = !isVerticalWhitespace(PrevChar); 1930 1931 Result.setFlagValue(Token::LeadingSpace, HasLeadingSpace); 1932 if (SawNewline) 1933 Result.setFlag(Token::StartOfLine); 1934 1935 BufferPtr = CurPtr; 1936 return false; 1937} 1938 1939/// We have just read the // characters from input. Skip until we find the 1940/// newline character thats terminate the comment. Then update BufferPtr and 1941/// return. 1942/// 1943/// If we're in KeepCommentMode or any CommentHandler has inserted 1944/// some tokens, this will store the first token and return true. 1945bool Lexer::SkipLineComment(Token &Result, const char *CurPtr) { 1946 // If Line comments aren't explicitly enabled for this language, emit an 1947 // extension warning. 1948 if (!LangOpts.LineComment && !isLexingRawMode()) { 1949 Diag(BufferPtr, diag::ext_line_comment); 1950 1951 // Mark them enabled so we only emit one warning for this translation 1952 // unit. 1953 LangOpts.LineComment = true; 1954 } 1955 1956 // Scan over the body of the comment. The common case, when scanning, is that 1957 // the comment contains normal ascii characters with nothing interesting in 1958 // them. As such, optimize for this case with the inner loop. 1959 char C; 1960 do { 1961 C = *CurPtr; 1962 // Skip over characters in the fast loop. 1963 while (C != 0 && // Potentially EOF. 1964 C != '\n' && C != '\r') // Newline or DOS-style newline. 1965 C = *++CurPtr; 1966 1967 const char *NextLine = CurPtr; 1968 if (C != 0) { 1969 // We found a newline, see if it's escaped. 1970 const char *EscapePtr = CurPtr-1; 1971 while (isHorizontalWhitespace(*EscapePtr)) // Skip whitespace. 1972 --EscapePtr; 1973 1974 if (*EscapePtr == '\\') // Escaped newline. 1975 CurPtr = EscapePtr; 1976 else if (EscapePtr[0] == '/' && EscapePtr[-1] == '?' && 1977 EscapePtr[-2] == '?') // Trigraph-escaped newline. 1978 CurPtr = EscapePtr-2; 1979 else 1980 break; // This is a newline, we're done. 1981 } 1982 1983 // Otherwise, this is a hard case. Fall back on getAndAdvanceChar to 1984 // properly decode the character. Read it in raw mode to avoid emitting 1985 // diagnostics about things like trigraphs. If we see an escaped newline, 1986 // we'll handle it below. 1987 const char *OldPtr = CurPtr; 1988 bool OldRawMode = isLexingRawMode(); 1989 LexingRawMode = true; 1990 C = getAndAdvanceChar(CurPtr, Result); 1991 LexingRawMode = OldRawMode; 1992 1993 // If we only read only one character, then no special handling is needed. 1994 // We're done and can skip forward to the newline. 1995 if (C != 0 && CurPtr == OldPtr+1) { 1996 CurPtr = NextLine; 1997 break; 1998 } 1999 2000 // If we read multiple characters, and one of those characters was a \r or 2001 // \n, then we had an escaped newline within the comment. Emit diagnostic 2002 // unless the next line is also a // comment. 2003 if (CurPtr != OldPtr+1 && C != '/' && CurPtr[0] != '/') { 2004 for (; OldPtr != CurPtr; ++OldPtr) 2005 if (OldPtr[0] == '\n' || OldPtr[0] == '\r') { 2006 // Okay, we found a // comment that ends in a newline, if the next 2007 // line is also a // comment, but has spaces, don't emit a diagnostic. 2008 if (isWhitespace(C)) { 2009 const char *ForwardPtr = CurPtr; 2010 while (isWhitespace(*ForwardPtr)) // Skip whitespace. 2011 ++ForwardPtr; 2012 if (ForwardPtr[0] == '/' && ForwardPtr[1] == '/') 2013 break; 2014 } 2015 2016 if (!isLexingRawMode()) 2017 Diag(OldPtr-1, diag::ext_multi_line_line_comment); 2018 break; 2019 } 2020 } 2021 2022 if (CurPtr == BufferEnd+1) { 2023 --CurPtr; 2024 break; 2025 } 2026 2027 if (C == '\0' && isCodeCompletionPoint(CurPtr-1)) { 2028 PP->CodeCompleteNaturalLanguage(); 2029 cutOffLexing(); 2030 return false; 2031 } 2032 2033 } while (C != '\n' && C != '\r'); 2034 2035 // Found but did not consume the newline. Notify comment handlers about the 2036 // comment unless we're in a #if 0 block. 2037 if (PP && !isLexingRawMode() && 2038 PP->HandleComment(Result, SourceRange(getSourceLocation(BufferPtr), 2039 getSourceLocation(CurPtr)))) { 2040 BufferPtr = CurPtr; 2041 return true; // A token has to be returned. 2042 } 2043 2044 // If we are returning comments as tokens, return this comment as a token. 2045 if (inKeepCommentMode()) 2046 return SaveLineComment(Result, CurPtr); 2047 2048 // If we are inside a preprocessor directive and we see the end of line, 2049 // return immediately, so that the lexer can return this as an EOD token. 2050 if (ParsingPreprocessorDirective || CurPtr == BufferEnd) { 2051 BufferPtr = CurPtr; 2052 return false; 2053 } 2054 2055 // Otherwise, eat the \n character. We don't care if this is a \n\r or 2056 // \r\n sequence. This is an efficiency hack (because we know the \n can't 2057 // contribute to another token), it isn't needed for correctness. Note that 2058 // this is ok even in KeepWhitespaceMode, because we would have returned the 2059 /// comment above in that mode. 2060 ++CurPtr; 2061 2062 // The next returned token is at the start of the line. 2063 Result.setFlag(Token::StartOfLine); 2064 // No leading whitespace seen so far. 2065 Result.clearFlag(Token::LeadingSpace); 2066 BufferPtr = CurPtr; 2067 return false; 2068} 2069 2070/// If in save-comment mode, package up this Line comment in an appropriate 2071/// way and return it. 2072bool Lexer::SaveLineComment(Token &Result, const char *CurPtr) { 2073 // If we're not in a preprocessor directive, just return the // comment 2074 // directly. 2075 FormTokenWithChars(Result, CurPtr, tok::comment); 2076 2077 if (!ParsingPreprocessorDirective || LexingRawMode) 2078 return true; 2079 2080 // If this Line-style comment is in a macro definition, transmogrify it into 2081 // a C-style block comment. 2082 bool Invalid = false; 2083 std::string Spelling = PP->getSpelling(Result, &Invalid); 2084 if (Invalid) 2085 return true; 2086 2087 assert(Spelling[0] == '/' && Spelling[1] == '/' && "Not line comment?"); 2088 Spelling[1] = '*'; // Change prefix to "/*". 2089 Spelling += "*/"; // add suffix. 2090 2091 Result.setKind(tok::comment); 2092 PP->CreateString(Spelling, Result, 2093 Result.getLocation(), Result.getLocation()); 2094 return true; 2095} 2096 2097/// isBlockCommentEndOfEscapedNewLine - Return true if the specified newline 2098/// character (either \\n or \\r) is part of an escaped newline sequence. Issue 2099/// a diagnostic if so. We know that the newline is inside of a block comment. 2100static bool isEndOfBlockCommentWithEscapedNewLine(const char *CurPtr, 2101 Lexer *L) { 2102 assert(CurPtr[0] == '\n' || CurPtr[0] == '\r'); 2103 2104 // Back up off the newline. 2105 --CurPtr; 2106 2107 // If this is a two-character newline sequence, skip the other character. 2108 if (CurPtr[0] == '\n' || CurPtr[0] == '\r') { 2109 // \n\n or \r\r -> not escaped newline. 2110 if (CurPtr[0] == CurPtr[1]) 2111 return false; 2112 // \n\r or \r\n -> skip the newline. 2113 --CurPtr; 2114 } 2115 2116 // If we have horizontal whitespace, skip over it. We allow whitespace 2117 // between the slash and newline. 2118 bool HasSpace = false; 2119 while (isHorizontalWhitespace(*CurPtr) || *CurPtr == 0) { 2120 --CurPtr; 2121 HasSpace = true; 2122 } 2123 2124 // If we have a slash, we know this is an escaped newline. 2125 if (*CurPtr == '\\') { 2126 if (CurPtr[-1] != '*') return false; 2127 } else { 2128 // It isn't a slash, is it the ?? / trigraph? 2129 if (CurPtr[0] != '/' || CurPtr[-1] != '?' || CurPtr[-2] != '?' || 2130 CurPtr[-3] != '*') 2131 return false; 2132 2133 // This is the trigraph ending the comment. Emit a stern warning! 2134 CurPtr -= 2; 2135 2136 // If no trigraphs are enabled, warn that we ignored this trigraph and 2137 // ignore this * character. 2138 if (!L->getLangOpts().Trigraphs) { 2139 if (!L->isLexingRawMode()) 2140 L->Diag(CurPtr, diag::trigraph_ignored_block_comment); 2141 return false; 2142 } 2143 if (!L->isLexingRawMode()) 2144 L->Diag(CurPtr, diag::trigraph_ends_block_comment); 2145 } 2146 2147 // Warn about having an escaped newline between the */ characters. 2148 if (!L->isLexingRawMode()) 2149 L->Diag(CurPtr, diag::escaped_newline_block_comment_end); 2150 2151 // If there was space between the backslash and newline, warn about it. 2152 if (HasSpace && !L->isLexingRawMode()) 2153 L->Diag(CurPtr, diag::backslash_newline_space); 2154 2155 return true; 2156} 2157 2158#ifdef __SSE2__ 2159#include <emmintrin.h> 2160#elif __ALTIVEC__ 2161#include <altivec.h> 2162#undef bool 2163#endif 2164 2165/// We have just read from input the / and * characters that started a comment. 2166/// Read until we find the * and / characters that terminate the comment. 2167/// Note that we don't bother decoding trigraphs or escaped newlines in block 2168/// comments, because they cannot cause the comment to end. The only thing 2169/// that can happen is the comment could end with an escaped newline between 2170/// the terminating * and /. 2171/// 2172/// If we're in KeepCommentMode or any CommentHandler has inserted 2173/// some tokens, this will store the first token and return true. 2174bool Lexer::SkipBlockComment(Token &Result, const char *CurPtr) { 2175 // Scan one character past where we should, looking for a '/' character. Once 2176 // we find it, check to see if it was preceded by a *. This common 2177 // optimization helps people who like to put a lot of * characters in their 2178 // comments. 2179 2180 // The first character we get with newlines and trigraphs skipped to handle 2181 // the degenerate /*/ case below correctly if the * has an escaped newline 2182 // after it. 2183 unsigned CharSize; 2184 unsigned char C = getCharAndSize(CurPtr, CharSize); 2185 CurPtr += CharSize; 2186 if (C == 0 && CurPtr == BufferEnd+1) { 2187 if (!isLexingRawMode()) 2188 Diag(BufferPtr, diag::err_unterminated_block_comment); 2189 --CurPtr; 2190 2191 // KeepWhitespaceMode should return this broken comment as a token. Since 2192 // it isn't a well formed comment, just return it as an 'unknown' token. 2193 if (isKeepWhitespaceMode()) { 2194 FormTokenWithChars(Result, CurPtr, tok::unknown); 2195 return true; 2196 } 2197 2198 BufferPtr = CurPtr; 2199 return false; 2200 } 2201 2202 // Check to see if the first character after the '/*' is another /. If so, 2203 // then this slash does not end the block comment, it is part of it. 2204 if (C == '/') 2205 C = *CurPtr++; 2206 2207 while (1) { 2208 // Skip over all non-interesting characters until we find end of buffer or a 2209 // (probably ending) '/' character. 2210 if (CurPtr + 24 < BufferEnd && 2211 // If there is a code-completion point avoid the fast scan because it 2212 // doesn't check for '\0'. 2213 !(PP && PP->getCodeCompletionFileLoc() == FileLoc)) { 2214 // While not aligned to a 16-byte boundary. 2215 while (C != '/' && ((intptr_t)CurPtr & 0x0F) != 0) 2216 C = *CurPtr++; 2217 2218 if (C == '/') goto FoundSlash; 2219 2220#ifdef __SSE2__ 2221 __m128i Slashes = _mm_set1_epi8('/'); 2222 while (CurPtr+16 <= BufferEnd) { 2223 int cmp = _mm_movemask_epi8(_mm_cmpeq_epi8(*(const __m128i*)CurPtr, 2224 Slashes)); 2225 if (cmp != 0) { 2226 // Adjust the pointer to point directly after the first slash. It's 2227 // not necessary to set C here, it will be overwritten at the end of 2228 // the outer loop. 2229 CurPtr += llvm::countTrailingZeros<unsigned>(cmp) + 1; 2230 goto FoundSlash; 2231 } 2232 CurPtr += 16; 2233 } 2234#elif __ALTIVEC__ 2235 __vector unsigned char Slashes = { 2236 '/', '/', '/', '/', '/', '/', '/', '/', 2237 '/', '/', '/', '/', '/', '/', '/', '/' 2238 }; 2239 while (CurPtr+16 <= BufferEnd && 2240 !vec_any_eq(*(vector unsigned char*)CurPtr, Slashes)) 2241 CurPtr += 16; 2242#else 2243 // Scan for '/' quickly. Many block comments are very large. 2244 while (CurPtr[0] != '/' && 2245 CurPtr[1] != '/' && 2246 CurPtr[2] != '/' && 2247 CurPtr[3] != '/' && 2248 CurPtr+4 < BufferEnd) { 2249 CurPtr += 4; 2250 } 2251#endif 2252 2253 // It has to be one of the bytes scanned, increment to it and read one. 2254 C = *CurPtr++; 2255 } 2256 2257 // Loop to scan the remainder. 2258 while (C != '/' && C != '\0') 2259 C = *CurPtr++; 2260 2261 if (C == '/') { 2262 FoundSlash: 2263 if (CurPtr[-2] == '*') // We found the final */. We're done! 2264 break; 2265 2266 if ((CurPtr[-2] == '\n' || CurPtr[-2] == '\r')) { 2267 if (isEndOfBlockCommentWithEscapedNewLine(CurPtr-2, this)) { 2268 // We found the final */, though it had an escaped newline between the 2269 // * and /. We're done! 2270 break; 2271 } 2272 } 2273 if (CurPtr[0] == '*' && CurPtr[1] != '/') { 2274 // If this is a /* inside of the comment, emit a warning. Don't do this 2275 // if this is a /*/, which will end the comment. This misses cases with 2276 // embedded escaped newlines, but oh well. 2277 if (!isLexingRawMode()) 2278 Diag(CurPtr-1, diag::warn_nested_block_comment); 2279 } 2280 } else if (C == 0 && CurPtr == BufferEnd+1) { 2281 if (!isLexingRawMode()) 2282 Diag(BufferPtr, diag::err_unterminated_block_comment); 2283 // Note: the user probably forgot a */. We could continue immediately 2284 // after the /*, but this would involve lexing a lot of what really is the 2285 // comment, which surely would confuse the parser. 2286 --CurPtr; 2287 2288 // KeepWhitespaceMode should return this broken comment as a token. Since 2289 // it isn't a well formed comment, just return it as an 'unknown' token. 2290 if (isKeepWhitespaceMode()) { 2291 FormTokenWithChars(Result, CurPtr, tok::unknown); 2292 return true; 2293 } 2294 2295 BufferPtr = CurPtr; 2296 return false; 2297 } else if (C == '\0' && isCodeCompletionPoint(CurPtr-1)) { 2298 PP->CodeCompleteNaturalLanguage(); 2299 cutOffLexing(); 2300 return false; 2301 } 2302 2303 C = *CurPtr++; 2304 } 2305 2306 // Notify comment handlers about the comment unless we're in a #if 0 block. 2307 if (PP && !isLexingRawMode() && 2308 PP->HandleComment(Result, SourceRange(getSourceLocation(BufferPtr), 2309 getSourceLocation(CurPtr)))) { 2310 BufferPtr = CurPtr; 2311 return true; // A token has to be returned. 2312 } 2313 2314 // If we are returning comments as tokens, return this comment as a token. 2315 if (inKeepCommentMode()) { 2316 FormTokenWithChars(Result, CurPtr, tok::comment); 2317 return true; 2318 } 2319 2320 // It is common for the tokens immediately after a /**/ comment to be 2321 // whitespace. Instead of going through the big switch, handle it 2322 // efficiently now. This is safe even in KeepWhitespaceMode because we would 2323 // have already returned above with the comment as a token. 2324 if (isHorizontalWhitespace(*CurPtr)) { 2325 SkipWhitespace(Result, CurPtr+1); 2326 return false; 2327 } 2328 2329 // Otherwise, just return so that the next character will be lexed as a token. 2330 BufferPtr = CurPtr; 2331 Result.setFlag(Token::LeadingSpace); 2332 return false; 2333} 2334 2335//===----------------------------------------------------------------------===// 2336// Primary Lexing Entry Points 2337//===----------------------------------------------------------------------===// 2338 2339/// ReadToEndOfLine - Read the rest of the current preprocessor line as an 2340/// uninterpreted string. This switches the lexer out of directive mode. 2341void Lexer::ReadToEndOfLine(SmallVectorImpl<char> *Result) { 2342 assert(ParsingPreprocessorDirective && ParsingFilename == false && 2343 "Must be in a preprocessing directive!"); 2344 Token Tmp; 2345 2346 // CurPtr - Cache BufferPtr in an automatic variable. 2347 const char *CurPtr = BufferPtr; 2348 while (1) { 2349 char Char = getAndAdvanceChar(CurPtr, Tmp); 2350 switch (Char) { 2351 default: 2352 if (Result) 2353 Result->push_back(Char); 2354 break; 2355 case 0: // Null. 2356 // Found end of file? 2357 if (CurPtr-1 != BufferEnd) { 2358 if (isCodeCompletionPoint(CurPtr-1)) { 2359 PP->CodeCompleteNaturalLanguage(); 2360 cutOffLexing(); 2361 return; 2362 } 2363 2364 // Nope, normal character, continue. 2365 if (Result) 2366 Result->push_back(Char); 2367 break; 2368 } 2369 // FALL THROUGH. 2370 case '\r': 2371 case '\n': 2372 // Okay, we found the end of the line. First, back up past the \0, \r, \n. 2373 assert(CurPtr[-1] == Char && "Trigraphs for newline?"); 2374 BufferPtr = CurPtr-1; 2375 2376 // Next, lex the character, which should handle the EOD transition. 2377 Lex(Tmp); 2378 if (Tmp.is(tok::code_completion)) { 2379 if (PP) 2380 PP->CodeCompleteNaturalLanguage(); 2381 Lex(Tmp); 2382 } 2383 assert(Tmp.is(tok::eod) && "Unexpected token!"); 2384 2385 // Finally, we're done; 2386 return; 2387 } 2388 } 2389} 2390 2391/// LexEndOfFile - CurPtr points to the end of this file. Handle this 2392/// condition, reporting diagnostics and handling other edge cases as required. 2393/// This returns true if Result contains a token, false if PP.Lex should be 2394/// called again. 2395bool Lexer::LexEndOfFile(Token &Result, const char *CurPtr) { 2396 // If we hit the end of the file while parsing a preprocessor directive, 2397 // end the preprocessor directive first. The next token returned will 2398 // then be the end of file. 2399 if (ParsingPreprocessorDirective) { 2400 // Done parsing the "line". 2401 ParsingPreprocessorDirective = false; 2402 // Update the location of token as well as BufferPtr. 2403 FormTokenWithChars(Result, CurPtr, tok::eod); 2404 2405 // Restore comment saving mode, in case it was disabled for directive. 2406 resetExtendedTokenMode(); 2407 return true; // Have a token. 2408 } 2409 2410 // If we are in raw mode, return this event as an EOF token. Let the caller 2411 // that put us in raw mode handle the event. 2412 if (isLexingRawMode()) { 2413 Result.startToken(); 2414 BufferPtr = BufferEnd; 2415 FormTokenWithChars(Result, BufferEnd, tok::eof); 2416 return true; 2417 } 2418 2419 // Issue diagnostics for unterminated #if and missing newline. 2420 2421 // If we are in a #if directive, emit an error. 2422 while (!ConditionalStack.empty()) { 2423 if (PP->getCodeCompletionFileLoc() != FileLoc) 2424 PP->Diag(ConditionalStack.back().IfLoc, 2425 diag::err_pp_unterminated_conditional); 2426 ConditionalStack.pop_back(); 2427 } 2428 2429 // C99 5.1.1.2p2: If the file is non-empty and didn't end in a newline, issue 2430 // a pedwarn. 2431 if (CurPtr != BufferStart && (CurPtr[-1] != '\n' && CurPtr[-1] != '\r')) { 2432 DiagnosticsEngine &Diags = PP->getDiagnostics(); 2433 SourceLocation EndLoc = getSourceLocation(BufferEnd); 2434 unsigned DiagID; 2435 2436 if (LangOpts.CPlusPlus11) { 2437 // C++11 [lex.phases] 2.2 p2 2438 // Prefer the C++98 pedantic compatibility warning over the generic, 2439 // non-extension, user-requested "missing newline at EOF" warning. 2440 if (Diags.getDiagnosticLevel(diag::warn_cxx98_compat_no_newline_eof, 2441 EndLoc) != DiagnosticsEngine::Ignored) { 2442 DiagID = diag::warn_cxx98_compat_no_newline_eof; 2443 } else { 2444 DiagID = diag::warn_no_newline_eof; 2445 } 2446 } else { 2447 DiagID = diag::ext_no_newline_eof; 2448 } 2449 2450 Diag(BufferEnd, DiagID) 2451 << FixItHint::CreateInsertion(EndLoc, "\n"); 2452 } 2453 2454 BufferPtr = CurPtr; 2455 2456 // Finally, let the preprocessor handle this. 2457 return PP->HandleEndOfFile(Result, isPragmaLexer()); 2458} 2459 2460/// isNextPPTokenLParen - Return 1 if the next unexpanded token lexed from 2461/// the specified lexer will return a tok::l_paren token, 0 if it is something 2462/// else and 2 if there are no more tokens in the buffer controlled by the 2463/// lexer. 2464unsigned Lexer::isNextPPTokenLParen() { 2465 assert(!LexingRawMode && "How can we expand a macro from a skipping buffer?"); 2466 2467 // Switch to 'skipping' mode. This will ensure that we can lex a token 2468 // without emitting diagnostics, disables macro expansion, and will cause EOF 2469 // to return an EOF token instead of popping the include stack. 2470 LexingRawMode = true; 2471 2472 // Save state that can be changed while lexing so that we can restore it. 2473 const char *TmpBufferPtr = BufferPtr; 2474 bool inPPDirectiveMode = ParsingPreprocessorDirective; 2475 2476 Token Tok; 2477 Tok.startToken(); 2478 LexTokenInternal(Tok); 2479 2480 // Restore state that may have changed. 2481 BufferPtr = TmpBufferPtr; 2482 ParsingPreprocessorDirective = inPPDirectiveMode; 2483 2484 // Restore the lexer back to non-skipping mode. 2485 LexingRawMode = false; 2486 2487 if (Tok.is(tok::eof)) 2488 return 2; 2489 return Tok.is(tok::l_paren); 2490} 2491 2492/// \brief Find the end of a version control conflict marker. 2493static const char *FindConflictEnd(const char *CurPtr, const char *BufferEnd, 2494 ConflictMarkerKind CMK) { 2495 const char *Terminator = CMK == CMK_Perforce ? "<<<<\n" : ">>>>>>>"; 2496 size_t TermLen = CMK == CMK_Perforce ? 5 : 7; 2497 StringRef RestOfBuffer(CurPtr+TermLen, BufferEnd-CurPtr-TermLen); 2498 size_t Pos = RestOfBuffer.find(Terminator); 2499 while (Pos != StringRef::npos) { 2500 // Must occur at start of line. 2501 if (RestOfBuffer[Pos-1] != '\r' && 2502 RestOfBuffer[Pos-1] != '\n') { 2503 RestOfBuffer = RestOfBuffer.substr(Pos+TermLen); 2504 Pos = RestOfBuffer.find(Terminator); 2505 continue; 2506 } 2507 return RestOfBuffer.data()+Pos; 2508 } 2509 return 0; 2510} 2511 2512/// IsStartOfConflictMarker - If the specified pointer is the start of a version 2513/// control conflict marker like '<<<<<<<', recognize it as such, emit an error 2514/// and recover nicely. This returns true if it is a conflict marker and false 2515/// if not. 2516bool Lexer::IsStartOfConflictMarker(const char *CurPtr) { 2517 // Only a conflict marker if it starts at the beginning of a line. 2518 if (CurPtr != BufferStart && 2519 CurPtr[-1] != '\n' && CurPtr[-1] != '\r') 2520 return false; 2521 2522 // Check to see if we have <<<<<<< or >>>>. 2523 if ((BufferEnd-CurPtr < 8 || StringRef(CurPtr, 7) != "<<<<<<<") && 2524 (BufferEnd-CurPtr < 6 || StringRef(CurPtr, 5) != ">>>> ")) 2525 return false; 2526 2527 // If we have a situation where we don't care about conflict markers, ignore 2528 // it. 2529 if (CurrentConflictMarkerState || isLexingRawMode()) 2530 return false; 2531 2532 ConflictMarkerKind Kind = *CurPtr == '<' ? CMK_Normal : CMK_Perforce; 2533 2534 // Check to see if there is an ending marker somewhere in the buffer at the 2535 // start of a line to terminate this conflict marker. 2536 if (FindConflictEnd(CurPtr, BufferEnd, Kind)) { 2537 // We found a match. We are really in a conflict marker. 2538 // Diagnose this, and ignore to the end of line. 2539 Diag(CurPtr, diag::err_conflict_marker); 2540 CurrentConflictMarkerState = Kind; 2541 2542 // Skip ahead to the end of line. We know this exists because the 2543 // end-of-conflict marker starts with \r or \n. 2544 while (*CurPtr != '\r' && *CurPtr != '\n') { 2545 assert(CurPtr != BufferEnd && "Didn't find end of line"); 2546 ++CurPtr; 2547 } 2548 BufferPtr = CurPtr; 2549 return true; 2550 } 2551 2552 // No end of conflict marker found. 2553 return false; 2554} 2555 2556 2557/// HandleEndOfConflictMarker - If this is a '====' or '||||' or '>>>>', or if 2558/// it is '<<<<' and the conflict marker started with a '>>>>' marker, then it 2559/// is the end of a conflict marker. Handle it by ignoring up until the end of 2560/// the line. This returns true if it is a conflict marker and false if not. 2561bool Lexer::HandleEndOfConflictMarker(const char *CurPtr) { 2562 // Only a conflict marker if it starts at the beginning of a line. 2563 if (CurPtr != BufferStart && 2564 CurPtr[-1] != '\n' && CurPtr[-1] != '\r') 2565 return false; 2566 2567 // If we have a situation where we don't care about conflict markers, ignore 2568 // it. 2569 if (!CurrentConflictMarkerState || isLexingRawMode()) 2570 return false; 2571 2572 // Check to see if we have the marker (4 characters in a row). 2573 for (unsigned i = 1; i != 4; ++i) 2574 if (CurPtr[i] != CurPtr[0]) 2575 return false; 2576 2577 // If we do have it, search for the end of the conflict marker. This could 2578 // fail if it got skipped with a '#if 0' or something. Note that CurPtr might 2579 // be the end of conflict marker. 2580 if (const char *End = FindConflictEnd(CurPtr, BufferEnd, 2581 CurrentConflictMarkerState)) { 2582 CurPtr = End; 2583 2584 // Skip ahead to the end of line. 2585 while (CurPtr != BufferEnd && *CurPtr != '\r' && *CurPtr != '\n') 2586 ++CurPtr; 2587 2588 BufferPtr = CurPtr; 2589 2590 // No longer in the conflict marker. 2591 CurrentConflictMarkerState = CMK_None; 2592 return true; 2593 } 2594 2595 return false; 2596} 2597 2598bool Lexer::isCodeCompletionPoint(const char *CurPtr) const { 2599 if (PP && PP->isCodeCompletionEnabled()) { 2600 SourceLocation Loc = FileLoc.getLocWithOffset(CurPtr-BufferStart); 2601 return Loc == PP->getCodeCompletionLoc(); 2602 } 2603 2604 return false; 2605} 2606 2607uint32_t Lexer::tryReadUCN(const char *&StartPtr, const char *SlashLoc, 2608 Token *Result) { 2609 unsigned CharSize; 2610 char Kind = getCharAndSize(StartPtr, CharSize); 2611 2612 unsigned NumHexDigits; 2613 if (Kind == 'u') 2614 NumHexDigits = 4; 2615 else if (Kind == 'U') 2616 NumHexDigits = 8; 2617 else 2618 return 0; 2619 2620 if (!LangOpts.CPlusPlus && !LangOpts.C99) { 2621 if (Result && !isLexingRawMode()) 2622 Diag(SlashLoc, diag::warn_ucn_not_valid_in_c89); 2623 return 0; 2624 } 2625 2626 const char *CurPtr = StartPtr + CharSize; 2627 const char *KindLoc = &CurPtr[-1]; 2628 2629 uint32_t CodePoint = 0; 2630 for (unsigned i = 0; i < NumHexDigits; ++i) { 2631 char C = getCharAndSize(CurPtr, CharSize); 2632 2633 unsigned Value = llvm::hexDigitValue(C); 2634 if (Value == -1U) { 2635 if (Result && !isLexingRawMode()) { 2636 if (i == 0) { 2637 Diag(BufferPtr, diag::warn_ucn_escape_no_digits) 2638 << StringRef(KindLoc, 1); 2639 } else { 2640 Diag(BufferPtr, diag::warn_ucn_escape_incomplete); 2641 2642 // If the user wrote \U1234, suggest a fixit to \u. 2643 if (i == 4 && NumHexDigits == 8) { 2644 CharSourceRange URange = makeCharRange(*this, KindLoc, KindLoc + 1); 2645 Diag(KindLoc, diag::note_ucn_four_not_eight) 2646 << FixItHint::CreateReplacement(URange, "u"); 2647 } 2648 } 2649 } 2650 2651 return 0; 2652 } 2653 2654 CodePoint <<= 4; 2655 CodePoint += Value; 2656 2657 CurPtr += CharSize; 2658 } 2659 2660 if (Result) { 2661 Result->setFlag(Token::HasUCN); 2662 if (CurPtr - StartPtr == (ptrdiff_t)NumHexDigits + 2) 2663 StartPtr = CurPtr; 2664 else 2665 while (StartPtr != CurPtr) 2666 (void)getAndAdvanceChar(StartPtr, *Result); 2667 } else { 2668 StartPtr = CurPtr; 2669 } 2670 2671 // C99 6.4.3p2: A universal character name shall not specify a character whose 2672 // short identifier is less than 00A0 other than 0024 ($), 0040 (@), or 2673 // 0060 (`), nor one in the range D800 through DFFF inclusive.) 2674 // C++11 [lex.charset]p2: If the hexadecimal value for a 2675 // universal-character-name corresponds to a surrogate code point (in the 2676 // range 0xD800-0xDFFF, inclusive), the program is ill-formed. Additionally, 2677 // if the hexadecimal value for a universal-character-name outside the 2678 // c-char-sequence, s-char-sequence, or r-char-sequence of a character or 2679 // string literal corresponds to a control character (in either of the 2680 // ranges 0x00-0x1F or 0x7F-0x9F, both inclusive) or to a character in the 2681 // basic source character set, the program is ill-formed. 2682 if (CodePoint < 0xA0) { 2683 if (CodePoint == 0x24 || CodePoint == 0x40 || CodePoint == 0x60) 2684 return CodePoint; 2685 2686 // We don't use isLexingRawMode() here because we need to warn about bad 2687 // UCNs even when skipping preprocessing tokens in a #if block. 2688 if (Result && PP) { 2689 if (CodePoint < 0x20 || CodePoint >= 0x7F) 2690 Diag(BufferPtr, diag::err_ucn_control_character); 2691 else { 2692 char C = static_cast<char>(CodePoint); 2693 Diag(BufferPtr, diag::err_ucn_escape_basic_scs) << StringRef(&C, 1); 2694 } 2695 } 2696 2697 return 0; 2698 2699 } else if (CodePoint >= 0xD800 && CodePoint <= 0xDFFF) { 2700 // C++03 allows UCNs representing surrogate characters. C99 and C++11 don't. 2701 // We don't use isLexingRawMode() here because we need to diagnose bad 2702 // UCNs even when skipping preprocessing tokens in a #if block. 2703 if (Result && PP) { 2704 if (LangOpts.CPlusPlus && !LangOpts.CPlusPlus11) 2705 Diag(BufferPtr, diag::warn_ucn_escape_surrogate); 2706 else 2707 Diag(BufferPtr, diag::err_ucn_escape_invalid); 2708 } 2709 return 0; 2710 } 2711 2712 return CodePoint; 2713} 2714 2715void Lexer::LexUnicode(Token &Result, uint32_t C, const char *CurPtr) { 2716 static const llvm::sys::UnicodeCharSet UnicodeWhitespaceChars( 2717 UnicodeWhitespaceCharRanges); 2718 if (!isLexingRawMode() && !PP->isPreprocessedOutput() && 2719 UnicodeWhitespaceChars.contains(C)) { 2720 Diag(BufferPtr, diag::ext_unicode_whitespace) 2721 << makeCharRange(*this, BufferPtr, CurPtr); 2722 2723 Result.setFlag(Token::LeadingSpace); 2724 if (SkipWhitespace(Result, CurPtr)) 2725 return; // KeepWhitespaceMode 2726 2727 return LexTokenInternal(Result); 2728 } 2729 2730 if (isAllowedIDChar(C, LangOpts) && isAllowedInitiallyIDChar(C, LangOpts)) { 2731 if (!isLexingRawMode() && !ParsingPreprocessorDirective && 2732 !PP->isPreprocessedOutput()) { 2733 maybeDiagnoseIDCharCompat(PP->getDiagnostics(), C, 2734 makeCharRange(*this, BufferPtr, CurPtr), 2735 /*IsFirst=*/true); 2736 } 2737 2738 MIOpt.ReadToken(); 2739 return LexIdentifier(Result, CurPtr); 2740 } 2741 2742 if (!isLexingRawMode() && !ParsingPreprocessorDirective && 2743 !PP->isPreprocessedOutput() && 2744 !isASCII(*BufferPtr) && !isAllowedIDChar(C, LangOpts)) { 2745 // Non-ASCII characters tend to creep into source code unintentionally. 2746 // Instead of letting the parser complain about the unknown token, 2747 // just drop the character. 2748 // Note that we can /only/ do this when the non-ASCII character is actually 2749 // spelled as Unicode, not written as a UCN. The standard requires that 2750 // we not throw away any possible preprocessor tokens, but there's a 2751 // loophole in the mapping of Unicode characters to basic character set 2752 // characters that allows us to map these particular characters to, say, 2753 // whitespace. 2754 Diag(BufferPtr, diag::err_non_ascii) 2755 << FixItHint::CreateRemoval(makeCharRange(*this, BufferPtr, CurPtr)); 2756 2757 BufferPtr = CurPtr; 2758 return LexTokenInternal(Result); 2759 } 2760 2761 // Otherwise, we have an explicit UCN or a character that's unlikely to show 2762 // up by accident. 2763 MIOpt.ReadToken(); 2764 FormTokenWithChars(Result, CurPtr, tok::unknown); 2765} 2766 2767 2768/// LexTokenInternal - This implements a simple C family lexer. It is an 2769/// extremely performance critical piece of code. This assumes that the buffer 2770/// has a null character at the end of the file. This returns a preprocessing 2771/// token, not a normal token, as such, it is an internal interface. It assumes 2772/// that the Flags of result have been cleared before calling this. 2773void Lexer::LexTokenInternal(Token &Result) { 2774LexNextToken: 2775 // New token, can't need cleaning yet. 2776 Result.clearFlag(Token::NeedsCleaning); 2777 Result.setIdentifierInfo(0); 2778 2779 // CurPtr - Cache BufferPtr in an automatic variable. 2780 const char *CurPtr = BufferPtr; 2781 2782 // Small amounts of horizontal whitespace is very common between tokens. 2783 if ((*CurPtr == ' ') || (*CurPtr == '\t')) { 2784 ++CurPtr; 2785 while ((*CurPtr == ' ') || (*CurPtr == '\t')) 2786 ++CurPtr; 2787 2788 // If we are keeping whitespace and other tokens, just return what we just 2789 // skipped. The next lexer invocation will return the token after the 2790 // whitespace. 2791 if (isKeepWhitespaceMode()) { 2792 FormTokenWithChars(Result, CurPtr, tok::unknown); 2793 // FIXME: The next token will not have LeadingSpace set. 2794 return; 2795 } 2796 2797 BufferPtr = CurPtr; 2798 Result.setFlag(Token::LeadingSpace); 2799 } 2800 2801 unsigned SizeTmp, SizeTmp2; // Temporaries for use in cases below. 2802 2803 // Read a character, advancing over it. 2804 char Char = getAndAdvanceChar(CurPtr, Result); 2805 tok::TokenKind Kind; 2806 2807 switch (Char) { 2808 case 0: // Null. 2809 // Found end of file? 2810 if (CurPtr-1 == BufferEnd) { 2811 // Read the PP instance variable into an automatic variable, because 2812 // LexEndOfFile will often delete 'this'. 2813 Preprocessor *PPCache = PP; 2814 if (LexEndOfFile(Result, CurPtr-1)) // Retreat back into the file. 2815 return; // Got a token to return. 2816 assert(PPCache && "Raw buffer::LexEndOfFile should return a token"); 2817 return PPCache->Lex(Result); 2818 } 2819 2820 // Check if we are performing code completion. 2821 if (isCodeCompletionPoint(CurPtr-1)) { 2822 // Return the code-completion token. 2823 Result.startToken(); 2824 FormTokenWithChars(Result, CurPtr, tok::code_completion); 2825 return; 2826 } 2827 2828 if (!isLexingRawMode()) 2829 Diag(CurPtr-1, diag::null_in_file); 2830 Result.setFlag(Token::LeadingSpace); 2831 if (SkipWhitespace(Result, CurPtr)) 2832 return; // KeepWhitespaceMode 2833 2834 goto LexNextToken; // GCC isn't tail call eliminating. 2835 2836 case 26: // DOS & CP/M EOF: "^Z". 2837 // If we're in Microsoft extensions mode, treat this as end of file. 2838 if (LangOpts.MicrosoftExt) { 2839 // Read the PP instance variable into an automatic variable, because 2840 // LexEndOfFile will often delete 'this'. 2841 Preprocessor *PPCache = PP; 2842 if (LexEndOfFile(Result, CurPtr-1)) // Retreat back into the file. 2843 return; // Got a token to return. 2844 assert(PPCache && "Raw buffer::LexEndOfFile should return a token"); 2845 return PPCache->Lex(Result); 2846 } 2847 // If Microsoft extensions are disabled, this is just random garbage. 2848 Kind = tok::unknown; 2849 break; 2850 2851 case '\n': 2852 case '\r': 2853 // If we are inside a preprocessor directive and we see the end of line, 2854 // we know we are done with the directive, so return an EOD token. 2855 if (ParsingPreprocessorDirective) { 2856 // Done parsing the "line". 2857 ParsingPreprocessorDirective = false; 2858 2859 // Restore comment saving mode, in case it was disabled for directive. 2860 if (PP) 2861 resetExtendedTokenMode(); 2862 2863 // Since we consumed a newline, we are back at the start of a line. 2864 IsAtStartOfLine = true; 2865 2866 Kind = tok::eod; 2867 break; 2868 } 2869 2870 // No leading whitespace seen so far. 2871 Result.clearFlag(Token::LeadingSpace); 2872 2873 if (SkipWhitespace(Result, CurPtr)) 2874 return; // KeepWhitespaceMode 2875 goto LexNextToken; // GCC isn't tail call eliminating. 2876 case ' ': 2877 case '\t': 2878 case '\f': 2879 case '\v': 2880 SkipHorizontalWhitespace: 2881 Result.setFlag(Token::LeadingSpace); 2882 if (SkipWhitespace(Result, CurPtr)) 2883 return; // KeepWhitespaceMode 2884 2885 SkipIgnoredUnits: 2886 CurPtr = BufferPtr; 2887 2888 // If the next token is obviously a // or /* */ comment, skip it efficiently 2889 // too (without going through the big switch stmt). 2890 if (CurPtr[0] == '/' && CurPtr[1] == '/' && !inKeepCommentMode() && 2891 LangOpts.LineComment && 2892 (LangOpts.CPlusPlus || !LangOpts.TraditionalCPP)) { 2893 if (SkipLineComment(Result, CurPtr+2)) 2894 return; // There is a token to return. 2895 goto SkipIgnoredUnits; 2896 } else if (CurPtr[0] == '/' && CurPtr[1] == '*' && !inKeepCommentMode()) { 2897 if (SkipBlockComment(Result, CurPtr+2)) 2898 return; // There is a token to return. 2899 goto SkipIgnoredUnits; 2900 } else if (isHorizontalWhitespace(*CurPtr)) { 2901 goto SkipHorizontalWhitespace; 2902 } 2903 goto LexNextToken; // GCC isn't tail call eliminating. 2904 2905 // C99 6.4.4.1: Integer Constants. 2906 // C99 6.4.4.2: Floating Constants. 2907 case '0': case '1': case '2': case '3': case '4': 2908 case '5': case '6': case '7': case '8': case '9': 2909 // Notify MIOpt that we read a non-whitespace/non-comment token. 2910 MIOpt.ReadToken(); 2911 return LexNumericConstant(Result, CurPtr); 2912 2913 case 'u': // Identifier (uber) or C11/C++11 UTF-8 or UTF-16 string literal 2914 // Notify MIOpt that we read a non-whitespace/non-comment token. 2915 MIOpt.ReadToken(); 2916 2917 if (LangOpts.CPlusPlus11 || LangOpts.C11) { 2918 Char = getCharAndSize(CurPtr, SizeTmp); 2919 2920 // UTF-16 string literal 2921 if (Char == '"') 2922 return LexStringLiteral(Result, ConsumeChar(CurPtr, SizeTmp, Result), 2923 tok::utf16_string_literal); 2924 2925 // UTF-16 character constant 2926 if (Char == '\'') 2927 return LexCharConstant(Result, ConsumeChar(CurPtr, SizeTmp, Result), 2928 tok::utf16_char_constant); 2929 2930 // UTF-16 raw string literal 2931 if (Char == 'R' && LangOpts.CPlusPlus11 && 2932 getCharAndSize(CurPtr + SizeTmp, SizeTmp2) == '"') 2933 return LexRawStringLiteral(Result, 2934 ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 2935 SizeTmp2, Result), 2936 tok::utf16_string_literal); 2937 2938 if (Char == '8') { 2939 char Char2 = getCharAndSize(CurPtr + SizeTmp, SizeTmp2); 2940 2941 // UTF-8 string literal 2942 if (Char2 == '"') 2943 return LexStringLiteral(Result, 2944 ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 2945 SizeTmp2, Result), 2946 tok::utf8_string_literal); 2947 2948 if (Char2 == 'R' && LangOpts.CPlusPlus11) { 2949 unsigned SizeTmp3; 2950 char Char3 = getCharAndSize(CurPtr + SizeTmp + SizeTmp2, SizeTmp3); 2951 // UTF-8 raw string literal 2952 if (Char3 == '"') { 2953 return LexRawStringLiteral(Result, 2954 ConsumeChar(ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 2955 SizeTmp2, Result), 2956 SizeTmp3, Result), 2957 tok::utf8_string_literal); 2958 } 2959 } 2960 } 2961 } 2962 2963 // treat u like the start of an identifier. 2964 return LexIdentifier(Result, CurPtr); 2965 2966 case 'U': // Identifier (Uber) or C11/C++11 UTF-32 string literal 2967 // Notify MIOpt that we read a non-whitespace/non-comment token. 2968 MIOpt.ReadToken(); 2969 2970 if (LangOpts.CPlusPlus11 || LangOpts.C11) { 2971 Char = getCharAndSize(CurPtr, SizeTmp); 2972 2973 // UTF-32 string literal 2974 if (Char == '"') 2975 return LexStringLiteral(Result, ConsumeChar(CurPtr, SizeTmp, Result), 2976 tok::utf32_string_literal); 2977 2978 // UTF-32 character constant 2979 if (Char == '\'') 2980 return LexCharConstant(Result, ConsumeChar(CurPtr, SizeTmp, Result), 2981 tok::utf32_char_constant); 2982 2983 // UTF-32 raw string literal 2984 if (Char == 'R' && LangOpts.CPlusPlus11 && 2985 getCharAndSize(CurPtr + SizeTmp, SizeTmp2) == '"') 2986 return LexRawStringLiteral(Result, 2987 ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 2988 SizeTmp2, Result), 2989 tok::utf32_string_literal); 2990 } 2991 2992 // treat U like the start of an identifier. 2993 return LexIdentifier(Result, CurPtr); 2994 2995 case 'R': // Identifier or C++0x raw string literal 2996 // Notify MIOpt that we read a non-whitespace/non-comment token. 2997 MIOpt.ReadToken(); 2998 2999 if (LangOpts.CPlusPlus11) { 3000 Char = getCharAndSize(CurPtr, SizeTmp); 3001 3002 if (Char == '"') 3003 return LexRawStringLiteral(Result, 3004 ConsumeChar(CurPtr, SizeTmp, Result), 3005 tok::string_literal); 3006 } 3007 3008 // treat R like the start of an identifier. 3009 return LexIdentifier(Result, CurPtr); 3010 3011 case 'L': // Identifier (Loony) or wide literal (L'x' or L"xyz"). 3012 // Notify MIOpt that we read a non-whitespace/non-comment token. 3013 MIOpt.ReadToken(); 3014 Char = getCharAndSize(CurPtr, SizeTmp); 3015 3016 // Wide string literal. 3017 if (Char == '"') 3018 return LexStringLiteral(Result, ConsumeChar(CurPtr, SizeTmp, Result), 3019 tok::wide_string_literal); 3020 3021 // Wide raw string literal. 3022 if (LangOpts.CPlusPlus11 && Char == 'R' && 3023 getCharAndSize(CurPtr + SizeTmp, SizeTmp2) == '"') 3024 return LexRawStringLiteral(Result, 3025 ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 3026 SizeTmp2, Result), 3027 tok::wide_string_literal); 3028 3029 // Wide character constant. 3030 if (Char == '\'') 3031 return LexCharConstant(Result, ConsumeChar(CurPtr, SizeTmp, Result), 3032 tok::wide_char_constant); 3033 // FALL THROUGH, treating L like the start of an identifier. 3034 3035 // C99 6.4.2: Identifiers. 3036 case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': case 'G': 3037 case 'H': case 'I': case 'J': case 'K': /*'L'*/case 'M': case 'N': 3038 case 'O': case 'P': case 'Q': /*'R'*/case 'S': case 'T': /*'U'*/ 3039 case 'V': case 'W': case 'X': case 'Y': case 'Z': 3040 case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': case 'g': 3041 case 'h': case 'i': case 'j': case 'k': case 'l': case 'm': case 'n': 3042 case 'o': case 'p': case 'q': case 'r': case 's': case 't': /*'u'*/ 3043 case 'v': case 'w': case 'x': case 'y': case 'z': 3044 case '_': 3045 // Notify MIOpt that we read a non-whitespace/non-comment token. 3046 MIOpt.ReadToken(); 3047 return LexIdentifier(Result, CurPtr); 3048 3049 case '$': // $ in identifiers. 3050 if (LangOpts.DollarIdents) { 3051 if (!isLexingRawMode()) 3052 Diag(CurPtr-1, diag::ext_dollar_in_identifier); 3053 // Notify MIOpt that we read a non-whitespace/non-comment token. 3054 MIOpt.ReadToken(); 3055 return LexIdentifier(Result, CurPtr); 3056 } 3057 3058 Kind = tok::unknown; 3059 break; 3060 3061 // C99 6.4.4: Character Constants. 3062 case '\'': 3063 // Notify MIOpt that we read a non-whitespace/non-comment token. 3064 MIOpt.ReadToken(); 3065 return LexCharConstant(Result, CurPtr, tok::char_constant); 3066 3067 // C99 6.4.5: String Literals. 3068 case '"': 3069 // Notify MIOpt that we read a non-whitespace/non-comment token. 3070 MIOpt.ReadToken(); 3071 return LexStringLiteral(Result, CurPtr, tok::string_literal); 3072 3073 // C99 6.4.6: Punctuators. 3074 case '?': 3075 Kind = tok::question; 3076 break; 3077 case '[': 3078 Kind = tok::l_square; 3079 break; 3080 case ']': 3081 Kind = tok::r_square; 3082 break; 3083 case '(': 3084 Kind = tok::l_paren; 3085 break; 3086 case ')': 3087 Kind = tok::r_paren; 3088 break; 3089 case '{': 3090 Kind = tok::l_brace; 3091 break; 3092 case '}': 3093 Kind = tok::r_brace; 3094 break; 3095 case '.': 3096 Char = getCharAndSize(CurPtr, SizeTmp); 3097 if (Char >= '0' && Char <= '9') { 3098 // Notify MIOpt that we read a non-whitespace/non-comment token. 3099 MIOpt.ReadToken(); 3100 3101 return LexNumericConstant(Result, ConsumeChar(CurPtr, SizeTmp, Result)); 3102 } else if (LangOpts.CPlusPlus && Char == '*') { 3103 Kind = tok::periodstar; 3104 CurPtr += SizeTmp; 3105 } else if (Char == '.' && 3106 getCharAndSize(CurPtr+SizeTmp, SizeTmp2) == '.') { 3107 Kind = tok::ellipsis; 3108 CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 3109 SizeTmp2, Result); 3110 } else { 3111 Kind = tok::period; 3112 } 3113 break; 3114 case '&': 3115 Char = getCharAndSize(CurPtr, SizeTmp); 3116 if (Char == '&') { 3117 Kind = tok::ampamp; 3118 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3119 } else if (Char == '=') { 3120 Kind = tok::ampequal; 3121 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3122 } else { 3123 Kind = tok::amp; 3124 } 3125 break; 3126 case '*': 3127 if (getCharAndSize(CurPtr, SizeTmp) == '=') { 3128 Kind = tok::starequal; 3129 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3130 } else { 3131 Kind = tok::star; 3132 } 3133 break; 3134 case '+': 3135 Char = getCharAndSize(CurPtr, SizeTmp); 3136 if (Char == '+') { 3137 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3138 Kind = tok::plusplus; 3139 } else if (Char == '=') { 3140 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3141 Kind = tok::plusequal; 3142 } else { 3143 Kind = tok::plus; 3144 } 3145 break; 3146 case '-': 3147 Char = getCharAndSize(CurPtr, SizeTmp); 3148 if (Char == '-') { // -- 3149 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3150 Kind = tok::minusminus; 3151 } else if (Char == '>' && LangOpts.CPlusPlus && 3152 getCharAndSize(CurPtr+SizeTmp, SizeTmp2) == '*') { // C++ ->* 3153 CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 3154 SizeTmp2, Result); 3155 Kind = tok::arrowstar; 3156 } else if (Char == '>') { // -> 3157 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3158 Kind = tok::arrow; 3159 } else if (Char == '=') { // -= 3160 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3161 Kind = tok::minusequal; 3162 } else { 3163 Kind = tok::minus; 3164 } 3165 break; 3166 case '~': 3167 Kind = tok::tilde; 3168 break; 3169 case '!': 3170 if (getCharAndSize(CurPtr, SizeTmp) == '=') { 3171 Kind = tok::exclaimequal; 3172 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3173 } else { 3174 Kind = tok::exclaim; 3175 } 3176 break; 3177 case '/': 3178 // 6.4.9: Comments 3179 Char = getCharAndSize(CurPtr, SizeTmp); 3180 if (Char == '/') { // Line comment. 3181 // Even if Line comments are disabled (e.g. in C89 mode), we generally 3182 // want to lex this as a comment. There is one problem with this though, 3183 // that in one particular corner case, this can change the behavior of the 3184 // resultant program. For example, In "foo //**/ bar", C89 would lex 3185 // this as "foo / bar" and langauges with Line comments would lex it as 3186 // "foo". Check to see if the character after the second slash is a '*'. 3187 // If so, we will lex that as a "/" instead of the start of a comment. 3188 // However, we never do this if we are just preprocessing. 3189 bool TreatAsComment = LangOpts.LineComment && 3190 (LangOpts.CPlusPlus || !LangOpts.TraditionalCPP); 3191 if (!TreatAsComment) 3192 if (!(PP && PP->isPreprocessedOutput())) 3193 TreatAsComment = getCharAndSize(CurPtr+SizeTmp, SizeTmp2) != '*'; 3194 3195 if (TreatAsComment) { 3196 if (SkipLineComment(Result, ConsumeChar(CurPtr, SizeTmp, Result))) 3197 return; // There is a token to return. 3198 3199 // It is common for the tokens immediately after a // comment to be 3200 // whitespace (indentation for the next line). Instead of going through 3201 // the big switch, handle it efficiently now. 3202 goto SkipIgnoredUnits; 3203 } 3204 } 3205 3206 if (Char == '*') { // /**/ comment. 3207 if (SkipBlockComment(Result, ConsumeChar(CurPtr, SizeTmp, Result))) 3208 return; // There is a token to return. 3209 goto LexNextToken; // GCC isn't tail call eliminating. 3210 } 3211 3212 if (Char == '=') { 3213 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3214 Kind = tok::slashequal; 3215 } else { 3216 Kind = tok::slash; 3217 } 3218 break; 3219 case '%': 3220 Char = getCharAndSize(CurPtr, SizeTmp); 3221 if (Char == '=') { 3222 Kind = tok::percentequal; 3223 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3224 } else if (LangOpts.Digraphs && Char == '>') { 3225 Kind = tok::r_brace; // '%>' -> '}' 3226 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3227 } else if (LangOpts.Digraphs && Char == ':') { 3228 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3229 Char = getCharAndSize(CurPtr, SizeTmp); 3230 if (Char == '%' && getCharAndSize(CurPtr+SizeTmp, SizeTmp2) == ':') { 3231 Kind = tok::hashhash; // '%:%:' -> '##' 3232 CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 3233 SizeTmp2, Result); 3234 } else if (Char == '@' && LangOpts.MicrosoftExt) {// %:@ -> #@ -> Charize 3235 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3236 if (!isLexingRawMode()) 3237 Diag(BufferPtr, diag::ext_charize_microsoft); 3238 Kind = tok::hashat; 3239 } else { // '%:' -> '#' 3240 // We parsed a # character. If this occurs at the start of the line, 3241 // it's actually the start of a preprocessing directive. Callback to 3242 // the preprocessor to handle it. 3243 // FIXME: -fpreprocessed mode?? 3244 if (Result.isAtStartOfLine() && !LexingRawMode && !Is_PragmaLexer) 3245 goto HandleDirective; 3246 3247 Kind = tok::hash; 3248 } 3249 } else { 3250 Kind = tok::percent; 3251 } 3252 break; 3253 case '<': 3254 Char = getCharAndSize(CurPtr, SizeTmp); 3255 if (ParsingFilename) { 3256 return LexAngledStringLiteral(Result, CurPtr); 3257 } else if (Char == '<') { 3258 char After = getCharAndSize(CurPtr+SizeTmp, SizeTmp2); 3259 if (After == '=') { 3260 Kind = tok::lesslessequal; 3261 CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 3262 SizeTmp2, Result); 3263 } else if (After == '<' && IsStartOfConflictMarker(CurPtr-1)) { 3264 // If this is actually a '<<<<<<<' version control conflict marker, 3265 // recognize it as such and recover nicely. 3266 goto LexNextToken; 3267 } else if (After == '<' && HandleEndOfConflictMarker(CurPtr-1)) { 3268 // If this is '<<<<' and we're in a Perforce-style conflict marker, 3269 // ignore it. 3270 goto LexNextToken; 3271 } else if (LangOpts.CUDA && After == '<') { 3272 Kind = tok::lesslessless; 3273 CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 3274 SizeTmp2, Result); 3275 } else { 3276 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3277 Kind = tok::lessless; 3278 } 3279 } else if (Char == '=') { 3280 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3281 Kind = tok::lessequal; 3282 } else if (LangOpts.Digraphs && Char == ':') { // '<:' -> '[' 3283 if (LangOpts.CPlusPlus11 && 3284 getCharAndSize(CurPtr + SizeTmp, SizeTmp2) == ':') { 3285 // C++0x [lex.pptoken]p3: 3286 // Otherwise, if the next three characters are <:: and the subsequent 3287 // character is neither : nor >, the < is treated as a preprocessor 3288 // token by itself and not as the first character of the alternative 3289 // token <:. 3290 unsigned SizeTmp3; 3291 char After = getCharAndSize(CurPtr + SizeTmp + SizeTmp2, SizeTmp3); 3292 if (After != ':' && After != '>') { 3293 Kind = tok::less; 3294 if (!isLexingRawMode()) 3295 Diag(BufferPtr, diag::warn_cxx98_compat_less_colon_colon); 3296 break; 3297 } 3298 } 3299 3300 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3301 Kind = tok::l_square; 3302 } else if (LangOpts.Digraphs && Char == '%') { // '<%' -> '{' 3303 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3304 Kind = tok::l_brace; 3305 } else { 3306 Kind = tok::less; 3307 } 3308 break; 3309 case '>': 3310 Char = getCharAndSize(CurPtr, SizeTmp); 3311 if (Char == '=') { 3312 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3313 Kind = tok::greaterequal; 3314 } else if (Char == '>') { 3315 char After = getCharAndSize(CurPtr+SizeTmp, SizeTmp2); 3316 if (After == '=') { 3317 CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 3318 SizeTmp2, Result); 3319 Kind = tok::greatergreaterequal; 3320 } else if (After == '>' && IsStartOfConflictMarker(CurPtr-1)) { 3321 // If this is actually a '>>>>' conflict marker, recognize it as such 3322 // and recover nicely. 3323 goto LexNextToken; 3324 } else if (After == '>' && HandleEndOfConflictMarker(CurPtr-1)) { 3325 // If this is '>>>>>>>' and we're in a conflict marker, ignore it. 3326 goto LexNextToken; 3327 } else if (LangOpts.CUDA && After == '>') { 3328 Kind = tok::greatergreatergreater; 3329 CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 3330 SizeTmp2, Result); 3331 } else { 3332 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3333 Kind = tok::greatergreater; 3334 } 3335 3336 } else { 3337 Kind = tok::greater; 3338 } 3339 break; 3340 case '^': 3341 Char = getCharAndSize(CurPtr, SizeTmp); 3342 if (Char == '=') { 3343 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3344 Kind = tok::caretequal; 3345 } else { 3346 Kind = tok::caret; 3347 } 3348 break; 3349 case '|': 3350 Char = getCharAndSize(CurPtr, SizeTmp); 3351 if (Char == '=') { 3352 Kind = tok::pipeequal; 3353 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3354 } else if (Char == '|') { 3355 // If this is '|||||||' and we're in a conflict marker, ignore it. 3356 if (CurPtr[1] == '|' && HandleEndOfConflictMarker(CurPtr-1)) 3357 goto LexNextToken; 3358 Kind = tok::pipepipe; 3359 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3360 } else { 3361 Kind = tok::pipe; 3362 } 3363 break; 3364 case ':': 3365 Char = getCharAndSize(CurPtr, SizeTmp); 3366 if (LangOpts.Digraphs && Char == '>') { 3367 Kind = tok::r_square; // ':>' -> ']' 3368 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3369 } else if (LangOpts.CPlusPlus && Char == ':') { 3370 Kind = tok::coloncolon; 3371 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3372 } else { 3373 Kind = tok::colon; 3374 } 3375 break; 3376 case ';': 3377 Kind = tok::semi; 3378 break; 3379 case '=': 3380 Char = getCharAndSize(CurPtr, SizeTmp); 3381 if (Char == '=') { 3382 // If this is '====' and we're in a conflict marker, ignore it. 3383 if (CurPtr[1] == '=' && HandleEndOfConflictMarker(CurPtr-1)) 3384 goto LexNextToken; 3385 3386 Kind = tok::equalequal; 3387 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3388 } else { 3389 Kind = tok::equal; 3390 } 3391 break; 3392 case ',': 3393 Kind = tok::comma; 3394 break; 3395 case '#': 3396 Char = getCharAndSize(CurPtr, SizeTmp); 3397 if (Char == '#') { 3398 Kind = tok::hashhash; 3399 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3400 } else if (Char == '@' && LangOpts.MicrosoftExt) { // #@ -> Charize 3401 Kind = tok::hashat; 3402 if (!isLexingRawMode()) 3403 Diag(BufferPtr, diag::ext_charize_microsoft); 3404 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3405 } else { 3406 // We parsed a # character. If this occurs at the start of the line, 3407 // it's actually the start of a preprocessing directive. Callback to 3408 // the preprocessor to handle it. 3409 // FIXME: -fpreprocessed mode?? 3410 if (Result.isAtStartOfLine() && !LexingRawMode && !Is_PragmaLexer) 3411 goto HandleDirective; 3412 3413 Kind = tok::hash; 3414 } 3415 break; 3416 3417 case '@': 3418 // Objective C support. 3419 if (CurPtr[-1] == '@' && LangOpts.ObjC1) 3420 Kind = tok::at; 3421 else 3422 Kind = tok::unknown; 3423 break; 3424 3425 // UCNs (C99 6.4.3, C++11 [lex.charset]p2) 3426 case '\\': 3427 if (uint32_t CodePoint = tryReadUCN(CurPtr, BufferPtr, &Result)) 3428 return LexUnicode(Result, CodePoint, CurPtr); 3429 3430 Kind = tok::unknown; 3431 break; 3432 3433 default: { 3434 if (isASCII(Char)) { 3435 Kind = tok::unknown; 3436 break; 3437 } 3438 3439 UTF32 CodePoint; 3440 3441 // We can't just reset CurPtr to BufferPtr because BufferPtr may point to 3442 // an escaped newline. 3443 --CurPtr; 3444 ConversionResult Status = 3445 llvm::convertUTF8Sequence((const UTF8 **)&CurPtr, 3446 (const UTF8 *)BufferEnd, 3447 &CodePoint, 3448 strictConversion); 3449 if (Status == conversionOK) 3450 return LexUnicode(Result, CodePoint, CurPtr); 3451 3452 if (isLexingRawMode() || ParsingPreprocessorDirective || 3453 PP->isPreprocessedOutput()) { 3454 ++CurPtr; 3455 Kind = tok::unknown; 3456 break; 3457 } 3458 3459 // Non-ASCII characters tend to creep into source code unintentionally. 3460 // Instead of letting the parser complain about the unknown token, 3461 // just diagnose the invalid UTF-8, then drop the character. 3462 Diag(CurPtr, diag::err_invalid_utf8); 3463 3464 BufferPtr = CurPtr+1; 3465 goto LexNextToken; 3466 } 3467 } 3468 3469 // Notify MIOpt that we read a non-whitespace/non-comment token. 3470 MIOpt.ReadToken(); 3471 3472 // Update the location of token as well as BufferPtr. 3473 FormTokenWithChars(Result, CurPtr, Kind); 3474 return; 3475 3476HandleDirective: 3477 // We parsed a # character and it's the start of a preprocessing directive. 3478 3479 FormTokenWithChars(Result, CurPtr, tok::hash); 3480 PP->HandleDirective(Result); 3481 3482 if (PP->hadModuleLoaderFatalFailure()) { 3483 // With a fatal failure in the module loader, we abort parsing. 3484 assert(Result.is(tok::eof) && "Preprocessor did not set tok:eof"); 3485 return; 3486 } 3487 3488 // As an optimization, if the preprocessor didn't switch lexers, tail 3489 // recurse. 3490 if (PP->isCurrentLexer(this)) { 3491 // Start a new token. If this is a #include or something, the PP may 3492 // want us starting at the beginning of the line again. If so, set 3493 // the StartOfLine flag and clear LeadingSpace. 3494 if (IsAtStartOfLine) { 3495 Result.setFlag(Token::StartOfLine); 3496 Result.clearFlag(Token::LeadingSpace); 3497 IsAtStartOfLine = false; 3498 } 3499 goto LexNextToken; // GCC isn't tail call eliminating. 3500 } 3501 return PP->Lex(Result); 3502} 3503