Lexer.cpp revision c50c6ff49aa3648ae031349de6f09439f52425f0
1//===--- Lexer.cpp - C Language Family Lexer ------------------------------===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file implements the Lexer and Token interfaces. 11// 12//===----------------------------------------------------------------------===// 13// 14// TODO: GCC Diagnostics emitted by the lexer: 15// PEDWARN: (form feed|vertical tab) in preprocessing directive 16// 17// Universal characters, unicode, char mapping: 18// WARNING: `%.*s' is not in NFKC 19// WARNING: `%.*s' is not in NFC 20// 21// Other: 22// TODO: Options to support: 23// -fexec-charset,-fwide-exec-charset 24// 25//===----------------------------------------------------------------------===// 26 27#include "clang/Lex/Lexer.h" 28#include "clang/Basic/CharInfo.h" 29#include "clang/Basic/SourceManager.h" 30#include "clang/Lex/CodeCompletionHandler.h" 31#include "clang/Lex/LexDiagnostic.h" 32#include "clang/Lex/Preprocessor.h" 33#include "llvm/ADT/STLExtras.h" 34#include "llvm/ADT/StringExtras.h" 35#include "llvm/ADT/StringSwitch.h" 36#include "llvm/Support/Compiler.h" 37#include "llvm/Support/ConvertUTF.h" 38#include "llvm/Support/MemoryBuffer.h" 39#include "UnicodeCharSets.h" 40#include <cstring> 41using namespace clang; 42 43//===----------------------------------------------------------------------===// 44// Token Class Implementation 45//===----------------------------------------------------------------------===// 46 47/// isObjCAtKeyword - Return true if we have an ObjC keyword identifier. 48bool Token::isObjCAtKeyword(tok::ObjCKeywordKind objcKey) const { 49 if (IdentifierInfo *II = getIdentifierInfo()) 50 return II->getObjCKeywordID() == objcKey; 51 return false; 52} 53 54/// getObjCKeywordID - Return the ObjC keyword kind. 55tok::ObjCKeywordKind Token::getObjCKeywordID() const { 56 IdentifierInfo *specId = getIdentifierInfo(); 57 return specId ? specId->getObjCKeywordID() : tok::objc_not_keyword; 58} 59 60 61//===----------------------------------------------------------------------===// 62// Lexer Class Implementation 63//===----------------------------------------------------------------------===// 64 65void Lexer::anchor() { } 66 67void Lexer::InitLexer(const char *BufStart, const char *BufPtr, 68 const char *BufEnd) { 69 BufferStart = BufStart; 70 BufferPtr = BufPtr; 71 BufferEnd = BufEnd; 72 73 assert(BufEnd[0] == 0 && 74 "We assume that the input buffer has a null character at the end" 75 " to simplify lexing!"); 76 77 // Check whether we have a BOM in the beginning of the buffer. If yes - act 78 // accordingly. Right now we support only UTF-8 with and without BOM, so, just 79 // skip the UTF-8 BOM if it's present. 80 if (BufferStart == BufferPtr) { 81 // Determine the size of the BOM. 82 StringRef Buf(BufferStart, BufferEnd - BufferStart); 83 size_t BOMLength = llvm::StringSwitch<size_t>(Buf) 84 .StartsWith("\xEF\xBB\xBF", 3) // UTF-8 BOM 85 .Default(0); 86 87 // Skip the BOM. 88 BufferPtr += BOMLength; 89 } 90 91 Is_PragmaLexer = false; 92 CurrentConflictMarkerState = CMK_None; 93 94 // Start of the file is a start of line. 95 IsAtStartOfLine = true; 96 97 // We are not after parsing a #. 98 ParsingPreprocessorDirective = false; 99 100 // We are not after parsing #include. 101 ParsingFilename = false; 102 103 // We are not in raw mode. Raw mode disables diagnostics and interpretation 104 // of tokens (e.g. identifiers, thus disabling macro expansion). It is used 105 // to quickly lex the tokens of the buffer, e.g. when handling a "#if 0" block 106 // or otherwise skipping over tokens. 107 LexingRawMode = false; 108 109 // Default to not keeping comments. 110 ExtendedTokenMode = 0; 111} 112 113/// Lexer constructor - Create a new lexer object for the specified buffer 114/// with the specified preprocessor managing the lexing process. This lexer 115/// assumes that the associated file buffer and Preprocessor objects will 116/// outlive it, so it doesn't take ownership of either of them. 117Lexer::Lexer(FileID FID, const llvm::MemoryBuffer *InputFile, Preprocessor &PP) 118 : PreprocessorLexer(&PP, FID), 119 FileLoc(PP.getSourceManager().getLocForStartOfFile(FID)), 120 LangOpts(PP.getLangOpts()) { 121 122 InitLexer(InputFile->getBufferStart(), InputFile->getBufferStart(), 123 InputFile->getBufferEnd()); 124 125 resetExtendedTokenMode(); 126} 127 128void Lexer::resetExtendedTokenMode() { 129 assert(PP && "Cannot reset token mode without a preprocessor"); 130 if (LangOpts.TraditionalCPP) 131 SetKeepWhitespaceMode(true); 132 else 133 SetCommentRetentionState(PP->getCommentRetentionState()); 134} 135 136/// Lexer constructor - Create a new raw lexer object. This object is only 137/// suitable for calls to 'LexFromRawLexer'. This lexer assumes that the text 138/// range will outlive it, so it doesn't take ownership of it. 139Lexer::Lexer(SourceLocation fileloc, const LangOptions &langOpts, 140 const char *BufStart, const char *BufPtr, const char *BufEnd) 141 : FileLoc(fileloc), LangOpts(langOpts) { 142 143 InitLexer(BufStart, BufPtr, BufEnd); 144 145 // We *are* in raw mode. 146 LexingRawMode = true; 147} 148 149/// Lexer constructor - Create a new raw lexer object. This object is only 150/// suitable for calls to 'LexFromRawLexer'. This lexer assumes that the text 151/// range will outlive it, so it doesn't take ownership of it. 152Lexer::Lexer(FileID FID, const llvm::MemoryBuffer *FromFile, 153 const SourceManager &SM, const LangOptions &langOpts) 154 : FileLoc(SM.getLocForStartOfFile(FID)), LangOpts(langOpts) { 155 156 InitLexer(FromFile->getBufferStart(), FromFile->getBufferStart(), 157 FromFile->getBufferEnd()); 158 159 // We *are* in raw mode. 160 LexingRawMode = true; 161} 162 163/// Create_PragmaLexer: Lexer constructor - Create a new lexer object for 164/// _Pragma expansion. This has a variety of magic semantics that this method 165/// sets up. It returns a new'd Lexer that must be delete'd when done. 166/// 167/// On entrance to this routine, TokStartLoc is a macro location which has a 168/// spelling loc that indicates the bytes to be lexed for the token and an 169/// expansion location that indicates where all lexed tokens should be 170/// "expanded from". 171/// 172/// FIXME: It would really be nice to make _Pragma just be a wrapper around a 173/// normal lexer that remaps tokens as they fly by. This would require making 174/// Preprocessor::Lex virtual. Given that, we could just dump in a magic lexer 175/// interface that could handle this stuff. This would pull GetMappedTokenLoc 176/// out of the critical path of the lexer! 177/// 178Lexer *Lexer::Create_PragmaLexer(SourceLocation SpellingLoc, 179 SourceLocation ExpansionLocStart, 180 SourceLocation ExpansionLocEnd, 181 unsigned TokLen, Preprocessor &PP) { 182 SourceManager &SM = PP.getSourceManager(); 183 184 // Create the lexer as if we were going to lex the file normally. 185 FileID SpellingFID = SM.getFileID(SpellingLoc); 186 const llvm::MemoryBuffer *InputFile = SM.getBuffer(SpellingFID); 187 Lexer *L = new Lexer(SpellingFID, InputFile, PP); 188 189 // Now that the lexer is created, change the start/end locations so that we 190 // just lex the subsection of the file that we want. This is lexing from a 191 // scratch buffer. 192 const char *StrData = SM.getCharacterData(SpellingLoc); 193 194 L->BufferPtr = StrData; 195 L->BufferEnd = StrData+TokLen; 196 assert(L->BufferEnd[0] == 0 && "Buffer is not nul terminated!"); 197 198 // Set the SourceLocation with the remapping information. This ensures that 199 // GetMappedTokenLoc will remap the tokens as they are lexed. 200 L->FileLoc = SM.createExpansionLoc(SM.getLocForStartOfFile(SpellingFID), 201 ExpansionLocStart, 202 ExpansionLocEnd, TokLen); 203 204 // Ensure that the lexer thinks it is inside a directive, so that end \n will 205 // return an EOD token. 206 L->ParsingPreprocessorDirective = true; 207 208 // This lexer really is for _Pragma. 209 L->Is_PragmaLexer = true; 210 return L; 211} 212 213 214/// Stringify - Convert the specified string into a C string, with surrounding 215/// ""'s, and with escaped \ and " characters. 216std::string Lexer::Stringify(const std::string &Str, bool Charify) { 217 std::string Result = Str; 218 char Quote = Charify ? '\'' : '"'; 219 for (unsigned i = 0, e = Result.size(); i != e; ++i) { 220 if (Result[i] == '\\' || Result[i] == Quote) { 221 Result.insert(Result.begin()+i, '\\'); 222 ++i; ++e; 223 } 224 } 225 return Result; 226} 227 228/// Stringify - Convert the specified string into a C string by escaping '\' 229/// and " characters. This does not add surrounding ""'s to the string. 230void Lexer::Stringify(SmallVectorImpl<char> &Str) { 231 for (unsigned i = 0, e = Str.size(); i != e; ++i) { 232 if (Str[i] == '\\' || Str[i] == '"') { 233 Str.insert(Str.begin()+i, '\\'); 234 ++i; ++e; 235 } 236 } 237} 238 239//===----------------------------------------------------------------------===// 240// Token Spelling 241//===----------------------------------------------------------------------===// 242 243/// \brief Slow case of getSpelling. Extract the characters comprising the 244/// spelling of this token from the provided input buffer. 245static size_t getSpellingSlow(const Token &Tok, const char *BufPtr, 246 const LangOptions &LangOpts, char *Spelling) { 247 assert(Tok.needsCleaning() && "getSpellingSlow called on simple token"); 248 249 size_t Length = 0; 250 const char *BufEnd = BufPtr + Tok.getLength(); 251 252 if (Tok.is(tok::string_literal)) { 253 // Munch the encoding-prefix and opening double-quote. 254 while (BufPtr < BufEnd) { 255 unsigned Size; 256 Spelling[Length++] = Lexer::getCharAndSizeNoWarn(BufPtr, Size, LangOpts); 257 BufPtr += Size; 258 259 if (Spelling[Length - 1] == '"') 260 break; 261 } 262 263 // Raw string literals need special handling; trigraph expansion and line 264 // splicing do not occur within their d-char-sequence nor within their 265 // r-char-sequence. 266 if (Length >= 2 && 267 Spelling[Length - 2] == 'R' && Spelling[Length - 1] == '"') { 268 // Search backwards from the end of the token to find the matching closing 269 // quote. 270 const char *RawEnd = BufEnd; 271 do --RawEnd; while (*RawEnd != '"'); 272 size_t RawLength = RawEnd - BufPtr + 1; 273 274 // Everything between the quotes is included verbatim in the spelling. 275 memcpy(Spelling + Length, BufPtr, RawLength); 276 Length += RawLength; 277 BufPtr += RawLength; 278 279 // The rest of the token is lexed normally. 280 } 281 } 282 283 while (BufPtr < BufEnd) { 284 unsigned Size; 285 Spelling[Length++] = Lexer::getCharAndSizeNoWarn(BufPtr, Size, LangOpts); 286 BufPtr += Size; 287 } 288 289 assert(Length < Tok.getLength() && 290 "NeedsCleaning flag set on token that didn't need cleaning!"); 291 return Length; 292} 293 294/// getSpelling() - Return the 'spelling' of this token. The spelling of a 295/// token are the characters used to represent the token in the source file 296/// after trigraph expansion and escaped-newline folding. In particular, this 297/// wants to get the true, uncanonicalized, spelling of things like digraphs 298/// UCNs, etc. 299StringRef Lexer::getSpelling(SourceLocation loc, 300 SmallVectorImpl<char> &buffer, 301 const SourceManager &SM, 302 const LangOptions &options, 303 bool *invalid) { 304 // Break down the source location. 305 std::pair<FileID, unsigned> locInfo = SM.getDecomposedLoc(loc); 306 307 // Try to the load the file buffer. 308 bool invalidTemp = false; 309 StringRef file = SM.getBufferData(locInfo.first, &invalidTemp); 310 if (invalidTemp) { 311 if (invalid) *invalid = true; 312 return StringRef(); 313 } 314 315 const char *tokenBegin = file.data() + locInfo.second; 316 317 // Lex from the start of the given location. 318 Lexer lexer(SM.getLocForStartOfFile(locInfo.first), options, 319 file.begin(), tokenBegin, file.end()); 320 Token token; 321 lexer.LexFromRawLexer(token); 322 323 unsigned length = token.getLength(); 324 325 // Common case: no need for cleaning. 326 if (!token.needsCleaning()) 327 return StringRef(tokenBegin, length); 328 329 // Hard case, we need to relex the characters into the string. 330 buffer.resize(length); 331 buffer.resize(getSpellingSlow(token, tokenBegin, options, buffer.data())); 332 return StringRef(buffer.data(), buffer.size()); 333} 334 335/// getSpelling() - Return the 'spelling' of this token. The spelling of a 336/// token are the characters used to represent the token in the source file 337/// after trigraph expansion and escaped-newline folding. In particular, this 338/// wants to get the true, uncanonicalized, spelling of things like digraphs 339/// UCNs, etc. 340std::string Lexer::getSpelling(const Token &Tok, const SourceManager &SourceMgr, 341 const LangOptions &LangOpts, bool *Invalid) { 342 assert((int)Tok.getLength() >= 0 && "Token character range is bogus!"); 343 344 bool CharDataInvalid = false; 345 const char *TokStart = SourceMgr.getCharacterData(Tok.getLocation(), 346 &CharDataInvalid); 347 if (Invalid) 348 *Invalid = CharDataInvalid; 349 if (CharDataInvalid) 350 return std::string(); 351 352 // If this token contains nothing interesting, return it directly. 353 if (!Tok.needsCleaning()) 354 return std::string(TokStart, TokStart + Tok.getLength()); 355 356 std::string Result; 357 Result.resize(Tok.getLength()); 358 Result.resize(getSpellingSlow(Tok, TokStart, LangOpts, &*Result.begin())); 359 return Result; 360} 361 362/// getSpelling - This method is used to get the spelling of a token into a 363/// preallocated buffer, instead of as an std::string. The caller is required 364/// to allocate enough space for the token, which is guaranteed to be at least 365/// Tok.getLength() bytes long. The actual length of the token is returned. 366/// 367/// Note that this method may do two possible things: it may either fill in 368/// the buffer specified with characters, or it may *change the input pointer* 369/// to point to a constant buffer with the data already in it (avoiding a 370/// copy). The caller is not allowed to modify the returned buffer pointer 371/// if an internal buffer is returned. 372unsigned Lexer::getSpelling(const Token &Tok, const char *&Buffer, 373 const SourceManager &SourceMgr, 374 const LangOptions &LangOpts, bool *Invalid) { 375 assert((int)Tok.getLength() >= 0 && "Token character range is bogus!"); 376 377 const char *TokStart = 0; 378 // NOTE: this has to be checked *before* testing for an IdentifierInfo. 379 if (Tok.is(tok::raw_identifier)) 380 TokStart = Tok.getRawIdentifierData(); 381 else if (!Tok.hasUCN()) { 382 if (const IdentifierInfo *II = Tok.getIdentifierInfo()) { 383 // Just return the string from the identifier table, which is very quick. 384 Buffer = II->getNameStart(); 385 return II->getLength(); 386 } 387 } 388 389 // NOTE: this can be checked even after testing for an IdentifierInfo. 390 if (Tok.isLiteral()) 391 TokStart = Tok.getLiteralData(); 392 393 if (TokStart == 0) { 394 // Compute the start of the token in the input lexer buffer. 395 bool CharDataInvalid = false; 396 TokStart = SourceMgr.getCharacterData(Tok.getLocation(), &CharDataInvalid); 397 if (Invalid) 398 *Invalid = CharDataInvalid; 399 if (CharDataInvalid) { 400 Buffer = ""; 401 return 0; 402 } 403 } 404 405 // If this token contains nothing interesting, return it directly. 406 if (!Tok.needsCleaning()) { 407 Buffer = TokStart; 408 return Tok.getLength(); 409 } 410 411 // Otherwise, hard case, relex the characters into the string. 412 return getSpellingSlow(Tok, TokStart, LangOpts, const_cast<char*>(Buffer)); 413} 414 415 416/// MeasureTokenLength - Relex the token at the specified location and return 417/// its length in bytes in the input file. If the token needs cleaning (e.g. 418/// includes a trigraph or an escaped newline) then this count includes bytes 419/// that are part of that. 420unsigned Lexer::MeasureTokenLength(SourceLocation Loc, 421 const SourceManager &SM, 422 const LangOptions &LangOpts) { 423 Token TheTok; 424 if (getRawToken(Loc, TheTok, SM, LangOpts)) 425 return 0; 426 return TheTok.getLength(); 427} 428 429/// \brief Relex the token at the specified location. 430/// \returns true if there was a failure, false on success. 431bool Lexer::getRawToken(SourceLocation Loc, Token &Result, 432 const SourceManager &SM, 433 const LangOptions &LangOpts) { 434 // TODO: this could be special cased for common tokens like identifiers, ')', 435 // etc to make this faster, if it mattered. Just look at StrData[0] to handle 436 // all obviously single-char tokens. This could use 437 // Lexer::isObviouslySimpleCharacter for example to handle identifiers or 438 // something. 439 440 // If this comes from a macro expansion, we really do want the macro name, not 441 // the token this macro expanded to. 442 Loc = SM.getExpansionLoc(Loc); 443 std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(Loc); 444 bool Invalid = false; 445 StringRef Buffer = SM.getBufferData(LocInfo.first, &Invalid); 446 if (Invalid) 447 return true; 448 449 const char *StrData = Buffer.data()+LocInfo.second; 450 451 if (isWhitespace(StrData[0])) 452 return true; 453 454 // Create a lexer starting at the beginning of this token. 455 Lexer TheLexer(SM.getLocForStartOfFile(LocInfo.first), LangOpts, 456 Buffer.begin(), StrData, Buffer.end()); 457 TheLexer.SetCommentRetentionState(true); 458 TheLexer.LexFromRawLexer(Result); 459 return false; 460} 461 462static SourceLocation getBeginningOfFileToken(SourceLocation Loc, 463 const SourceManager &SM, 464 const LangOptions &LangOpts) { 465 assert(Loc.isFileID()); 466 std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(Loc); 467 if (LocInfo.first.isInvalid()) 468 return Loc; 469 470 bool Invalid = false; 471 StringRef Buffer = SM.getBufferData(LocInfo.first, &Invalid); 472 if (Invalid) 473 return Loc; 474 475 // Back up from the current location until we hit the beginning of a line 476 // (or the buffer). We'll relex from that point. 477 const char *BufStart = Buffer.data(); 478 if (LocInfo.second >= Buffer.size()) 479 return Loc; 480 481 const char *StrData = BufStart+LocInfo.second; 482 if (StrData[0] == '\n' || StrData[0] == '\r') 483 return Loc; 484 485 const char *LexStart = StrData; 486 while (LexStart != BufStart) { 487 if (LexStart[0] == '\n' || LexStart[0] == '\r') { 488 ++LexStart; 489 break; 490 } 491 492 --LexStart; 493 } 494 495 // Create a lexer starting at the beginning of this token. 496 SourceLocation LexerStartLoc = Loc.getLocWithOffset(-LocInfo.second); 497 Lexer TheLexer(LexerStartLoc, LangOpts, BufStart, LexStart, Buffer.end()); 498 TheLexer.SetCommentRetentionState(true); 499 500 // Lex tokens until we find the token that contains the source location. 501 Token TheTok; 502 do { 503 TheLexer.LexFromRawLexer(TheTok); 504 505 if (TheLexer.getBufferLocation() > StrData) { 506 // Lexing this token has taken the lexer past the source location we're 507 // looking for. If the current token encompasses our source location, 508 // return the beginning of that token. 509 if (TheLexer.getBufferLocation() - TheTok.getLength() <= StrData) 510 return TheTok.getLocation(); 511 512 // We ended up skipping over the source location entirely, which means 513 // that it points into whitespace. We're done here. 514 break; 515 } 516 } while (TheTok.getKind() != tok::eof); 517 518 // We've passed our source location; just return the original source location. 519 return Loc; 520} 521 522SourceLocation Lexer::GetBeginningOfToken(SourceLocation Loc, 523 const SourceManager &SM, 524 const LangOptions &LangOpts) { 525 if (Loc.isFileID()) 526 return getBeginningOfFileToken(Loc, SM, LangOpts); 527 528 if (!SM.isMacroArgExpansion(Loc)) 529 return Loc; 530 531 SourceLocation FileLoc = SM.getSpellingLoc(Loc); 532 SourceLocation BeginFileLoc = getBeginningOfFileToken(FileLoc, SM, LangOpts); 533 std::pair<FileID, unsigned> FileLocInfo = SM.getDecomposedLoc(FileLoc); 534 std::pair<FileID, unsigned> BeginFileLocInfo 535 = SM.getDecomposedLoc(BeginFileLoc); 536 assert(FileLocInfo.first == BeginFileLocInfo.first && 537 FileLocInfo.second >= BeginFileLocInfo.second); 538 return Loc.getLocWithOffset(BeginFileLocInfo.second - FileLocInfo.second); 539} 540 541namespace { 542 enum PreambleDirectiveKind { 543 PDK_Skipped, 544 PDK_StartIf, 545 PDK_EndIf, 546 PDK_Unknown 547 }; 548} 549 550std::pair<unsigned, bool> 551Lexer::ComputePreamble(const llvm::MemoryBuffer *Buffer, 552 const LangOptions &LangOpts, unsigned MaxLines) { 553 // Create a lexer starting at the beginning of the file. Note that we use a 554 // "fake" file source location at offset 1 so that the lexer will track our 555 // position within the file. 556 const unsigned StartOffset = 1; 557 SourceLocation FileLoc = SourceLocation::getFromRawEncoding(StartOffset); 558 Lexer TheLexer(FileLoc, LangOpts, Buffer->getBufferStart(), 559 Buffer->getBufferStart(), Buffer->getBufferEnd()); 560 TheLexer.SetCommentRetentionState(true); 561 562 // StartLoc will differ from FileLoc if there is a BOM that was skipped. 563 SourceLocation StartLoc = TheLexer.getSourceLocation(); 564 565 bool InPreprocessorDirective = false; 566 Token TheTok; 567 Token IfStartTok; 568 unsigned IfCount = 0; 569 SourceLocation ActiveCommentLoc; 570 571 unsigned MaxLineOffset = 0; 572 if (MaxLines) { 573 const char *CurPtr = Buffer->getBufferStart(); 574 unsigned CurLine = 0; 575 while (CurPtr != Buffer->getBufferEnd()) { 576 char ch = *CurPtr++; 577 if (ch == '\n') { 578 ++CurLine; 579 if (CurLine == MaxLines) 580 break; 581 } 582 } 583 if (CurPtr != Buffer->getBufferEnd()) 584 MaxLineOffset = CurPtr - Buffer->getBufferStart(); 585 } 586 587 do { 588 TheLexer.LexFromRawLexer(TheTok); 589 590 if (InPreprocessorDirective) { 591 // If we've hit the end of the file, we're done. 592 if (TheTok.getKind() == tok::eof) { 593 break; 594 } 595 596 // If we haven't hit the end of the preprocessor directive, skip this 597 // token. 598 if (!TheTok.isAtStartOfLine()) 599 continue; 600 601 // We've passed the end of the preprocessor directive, and will look 602 // at this token again below. 603 InPreprocessorDirective = false; 604 } 605 606 // Keep track of the # of lines in the preamble. 607 if (TheTok.isAtStartOfLine()) { 608 unsigned TokOffset = TheTok.getLocation().getRawEncoding() - StartOffset; 609 610 // If we were asked to limit the number of lines in the preamble, 611 // and we're about to exceed that limit, we're done. 612 if (MaxLineOffset && TokOffset >= MaxLineOffset) 613 break; 614 } 615 616 // Comments are okay; skip over them. 617 if (TheTok.getKind() == tok::comment) { 618 if (ActiveCommentLoc.isInvalid()) 619 ActiveCommentLoc = TheTok.getLocation(); 620 continue; 621 } 622 623 if (TheTok.isAtStartOfLine() && TheTok.getKind() == tok::hash) { 624 // This is the start of a preprocessor directive. 625 Token HashTok = TheTok; 626 InPreprocessorDirective = true; 627 ActiveCommentLoc = SourceLocation(); 628 629 // Figure out which directive this is. Since we're lexing raw tokens, 630 // we don't have an identifier table available. Instead, just look at 631 // the raw identifier to recognize and categorize preprocessor directives. 632 TheLexer.LexFromRawLexer(TheTok); 633 if (TheTok.getKind() == tok::raw_identifier && !TheTok.needsCleaning()) { 634 StringRef Keyword(TheTok.getRawIdentifierData(), 635 TheTok.getLength()); 636 PreambleDirectiveKind PDK 637 = llvm::StringSwitch<PreambleDirectiveKind>(Keyword) 638 .Case("include", PDK_Skipped) 639 .Case("__include_macros", PDK_Skipped) 640 .Case("define", PDK_Skipped) 641 .Case("undef", PDK_Skipped) 642 .Case("line", PDK_Skipped) 643 .Case("error", PDK_Skipped) 644 .Case("pragma", PDK_Skipped) 645 .Case("import", PDK_Skipped) 646 .Case("include_next", PDK_Skipped) 647 .Case("warning", PDK_Skipped) 648 .Case("ident", PDK_Skipped) 649 .Case("sccs", PDK_Skipped) 650 .Case("assert", PDK_Skipped) 651 .Case("unassert", PDK_Skipped) 652 .Case("if", PDK_StartIf) 653 .Case("ifdef", PDK_StartIf) 654 .Case("ifndef", PDK_StartIf) 655 .Case("elif", PDK_Skipped) 656 .Case("else", PDK_Skipped) 657 .Case("endif", PDK_EndIf) 658 .Default(PDK_Unknown); 659 660 switch (PDK) { 661 case PDK_Skipped: 662 continue; 663 664 case PDK_StartIf: 665 if (IfCount == 0) 666 IfStartTok = HashTok; 667 668 ++IfCount; 669 continue; 670 671 case PDK_EndIf: 672 // Mismatched #endif. The preamble ends here. 673 if (IfCount == 0) 674 break; 675 676 --IfCount; 677 continue; 678 679 case PDK_Unknown: 680 // We don't know what this directive is; stop at the '#'. 681 break; 682 } 683 } 684 685 // We only end up here if we didn't recognize the preprocessor 686 // directive or it was one that can't occur in the preamble at this 687 // point. Roll back the current token to the location of the '#'. 688 InPreprocessorDirective = false; 689 TheTok = HashTok; 690 } 691 692 // We hit a token that we don't recognize as being in the 693 // "preprocessing only" part of the file, so we're no longer in 694 // the preamble. 695 break; 696 } while (true); 697 698 SourceLocation End; 699 if (IfCount) 700 End = IfStartTok.getLocation(); 701 else if (ActiveCommentLoc.isValid()) 702 End = ActiveCommentLoc; // don't truncate a decl comment. 703 else 704 End = TheTok.getLocation(); 705 706 return std::make_pair(End.getRawEncoding() - StartLoc.getRawEncoding(), 707 IfCount? IfStartTok.isAtStartOfLine() 708 : TheTok.isAtStartOfLine()); 709} 710 711 712/// AdvanceToTokenCharacter - Given a location that specifies the start of a 713/// token, return a new location that specifies a character within the token. 714SourceLocation Lexer::AdvanceToTokenCharacter(SourceLocation TokStart, 715 unsigned CharNo, 716 const SourceManager &SM, 717 const LangOptions &LangOpts) { 718 // Figure out how many physical characters away the specified expansion 719 // character is. This needs to take into consideration newlines and 720 // trigraphs. 721 bool Invalid = false; 722 const char *TokPtr = SM.getCharacterData(TokStart, &Invalid); 723 724 // If they request the first char of the token, we're trivially done. 725 if (Invalid || (CharNo == 0 && Lexer::isObviouslySimpleCharacter(*TokPtr))) 726 return TokStart; 727 728 unsigned PhysOffset = 0; 729 730 // The usual case is that tokens don't contain anything interesting. Skip 731 // over the uninteresting characters. If a token only consists of simple 732 // chars, this method is extremely fast. 733 while (Lexer::isObviouslySimpleCharacter(*TokPtr)) { 734 if (CharNo == 0) 735 return TokStart.getLocWithOffset(PhysOffset); 736 ++TokPtr, --CharNo, ++PhysOffset; 737 } 738 739 // If we have a character that may be a trigraph or escaped newline, use a 740 // lexer to parse it correctly. 741 for (; CharNo; --CharNo) { 742 unsigned Size; 743 Lexer::getCharAndSizeNoWarn(TokPtr, Size, LangOpts); 744 TokPtr += Size; 745 PhysOffset += Size; 746 } 747 748 // Final detail: if we end up on an escaped newline, we want to return the 749 // location of the actual byte of the token. For example foo\<newline>bar 750 // advanced by 3 should return the location of b, not of \\. One compounding 751 // detail of this is that the escape may be made by a trigraph. 752 if (!Lexer::isObviouslySimpleCharacter(*TokPtr)) 753 PhysOffset += Lexer::SkipEscapedNewLines(TokPtr)-TokPtr; 754 755 return TokStart.getLocWithOffset(PhysOffset); 756} 757 758/// \brief Computes the source location just past the end of the 759/// token at this source location. 760/// 761/// This routine can be used to produce a source location that 762/// points just past the end of the token referenced by \p Loc, and 763/// is generally used when a diagnostic needs to point just after a 764/// token where it expected something different that it received. If 765/// the returned source location would not be meaningful (e.g., if 766/// it points into a macro), this routine returns an invalid 767/// source location. 768/// 769/// \param Offset an offset from the end of the token, where the source 770/// location should refer to. The default offset (0) produces a source 771/// location pointing just past the end of the token; an offset of 1 produces 772/// a source location pointing to the last character in the token, etc. 773SourceLocation Lexer::getLocForEndOfToken(SourceLocation Loc, unsigned Offset, 774 const SourceManager &SM, 775 const LangOptions &LangOpts) { 776 if (Loc.isInvalid()) 777 return SourceLocation(); 778 779 if (Loc.isMacroID()) { 780 if (Offset > 0 || !isAtEndOfMacroExpansion(Loc, SM, LangOpts, &Loc)) 781 return SourceLocation(); // Points inside the macro expansion. 782 } 783 784 unsigned Len = Lexer::MeasureTokenLength(Loc, SM, LangOpts); 785 if (Len > Offset) 786 Len = Len - Offset; 787 else 788 return Loc; 789 790 return Loc.getLocWithOffset(Len); 791} 792 793/// \brief Returns true if the given MacroID location points at the first 794/// token of the macro expansion. 795bool Lexer::isAtStartOfMacroExpansion(SourceLocation loc, 796 const SourceManager &SM, 797 const LangOptions &LangOpts, 798 SourceLocation *MacroBegin) { 799 assert(loc.isValid() && loc.isMacroID() && "Expected a valid macro loc"); 800 801 SourceLocation expansionLoc; 802 if (!SM.isAtStartOfImmediateMacroExpansion(loc, &expansionLoc)) 803 return false; 804 805 if (expansionLoc.isFileID()) { 806 // No other macro expansions, this is the first. 807 if (MacroBegin) 808 *MacroBegin = expansionLoc; 809 return true; 810 } 811 812 return isAtStartOfMacroExpansion(expansionLoc, SM, LangOpts, MacroBegin); 813} 814 815/// \brief Returns true if the given MacroID location points at the last 816/// token of the macro expansion. 817bool Lexer::isAtEndOfMacroExpansion(SourceLocation loc, 818 const SourceManager &SM, 819 const LangOptions &LangOpts, 820 SourceLocation *MacroEnd) { 821 assert(loc.isValid() && loc.isMacroID() && "Expected a valid macro loc"); 822 823 SourceLocation spellLoc = SM.getSpellingLoc(loc); 824 unsigned tokLen = MeasureTokenLength(spellLoc, SM, LangOpts); 825 if (tokLen == 0) 826 return false; 827 828 SourceLocation afterLoc = loc.getLocWithOffset(tokLen); 829 SourceLocation expansionLoc; 830 if (!SM.isAtEndOfImmediateMacroExpansion(afterLoc, &expansionLoc)) 831 return false; 832 833 if (expansionLoc.isFileID()) { 834 // No other macro expansions. 835 if (MacroEnd) 836 *MacroEnd = expansionLoc; 837 return true; 838 } 839 840 return isAtEndOfMacroExpansion(expansionLoc, SM, LangOpts, MacroEnd); 841} 842 843static CharSourceRange makeRangeFromFileLocs(CharSourceRange Range, 844 const SourceManager &SM, 845 const LangOptions &LangOpts) { 846 SourceLocation Begin = Range.getBegin(); 847 SourceLocation End = Range.getEnd(); 848 assert(Begin.isFileID() && End.isFileID()); 849 if (Range.isTokenRange()) { 850 End = Lexer::getLocForEndOfToken(End, 0, SM,LangOpts); 851 if (End.isInvalid()) 852 return CharSourceRange(); 853 } 854 855 // Break down the source locations. 856 FileID FID; 857 unsigned BeginOffs; 858 llvm::tie(FID, BeginOffs) = SM.getDecomposedLoc(Begin); 859 if (FID.isInvalid()) 860 return CharSourceRange(); 861 862 unsigned EndOffs; 863 if (!SM.isInFileID(End, FID, &EndOffs) || 864 BeginOffs > EndOffs) 865 return CharSourceRange(); 866 867 return CharSourceRange::getCharRange(Begin, End); 868} 869 870CharSourceRange Lexer::makeFileCharRange(CharSourceRange Range, 871 const SourceManager &SM, 872 const LangOptions &LangOpts) { 873 SourceLocation Begin = Range.getBegin(); 874 SourceLocation End = Range.getEnd(); 875 if (Begin.isInvalid() || End.isInvalid()) 876 return CharSourceRange(); 877 878 if (Begin.isFileID() && End.isFileID()) 879 return makeRangeFromFileLocs(Range, SM, LangOpts); 880 881 if (Begin.isMacroID() && End.isFileID()) { 882 if (!isAtStartOfMacroExpansion(Begin, SM, LangOpts, &Begin)) 883 return CharSourceRange(); 884 Range.setBegin(Begin); 885 return makeRangeFromFileLocs(Range, SM, LangOpts); 886 } 887 888 if (Begin.isFileID() && End.isMacroID()) { 889 if ((Range.isTokenRange() && !isAtEndOfMacroExpansion(End, SM, LangOpts, 890 &End)) || 891 (Range.isCharRange() && !isAtStartOfMacroExpansion(End, SM, LangOpts, 892 &End))) 893 return CharSourceRange(); 894 Range.setEnd(End); 895 return makeRangeFromFileLocs(Range, SM, LangOpts); 896 } 897 898 assert(Begin.isMacroID() && End.isMacroID()); 899 SourceLocation MacroBegin, MacroEnd; 900 if (isAtStartOfMacroExpansion(Begin, SM, LangOpts, &MacroBegin) && 901 ((Range.isTokenRange() && isAtEndOfMacroExpansion(End, SM, LangOpts, 902 &MacroEnd)) || 903 (Range.isCharRange() && isAtStartOfMacroExpansion(End, SM, LangOpts, 904 &MacroEnd)))) { 905 Range.setBegin(MacroBegin); 906 Range.setEnd(MacroEnd); 907 return makeRangeFromFileLocs(Range, SM, LangOpts); 908 } 909 910 bool Invalid = false; 911 const SrcMgr::SLocEntry &BeginEntry = SM.getSLocEntry(SM.getFileID(Begin), 912 &Invalid); 913 if (Invalid) 914 return CharSourceRange(); 915 916 if (BeginEntry.getExpansion().isMacroArgExpansion()) { 917 const SrcMgr::SLocEntry &EndEntry = SM.getSLocEntry(SM.getFileID(End), 918 &Invalid); 919 if (Invalid) 920 return CharSourceRange(); 921 922 if (EndEntry.getExpansion().isMacroArgExpansion() && 923 BeginEntry.getExpansion().getExpansionLocStart() == 924 EndEntry.getExpansion().getExpansionLocStart()) { 925 Range.setBegin(SM.getImmediateSpellingLoc(Begin)); 926 Range.setEnd(SM.getImmediateSpellingLoc(End)); 927 return makeFileCharRange(Range, SM, LangOpts); 928 } 929 } 930 931 return CharSourceRange(); 932} 933 934StringRef Lexer::getSourceText(CharSourceRange Range, 935 const SourceManager &SM, 936 const LangOptions &LangOpts, 937 bool *Invalid) { 938 Range = makeFileCharRange(Range, SM, LangOpts); 939 if (Range.isInvalid()) { 940 if (Invalid) *Invalid = true; 941 return StringRef(); 942 } 943 944 // Break down the source location. 945 std::pair<FileID, unsigned> beginInfo = SM.getDecomposedLoc(Range.getBegin()); 946 if (beginInfo.first.isInvalid()) { 947 if (Invalid) *Invalid = true; 948 return StringRef(); 949 } 950 951 unsigned EndOffs; 952 if (!SM.isInFileID(Range.getEnd(), beginInfo.first, &EndOffs) || 953 beginInfo.second > EndOffs) { 954 if (Invalid) *Invalid = true; 955 return StringRef(); 956 } 957 958 // Try to the load the file buffer. 959 bool invalidTemp = false; 960 StringRef file = SM.getBufferData(beginInfo.first, &invalidTemp); 961 if (invalidTemp) { 962 if (Invalid) *Invalid = true; 963 return StringRef(); 964 } 965 966 if (Invalid) *Invalid = false; 967 return file.substr(beginInfo.second, EndOffs - beginInfo.second); 968} 969 970StringRef Lexer::getImmediateMacroName(SourceLocation Loc, 971 const SourceManager &SM, 972 const LangOptions &LangOpts) { 973 assert(Loc.isMacroID() && "Only reasonble to call this on macros"); 974 975 // Find the location of the immediate macro expansion. 976 while (1) { 977 FileID FID = SM.getFileID(Loc); 978 const SrcMgr::SLocEntry *E = &SM.getSLocEntry(FID); 979 const SrcMgr::ExpansionInfo &Expansion = E->getExpansion(); 980 Loc = Expansion.getExpansionLocStart(); 981 if (!Expansion.isMacroArgExpansion()) 982 break; 983 984 // For macro arguments we need to check that the argument did not come 985 // from an inner macro, e.g: "MAC1( MAC2(foo) )" 986 987 // Loc points to the argument id of the macro definition, move to the 988 // macro expansion. 989 Loc = SM.getImmediateExpansionRange(Loc).first; 990 SourceLocation SpellLoc = Expansion.getSpellingLoc(); 991 if (SpellLoc.isFileID()) 992 break; // No inner macro. 993 994 // If spelling location resides in the same FileID as macro expansion 995 // location, it means there is no inner macro. 996 FileID MacroFID = SM.getFileID(Loc); 997 if (SM.isInFileID(SpellLoc, MacroFID)) 998 break; 999 1000 // Argument came from inner macro. 1001 Loc = SpellLoc; 1002 } 1003 1004 // Find the spelling location of the start of the non-argument expansion 1005 // range. This is where the macro name was spelled in order to begin 1006 // expanding this macro. 1007 Loc = SM.getSpellingLoc(Loc); 1008 1009 // Dig out the buffer where the macro name was spelled and the extents of the 1010 // name so that we can render it into the expansion note. 1011 std::pair<FileID, unsigned> ExpansionInfo = SM.getDecomposedLoc(Loc); 1012 unsigned MacroTokenLength = Lexer::MeasureTokenLength(Loc, SM, LangOpts); 1013 StringRef ExpansionBuffer = SM.getBufferData(ExpansionInfo.first); 1014 return ExpansionBuffer.substr(ExpansionInfo.second, MacroTokenLength); 1015} 1016 1017bool Lexer::isIdentifierBodyChar(char c, const LangOptions &LangOpts) { 1018 return isIdentifierBody(c, LangOpts.DollarIdents); 1019} 1020 1021 1022//===----------------------------------------------------------------------===// 1023// Diagnostics forwarding code. 1024//===----------------------------------------------------------------------===// 1025 1026/// GetMappedTokenLoc - If lexing out of a 'mapped buffer', where we pretend the 1027/// lexer buffer was all expanded at a single point, perform the mapping. 1028/// This is currently only used for _Pragma implementation, so it is the slow 1029/// path of the hot getSourceLocation method. Do not allow it to be inlined. 1030static LLVM_ATTRIBUTE_NOINLINE SourceLocation GetMappedTokenLoc( 1031 Preprocessor &PP, SourceLocation FileLoc, unsigned CharNo, unsigned TokLen); 1032static SourceLocation GetMappedTokenLoc(Preprocessor &PP, 1033 SourceLocation FileLoc, 1034 unsigned CharNo, unsigned TokLen) { 1035 assert(FileLoc.isMacroID() && "Must be a macro expansion"); 1036 1037 // Otherwise, we're lexing "mapped tokens". This is used for things like 1038 // _Pragma handling. Combine the expansion location of FileLoc with the 1039 // spelling location. 1040 SourceManager &SM = PP.getSourceManager(); 1041 1042 // Create a new SLoc which is expanded from Expansion(FileLoc) but whose 1043 // characters come from spelling(FileLoc)+Offset. 1044 SourceLocation SpellingLoc = SM.getSpellingLoc(FileLoc); 1045 SpellingLoc = SpellingLoc.getLocWithOffset(CharNo); 1046 1047 // Figure out the expansion loc range, which is the range covered by the 1048 // original _Pragma(...) sequence. 1049 std::pair<SourceLocation,SourceLocation> II = 1050 SM.getImmediateExpansionRange(FileLoc); 1051 1052 return SM.createExpansionLoc(SpellingLoc, II.first, II.second, TokLen); 1053} 1054 1055/// getSourceLocation - Return a source location identifier for the specified 1056/// offset in the current file. 1057SourceLocation Lexer::getSourceLocation(const char *Loc, 1058 unsigned TokLen) const { 1059 assert(Loc >= BufferStart && Loc <= BufferEnd && 1060 "Location out of range for this buffer!"); 1061 1062 // In the normal case, we're just lexing from a simple file buffer, return 1063 // the file id from FileLoc with the offset specified. 1064 unsigned CharNo = Loc-BufferStart; 1065 if (FileLoc.isFileID()) 1066 return FileLoc.getLocWithOffset(CharNo); 1067 1068 // Otherwise, this is the _Pragma lexer case, which pretends that all of the 1069 // tokens are lexed from where the _Pragma was defined. 1070 assert(PP && "This doesn't work on raw lexers"); 1071 return GetMappedTokenLoc(*PP, FileLoc, CharNo, TokLen); 1072} 1073 1074/// Diag - Forwarding function for diagnostics. This translate a source 1075/// position in the current buffer into a SourceLocation object for rendering. 1076DiagnosticBuilder Lexer::Diag(const char *Loc, unsigned DiagID) const { 1077 return PP->Diag(getSourceLocation(Loc), DiagID); 1078} 1079 1080//===----------------------------------------------------------------------===// 1081// Trigraph and Escaped Newline Handling Code. 1082//===----------------------------------------------------------------------===// 1083 1084/// GetTrigraphCharForLetter - Given a character that occurs after a ?? pair, 1085/// return the decoded trigraph letter it corresponds to, or '\0' if nothing. 1086static char GetTrigraphCharForLetter(char Letter) { 1087 switch (Letter) { 1088 default: return 0; 1089 case '=': return '#'; 1090 case ')': return ']'; 1091 case '(': return '['; 1092 case '!': return '|'; 1093 case '\'': return '^'; 1094 case '>': return '}'; 1095 case '/': return '\\'; 1096 case '<': return '{'; 1097 case '-': return '~'; 1098 } 1099} 1100 1101/// DecodeTrigraphChar - If the specified character is a legal trigraph when 1102/// prefixed with ??, emit a trigraph warning. If trigraphs are enabled, 1103/// return the result character. Finally, emit a warning about trigraph use 1104/// whether trigraphs are enabled or not. 1105static char DecodeTrigraphChar(const char *CP, Lexer *L) { 1106 char Res = GetTrigraphCharForLetter(*CP); 1107 if (!Res || !L) return Res; 1108 1109 if (!L->getLangOpts().Trigraphs) { 1110 if (!L->isLexingRawMode()) 1111 L->Diag(CP-2, diag::trigraph_ignored); 1112 return 0; 1113 } 1114 1115 if (!L->isLexingRawMode()) 1116 L->Diag(CP-2, diag::trigraph_converted) << StringRef(&Res, 1); 1117 return Res; 1118} 1119 1120/// getEscapedNewLineSize - Return the size of the specified escaped newline, 1121/// or 0 if it is not an escaped newline. P[-1] is known to be a "\" or a 1122/// trigraph equivalent on entry to this function. 1123unsigned Lexer::getEscapedNewLineSize(const char *Ptr) { 1124 unsigned Size = 0; 1125 while (isWhitespace(Ptr[Size])) { 1126 ++Size; 1127 1128 if (Ptr[Size-1] != '\n' && Ptr[Size-1] != '\r') 1129 continue; 1130 1131 // If this is a \r\n or \n\r, skip the other half. 1132 if ((Ptr[Size] == '\r' || Ptr[Size] == '\n') && 1133 Ptr[Size-1] != Ptr[Size]) 1134 ++Size; 1135 1136 return Size; 1137 } 1138 1139 // Not an escaped newline, must be a \t or something else. 1140 return 0; 1141} 1142 1143/// SkipEscapedNewLines - If P points to an escaped newline (or a series of 1144/// them), skip over them and return the first non-escaped-newline found, 1145/// otherwise return P. 1146const char *Lexer::SkipEscapedNewLines(const char *P) { 1147 while (1) { 1148 const char *AfterEscape; 1149 if (*P == '\\') { 1150 AfterEscape = P+1; 1151 } else if (*P == '?') { 1152 // If not a trigraph for escape, bail out. 1153 if (P[1] != '?' || P[2] != '/') 1154 return P; 1155 AfterEscape = P+3; 1156 } else { 1157 return P; 1158 } 1159 1160 unsigned NewLineSize = Lexer::getEscapedNewLineSize(AfterEscape); 1161 if (NewLineSize == 0) return P; 1162 P = AfterEscape+NewLineSize; 1163 } 1164} 1165 1166/// \brief Checks that the given token is the first token that occurs after the 1167/// given location (this excludes comments and whitespace). Returns the location 1168/// immediately after the specified token. If the token is not found or the 1169/// location is inside a macro, the returned source location will be invalid. 1170SourceLocation Lexer::findLocationAfterToken(SourceLocation Loc, 1171 tok::TokenKind TKind, 1172 const SourceManager &SM, 1173 const LangOptions &LangOpts, 1174 bool SkipTrailingWhitespaceAndNewLine) { 1175 if (Loc.isMacroID()) { 1176 if (!Lexer::isAtEndOfMacroExpansion(Loc, SM, LangOpts, &Loc)) 1177 return SourceLocation(); 1178 } 1179 Loc = Lexer::getLocForEndOfToken(Loc, 0, SM, LangOpts); 1180 1181 // Break down the source location. 1182 std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(Loc); 1183 1184 // Try to load the file buffer. 1185 bool InvalidTemp = false; 1186 StringRef File = SM.getBufferData(LocInfo.first, &InvalidTemp); 1187 if (InvalidTemp) 1188 return SourceLocation(); 1189 1190 const char *TokenBegin = File.data() + LocInfo.second; 1191 1192 // Lex from the start of the given location. 1193 Lexer lexer(SM.getLocForStartOfFile(LocInfo.first), LangOpts, File.begin(), 1194 TokenBegin, File.end()); 1195 // Find the token. 1196 Token Tok; 1197 lexer.LexFromRawLexer(Tok); 1198 if (Tok.isNot(TKind)) 1199 return SourceLocation(); 1200 SourceLocation TokenLoc = Tok.getLocation(); 1201 1202 // Calculate how much whitespace needs to be skipped if any. 1203 unsigned NumWhitespaceChars = 0; 1204 if (SkipTrailingWhitespaceAndNewLine) { 1205 const char *TokenEnd = SM.getCharacterData(TokenLoc) + 1206 Tok.getLength(); 1207 unsigned char C = *TokenEnd; 1208 while (isHorizontalWhitespace(C)) { 1209 C = *(++TokenEnd); 1210 NumWhitespaceChars++; 1211 } 1212 1213 // Skip \r, \n, \r\n, or \n\r 1214 if (C == '\n' || C == '\r') { 1215 char PrevC = C; 1216 C = *(++TokenEnd); 1217 NumWhitespaceChars++; 1218 if ((C == '\n' || C == '\r') && C != PrevC) 1219 NumWhitespaceChars++; 1220 } 1221 } 1222 1223 return TokenLoc.getLocWithOffset(Tok.getLength() + NumWhitespaceChars); 1224} 1225 1226/// getCharAndSizeSlow - Peek a single 'character' from the specified buffer, 1227/// get its size, and return it. This is tricky in several cases: 1228/// 1. If currently at the start of a trigraph, we warn about the trigraph, 1229/// then either return the trigraph (skipping 3 chars) or the '?', 1230/// depending on whether trigraphs are enabled or not. 1231/// 2. If this is an escaped newline (potentially with whitespace between 1232/// the backslash and newline), implicitly skip the newline and return 1233/// the char after it. 1234/// 1235/// This handles the slow/uncommon case of the getCharAndSize method. Here we 1236/// know that we can accumulate into Size, and that we have already incremented 1237/// Ptr by Size bytes. 1238/// 1239/// NOTE: When this method is updated, getCharAndSizeSlowNoWarn (below) should 1240/// be updated to match. 1241/// 1242char Lexer::getCharAndSizeSlow(const char *Ptr, unsigned &Size, 1243 Token *Tok) { 1244 // If we have a slash, look for an escaped newline. 1245 if (Ptr[0] == '\\') { 1246 ++Size; 1247 ++Ptr; 1248Slash: 1249 // Common case, backslash-char where the char is not whitespace. 1250 if (!isWhitespace(Ptr[0])) return '\\'; 1251 1252 // See if we have optional whitespace characters between the slash and 1253 // newline. 1254 if (unsigned EscapedNewLineSize = getEscapedNewLineSize(Ptr)) { 1255 // Remember that this token needs to be cleaned. 1256 if (Tok) Tok->setFlag(Token::NeedsCleaning); 1257 1258 // Warn if there was whitespace between the backslash and newline. 1259 if (Ptr[0] != '\n' && Ptr[0] != '\r' && Tok && !isLexingRawMode()) 1260 Diag(Ptr, diag::backslash_newline_space); 1261 1262 // Found backslash<whitespace><newline>. Parse the char after it. 1263 Size += EscapedNewLineSize; 1264 Ptr += EscapedNewLineSize; 1265 1266 // If the char that we finally got was a \n, then we must have had 1267 // something like \<newline><newline>. We don't want to consume the 1268 // second newline. 1269 if (*Ptr == '\n' || *Ptr == '\r' || *Ptr == '\0') 1270 return ' '; 1271 1272 // Use slow version to accumulate a correct size field. 1273 return getCharAndSizeSlow(Ptr, Size, Tok); 1274 } 1275 1276 // Otherwise, this is not an escaped newline, just return the slash. 1277 return '\\'; 1278 } 1279 1280 // If this is a trigraph, process it. 1281 if (Ptr[0] == '?' && Ptr[1] == '?') { 1282 // If this is actually a legal trigraph (not something like "??x"), emit 1283 // a trigraph warning. If so, and if trigraphs are enabled, return it. 1284 if (char C = DecodeTrigraphChar(Ptr+2, Tok ? this : 0)) { 1285 // Remember that this token needs to be cleaned. 1286 if (Tok) Tok->setFlag(Token::NeedsCleaning); 1287 1288 Ptr += 3; 1289 Size += 3; 1290 if (C == '\\') goto Slash; 1291 return C; 1292 } 1293 } 1294 1295 // If this is neither, return a single character. 1296 ++Size; 1297 return *Ptr; 1298} 1299 1300 1301/// getCharAndSizeSlowNoWarn - Handle the slow/uncommon case of the 1302/// getCharAndSizeNoWarn method. Here we know that we can accumulate into Size, 1303/// and that we have already incremented Ptr by Size bytes. 1304/// 1305/// NOTE: When this method is updated, getCharAndSizeSlow (above) should 1306/// be updated to match. 1307char Lexer::getCharAndSizeSlowNoWarn(const char *Ptr, unsigned &Size, 1308 const LangOptions &LangOpts) { 1309 // If we have a slash, look for an escaped newline. 1310 if (Ptr[0] == '\\') { 1311 ++Size; 1312 ++Ptr; 1313Slash: 1314 // Common case, backslash-char where the char is not whitespace. 1315 if (!isWhitespace(Ptr[0])) return '\\'; 1316 1317 // See if we have optional whitespace characters followed by a newline. 1318 if (unsigned EscapedNewLineSize = getEscapedNewLineSize(Ptr)) { 1319 // Found backslash<whitespace><newline>. Parse the char after it. 1320 Size += EscapedNewLineSize; 1321 Ptr += EscapedNewLineSize; 1322 1323 // If the char that we finally got was a \n, then we must have had 1324 // something like \<newline><newline>. We don't want to consume the 1325 // second newline. 1326 if (*Ptr == '\n' || *Ptr == '\r' || *Ptr == '\0') 1327 return ' '; 1328 1329 // Use slow version to accumulate a correct size field. 1330 return getCharAndSizeSlowNoWarn(Ptr, Size, LangOpts); 1331 } 1332 1333 // Otherwise, this is not an escaped newline, just return the slash. 1334 return '\\'; 1335 } 1336 1337 // If this is a trigraph, process it. 1338 if (LangOpts.Trigraphs && Ptr[0] == '?' && Ptr[1] == '?') { 1339 // If this is actually a legal trigraph (not something like "??x"), return 1340 // it. 1341 if (char C = GetTrigraphCharForLetter(Ptr[2])) { 1342 Ptr += 3; 1343 Size += 3; 1344 if (C == '\\') goto Slash; 1345 return C; 1346 } 1347 } 1348 1349 // If this is neither, return a single character. 1350 ++Size; 1351 return *Ptr; 1352} 1353 1354//===----------------------------------------------------------------------===// 1355// Helper methods for lexing. 1356//===----------------------------------------------------------------------===// 1357 1358/// \brief Routine that indiscriminately skips bytes in the source file. 1359void Lexer::SkipBytes(unsigned Bytes, bool StartOfLine) { 1360 BufferPtr += Bytes; 1361 if (BufferPtr > BufferEnd) 1362 BufferPtr = BufferEnd; 1363 IsAtStartOfLine = StartOfLine; 1364} 1365 1366static bool isAllowedIDChar(uint32_t C, const LangOptions &LangOpts) { 1367 if (LangOpts.CPlusPlus11 || LangOpts.C11) 1368 return isCharInSet(C, C11AllowedIDChars); 1369 else if (LangOpts.CPlusPlus) 1370 return isCharInSet(C, CXX03AllowedIDChars); 1371 else 1372 return isCharInSet(C, C99AllowedIDChars); 1373} 1374 1375static bool isAllowedInitiallyIDChar(uint32_t C, const LangOptions &LangOpts) { 1376 assert(isAllowedIDChar(C, LangOpts)); 1377 if (LangOpts.CPlusPlus11 || LangOpts.C11) 1378 return !isCharInSet(C, C11DisallowedInitialIDChars); 1379 else if (LangOpts.CPlusPlus) 1380 return true; 1381 else 1382 return !isCharInSet(C, C99DisallowedInitialIDChars); 1383} 1384 1385static inline CharSourceRange makeCharRange(Lexer &L, const char *Begin, 1386 const char *End) { 1387 return CharSourceRange::getCharRange(L.getSourceLocation(Begin), 1388 L.getSourceLocation(End)); 1389} 1390 1391static void maybeDiagnoseIDCharCompat(DiagnosticsEngine &Diags, uint32_t C, 1392 CharSourceRange Range, bool IsFirst) { 1393 // Check C99 compatibility. 1394 if (Diags.getDiagnosticLevel(diag::warn_c99_compat_unicode_id, 1395 Range.getBegin()) > DiagnosticsEngine::Ignored) { 1396 enum { 1397 CannotAppearInIdentifier = 0, 1398 CannotStartIdentifier 1399 }; 1400 1401 if (!isCharInSet(C, C99AllowedIDChars)) { 1402 Diags.Report(Range.getBegin(), diag::warn_c99_compat_unicode_id) 1403 << Range 1404 << CannotAppearInIdentifier; 1405 } else if (IsFirst && isCharInSet(C, C99DisallowedInitialIDChars)) { 1406 Diags.Report(Range.getBegin(), diag::warn_c99_compat_unicode_id) 1407 << Range 1408 << CannotStartIdentifier; 1409 } 1410 } 1411 1412 // Check C++98 compatibility. 1413 if (Diags.getDiagnosticLevel(diag::warn_cxx98_compat_unicode_id, 1414 Range.getBegin()) > DiagnosticsEngine::Ignored) { 1415 if (!isCharInSet(C, CXX03AllowedIDChars)) { 1416 Diags.Report(Range.getBegin(), diag::warn_cxx98_compat_unicode_id) 1417 << Range; 1418 } 1419 } 1420 } 1421 1422void Lexer::LexIdentifier(Token &Result, const char *CurPtr) { 1423 // Match [_A-Za-z0-9]*, we have already matched [_A-Za-z$] 1424 unsigned Size; 1425 unsigned char C = *CurPtr++; 1426 while (isIdentifierBody(C)) 1427 C = *CurPtr++; 1428 1429 --CurPtr; // Back up over the skipped character. 1430 1431 // Fast path, no $,\,? in identifier found. '\' might be an escaped newline 1432 // or UCN, and ? might be a trigraph for '\', an escaped newline or UCN. 1433 // 1434 // TODO: Could merge these checks into an InfoTable flag to make the 1435 // comparison cheaper 1436 if (isASCII(C) && C != '\\' && C != '?' && 1437 (C != '$' || !LangOpts.DollarIdents)) { 1438FinishIdentifier: 1439 const char *IdStart = BufferPtr; 1440 FormTokenWithChars(Result, CurPtr, tok::raw_identifier); 1441 Result.setRawIdentifierData(IdStart); 1442 1443 // If we are in raw mode, return this identifier raw. There is no need to 1444 // look up identifier information or attempt to macro expand it. 1445 if (LexingRawMode) 1446 return; 1447 1448 // Fill in Result.IdentifierInfo and update the token kind, 1449 // looking up the identifier in the identifier table. 1450 IdentifierInfo *II = PP->LookUpIdentifierInfo(Result); 1451 1452 // Finally, now that we know we have an identifier, pass this off to the 1453 // preprocessor, which may macro expand it or something. 1454 if (II->isHandleIdentifierCase()) 1455 PP->HandleIdentifier(Result); 1456 1457 return; 1458 } 1459 1460 // Otherwise, $,\,? in identifier found. Enter slower path. 1461 1462 C = getCharAndSize(CurPtr, Size); 1463 while (1) { 1464 if (C == '$') { 1465 // If we hit a $ and they are not supported in identifiers, we are done. 1466 if (!LangOpts.DollarIdents) goto FinishIdentifier; 1467 1468 // Otherwise, emit a diagnostic and continue. 1469 if (!isLexingRawMode()) 1470 Diag(CurPtr, diag::ext_dollar_in_identifier); 1471 CurPtr = ConsumeChar(CurPtr, Size, Result); 1472 C = getCharAndSize(CurPtr, Size); 1473 continue; 1474 1475 } else if (C == '\\') { 1476 const char *UCNPtr = CurPtr + Size; 1477 uint32_t CodePoint = tryReadUCN(UCNPtr, CurPtr, /*Token=*/0); 1478 if (CodePoint == 0 || !isAllowedIDChar(CodePoint, LangOpts)) 1479 goto FinishIdentifier; 1480 1481 if (!isLexingRawMode()) { 1482 maybeDiagnoseIDCharCompat(PP->getDiagnostics(), CodePoint, 1483 makeCharRange(*this, CurPtr, UCNPtr), 1484 /*IsFirst=*/false); 1485 } 1486 1487 Result.setFlag(Token::HasUCN); 1488 if ((UCNPtr - CurPtr == 6 && CurPtr[1] == 'u') || 1489 (UCNPtr - CurPtr == 10 && CurPtr[1] == 'U')) 1490 CurPtr = UCNPtr; 1491 else 1492 while (CurPtr != UCNPtr) 1493 (void)getAndAdvanceChar(CurPtr, Result); 1494 1495 C = getCharAndSize(CurPtr, Size); 1496 continue; 1497 } else if (!isASCII(C)) { 1498 const char *UnicodePtr = CurPtr; 1499 UTF32 CodePoint; 1500 ConversionResult Result = 1501 llvm::convertUTF8Sequence((const UTF8 **)&UnicodePtr, 1502 (const UTF8 *)BufferEnd, 1503 &CodePoint, 1504 strictConversion); 1505 if (Result != conversionOK || 1506 !isAllowedIDChar(static_cast<uint32_t>(CodePoint), LangOpts)) 1507 goto FinishIdentifier; 1508 1509 if (!isLexingRawMode()) { 1510 maybeDiagnoseIDCharCompat(PP->getDiagnostics(), CodePoint, 1511 makeCharRange(*this, CurPtr, UnicodePtr), 1512 /*IsFirst=*/false); 1513 } 1514 1515 CurPtr = UnicodePtr; 1516 C = getCharAndSize(CurPtr, Size); 1517 continue; 1518 } else if (!isIdentifierBody(C)) { 1519 goto FinishIdentifier; 1520 } 1521 1522 // Otherwise, this character is good, consume it. 1523 CurPtr = ConsumeChar(CurPtr, Size, Result); 1524 1525 C = getCharAndSize(CurPtr, Size); 1526 while (isIdentifierBody(C)) { 1527 CurPtr = ConsumeChar(CurPtr, Size, Result); 1528 C = getCharAndSize(CurPtr, Size); 1529 } 1530 } 1531} 1532 1533/// isHexaLiteral - Return true if Start points to a hex constant. 1534/// in microsoft mode (where this is supposed to be several different tokens). 1535bool Lexer::isHexaLiteral(const char *Start, const LangOptions &LangOpts) { 1536 unsigned Size; 1537 char C1 = Lexer::getCharAndSizeNoWarn(Start, Size, LangOpts); 1538 if (C1 != '0') 1539 return false; 1540 char C2 = Lexer::getCharAndSizeNoWarn(Start + Size, Size, LangOpts); 1541 return (C2 == 'x' || C2 == 'X'); 1542} 1543 1544/// LexNumericConstant - Lex the remainder of a integer or floating point 1545/// constant. From[-1] is the first character lexed. Return the end of the 1546/// constant. 1547void Lexer::LexNumericConstant(Token &Result, const char *CurPtr) { 1548 unsigned Size; 1549 char C = getCharAndSize(CurPtr, Size); 1550 char PrevCh = 0; 1551 while (isPreprocessingNumberBody(C)) { // FIXME: UCNs in ud-suffix. 1552 CurPtr = ConsumeChar(CurPtr, Size, Result); 1553 PrevCh = C; 1554 C = getCharAndSize(CurPtr, Size); 1555 } 1556 1557 // If we fell out, check for a sign, due to 1e+12. If we have one, continue. 1558 if ((C == '-' || C == '+') && (PrevCh == 'E' || PrevCh == 'e')) { 1559 // If we are in Microsoft mode, don't continue if the constant is hex. 1560 // For example, MSVC will accept the following as 3 tokens: 0x1234567e+1 1561 if (!LangOpts.MicrosoftExt || !isHexaLiteral(BufferPtr, LangOpts)) 1562 return LexNumericConstant(Result, ConsumeChar(CurPtr, Size, Result)); 1563 } 1564 1565 // If we have a hex FP constant, continue. 1566 if ((C == '-' || C == '+') && (PrevCh == 'P' || PrevCh == 'p')) { 1567 // Outside C99, we accept hexadecimal floating point numbers as a 1568 // not-quite-conforming extension. Only do so if this looks like it's 1569 // actually meant to be a hexfloat, and not if it has a ud-suffix. 1570 bool IsHexFloat = true; 1571 if (!LangOpts.C99) { 1572 if (!isHexaLiteral(BufferPtr, LangOpts)) 1573 IsHexFloat = false; 1574 else if (std::find(BufferPtr, CurPtr, '_') != CurPtr) 1575 IsHexFloat = false; 1576 } 1577 if (IsHexFloat) 1578 return LexNumericConstant(Result, ConsumeChar(CurPtr, Size, Result)); 1579 } 1580 1581 // Update the location of token as well as BufferPtr. 1582 const char *TokStart = BufferPtr; 1583 FormTokenWithChars(Result, CurPtr, tok::numeric_constant); 1584 Result.setLiteralData(TokStart); 1585} 1586 1587/// LexUDSuffix - Lex the ud-suffix production for user-defined literal suffixes 1588/// in C++11, or warn on a ud-suffix in C++98. 1589const char *Lexer::LexUDSuffix(Token &Result, const char *CurPtr) { 1590 assert(getLangOpts().CPlusPlus); 1591 1592 // Maximally munch an identifier. FIXME: UCNs. 1593 unsigned Size; 1594 char C = getCharAndSize(CurPtr, Size); 1595 if (isIdentifierHead(C)) { 1596 if (!getLangOpts().CPlusPlus11) { 1597 if (!isLexingRawMode()) 1598 Diag(CurPtr, 1599 C == '_' ? diag::warn_cxx11_compat_user_defined_literal 1600 : diag::warn_cxx11_compat_reserved_user_defined_literal) 1601 << FixItHint::CreateInsertion(getSourceLocation(CurPtr), " "); 1602 return CurPtr; 1603 } 1604 1605 // C++11 [lex.ext]p10, [usrlit.suffix]p1: A program containing a ud-suffix 1606 // that does not start with an underscore is ill-formed. As a conforming 1607 // extension, we treat all such suffixes as if they had whitespace before 1608 // them. 1609 if (C != '_') { 1610 if (!isLexingRawMode()) 1611 Diag(CurPtr, getLangOpts().MicrosoftMode ? 1612 diag::ext_ms_reserved_user_defined_literal : 1613 diag::ext_reserved_user_defined_literal) 1614 << FixItHint::CreateInsertion(getSourceLocation(CurPtr), " "); 1615 return CurPtr; 1616 } 1617 1618 Result.setFlag(Token::HasUDSuffix); 1619 do { 1620 CurPtr = ConsumeChar(CurPtr, Size, Result); 1621 C = getCharAndSize(CurPtr, Size); 1622 } while (isIdentifierBody(C)); 1623 } 1624 return CurPtr; 1625} 1626 1627/// LexStringLiteral - Lex the remainder of a string literal, after having lexed 1628/// either " or L" or u8" or u" or U". 1629void Lexer::LexStringLiteral(Token &Result, const char *CurPtr, 1630 tok::TokenKind Kind) { 1631 const char *NulCharacter = 0; // Does this string contain the \0 character? 1632 1633 if (!isLexingRawMode() && 1634 (Kind == tok::utf8_string_literal || 1635 Kind == tok::utf16_string_literal || 1636 Kind == tok::utf32_string_literal)) 1637 Diag(BufferPtr, getLangOpts().CPlusPlus 1638 ? diag::warn_cxx98_compat_unicode_literal 1639 : diag::warn_c99_compat_unicode_literal); 1640 1641 char C = getAndAdvanceChar(CurPtr, Result); 1642 while (C != '"') { 1643 // Skip escaped characters. Escaped newlines will already be processed by 1644 // getAndAdvanceChar. 1645 if (C == '\\') 1646 C = getAndAdvanceChar(CurPtr, Result); 1647 1648 if (C == '\n' || C == '\r' || // Newline. 1649 (C == 0 && CurPtr-1 == BufferEnd)) { // End of file. 1650 if (!isLexingRawMode() && !LangOpts.AsmPreprocessor) 1651 Diag(BufferPtr, diag::ext_unterminated_string); 1652 FormTokenWithChars(Result, CurPtr-1, tok::unknown); 1653 return; 1654 } 1655 1656 if (C == 0) { 1657 if (isCodeCompletionPoint(CurPtr-1)) { 1658 PP->CodeCompleteNaturalLanguage(); 1659 FormTokenWithChars(Result, CurPtr-1, tok::unknown); 1660 return cutOffLexing(); 1661 } 1662 1663 NulCharacter = CurPtr-1; 1664 } 1665 C = getAndAdvanceChar(CurPtr, Result); 1666 } 1667 1668 // If we are in C++11, lex the optional ud-suffix. 1669 if (getLangOpts().CPlusPlus) 1670 CurPtr = LexUDSuffix(Result, CurPtr); 1671 1672 // If a nul character existed in the string, warn about it. 1673 if (NulCharacter && !isLexingRawMode()) 1674 Diag(NulCharacter, diag::null_in_string); 1675 1676 // Update the location of the token as well as the BufferPtr instance var. 1677 const char *TokStart = BufferPtr; 1678 FormTokenWithChars(Result, CurPtr, Kind); 1679 Result.setLiteralData(TokStart); 1680} 1681 1682/// LexRawStringLiteral - Lex the remainder of a raw string literal, after 1683/// having lexed R", LR", u8R", uR", or UR". 1684void Lexer::LexRawStringLiteral(Token &Result, const char *CurPtr, 1685 tok::TokenKind Kind) { 1686 // This function doesn't use getAndAdvanceChar because C++0x [lex.pptoken]p3: 1687 // Between the initial and final double quote characters of the raw string, 1688 // any transformations performed in phases 1 and 2 (trigraphs, 1689 // universal-character-names, and line splicing) are reverted. 1690 1691 if (!isLexingRawMode()) 1692 Diag(BufferPtr, diag::warn_cxx98_compat_raw_string_literal); 1693 1694 unsigned PrefixLen = 0; 1695 1696 while (PrefixLen != 16 && isRawStringDelimBody(CurPtr[PrefixLen])) 1697 ++PrefixLen; 1698 1699 // If the last character was not a '(', then we didn't lex a valid delimiter. 1700 if (CurPtr[PrefixLen] != '(') { 1701 if (!isLexingRawMode()) { 1702 const char *PrefixEnd = &CurPtr[PrefixLen]; 1703 if (PrefixLen == 16) { 1704 Diag(PrefixEnd, diag::err_raw_delim_too_long); 1705 } else { 1706 Diag(PrefixEnd, diag::err_invalid_char_raw_delim) 1707 << StringRef(PrefixEnd, 1); 1708 } 1709 } 1710 1711 // Search for the next '"' in hopes of salvaging the lexer. Unfortunately, 1712 // it's possible the '"' was intended to be part of the raw string, but 1713 // there's not much we can do about that. 1714 while (1) { 1715 char C = *CurPtr++; 1716 1717 if (C == '"') 1718 break; 1719 if (C == 0 && CurPtr-1 == BufferEnd) { 1720 --CurPtr; 1721 break; 1722 } 1723 } 1724 1725 FormTokenWithChars(Result, CurPtr, tok::unknown); 1726 return; 1727 } 1728 1729 // Save prefix and move CurPtr past it 1730 const char *Prefix = CurPtr; 1731 CurPtr += PrefixLen + 1; // skip over prefix and '(' 1732 1733 while (1) { 1734 char C = *CurPtr++; 1735 1736 if (C == ')') { 1737 // Check for prefix match and closing quote. 1738 if (strncmp(CurPtr, Prefix, PrefixLen) == 0 && CurPtr[PrefixLen] == '"') { 1739 CurPtr += PrefixLen + 1; // skip over prefix and '"' 1740 break; 1741 } 1742 } else if (C == 0 && CurPtr-1 == BufferEnd) { // End of file. 1743 if (!isLexingRawMode()) 1744 Diag(BufferPtr, diag::err_unterminated_raw_string) 1745 << StringRef(Prefix, PrefixLen); 1746 FormTokenWithChars(Result, CurPtr-1, tok::unknown); 1747 return; 1748 } 1749 } 1750 1751 // If we are in C++11, lex the optional ud-suffix. 1752 if (getLangOpts().CPlusPlus) 1753 CurPtr = LexUDSuffix(Result, CurPtr); 1754 1755 // Update the location of token as well as BufferPtr. 1756 const char *TokStart = BufferPtr; 1757 FormTokenWithChars(Result, CurPtr, Kind); 1758 Result.setLiteralData(TokStart); 1759} 1760 1761/// LexAngledStringLiteral - Lex the remainder of an angled string literal, 1762/// after having lexed the '<' character. This is used for #include filenames. 1763void Lexer::LexAngledStringLiteral(Token &Result, const char *CurPtr) { 1764 const char *NulCharacter = 0; // Does this string contain the \0 character? 1765 const char *AfterLessPos = CurPtr; 1766 char C = getAndAdvanceChar(CurPtr, Result); 1767 while (C != '>') { 1768 // Skip escaped characters. 1769 if (C == '\\') { 1770 // Skip the escaped character. 1771 getAndAdvanceChar(CurPtr, Result); 1772 } else if (C == '\n' || C == '\r' || // Newline. 1773 (C == 0 && (CurPtr-1 == BufferEnd || // End of file. 1774 isCodeCompletionPoint(CurPtr-1)))) { 1775 // If the filename is unterminated, then it must just be a lone < 1776 // character. Return this as such. 1777 FormTokenWithChars(Result, AfterLessPos, tok::less); 1778 return; 1779 } else if (C == 0) { 1780 NulCharacter = CurPtr-1; 1781 } 1782 C = getAndAdvanceChar(CurPtr, Result); 1783 } 1784 1785 // If a nul character existed in the string, warn about it. 1786 if (NulCharacter && !isLexingRawMode()) 1787 Diag(NulCharacter, diag::null_in_string); 1788 1789 // Update the location of token as well as BufferPtr. 1790 const char *TokStart = BufferPtr; 1791 FormTokenWithChars(Result, CurPtr, tok::angle_string_literal); 1792 Result.setLiteralData(TokStart); 1793} 1794 1795 1796/// LexCharConstant - Lex the remainder of a character constant, after having 1797/// lexed either ' or L' or u' or U'. 1798void Lexer::LexCharConstant(Token &Result, const char *CurPtr, 1799 tok::TokenKind Kind) { 1800 const char *NulCharacter = 0; // Does this character contain the \0 character? 1801 1802 if (!isLexingRawMode() && 1803 (Kind == tok::utf16_char_constant || Kind == tok::utf32_char_constant)) 1804 Diag(BufferPtr, getLangOpts().CPlusPlus 1805 ? diag::warn_cxx98_compat_unicode_literal 1806 : diag::warn_c99_compat_unicode_literal); 1807 1808 char C = getAndAdvanceChar(CurPtr, Result); 1809 if (C == '\'') { 1810 if (!isLexingRawMode() && !LangOpts.AsmPreprocessor) 1811 Diag(BufferPtr, diag::ext_empty_character); 1812 FormTokenWithChars(Result, CurPtr, tok::unknown); 1813 return; 1814 } 1815 1816 while (C != '\'') { 1817 // Skip escaped characters. 1818 if (C == '\\') 1819 C = getAndAdvanceChar(CurPtr, Result); 1820 1821 if (C == '\n' || C == '\r' || // Newline. 1822 (C == 0 && CurPtr-1 == BufferEnd)) { // End of file. 1823 if (!isLexingRawMode() && !LangOpts.AsmPreprocessor) 1824 Diag(BufferPtr, diag::ext_unterminated_char); 1825 FormTokenWithChars(Result, CurPtr-1, tok::unknown); 1826 return; 1827 } 1828 1829 if (C == 0) { 1830 if (isCodeCompletionPoint(CurPtr-1)) { 1831 PP->CodeCompleteNaturalLanguage(); 1832 FormTokenWithChars(Result, CurPtr-1, tok::unknown); 1833 return cutOffLexing(); 1834 } 1835 1836 NulCharacter = CurPtr-1; 1837 } 1838 C = getAndAdvanceChar(CurPtr, Result); 1839 } 1840 1841 // If we are in C++11, lex the optional ud-suffix. 1842 if (getLangOpts().CPlusPlus) 1843 CurPtr = LexUDSuffix(Result, CurPtr); 1844 1845 // If a nul character existed in the character, warn about it. 1846 if (NulCharacter && !isLexingRawMode()) 1847 Diag(NulCharacter, diag::null_in_char); 1848 1849 // Update the location of token as well as BufferPtr. 1850 const char *TokStart = BufferPtr; 1851 FormTokenWithChars(Result, CurPtr, Kind); 1852 Result.setLiteralData(TokStart); 1853} 1854 1855/// SkipWhitespace - Efficiently skip over a series of whitespace characters. 1856/// Update BufferPtr to point to the next non-whitespace character and return. 1857/// 1858/// This method forms a token and returns true if KeepWhitespaceMode is enabled. 1859/// 1860bool Lexer::SkipWhitespace(Token &Result, const char *CurPtr) { 1861 // Whitespace - Skip it, then return the token after the whitespace. 1862 bool SawNewline = isVerticalWhitespace(CurPtr[-1]); 1863 1864 unsigned char Char = *CurPtr; 1865 1866 // Skip consecutive spaces efficiently. 1867 while (1) { 1868 // Skip horizontal whitespace very aggressively. 1869 while (isHorizontalWhitespace(Char)) 1870 Char = *++CurPtr; 1871 1872 // Otherwise if we have something other than whitespace, we're done. 1873 if (!isVerticalWhitespace(Char)) 1874 break; 1875 1876 if (ParsingPreprocessorDirective) { 1877 // End of preprocessor directive line, let LexTokenInternal handle this. 1878 BufferPtr = CurPtr; 1879 return false; 1880 } 1881 1882 // OK, but handle newline. 1883 SawNewline = true; 1884 Char = *++CurPtr; 1885 } 1886 1887 // If the client wants us to return whitespace, return it now. 1888 if (isKeepWhitespaceMode()) { 1889 FormTokenWithChars(Result, CurPtr, tok::unknown); 1890 if (SawNewline) 1891 IsAtStartOfLine = true; 1892 // FIXME: The next token will not have LeadingSpace set. 1893 return true; 1894 } 1895 1896 // If this isn't immediately after a newline, there is leading space. 1897 char PrevChar = CurPtr[-1]; 1898 bool HasLeadingSpace = !isVerticalWhitespace(PrevChar); 1899 1900 Result.setFlagValue(Token::LeadingSpace, HasLeadingSpace); 1901 if (SawNewline) 1902 Result.setFlag(Token::StartOfLine); 1903 1904 BufferPtr = CurPtr; 1905 return false; 1906} 1907 1908/// We have just read the // characters from input. Skip until we find the 1909/// newline character thats terminate the comment. Then update BufferPtr and 1910/// return. 1911/// 1912/// If we're in KeepCommentMode or any CommentHandler has inserted 1913/// some tokens, this will store the first token and return true. 1914bool Lexer::SkipLineComment(Token &Result, const char *CurPtr) { 1915 // If Line comments aren't explicitly enabled for this language, emit an 1916 // extension warning. 1917 if (!LangOpts.LineComment && !isLexingRawMode()) { 1918 Diag(BufferPtr, diag::ext_line_comment); 1919 1920 // Mark them enabled so we only emit one warning for this translation 1921 // unit. 1922 LangOpts.LineComment = true; 1923 } 1924 1925 // Scan over the body of the comment. The common case, when scanning, is that 1926 // the comment contains normal ascii characters with nothing interesting in 1927 // them. As such, optimize for this case with the inner loop. 1928 char C; 1929 do { 1930 C = *CurPtr; 1931 // Skip over characters in the fast loop. 1932 while (C != 0 && // Potentially EOF. 1933 C != '\n' && C != '\r') // Newline or DOS-style newline. 1934 C = *++CurPtr; 1935 1936 const char *NextLine = CurPtr; 1937 if (C != 0) { 1938 // We found a newline, see if it's escaped. 1939 const char *EscapePtr = CurPtr-1; 1940 while (isHorizontalWhitespace(*EscapePtr)) // Skip whitespace. 1941 --EscapePtr; 1942 1943 if (*EscapePtr == '\\') // Escaped newline. 1944 CurPtr = EscapePtr; 1945 else if (EscapePtr[0] == '/' && EscapePtr[-1] == '?' && 1946 EscapePtr[-2] == '?') // Trigraph-escaped newline. 1947 CurPtr = EscapePtr-2; 1948 else 1949 break; // This is a newline, we're done. 1950 } 1951 1952 // Otherwise, this is a hard case. Fall back on getAndAdvanceChar to 1953 // properly decode the character. Read it in raw mode to avoid emitting 1954 // diagnostics about things like trigraphs. If we see an escaped newline, 1955 // we'll handle it below. 1956 const char *OldPtr = CurPtr; 1957 bool OldRawMode = isLexingRawMode(); 1958 LexingRawMode = true; 1959 C = getAndAdvanceChar(CurPtr, Result); 1960 LexingRawMode = OldRawMode; 1961 1962 // If we only read only one character, then no special handling is needed. 1963 // We're done and can skip forward to the newline. 1964 if (C != 0 && CurPtr == OldPtr+1) { 1965 CurPtr = NextLine; 1966 break; 1967 } 1968 1969 // If we read multiple characters, and one of those characters was a \r or 1970 // \n, then we had an escaped newline within the comment. Emit diagnostic 1971 // unless the next line is also a // comment. 1972 if (CurPtr != OldPtr+1 && C != '/' && CurPtr[0] != '/') { 1973 for (; OldPtr != CurPtr; ++OldPtr) 1974 if (OldPtr[0] == '\n' || OldPtr[0] == '\r') { 1975 // Okay, we found a // comment that ends in a newline, if the next 1976 // line is also a // comment, but has spaces, don't emit a diagnostic. 1977 if (isWhitespace(C)) { 1978 const char *ForwardPtr = CurPtr; 1979 while (isWhitespace(*ForwardPtr)) // Skip whitespace. 1980 ++ForwardPtr; 1981 if (ForwardPtr[0] == '/' && ForwardPtr[1] == '/') 1982 break; 1983 } 1984 1985 if (!isLexingRawMode()) 1986 Diag(OldPtr-1, diag::ext_multi_line_line_comment); 1987 break; 1988 } 1989 } 1990 1991 if (CurPtr == BufferEnd+1) { 1992 --CurPtr; 1993 break; 1994 } 1995 1996 if (C == '\0' && isCodeCompletionPoint(CurPtr-1)) { 1997 PP->CodeCompleteNaturalLanguage(); 1998 cutOffLexing(); 1999 return false; 2000 } 2001 2002 } while (C != '\n' && C != '\r'); 2003 2004 // Found but did not consume the newline. Notify comment handlers about the 2005 // comment unless we're in a #if 0 block. 2006 if (PP && !isLexingRawMode() && 2007 PP->HandleComment(Result, SourceRange(getSourceLocation(BufferPtr), 2008 getSourceLocation(CurPtr)))) { 2009 BufferPtr = CurPtr; 2010 return true; // A token has to be returned. 2011 } 2012 2013 // If we are returning comments as tokens, return this comment as a token. 2014 if (inKeepCommentMode()) 2015 return SaveLineComment(Result, CurPtr); 2016 2017 // If we are inside a preprocessor directive and we see the end of line, 2018 // return immediately, so that the lexer can return this as an EOD token. 2019 if (ParsingPreprocessorDirective || CurPtr == BufferEnd) { 2020 BufferPtr = CurPtr; 2021 return false; 2022 } 2023 2024 // Otherwise, eat the \n character. We don't care if this is a \n\r or 2025 // \r\n sequence. This is an efficiency hack (because we know the \n can't 2026 // contribute to another token), it isn't needed for correctness. Note that 2027 // this is ok even in KeepWhitespaceMode, because we would have returned the 2028 /// comment above in that mode. 2029 ++CurPtr; 2030 2031 // The next returned token is at the start of the line. 2032 Result.setFlag(Token::StartOfLine); 2033 // No leading whitespace seen so far. 2034 Result.clearFlag(Token::LeadingSpace); 2035 BufferPtr = CurPtr; 2036 return false; 2037} 2038 2039/// If in save-comment mode, package up this Line comment in an appropriate 2040/// way and return it. 2041bool Lexer::SaveLineComment(Token &Result, const char *CurPtr) { 2042 // If we're not in a preprocessor directive, just return the // comment 2043 // directly. 2044 FormTokenWithChars(Result, CurPtr, tok::comment); 2045 2046 if (!ParsingPreprocessorDirective || LexingRawMode) 2047 return true; 2048 2049 // If this Line-style comment is in a macro definition, transmogrify it into 2050 // a C-style block comment. 2051 bool Invalid = false; 2052 std::string Spelling = PP->getSpelling(Result, &Invalid); 2053 if (Invalid) 2054 return true; 2055 2056 assert(Spelling[0] == '/' && Spelling[1] == '/' && "Not line comment?"); 2057 Spelling[1] = '*'; // Change prefix to "/*". 2058 Spelling += "*/"; // add suffix. 2059 2060 Result.setKind(tok::comment); 2061 PP->CreateString(Spelling, Result, 2062 Result.getLocation(), Result.getLocation()); 2063 return true; 2064} 2065 2066/// isBlockCommentEndOfEscapedNewLine - Return true if the specified newline 2067/// character (either \\n or \\r) is part of an escaped newline sequence. Issue 2068/// a diagnostic if so. We know that the newline is inside of a block comment. 2069static bool isEndOfBlockCommentWithEscapedNewLine(const char *CurPtr, 2070 Lexer *L) { 2071 assert(CurPtr[0] == '\n' || CurPtr[0] == '\r'); 2072 2073 // Back up off the newline. 2074 --CurPtr; 2075 2076 // If this is a two-character newline sequence, skip the other character. 2077 if (CurPtr[0] == '\n' || CurPtr[0] == '\r') { 2078 // \n\n or \r\r -> not escaped newline. 2079 if (CurPtr[0] == CurPtr[1]) 2080 return false; 2081 // \n\r or \r\n -> skip the newline. 2082 --CurPtr; 2083 } 2084 2085 // If we have horizontal whitespace, skip over it. We allow whitespace 2086 // between the slash and newline. 2087 bool HasSpace = false; 2088 while (isHorizontalWhitespace(*CurPtr) || *CurPtr == 0) { 2089 --CurPtr; 2090 HasSpace = true; 2091 } 2092 2093 // If we have a slash, we know this is an escaped newline. 2094 if (*CurPtr == '\\') { 2095 if (CurPtr[-1] != '*') return false; 2096 } else { 2097 // It isn't a slash, is it the ?? / trigraph? 2098 if (CurPtr[0] != '/' || CurPtr[-1] != '?' || CurPtr[-2] != '?' || 2099 CurPtr[-3] != '*') 2100 return false; 2101 2102 // This is the trigraph ending the comment. Emit a stern warning! 2103 CurPtr -= 2; 2104 2105 // If no trigraphs are enabled, warn that we ignored this trigraph and 2106 // ignore this * character. 2107 if (!L->getLangOpts().Trigraphs) { 2108 if (!L->isLexingRawMode()) 2109 L->Diag(CurPtr, diag::trigraph_ignored_block_comment); 2110 return false; 2111 } 2112 if (!L->isLexingRawMode()) 2113 L->Diag(CurPtr, diag::trigraph_ends_block_comment); 2114 } 2115 2116 // Warn about having an escaped newline between the */ characters. 2117 if (!L->isLexingRawMode()) 2118 L->Diag(CurPtr, diag::escaped_newline_block_comment_end); 2119 2120 // If there was space between the backslash and newline, warn about it. 2121 if (HasSpace && !L->isLexingRawMode()) 2122 L->Diag(CurPtr, diag::backslash_newline_space); 2123 2124 return true; 2125} 2126 2127#ifdef __SSE2__ 2128#include <emmintrin.h> 2129#elif __ALTIVEC__ 2130#include <altivec.h> 2131#undef bool 2132#endif 2133 2134/// We have just read from input the / and * characters that started a comment. 2135/// Read until we find the * and / characters that terminate the comment. 2136/// Note that we don't bother decoding trigraphs or escaped newlines in block 2137/// comments, because they cannot cause the comment to end. The only thing 2138/// that can happen is the comment could end with an escaped newline between 2139/// the terminating * and /. 2140/// 2141/// If we're in KeepCommentMode or any CommentHandler has inserted 2142/// some tokens, this will store the first token and return true. 2143bool Lexer::SkipBlockComment(Token &Result, const char *CurPtr) { 2144 // Scan one character past where we should, looking for a '/' character. Once 2145 // we find it, check to see if it was preceded by a *. This common 2146 // optimization helps people who like to put a lot of * characters in their 2147 // comments. 2148 2149 // The first character we get with newlines and trigraphs skipped to handle 2150 // the degenerate /*/ case below correctly if the * has an escaped newline 2151 // after it. 2152 unsigned CharSize; 2153 unsigned char C = getCharAndSize(CurPtr, CharSize); 2154 CurPtr += CharSize; 2155 if (C == 0 && CurPtr == BufferEnd+1) { 2156 if (!isLexingRawMode()) 2157 Diag(BufferPtr, diag::err_unterminated_block_comment); 2158 --CurPtr; 2159 2160 // KeepWhitespaceMode should return this broken comment as a token. Since 2161 // it isn't a well formed comment, just return it as an 'unknown' token. 2162 if (isKeepWhitespaceMode()) { 2163 FormTokenWithChars(Result, CurPtr, tok::unknown); 2164 return true; 2165 } 2166 2167 BufferPtr = CurPtr; 2168 return false; 2169 } 2170 2171 // Check to see if the first character after the '/*' is another /. If so, 2172 // then this slash does not end the block comment, it is part of it. 2173 if (C == '/') 2174 C = *CurPtr++; 2175 2176 while (1) { 2177 // Skip over all non-interesting characters until we find end of buffer or a 2178 // (probably ending) '/' character. 2179 if (CurPtr + 24 < BufferEnd && 2180 // If there is a code-completion point avoid the fast scan because it 2181 // doesn't check for '\0'. 2182 !(PP && PP->getCodeCompletionFileLoc() == FileLoc)) { 2183 // While not aligned to a 16-byte boundary. 2184 while (C != '/' && ((intptr_t)CurPtr & 0x0F) != 0) 2185 C = *CurPtr++; 2186 2187 if (C == '/') goto FoundSlash; 2188 2189#ifdef __SSE2__ 2190 __m128i Slashes = _mm_set1_epi8('/'); 2191 while (CurPtr+16 <= BufferEnd) { 2192 int cmp = _mm_movemask_epi8(_mm_cmpeq_epi8(*(const __m128i*)CurPtr, 2193 Slashes)); 2194 if (cmp != 0) { 2195 // Adjust the pointer to point directly after the first slash. It's 2196 // not necessary to set C here, it will be overwritten at the end of 2197 // the outer loop. 2198 CurPtr += llvm::CountTrailingZeros_32(cmp) + 1; 2199 goto FoundSlash; 2200 } 2201 CurPtr += 16; 2202 } 2203#elif __ALTIVEC__ 2204 __vector unsigned char Slashes = { 2205 '/', '/', '/', '/', '/', '/', '/', '/', 2206 '/', '/', '/', '/', '/', '/', '/', '/' 2207 }; 2208 while (CurPtr+16 <= BufferEnd && 2209 !vec_any_eq(*(vector unsigned char*)CurPtr, Slashes)) 2210 CurPtr += 16; 2211#else 2212 // Scan for '/' quickly. Many block comments are very large. 2213 while (CurPtr[0] != '/' && 2214 CurPtr[1] != '/' && 2215 CurPtr[2] != '/' && 2216 CurPtr[3] != '/' && 2217 CurPtr+4 < BufferEnd) { 2218 CurPtr += 4; 2219 } 2220#endif 2221 2222 // It has to be one of the bytes scanned, increment to it and read one. 2223 C = *CurPtr++; 2224 } 2225 2226 // Loop to scan the remainder. 2227 while (C != '/' && C != '\0') 2228 C = *CurPtr++; 2229 2230 if (C == '/') { 2231 FoundSlash: 2232 if (CurPtr[-2] == '*') // We found the final */. We're done! 2233 break; 2234 2235 if ((CurPtr[-2] == '\n' || CurPtr[-2] == '\r')) { 2236 if (isEndOfBlockCommentWithEscapedNewLine(CurPtr-2, this)) { 2237 // We found the final */, though it had an escaped newline between the 2238 // * and /. We're done! 2239 break; 2240 } 2241 } 2242 if (CurPtr[0] == '*' && CurPtr[1] != '/') { 2243 // If this is a /* inside of the comment, emit a warning. Don't do this 2244 // if this is a /*/, which will end the comment. This misses cases with 2245 // embedded escaped newlines, but oh well. 2246 if (!isLexingRawMode()) 2247 Diag(CurPtr-1, diag::warn_nested_block_comment); 2248 } 2249 } else if (C == 0 && CurPtr == BufferEnd+1) { 2250 if (!isLexingRawMode()) 2251 Diag(BufferPtr, diag::err_unterminated_block_comment); 2252 // Note: the user probably forgot a */. We could continue immediately 2253 // after the /*, but this would involve lexing a lot of what really is the 2254 // comment, which surely would confuse the parser. 2255 --CurPtr; 2256 2257 // KeepWhitespaceMode should return this broken comment as a token. Since 2258 // it isn't a well formed comment, just return it as an 'unknown' token. 2259 if (isKeepWhitespaceMode()) { 2260 FormTokenWithChars(Result, CurPtr, tok::unknown); 2261 return true; 2262 } 2263 2264 BufferPtr = CurPtr; 2265 return false; 2266 } else if (C == '\0' && isCodeCompletionPoint(CurPtr-1)) { 2267 PP->CodeCompleteNaturalLanguage(); 2268 cutOffLexing(); 2269 return false; 2270 } 2271 2272 C = *CurPtr++; 2273 } 2274 2275 // Notify comment handlers about the comment unless we're in a #if 0 block. 2276 if (PP && !isLexingRawMode() && 2277 PP->HandleComment(Result, SourceRange(getSourceLocation(BufferPtr), 2278 getSourceLocation(CurPtr)))) { 2279 BufferPtr = CurPtr; 2280 return true; // A token has to be returned. 2281 } 2282 2283 // If we are returning comments as tokens, return this comment as a token. 2284 if (inKeepCommentMode()) { 2285 FormTokenWithChars(Result, CurPtr, tok::comment); 2286 return true; 2287 } 2288 2289 // It is common for the tokens immediately after a /**/ comment to be 2290 // whitespace. Instead of going through the big switch, handle it 2291 // efficiently now. This is safe even in KeepWhitespaceMode because we would 2292 // have already returned above with the comment as a token. 2293 if (isHorizontalWhitespace(*CurPtr)) { 2294 SkipWhitespace(Result, CurPtr+1); 2295 return false; 2296 } 2297 2298 // Otherwise, just return so that the next character will be lexed as a token. 2299 BufferPtr = CurPtr; 2300 Result.setFlag(Token::LeadingSpace); 2301 return false; 2302} 2303 2304//===----------------------------------------------------------------------===// 2305// Primary Lexing Entry Points 2306//===----------------------------------------------------------------------===// 2307 2308/// ReadToEndOfLine - Read the rest of the current preprocessor line as an 2309/// uninterpreted string. This switches the lexer out of directive mode. 2310void Lexer::ReadToEndOfLine(SmallVectorImpl<char> *Result) { 2311 assert(ParsingPreprocessorDirective && ParsingFilename == false && 2312 "Must be in a preprocessing directive!"); 2313 Token Tmp; 2314 2315 // CurPtr - Cache BufferPtr in an automatic variable. 2316 const char *CurPtr = BufferPtr; 2317 while (1) { 2318 char Char = getAndAdvanceChar(CurPtr, Tmp); 2319 switch (Char) { 2320 default: 2321 if (Result) 2322 Result->push_back(Char); 2323 break; 2324 case 0: // Null. 2325 // Found end of file? 2326 if (CurPtr-1 != BufferEnd) { 2327 if (isCodeCompletionPoint(CurPtr-1)) { 2328 PP->CodeCompleteNaturalLanguage(); 2329 cutOffLexing(); 2330 return; 2331 } 2332 2333 // Nope, normal character, continue. 2334 if (Result) 2335 Result->push_back(Char); 2336 break; 2337 } 2338 // FALL THROUGH. 2339 case '\r': 2340 case '\n': 2341 // Okay, we found the end of the line. First, back up past the \0, \r, \n. 2342 assert(CurPtr[-1] == Char && "Trigraphs for newline?"); 2343 BufferPtr = CurPtr-1; 2344 2345 // Next, lex the character, which should handle the EOD transition. 2346 Lex(Tmp); 2347 if (Tmp.is(tok::code_completion)) { 2348 if (PP) 2349 PP->CodeCompleteNaturalLanguage(); 2350 Lex(Tmp); 2351 } 2352 assert(Tmp.is(tok::eod) && "Unexpected token!"); 2353 2354 // Finally, we're done; 2355 return; 2356 } 2357 } 2358} 2359 2360/// LexEndOfFile - CurPtr points to the end of this file. Handle this 2361/// condition, reporting diagnostics and handling other edge cases as required. 2362/// This returns true if Result contains a token, false if PP.Lex should be 2363/// called again. 2364bool Lexer::LexEndOfFile(Token &Result, const char *CurPtr) { 2365 // If we hit the end of the file while parsing a preprocessor directive, 2366 // end the preprocessor directive first. The next token returned will 2367 // then be the end of file. 2368 if (ParsingPreprocessorDirective) { 2369 // Done parsing the "line". 2370 ParsingPreprocessorDirective = false; 2371 // Update the location of token as well as BufferPtr. 2372 FormTokenWithChars(Result, CurPtr, tok::eod); 2373 2374 // Restore comment saving mode, in case it was disabled for directive. 2375 resetExtendedTokenMode(); 2376 return true; // Have a token. 2377 } 2378 2379 // If we are in raw mode, return this event as an EOF token. Let the caller 2380 // that put us in raw mode handle the event. 2381 if (isLexingRawMode()) { 2382 Result.startToken(); 2383 BufferPtr = BufferEnd; 2384 FormTokenWithChars(Result, BufferEnd, tok::eof); 2385 return true; 2386 } 2387 2388 // Issue diagnostics for unterminated #if and missing newline. 2389 2390 // If we are in a #if directive, emit an error. 2391 while (!ConditionalStack.empty()) { 2392 if (PP->getCodeCompletionFileLoc() != FileLoc) 2393 PP->Diag(ConditionalStack.back().IfLoc, 2394 diag::err_pp_unterminated_conditional); 2395 ConditionalStack.pop_back(); 2396 } 2397 2398 // C99 5.1.1.2p2: If the file is non-empty and didn't end in a newline, issue 2399 // a pedwarn. 2400 if (CurPtr != BufferStart && (CurPtr[-1] != '\n' && CurPtr[-1] != '\r')) 2401 Diag(BufferEnd, LangOpts.CPlusPlus11 ? // C++11 [lex.phases] 2.2 p2 2402 diag::warn_cxx98_compat_no_newline_eof : diag::ext_no_newline_eof) 2403 << FixItHint::CreateInsertion(getSourceLocation(BufferEnd), "\n"); 2404 2405 BufferPtr = CurPtr; 2406 2407 // Finally, let the preprocessor handle this. 2408 return PP->HandleEndOfFile(Result, isPragmaLexer()); 2409} 2410 2411/// isNextPPTokenLParen - Return 1 if the next unexpanded token lexed from 2412/// the specified lexer will return a tok::l_paren token, 0 if it is something 2413/// else and 2 if there are no more tokens in the buffer controlled by the 2414/// lexer. 2415unsigned Lexer::isNextPPTokenLParen() { 2416 assert(!LexingRawMode && "How can we expand a macro from a skipping buffer?"); 2417 2418 // Switch to 'skipping' mode. This will ensure that we can lex a token 2419 // without emitting diagnostics, disables macro expansion, and will cause EOF 2420 // to return an EOF token instead of popping the include stack. 2421 LexingRawMode = true; 2422 2423 // Save state that can be changed while lexing so that we can restore it. 2424 const char *TmpBufferPtr = BufferPtr; 2425 bool inPPDirectiveMode = ParsingPreprocessorDirective; 2426 2427 Token Tok; 2428 Tok.startToken(); 2429 LexTokenInternal(Tok); 2430 2431 // Restore state that may have changed. 2432 BufferPtr = TmpBufferPtr; 2433 ParsingPreprocessorDirective = inPPDirectiveMode; 2434 2435 // Restore the lexer back to non-skipping mode. 2436 LexingRawMode = false; 2437 2438 if (Tok.is(tok::eof)) 2439 return 2; 2440 return Tok.is(tok::l_paren); 2441} 2442 2443/// \brief Find the end of a version control conflict marker. 2444static const char *FindConflictEnd(const char *CurPtr, const char *BufferEnd, 2445 ConflictMarkerKind CMK) { 2446 const char *Terminator = CMK == CMK_Perforce ? "<<<<\n" : ">>>>>>>"; 2447 size_t TermLen = CMK == CMK_Perforce ? 5 : 7; 2448 StringRef RestOfBuffer(CurPtr+TermLen, BufferEnd-CurPtr-TermLen); 2449 size_t Pos = RestOfBuffer.find(Terminator); 2450 while (Pos != StringRef::npos) { 2451 // Must occur at start of line. 2452 if (RestOfBuffer[Pos-1] != '\r' && 2453 RestOfBuffer[Pos-1] != '\n') { 2454 RestOfBuffer = RestOfBuffer.substr(Pos+TermLen); 2455 Pos = RestOfBuffer.find(Terminator); 2456 continue; 2457 } 2458 return RestOfBuffer.data()+Pos; 2459 } 2460 return 0; 2461} 2462 2463/// IsStartOfConflictMarker - If the specified pointer is the start of a version 2464/// control conflict marker like '<<<<<<<', recognize it as such, emit an error 2465/// and recover nicely. This returns true if it is a conflict marker and false 2466/// if not. 2467bool Lexer::IsStartOfConflictMarker(const char *CurPtr) { 2468 // Only a conflict marker if it starts at the beginning of a line. 2469 if (CurPtr != BufferStart && 2470 CurPtr[-1] != '\n' && CurPtr[-1] != '\r') 2471 return false; 2472 2473 // Check to see if we have <<<<<<< or >>>>. 2474 if ((BufferEnd-CurPtr < 8 || StringRef(CurPtr, 7) != "<<<<<<<") && 2475 (BufferEnd-CurPtr < 6 || StringRef(CurPtr, 5) != ">>>> ")) 2476 return false; 2477 2478 // If we have a situation where we don't care about conflict markers, ignore 2479 // it. 2480 if (CurrentConflictMarkerState || isLexingRawMode()) 2481 return false; 2482 2483 ConflictMarkerKind Kind = *CurPtr == '<' ? CMK_Normal : CMK_Perforce; 2484 2485 // Check to see if there is an ending marker somewhere in the buffer at the 2486 // start of a line to terminate this conflict marker. 2487 if (FindConflictEnd(CurPtr, BufferEnd, Kind)) { 2488 // We found a match. We are really in a conflict marker. 2489 // Diagnose this, and ignore to the end of line. 2490 Diag(CurPtr, diag::err_conflict_marker); 2491 CurrentConflictMarkerState = Kind; 2492 2493 // Skip ahead to the end of line. We know this exists because the 2494 // end-of-conflict marker starts with \r or \n. 2495 while (*CurPtr != '\r' && *CurPtr != '\n') { 2496 assert(CurPtr != BufferEnd && "Didn't find end of line"); 2497 ++CurPtr; 2498 } 2499 BufferPtr = CurPtr; 2500 return true; 2501 } 2502 2503 // No end of conflict marker found. 2504 return false; 2505} 2506 2507 2508/// HandleEndOfConflictMarker - If this is a '====' or '||||' or '>>>>', or if 2509/// it is '<<<<' and the conflict marker started with a '>>>>' marker, then it 2510/// is the end of a conflict marker. Handle it by ignoring up until the end of 2511/// the line. This returns true if it is a conflict marker and false if not. 2512bool Lexer::HandleEndOfConflictMarker(const char *CurPtr) { 2513 // Only a conflict marker if it starts at the beginning of a line. 2514 if (CurPtr != BufferStart && 2515 CurPtr[-1] != '\n' && CurPtr[-1] != '\r') 2516 return false; 2517 2518 // If we have a situation where we don't care about conflict markers, ignore 2519 // it. 2520 if (!CurrentConflictMarkerState || isLexingRawMode()) 2521 return false; 2522 2523 // Check to see if we have the marker (4 characters in a row). 2524 for (unsigned i = 1; i != 4; ++i) 2525 if (CurPtr[i] != CurPtr[0]) 2526 return false; 2527 2528 // If we do have it, search for the end of the conflict marker. This could 2529 // fail if it got skipped with a '#if 0' or something. Note that CurPtr might 2530 // be the end of conflict marker. 2531 if (const char *End = FindConflictEnd(CurPtr, BufferEnd, 2532 CurrentConflictMarkerState)) { 2533 CurPtr = End; 2534 2535 // Skip ahead to the end of line. 2536 while (CurPtr != BufferEnd && *CurPtr != '\r' && *CurPtr != '\n') 2537 ++CurPtr; 2538 2539 BufferPtr = CurPtr; 2540 2541 // No longer in the conflict marker. 2542 CurrentConflictMarkerState = CMK_None; 2543 return true; 2544 } 2545 2546 return false; 2547} 2548 2549bool Lexer::isCodeCompletionPoint(const char *CurPtr) const { 2550 if (PP && PP->isCodeCompletionEnabled()) { 2551 SourceLocation Loc = FileLoc.getLocWithOffset(CurPtr-BufferStart); 2552 return Loc == PP->getCodeCompletionLoc(); 2553 } 2554 2555 return false; 2556} 2557 2558uint32_t Lexer::tryReadUCN(const char *&StartPtr, const char *SlashLoc, 2559 Token *Result) { 2560 unsigned CharSize; 2561 char Kind = getCharAndSize(StartPtr, CharSize); 2562 2563 unsigned NumHexDigits; 2564 if (Kind == 'u') 2565 NumHexDigits = 4; 2566 else if (Kind == 'U') 2567 NumHexDigits = 8; 2568 else 2569 return 0; 2570 2571 if (!LangOpts.CPlusPlus && !LangOpts.C99) { 2572 if (Result && !isLexingRawMode()) 2573 Diag(SlashLoc, diag::warn_ucn_not_valid_in_c89); 2574 return 0; 2575 } 2576 2577 const char *CurPtr = StartPtr + CharSize; 2578 const char *KindLoc = &CurPtr[-1]; 2579 2580 uint32_t CodePoint = 0; 2581 for (unsigned i = 0; i < NumHexDigits; ++i) { 2582 char C = getCharAndSize(CurPtr, CharSize); 2583 2584 unsigned Value = llvm::hexDigitValue(C); 2585 if (Value == -1U) { 2586 if (Result && !isLexingRawMode()) { 2587 if (i == 0) { 2588 Diag(BufferPtr, diag::warn_ucn_escape_no_digits) 2589 << StringRef(KindLoc, 1); 2590 } else { 2591 Diag(BufferPtr, diag::warn_ucn_escape_incomplete); 2592 2593 // If the user wrote \U1234, suggest a fixit to \u. 2594 if (i == 4 && NumHexDigits == 8) { 2595 CharSourceRange URange = makeCharRange(*this, KindLoc, KindLoc + 1); 2596 Diag(KindLoc, diag::note_ucn_four_not_eight) 2597 << FixItHint::CreateReplacement(URange, "u"); 2598 } 2599 } 2600 } 2601 2602 return 0; 2603 } 2604 2605 CodePoint <<= 4; 2606 CodePoint += Value; 2607 2608 CurPtr += CharSize; 2609 } 2610 2611 if (Result) { 2612 Result->setFlag(Token::HasUCN); 2613 if (CurPtr - StartPtr == (ptrdiff_t)NumHexDigits + 2) 2614 StartPtr = CurPtr; 2615 else 2616 while (StartPtr != CurPtr) 2617 (void)getAndAdvanceChar(StartPtr, *Result); 2618 } else { 2619 StartPtr = CurPtr; 2620 } 2621 2622 // C99 6.4.3p2: A universal character name shall not specify a character whose 2623 // short identifier is less than 00A0 other than 0024 ($), 0040 (@), or 2624 // 0060 (`), nor one in the range D800 through DFFF inclusive.) 2625 // C++11 [lex.charset]p2: If the hexadecimal value for a 2626 // universal-character-name corresponds to a surrogate code point (in the 2627 // range 0xD800-0xDFFF, inclusive), the program is ill-formed. Additionally, 2628 // if the hexadecimal value for a universal-character-name outside the 2629 // c-char-sequence, s-char-sequence, or r-char-sequence of a character or 2630 // string literal corresponds to a control character (in either of the 2631 // ranges 0x00-0x1F or 0x7F-0x9F, both inclusive) or to a character in the 2632 // basic source character set, the program is ill-formed. 2633 if (CodePoint < 0xA0) { 2634 if (CodePoint == 0x24 || CodePoint == 0x40 || CodePoint == 0x60) 2635 return CodePoint; 2636 2637 // We don't use isLexingRawMode() here because we need to warn about bad 2638 // UCNs even when skipping preprocessing tokens in a #if block. 2639 if (Result && PP) { 2640 if (CodePoint < 0x20 || CodePoint >= 0x7F) 2641 Diag(BufferPtr, diag::err_ucn_control_character); 2642 else { 2643 char C = static_cast<char>(CodePoint); 2644 Diag(BufferPtr, diag::err_ucn_escape_basic_scs) << StringRef(&C, 1); 2645 } 2646 } 2647 2648 return 0; 2649 2650 } else if (CodePoint >= 0xD800 && CodePoint <= 0xDFFF) { 2651 // C++03 allows UCNs representing surrogate characters. C99 and C++11 don't. 2652 // We don't use isLexingRawMode() here because we need to diagnose bad 2653 // UCNs even when skipping preprocessing tokens in a #if block. 2654 if (Result && PP) { 2655 if (LangOpts.CPlusPlus && !LangOpts.CPlusPlus11) 2656 Diag(BufferPtr, diag::warn_ucn_escape_surrogate); 2657 else 2658 Diag(BufferPtr, diag::err_ucn_escape_invalid); 2659 } 2660 return 0; 2661 } 2662 2663 return CodePoint; 2664} 2665 2666void Lexer::LexUnicode(Token &Result, uint32_t C, const char *CurPtr) { 2667 if (!isLexingRawMode() && !PP->isPreprocessedOutput() && 2668 isCharInSet(C, UnicodeWhitespaceChars)) { 2669 Diag(BufferPtr, diag::ext_unicode_whitespace) 2670 << makeCharRange(*this, BufferPtr, CurPtr); 2671 2672 Result.setFlag(Token::LeadingSpace); 2673 if (SkipWhitespace(Result, CurPtr)) 2674 return; // KeepWhitespaceMode 2675 2676 return LexTokenInternal(Result); 2677 } 2678 2679 if (isAllowedIDChar(C, LangOpts) && isAllowedInitiallyIDChar(C, LangOpts)) { 2680 if (!isLexingRawMode() && !ParsingPreprocessorDirective && 2681 !PP->isPreprocessedOutput()) { 2682 maybeDiagnoseIDCharCompat(PP->getDiagnostics(), C, 2683 makeCharRange(*this, BufferPtr, CurPtr), 2684 /*IsFirst=*/true); 2685 } 2686 2687 MIOpt.ReadToken(); 2688 return LexIdentifier(Result, CurPtr); 2689 } 2690 2691 if (!isLexingRawMode() && !ParsingPreprocessorDirective && 2692 !PP->isPreprocessedOutput() && 2693 !isASCII(*BufferPtr) && !isAllowedIDChar(C, LangOpts)) { 2694 // Non-ASCII characters tend to creep into source code unintentionally. 2695 // Instead of letting the parser complain about the unknown token, 2696 // just drop the character. 2697 // Note that we can /only/ do this when the non-ASCII character is actually 2698 // spelled as Unicode, not written as a UCN. The standard requires that 2699 // we not throw away any possible preprocessor tokens, but there's a 2700 // loophole in the mapping of Unicode characters to basic character set 2701 // characters that allows us to map these particular characters to, say, 2702 // whitespace. 2703 Diag(BufferPtr, diag::err_non_ascii) 2704 << FixItHint::CreateRemoval(makeCharRange(*this, BufferPtr, CurPtr)); 2705 2706 BufferPtr = CurPtr; 2707 return LexTokenInternal(Result); 2708 } 2709 2710 // Otherwise, we have an explicit UCN or a character that's unlikely to show 2711 // up by accident. 2712 MIOpt.ReadToken(); 2713 FormTokenWithChars(Result, CurPtr, tok::unknown); 2714} 2715 2716 2717/// LexTokenInternal - This implements a simple C family lexer. It is an 2718/// extremely performance critical piece of code. This assumes that the buffer 2719/// has a null character at the end of the file. This returns a preprocessing 2720/// token, not a normal token, as such, it is an internal interface. It assumes 2721/// that the Flags of result have been cleared before calling this. 2722void Lexer::LexTokenInternal(Token &Result) { 2723LexNextToken: 2724 // New token, can't need cleaning yet. 2725 Result.clearFlag(Token::NeedsCleaning); 2726 Result.setIdentifierInfo(0); 2727 2728 // CurPtr - Cache BufferPtr in an automatic variable. 2729 const char *CurPtr = BufferPtr; 2730 2731 // Small amounts of horizontal whitespace is very common between tokens. 2732 if ((*CurPtr == ' ') || (*CurPtr == '\t')) { 2733 ++CurPtr; 2734 while ((*CurPtr == ' ') || (*CurPtr == '\t')) 2735 ++CurPtr; 2736 2737 // If we are keeping whitespace and other tokens, just return what we just 2738 // skipped. The next lexer invocation will return the token after the 2739 // whitespace. 2740 if (isKeepWhitespaceMode()) { 2741 FormTokenWithChars(Result, CurPtr, tok::unknown); 2742 // FIXME: The next token will not have LeadingSpace set. 2743 return; 2744 } 2745 2746 BufferPtr = CurPtr; 2747 Result.setFlag(Token::LeadingSpace); 2748 } 2749 2750 unsigned SizeTmp, SizeTmp2; // Temporaries for use in cases below. 2751 2752 // Read a character, advancing over it. 2753 char Char = getAndAdvanceChar(CurPtr, Result); 2754 tok::TokenKind Kind; 2755 2756 switch (Char) { 2757 case 0: // Null. 2758 // Found end of file? 2759 if (CurPtr-1 == BufferEnd) { 2760 // Read the PP instance variable into an automatic variable, because 2761 // LexEndOfFile will often delete 'this'. 2762 Preprocessor *PPCache = PP; 2763 if (LexEndOfFile(Result, CurPtr-1)) // Retreat back into the file. 2764 return; // Got a token to return. 2765 assert(PPCache && "Raw buffer::LexEndOfFile should return a token"); 2766 return PPCache->Lex(Result); 2767 } 2768 2769 // Check if we are performing code completion. 2770 if (isCodeCompletionPoint(CurPtr-1)) { 2771 // Return the code-completion token. 2772 Result.startToken(); 2773 FormTokenWithChars(Result, CurPtr, tok::code_completion); 2774 return; 2775 } 2776 2777 if (!isLexingRawMode()) 2778 Diag(CurPtr-1, diag::null_in_file); 2779 Result.setFlag(Token::LeadingSpace); 2780 if (SkipWhitespace(Result, CurPtr)) 2781 return; // KeepWhitespaceMode 2782 2783 goto LexNextToken; // GCC isn't tail call eliminating. 2784 2785 case 26: // DOS & CP/M EOF: "^Z". 2786 // If we're in Microsoft extensions mode, treat this as end of file. 2787 if (LangOpts.MicrosoftExt) { 2788 // Read the PP instance variable into an automatic variable, because 2789 // LexEndOfFile will often delete 'this'. 2790 Preprocessor *PPCache = PP; 2791 if (LexEndOfFile(Result, CurPtr-1)) // Retreat back into the file. 2792 return; // Got a token to return. 2793 assert(PPCache && "Raw buffer::LexEndOfFile should return a token"); 2794 return PPCache->Lex(Result); 2795 } 2796 // If Microsoft extensions are disabled, this is just random garbage. 2797 Kind = tok::unknown; 2798 break; 2799 2800 case '\n': 2801 case '\r': 2802 // If we are inside a preprocessor directive and we see the end of line, 2803 // we know we are done with the directive, so return an EOD token. 2804 if (ParsingPreprocessorDirective) { 2805 // Done parsing the "line". 2806 ParsingPreprocessorDirective = false; 2807 2808 // Restore comment saving mode, in case it was disabled for directive. 2809 if (PP) 2810 resetExtendedTokenMode(); 2811 2812 // Since we consumed a newline, we are back at the start of a line. 2813 IsAtStartOfLine = true; 2814 2815 Kind = tok::eod; 2816 break; 2817 } 2818 2819 // No leading whitespace seen so far. 2820 Result.clearFlag(Token::LeadingSpace); 2821 2822 if (SkipWhitespace(Result, CurPtr)) 2823 return; // KeepWhitespaceMode 2824 goto LexNextToken; // GCC isn't tail call eliminating. 2825 case ' ': 2826 case '\t': 2827 case '\f': 2828 case '\v': 2829 SkipHorizontalWhitespace: 2830 Result.setFlag(Token::LeadingSpace); 2831 if (SkipWhitespace(Result, CurPtr)) 2832 return; // KeepWhitespaceMode 2833 2834 SkipIgnoredUnits: 2835 CurPtr = BufferPtr; 2836 2837 // If the next token is obviously a // or /* */ comment, skip it efficiently 2838 // too (without going through the big switch stmt). 2839 if (CurPtr[0] == '/' && CurPtr[1] == '/' && !inKeepCommentMode() && 2840 LangOpts.LineComment && !LangOpts.TraditionalCPP) { 2841 if (SkipLineComment(Result, CurPtr+2)) 2842 return; // There is a token to return. 2843 goto SkipIgnoredUnits; 2844 } else if (CurPtr[0] == '/' && CurPtr[1] == '*' && !inKeepCommentMode()) { 2845 if (SkipBlockComment(Result, CurPtr+2)) 2846 return; // There is a token to return. 2847 goto SkipIgnoredUnits; 2848 } else if (isHorizontalWhitespace(*CurPtr)) { 2849 goto SkipHorizontalWhitespace; 2850 } 2851 goto LexNextToken; // GCC isn't tail call eliminating. 2852 2853 // C99 6.4.4.1: Integer Constants. 2854 // C99 6.4.4.2: Floating Constants. 2855 case '0': case '1': case '2': case '3': case '4': 2856 case '5': case '6': case '7': case '8': case '9': 2857 // Notify MIOpt that we read a non-whitespace/non-comment token. 2858 MIOpt.ReadToken(); 2859 return LexNumericConstant(Result, CurPtr); 2860 2861 case 'u': // Identifier (uber) or C11/C++11 UTF-8 or UTF-16 string literal 2862 // Notify MIOpt that we read a non-whitespace/non-comment token. 2863 MIOpt.ReadToken(); 2864 2865 if (LangOpts.CPlusPlus11 || LangOpts.C11) { 2866 Char = getCharAndSize(CurPtr, SizeTmp); 2867 2868 // UTF-16 string literal 2869 if (Char == '"') 2870 return LexStringLiteral(Result, ConsumeChar(CurPtr, SizeTmp, Result), 2871 tok::utf16_string_literal); 2872 2873 // UTF-16 character constant 2874 if (Char == '\'') 2875 return LexCharConstant(Result, ConsumeChar(CurPtr, SizeTmp, Result), 2876 tok::utf16_char_constant); 2877 2878 // UTF-16 raw string literal 2879 if (Char == 'R' && LangOpts.CPlusPlus11 && 2880 getCharAndSize(CurPtr + SizeTmp, SizeTmp2) == '"') 2881 return LexRawStringLiteral(Result, 2882 ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 2883 SizeTmp2, Result), 2884 tok::utf16_string_literal); 2885 2886 if (Char == '8') { 2887 char Char2 = getCharAndSize(CurPtr + SizeTmp, SizeTmp2); 2888 2889 // UTF-8 string literal 2890 if (Char2 == '"') 2891 return LexStringLiteral(Result, 2892 ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 2893 SizeTmp2, Result), 2894 tok::utf8_string_literal); 2895 2896 if (Char2 == 'R' && LangOpts.CPlusPlus11) { 2897 unsigned SizeTmp3; 2898 char Char3 = getCharAndSize(CurPtr + SizeTmp + SizeTmp2, SizeTmp3); 2899 // UTF-8 raw string literal 2900 if (Char3 == '"') { 2901 return LexRawStringLiteral(Result, 2902 ConsumeChar(ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 2903 SizeTmp2, Result), 2904 SizeTmp3, Result), 2905 tok::utf8_string_literal); 2906 } 2907 } 2908 } 2909 } 2910 2911 // treat u like the start of an identifier. 2912 return LexIdentifier(Result, CurPtr); 2913 2914 case 'U': // Identifier (Uber) or C11/C++11 UTF-32 string literal 2915 // Notify MIOpt that we read a non-whitespace/non-comment token. 2916 MIOpt.ReadToken(); 2917 2918 if (LangOpts.CPlusPlus11 || LangOpts.C11) { 2919 Char = getCharAndSize(CurPtr, SizeTmp); 2920 2921 // UTF-32 string literal 2922 if (Char == '"') 2923 return LexStringLiteral(Result, ConsumeChar(CurPtr, SizeTmp, Result), 2924 tok::utf32_string_literal); 2925 2926 // UTF-32 character constant 2927 if (Char == '\'') 2928 return LexCharConstant(Result, ConsumeChar(CurPtr, SizeTmp, Result), 2929 tok::utf32_char_constant); 2930 2931 // UTF-32 raw string literal 2932 if (Char == 'R' && LangOpts.CPlusPlus11 && 2933 getCharAndSize(CurPtr + SizeTmp, SizeTmp2) == '"') 2934 return LexRawStringLiteral(Result, 2935 ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 2936 SizeTmp2, Result), 2937 tok::utf32_string_literal); 2938 } 2939 2940 // treat U like the start of an identifier. 2941 return LexIdentifier(Result, CurPtr); 2942 2943 case 'R': // Identifier or C++0x raw string literal 2944 // Notify MIOpt that we read a non-whitespace/non-comment token. 2945 MIOpt.ReadToken(); 2946 2947 if (LangOpts.CPlusPlus11) { 2948 Char = getCharAndSize(CurPtr, SizeTmp); 2949 2950 if (Char == '"') 2951 return LexRawStringLiteral(Result, 2952 ConsumeChar(CurPtr, SizeTmp, Result), 2953 tok::string_literal); 2954 } 2955 2956 // treat R like the start of an identifier. 2957 return LexIdentifier(Result, CurPtr); 2958 2959 case 'L': // Identifier (Loony) or wide literal (L'x' or L"xyz"). 2960 // Notify MIOpt that we read a non-whitespace/non-comment token. 2961 MIOpt.ReadToken(); 2962 Char = getCharAndSize(CurPtr, SizeTmp); 2963 2964 // Wide string literal. 2965 if (Char == '"') 2966 return LexStringLiteral(Result, ConsumeChar(CurPtr, SizeTmp, Result), 2967 tok::wide_string_literal); 2968 2969 // Wide raw string literal. 2970 if (LangOpts.CPlusPlus11 && Char == 'R' && 2971 getCharAndSize(CurPtr + SizeTmp, SizeTmp2) == '"') 2972 return LexRawStringLiteral(Result, 2973 ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 2974 SizeTmp2, Result), 2975 tok::wide_string_literal); 2976 2977 // Wide character constant. 2978 if (Char == '\'') 2979 return LexCharConstant(Result, ConsumeChar(CurPtr, SizeTmp, Result), 2980 tok::wide_char_constant); 2981 // FALL THROUGH, treating L like the start of an identifier. 2982 2983 // C99 6.4.2: Identifiers. 2984 case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': case 'G': 2985 case 'H': case 'I': case 'J': case 'K': /*'L'*/case 'M': case 'N': 2986 case 'O': case 'P': case 'Q': /*'R'*/case 'S': case 'T': /*'U'*/ 2987 case 'V': case 'W': case 'X': case 'Y': case 'Z': 2988 case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': case 'g': 2989 case 'h': case 'i': case 'j': case 'k': case 'l': case 'm': case 'n': 2990 case 'o': case 'p': case 'q': case 'r': case 's': case 't': /*'u'*/ 2991 case 'v': case 'w': case 'x': case 'y': case 'z': 2992 case '_': 2993 // Notify MIOpt that we read a non-whitespace/non-comment token. 2994 MIOpt.ReadToken(); 2995 return LexIdentifier(Result, CurPtr); 2996 2997 case '$': // $ in identifiers. 2998 if (LangOpts.DollarIdents) { 2999 if (!isLexingRawMode()) 3000 Diag(CurPtr-1, diag::ext_dollar_in_identifier); 3001 // Notify MIOpt that we read a non-whitespace/non-comment token. 3002 MIOpt.ReadToken(); 3003 return LexIdentifier(Result, CurPtr); 3004 } 3005 3006 Kind = tok::unknown; 3007 break; 3008 3009 // C99 6.4.4: Character Constants. 3010 case '\'': 3011 // Notify MIOpt that we read a non-whitespace/non-comment token. 3012 MIOpt.ReadToken(); 3013 return LexCharConstant(Result, CurPtr, tok::char_constant); 3014 3015 // C99 6.4.5: String Literals. 3016 case '"': 3017 // Notify MIOpt that we read a non-whitespace/non-comment token. 3018 MIOpt.ReadToken(); 3019 return LexStringLiteral(Result, CurPtr, tok::string_literal); 3020 3021 // C99 6.4.6: Punctuators. 3022 case '?': 3023 Kind = tok::question; 3024 break; 3025 case '[': 3026 Kind = tok::l_square; 3027 break; 3028 case ']': 3029 Kind = tok::r_square; 3030 break; 3031 case '(': 3032 Kind = tok::l_paren; 3033 break; 3034 case ')': 3035 Kind = tok::r_paren; 3036 break; 3037 case '{': 3038 Kind = tok::l_brace; 3039 break; 3040 case '}': 3041 Kind = tok::r_brace; 3042 break; 3043 case '.': 3044 Char = getCharAndSize(CurPtr, SizeTmp); 3045 if (Char >= '0' && Char <= '9') { 3046 // Notify MIOpt that we read a non-whitespace/non-comment token. 3047 MIOpt.ReadToken(); 3048 3049 return LexNumericConstant(Result, ConsumeChar(CurPtr, SizeTmp, Result)); 3050 } else if (LangOpts.CPlusPlus && Char == '*') { 3051 Kind = tok::periodstar; 3052 CurPtr += SizeTmp; 3053 } else if (Char == '.' && 3054 getCharAndSize(CurPtr+SizeTmp, SizeTmp2) == '.') { 3055 Kind = tok::ellipsis; 3056 CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 3057 SizeTmp2, Result); 3058 } else { 3059 Kind = tok::period; 3060 } 3061 break; 3062 case '&': 3063 Char = getCharAndSize(CurPtr, SizeTmp); 3064 if (Char == '&') { 3065 Kind = tok::ampamp; 3066 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3067 } else if (Char == '=') { 3068 Kind = tok::ampequal; 3069 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3070 } else { 3071 Kind = tok::amp; 3072 } 3073 break; 3074 case '*': 3075 if (getCharAndSize(CurPtr, SizeTmp) == '=') { 3076 Kind = tok::starequal; 3077 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3078 } else { 3079 Kind = tok::star; 3080 } 3081 break; 3082 case '+': 3083 Char = getCharAndSize(CurPtr, SizeTmp); 3084 if (Char == '+') { 3085 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3086 Kind = tok::plusplus; 3087 } else if (Char == '=') { 3088 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3089 Kind = tok::plusequal; 3090 } else { 3091 Kind = tok::plus; 3092 } 3093 break; 3094 case '-': 3095 Char = getCharAndSize(CurPtr, SizeTmp); 3096 if (Char == '-') { // -- 3097 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3098 Kind = tok::minusminus; 3099 } else if (Char == '>' && LangOpts.CPlusPlus && 3100 getCharAndSize(CurPtr+SizeTmp, SizeTmp2) == '*') { // C++ ->* 3101 CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 3102 SizeTmp2, Result); 3103 Kind = tok::arrowstar; 3104 } else if (Char == '>') { // -> 3105 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3106 Kind = tok::arrow; 3107 } else if (Char == '=') { // -= 3108 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3109 Kind = tok::minusequal; 3110 } else { 3111 Kind = tok::minus; 3112 } 3113 break; 3114 case '~': 3115 Kind = tok::tilde; 3116 break; 3117 case '!': 3118 if (getCharAndSize(CurPtr, SizeTmp) == '=') { 3119 Kind = tok::exclaimequal; 3120 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3121 } else { 3122 Kind = tok::exclaim; 3123 } 3124 break; 3125 case '/': 3126 // 6.4.9: Comments 3127 Char = getCharAndSize(CurPtr, SizeTmp); 3128 if (Char == '/') { // Line comment. 3129 // Even if Line comments are disabled (e.g. in C89 mode), we generally 3130 // want to lex this as a comment. There is one problem with this though, 3131 // that in one particular corner case, this can change the behavior of the 3132 // resultant program. For example, In "foo //**/ bar", C89 would lex 3133 // this as "foo / bar" and langauges with Line comments would lex it as 3134 // "foo". Check to see if the character after the second slash is a '*'. 3135 // If so, we will lex that as a "/" instead of the start of a comment. 3136 // However, we never do this if we are just preprocessing. 3137 bool TreatAsComment = LangOpts.LineComment && !LangOpts.TraditionalCPP; 3138 if (!TreatAsComment) 3139 if (!(PP && PP->isPreprocessedOutput())) 3140 TreatAsComment = getCharAndSize(CurPtr+SizeTmp, SizeTmp2) != '*'; 3141 3142 if (TreatAsComment) { 3143 if (SkipLineComment(Result, ConsumeChar(CurPtr, SizeTmp, Result))) 3144 return; // There is a token to return. 3145 3146 // It is common for the tokens immediately after a // comment to be 3147 // whitespace (indentation for the next line). Instead of going through 3148 // the big switch, handle it efficiently now. 3149 goto SkipIgnoredUnits; 3150 } 3151 } 3152 3153 if (Char == '*') { // /**/ comment. 3154 if (SkipBlockComment(Result, ConsumeChar(CurPtr, SizeTmp, Result))) 3155 return; // There is a token to return. 3156 goto LexNextToken; // GCC isn't tail call eliminating. 3157 } 3158 3159 if (Char == '=') { 3160 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3161 Kind = tok::slashequal; 3162 } else { 3163 Kind = tok::slash; 3164 } 3165 break; 3166 case '%': 3167 Char = getCharAndSize(CurPtr, SizeTmp); 3168 if (Char == '=') { 3169 Kind = tok::percentequal; 3170 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3171 } else if (LangOpts.Digraphs && Char == '>') { 3172 Kind = tok::r_brace; // '%>' -> '}' 3173 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3174 } else if (LangOpts.Digraphs && Char == ':') { 3175 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3176 Char = getCharAndSize(CurPtr, SizeTmp); 3177 if (Char == '%' && getCharAndSize(CurPtr+SizeTmp, SizeTmp2) == ':') { 3178 Kind = tok::hashhash; // '%:%:' -> '##' 3179 CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 3180 SizeTmp2, Result); 3181 } else if (Char == '@' && LangOpts.MicrosoftExt) {// %:@ -> #@ -> Charize 3182 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3183 if (!isLexingRawMode()) 3184 Diag(BufferPtr, diag::ext_charize_microsoft); 3185 Kind = tok::hashat; 3186 } else { // '%:' -> '#' 3187 // We parsed a # character. If this occurs at the start of the line, 3188 // it's actually the start of a preprocessing directive. Callback to 3189 // the preprocessor to handle it. 3190 // FIXME: -fpreprocessed mode?? 3191 if (Result.isAtStartOfLine() && !LexingRawMode && !Is_PragmaLexer) 3192 goto HandleDirective; 3193 3194 Kind = tok::hash; 3195 } 3196 } else { 3197 Kind = tok::percent; 3198 } 3199 break; 3200 case '<': 3201 Char = getCharAndSize(CurPtr, SizeTmp); 3202 if (ParsingFilename) { 3203 return LexAngledStringLiteral(Result, CurPtr); 3204 } else if (Char == '<') { 3205 char After = getCharAndSize(CurPtr+SizeTmp, SizeTmp2); 3206 if (After == '=') { 3207 Kind = tok::lesslessequal; 3208 CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 3209 SizeTmp2, Result); 3210 } else if (After == '<' && IsStartOfConflictMarker(CurPtr-1)) { 3211 // If this is actually a '<<<<<<<' version control conflict marker, 3212 // recognize it as such and recover nicely. 3213 goto LexNextToken; 3214 } else if (After == '<' && HandleEndOfConflictMarker(CurPtr-1)) { 3215 // If this is '<<<<' and we're in a Perforce-style conflict marker, 3216 // ignore it. 3217 goto LexNextToken; 3218 } else if (LangOpts.CUDA && After == '<') { 3219 Kind = tok::lesslessless; 3220 CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 3221 SizeTmp2, Result); 3222 } else { 3223 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3224 Kind = tok::lessless; 3225 } 3226 } else if (Char == '=') { 3227 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3228 Kind = tok::lessequal; 3229 } else if (LangOpts.Digraphs && Char == ':') { // '<:' -> '[' 3230 if (LangOpts.CPlusPlus11 && 3231 getCharAndSize(CurPtr + SizeTmp, SizeTmp2) == ':') { 3232 // C++0x [lex.pptoken]p3: 3233 // Otherwise, if the next three characters are <:: and the subsequent 3234 // character is neither : nor >, the < is treated as a preprocessor 3235 // token by itself and not as the first character of the alternative 3236 // token <:. 3237 unsigned SizeTmp3; 3238 char After = getCharAndSize(CurPtr + SizeTmp + SizeTmp2, SizeTmp3); 3239 if (After != ':' && After != '>') { 3240 Kind = tok::less; 3241 if (!isLexingRawMode()) 3242 Diag(BufferPtr, diag::warn_cxx98_compat_less_colon_colon); 3243 break; 3244 } 3245 } 3246 3247 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3248 Kind = tok::l_square; 3249 } else if (LangOpts.Digraphs && Char == '%') { // '<%' -> '{' 3250 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3251 Kind = tok::l_brace; 3252 } else { 3253 Kind = tok::less; 3254 } 3255 break; 3256 case '>': 3257 Char = getCharAndSize(CurPtr, SizeTmp); 3258 if (Char == '=') { 3259 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3260 Kind = tok::greaterequal; 3261 } else if (Char == '>') { 3262 char After = getCharAndSize(CurPtr+SizeTmp, SizeTmp2); 3263 if (After == '=') { 3264 CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 3265 SizeTmp2, Result); 3266 Kind = tok::greatergreaterequal; 3267 } else if (After == '>' && IsStartOfConflictMarker(CurPtr-1)) { 3268 // If this is actually a '>>>>' conflict marker, recognize it as such 3269 // and recover nicely. 3270 goto LexNextToken; 3271 } else if (After == '>' && HandleEndOfConflictMarker(CurPtr-1)) { 3272 // If this is '>>>>>>>' and we're in a conflict marker, ignore it. 3273 goto LexNextToken; 3274 } else if (LangOpts.CUDA && After == '>') { 3275 Kind = tok::greatergreatergreater; 3276 CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 3277 SizeTmp2, Result); 3278 } else { 3279 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3280 Kind = tok::greatergreater; 3281 } 3282 3283 } else { 3284 Kind = tok::greater; 3285 } 3286 break; 3287 case '^': 3288 Char = getCharAndSize(CurPtr, SizeTmp); 3289 if (Char == '=') { 3290 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3291 Kind = tok::caretequal; 3292 } else { 3293 Kind = tok::caret; 3294 } 3295 break; 3296 case '|': 3297 Char = getCharAndSize(CurPtr, SizeTmp); 3298 if (Char == '=') { 3299 Kind = tok::pipeequal; 3300 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3301 } else if (Char == '|') { 3302 // If this is '|||||||' and we're in a conflict marker, ignore it. 3303 if (CurPtr[1] == '|' && HandleEndOfConflictMarker(CurPtr-1)) 3304 goto LexNextToken; 3305 Kind = tok::pipepipe; 3306 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3307 } else { 3308 Kind = tok::pipe; 3309 } 3310 break; 3311 case ':': 3312 Char = getCharAndSize(CurPtr, SizeTmp); 3313 if (LangOpts.Digraphs && Char == '>') { 3314 Kind = tok::r_square; // ':>' -> ']' 3315 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3316 } else if (LangOpts.CPlusPlus && Char == ':') { 3317 Kind = tok::coloncolon; 3318 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3319 } else { 3320 Kind = tok::colon; 3321 } 3322 break; 3323 case ';': 3324 Kind = tok::semi; 3325 break; 3326 case '=': 3327 Char = getCharAndSize(CurPtr, SizeTmp); 3328 if (Char == '=') { 3329 // If this is '====' and we're in a conflict marker, ignore it. 3330 if (CurPtr[1] == '=' && HandleEndOfConflictMarker(CurPtr-1)) 3331 goto LexNextToken; 3332 3333 Kind = tok::equalequal; 3334 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3335 } else { 3336 Kind = tok::equal; 3337 } 3338 break; 3339 case ',': 3340 Kind = tok::comma; 3341 break; 3342 case '#': 3343 Char = getCharAndSize(CurPtr, SizeTmp); 3344 if (Char == '#') { 3345 Kind = tok::hashhash; 3346 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3347 } else if (Char == '@' && LangOpts.MicrosoftExt) { // #@ -> Charize 3348 Kind = tok::hashat; 3349 if (!isLexingRawMode()) 3350 Diag(BufferPtr, diag::ext_charize_microsoft); 3351 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3352 } else { 3353 // We parsed a # character. If this occurs at the start of the line, 3354 // it's actually the start of a preprocessing directive. Callback to 3355 // the preprocessor to handle it. 3356 // FIXME: -fpreprocessed mode?? 3357 if (Result.isAtStartOfLine() && !LexingRawMode && !Is_PragmaLexer) 3358 goto HandleDirective; 3359 3360 Kind = tok::hash; 3361 } 3362 break; 3363 3364 case '@': 3365 // Objective C support. 3366 if (CurPtr[-1] == '@' && LangOpts.ObjC1) 3367 Kind = tok::at; 3368 else 3369 Kind = tok::unknown; 3370 break; 3371 3372 // UCNs (C99 6.4.3, C++11 [lex.charset]p2) 3373 case '\\': 3374 if (uint32_t CodePoint = tryReadUCN(CurPtr, BufferPtr, &Result)) 3375 return LexUnicode(Result, CodePoint, CurPtr); 3376 3377 Kind = tok::unknown; 3378 break; 3379 3380 default: { 3381 if (isASCII(Char)) { 3382 Kind = tok::unknown; 3383 break; 3384 } 3385 3386 UTF32 CodePoint; 3387 3388 // We can't just reset CurPtr to BufferPtr because BufferPtr may point to 3389 // an escaped newline. 3390 --CurPtr; 3391 ConversionResult Status = 3392 llvm::convertUTF8Sequence((const UTF8 **)&CurPtr, 3393 (const UTF8 *)BufferEnd, 3394 &CodePoint, 3395 strictConversion); 3396 if (Status == conversionOK) 3397 return LexUnicode(Result, CodePoint, CurPtr); 3398 3399 if (isLexingRawMode() || ParsingPreprocessorDirective || 3400 PP->isPreprocessedOutput()) { 3401 ++CurPtr; 3402 Kind = tok::unknown; 3403 break; 3404 } 3405 3406 // Non-ASCII characters tend to creep into source code unintentionally. 3407 // Instead of letting the parser complain about the unknown token, 3408 // just diagnose the invalid UTF-8, then drop the character. 3409 Diag(CurPtr, diag::err_invalid_utf8); 3410 3411 BufferPtr = CurPtr+1; 3412 goto LexNextToken; 3413 } 3414 } 3415 3416 // Notify MIOpt that we read a non-whitespace/non-comment token. 3417 MIOpt.ReadToken(); 3418 3419 // Update the location of token as well as BufferPtr. 3420 FormTokenWithChars(Result, CurPtr, Kind); 3421 return; 3422 3423HandleDirective: 3424 // We parsed a # character and it's the start of a preprocessing directive. 3425 3426 FormTokenWithChars(Result, CurPtr, tok::hash); 3427 PP->HandleDirective(Result); 3428 3429 // As an optimization, if the preprocessor didn't switch lexers, tail 3430 // recurse. 3431 if (PP->isCurrentLexer(this)) { 3432 // Start a new token. If this is a #include or something, the PP may 3433 // want us starting at the beginning of the line again. If so, set 3434 // the StartOfLine flag and clear LeadingSpace. 3435 if (IsAtStartOfLine) { 3436 Result.setFlag(Token::StartOfLine); 3437 Result.clearFlag(Token::LeadingSpace); 3438 IsAtStartOfLine = false; 3439 } 3440 goto LexNextToken; // GCC isn't tail call eliminating. 3441 } 3442 return PP->Lex(Result); 3443} 3444