Lexer.cpp revision 9893902eceba7f01dd1521349d33866f77254d78
1//===--- Lexer.cpp - C Language Family Lexer ------------------------------===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file implements the Lexer and Token interfaces. 11// 12//===----------------------------------------------------------------------===// 13// 14// TODO: GCC Diagnostics emitted by the lexer: 15// PEDWARN: (form feed|vertical tab) in preprocessing directive 16// 17// Universal characters, unicode, char mapping: 18// WARNING: `%.*s' is not in NFKC 19// WARNING: `%.*s' is not in NFC 20// 21// Other: 22// TODO: Options to support: 23// -fexec-charset,-fwide-exec-charset 24// 25//===----------------------------------------------------------------------===// 26 27#include "clang/Lex/Lexer.h" 28#include "clang/Basic/CharInfo.h" 29#include "clang/Basic/SourceManager.h" 30#include "clang/Lex/CodeCompletionHandler.h" 31#include "clang/Lex/LexDiagnostic.h" 32#include "clang/Lex/Preprocessor.h" 33#include "llvm/ADT/STLExtras.h" 34#include "llvm/ADT/StringExtras.h" 35#include "llvm/ADT/StringSwitch.h" 36#include "llvm/Support/Compiler.h" 37#include "llvm/Support/ConvertUTF.h" 38#include "llvm/Support/MemoryBuffer.h" 39#include <cstring> 40using namespace clang; 41 42//===----------------------------------------------------------------------===// 43// Token Class Implementation 44//===----------------------------------------------------------------------===// 45 46/// isObjCAtKeyword - Return true if we have an ObjC keyword identifier. 47bool Token::isObjCAtKeyword(tok::ObjCKeywordKind objcKey) const { 48 if (IdentifierInfo *II = getIdentifierInfo()) 49 return II->getObjCKeywordID() == objcKey; 50 return false; 51} 52 53/// getObjCKeywordID - Return the ObjC keyword kind. 54tok::ObjCKeywordKind Token::getObjCKeywordID() const { 55 IdentifierInfo *specId = getIdentifierInfo(); 56 return specId ? specId->getObjCKeywordID() : tok::objc_not_keyword; 57} 58 59 60//===----------------------------------------------------------------------===// 61// Lexer Class Implementation 62//===----------------------------------------------------------------------===// 63 64void Lexer::anchor() { } 65 66void Lexer::InitLexer(const char *BufStart, const char *BufPtr, 67 const char *BufEnd) { 68 BufferStart = BufStart; 69 BufferPtr = BufPtr; 70 BufferEnd = BufEnd; 71 72 assert(BufEnd[0] == 0 && 73 "We assume that the input buffer has a null character at the end" 74 " to simplify lexing!"); 75 76 // Check whether we have a BOM in the beginning of the buffer. If yes - act 77 // accordingly. Right now we support only UTF-8 with and without BOM, so, just 78 // skip the UTF-8 BOM if it's present. 79 if (BufferStart == BufferPtr) { 80 // Determine the size of the BOM. 81 StringRef Buf(BufferStart, BufferEnd - BufferStart); 82 size_t BOMLength = llvm::StringSwitch<size_t>(Buf) 83 .StartsWith("\xEF\xBB\xBF", 3) // UTF-8 BOM 84 .Default(0); 85 86 // Skip the BOM. 87 BufferPtr += BOMLength; 88 } 89 90 Is_PragmaLexer = false; 91 CurrentConflictMarkerState = CMK_None; 92 93 // Start of the file is a start of line. 94 IsAtStartOfLine = true; 95 96 // We are not after parsing a #. 97 ParsingPreprocessorDirective = false; 98 99 // We are not after parsing #include. 100 ParsingFilename = false; 101 102 // We are not in raw mode. Raw mode disables diagnostics and interpretation 103 // of tokens (e.g. identifiers, thus disabling macro expansion). It is used 104 // to quickly lex the tokens of the buffer, e.g. when handling a "#if 0" block 105 // or otherwise skipping over tokens. 106 LexingRawMode = false; 107 108 // Default to not keeping comments. 109 ExtendedTokenMode = 0; 110} 111 112/// Lexer constructor - Create a new lexer object for the specified buffer 113/// with the specified preprocessor managing the lexing process. This lexer 114/// assumes that the associated file buffer and Preprocessor objects will 115/// outlive it, so it doesn't take ownership of either of them. 116Lexer::Lexer(FileID FID, const llvm::MemoryBuffer *InputFile, Preprocessor &PP) 117 : PreprocessorLexer(&PP, FID), 118 FileLoc(PP.getSourceManager().getLocForStartOfFile(FID)), 119 LangOpts(PP.getLangOpts()) { 120 121 InitLexer(InputFile->getBufferStart(), InputFile->getBufferStart(), 122 InputFile->getBufferEnd()); 123 124 // Default to keeping comments if the preprocessor wants them. 125 SetCommentRetentionState(PP.getCommentRetentionState()); 126} 127 128/// Lexer constructor - Create a new raw lexer object. This object is only 129/// suitable for calls to 'LexFromRawLexer'. This lexer assumes that the text 130/// range will outlive it, so it doesn't take ownership of it. 131Lexer::Lexer(SourceLocation fileloc, const LangOptions &langOpts, 132 const char *BufStart, const char *BufPtr, const char *BufEnd) 133 : FileLoc(fileloc), LangOpts(langOpts) { 134 135 InitLexer(BufStart, BufPtr, BufEnd); 136 137 // We *are* in raw mode. 138 LexingRawMode = true; 139} 140 141/// Lexer constructor - Create a new raw lexer object. This object is only 142/// suitable for calls to 'LexFromRawLexer'. This lexer assumes that the text 143/// range will outlive it, so it doesn't take ownership of it. 144Lexer::Lexer(FileID FID, const llvm::MemoryBuffer *FromFile, 145 const SourceManager &SM, const LangOptions &langOpts) 146 : FileLoc(SM.getLocForStartOfFile(FID)), LangOpts(langOpts) { 147 148 InitLexer(FromFile->getBufferStart(), FromFile->getBufferStart(), 149 FromFile->getBufferEnd()); 150 151 // We *are* in raw mode. 152 LexingRawMode = true; 153} 154 155/// Create_PragmaLexer: Lexer constructor - Create a new lexer object for 156/// _Pragma expansion. This has a variety of magic semantics that this method 157/// sets up. It returns a new'd Lexer that must be delete'd when done. 158/// 159/// On entrance to this routine, TokStartLoc is a macro location which has a 160/// spelling loc that indicates the bytes to be lexed for the token and an 161/// expansion location that indicates where all lexed tokens should be 162/// "expanded from". 163/// 164/// FIXME: It would really be nice to make _Pragma just be a wrapper around a 165/// normal lexer that remaps tokens as they fly by. This would require making 166/// Preprocessor::Lex virtual. Given that, we could just dump in a magic lexer 167/// interface that could handle this stuff. This would pull GetMappedTokenLoc 168/// out of the critical path of the lexer! 169/// 170Lexer *Lexer::Create_PragmaLexer(SourceLocation SpellingLoc, 171 SourceLocation ExpansionLocStart, 172 SourceLocation ExpansionLocEnd, 173 unsigned TokLen, Preprocessor &PP) { 174 SourceManager &SM = PP.getSourceManager(); 175 176 // Create the lexer as if we were going to lex the file normally. 177 FileID SpellingFID = SM.getFileID(SpellingLoc); 178 const llvm::MemoryBuffer *InputFile = SM.getBuffer(SpellingFID); 179 Lexer *L = new Lexer(SpellingFID, InputFile, PP); 180 181 // Now that the lexer is created, change the start/end locations so that we 182 // just lex the subsection of the file that we want. This is lexing from a 183 // scratch buffer. 184 const char *StrData = SM.getCharacterData(SpellingLoc); 185 186 L->BufferPtr = StrData; 187 L->BufferEnd = StrData+TokLen; 188 assert(L->BufferEnd[0] == 0 && "Buffer is not nul terminated!"); 189 190 // Set the SourceLocation with the remapping information. This ensures that 191 // GetMappedTokenLoc will remap the tokens as they are lexed. 192 L->FileLoc = SM.createExpansionLoc(SM.getLocForStartOfFile(SpellingFID), 193 ExpansionLocStart, 194 ExpansionLocEnd, TokLen); 195 196 // Ensure that the lexer thinks it is inside a directive, so that end \n will 197 // return an EOD token. 198 L->ParsingPreprocessorDirective = true; 199 200 // This lexer really is for _Pragma. 201 L->Is_PragmaLexer = true; 202 return L; 203} 204 205 206/// Stringify - Convert the specified string into a C string, with surrounding 207/// ""'s, and with escaped \ and " characters. 208std::string Lexer::Stringify(const std::string &Str, bool Charify) { 209 std::string Result = Str; 210 char Quote = Charify ? '\'' : '"'; 211 for (unsigned i = 0, e = Result.size(); i != e; ++i) { 212 if (Result[i] == '\\' || Result[i] == Quote) { 213 Result.insert(Result.begin()+i, '\\'); 214 ++i; ++e; 215 } 216 } 217 return Result; 218} 219 220/// Stringify - Convert the specified string into a C string by escaping '\' 221/// and " characters. This does not add surrounding ""'s to the string. 222void Lexer::Stringify(SmallVectorImpl<char> &Str) { 223 for (unsigned i = 0, e = Str.size(); i != e; ++i) { 224 if (Str[i] == '\\' || Str[i] == '"') { 225 Str.insert(Str.begin()+i, '\\'); 226 ++i; ++e; 227 } 228 } 229} 230 231//===----------------------------------------------------------------------===// 232// Token Spelling 233//===----------------------------------------------------------------------===// 234 235/// \brief Slow case of getSpelling. Extract the characters comprising the 236/// spelling of this token from the provided input buffer. 237static size_t getSpellingSlow(const Token &Tok, const char *BufPtr, 238 const LangOptions &LangOpts, char *Spelling) { 239 assert(Tok.needsCleaning() && "getSpellingSlow called on simple token"); 240 241 size_t Length = 0; 242 const char *BufEnd = BufPtr + Tok.getLength(); 243 244 if (Tok.is(tok::string_literal)) { 245 // Munch the encoding-prefix and opening double-quote. 246 while (BufPtr < BufEnd) { 247 unsigned Size; 248 Spelling[Length++] = Lexer::getCharAndSizeNoWarn(BufPtr, Size, LangOpts); 249 BufPtr += Size; 250 251 if (Spelling[Length - 1] == '"') 252 break; 253 } 254 255 // Raw string literals need special handling; trigraph expansion and line 256 // splicing do not occur within their d-char-sequence nor within their 257 // r-char-sequence. 258 if (Length >= 2 && 259 Spelling[Length - 2] == 'R' && Spelling[Length - 1] == '"') { 260 // Search backwards from the end of the token to find the matching closing 261 // quote. 262 const char *RawEnd = BufEnd; 263 do --RawEnd; while (*RawEnd != '"'); 264 size_t RawLength = RawEnd - BufPtr + 1; 265 266 // Everything between the quotes is included verbatim in the spelling. 267 memcpy(Spelling + Length, BufPtr, RawLength); 268 Length += RawLength; 269 BufPtr += RawLength; 270 271 // The rest of the token is lexed normally. 272 } 273 } 274 275 while (BufPtr < BufEnd) { 276 unsigned Size; 277 Spelling[Length++] = Lexer::getCharAndSizeNoWarn(BufPtr, Size, LangOpts); 278 BufPtr += Size; 279 } 280 281 assert(Length < Tok.getLength() && 282 "NeedsCleaning flag set on token that didn't need cleaning!"); 283 return Length; 284} 285 286/// getSpelling() - Return the 'spelling' of this token. The spelling of a 287/// token are the characters used to represent the token in the source file 288/// after trigraph expansion and escaped-newline folding. In particular, this 289/// wants to get the true, uncanonicalized, spelling of things like digraphs 290/// UCNs, etc. 291StringRef Lexer::getSpelling(SourceLocation loc, 292 SmallVectorImpl<char> &buffer, 293 const SourceManager &SM, 294 const LangOptions &options, 295 bool *invalid) { 296 // Break down the source location. 297 std::pair<FileID, unsigned> locInfo = SM.getDecomposedLoc(loc); 298 299 // Try to the load the file buffer. 300 bool invalidTemp = false; 301 StringRef file = SM.getBufferData(locInfo.first, &invalidTemp); 302 if (invalidTemp) { 303 if (invalid) *invalid = true; 304 return StringRef(); 305 } 306 307 const char *tokenBegin = file.data() + locInfo.second; 308 309 // Lex from the start of the given location. 310 Lexer lexer(SM.getLocForStartOfFile(locInfo.first), options, 311 file.begin(), tokenBegin, file.end()); 312 Token token; 313 lexer.LexFromRawLexer(token); 314 315 unsigned length = token.getLength(); 316 317 // Common case: no need for cleaning. 318 if (!token.needsCleaning()) 319 return StringRef(tokenBegin, length); 320 321 // Hard case, we need to relex the characters into the string. 322 buffer.resize(length); 323 buffer.resize(getSpellingSlow(token, tokenBegin, options, buffer.data())); 324 return StringRef(buffer.data(), buffer.size()); 325} 326 327/// getSpelling() - Return the 'spelling' of this token. The spelling of a 328/// token are the characters used to represent the token in the source file 329/// after trigraph expansion and escaped-newline folding. In particular, this 330/// wants to get the true, uncanonicalized, spelling of things like digraphs 331/// UCNs, etc. 332std::string Lexer::getSpelling(const Token &Tok, const SourceManager &SourceMgr, 333 const LangOptions &LangOpts, bool *Invalid) { 334 assert((int)Tok.getLength() >= 0 && "Token character range is bogus!"); 335 336 bool CharDataInvalid = false; 337 const char *TokStart = SourceMgr.getCharacterData(Tok.getLocation(), 338 &CharDataInvalid); 339 if (Invalid) 340 *Invalid = CharDataInvalid; 341 if (CharDataInvalid) 342 return std::string(); 343 344 // If this token contains nothing interesting, return it directly. 345 if (!Tok.needsCleaning()) 346 return std::string(TokStart, TokStart + Tok.getLength()); 347 348 std::string Result; 349 Result.resize(Tok.getLength()); 350 Result.resize(getSpellingSlow(Tok, TokStart, LangOpts, &*Result.begin())); 351 return Result; 352} 353 354/// getSpelling - This method is used to get the spelling of a token into a 355/// preallocated buffer, instead of as an std::string. The caller is required 356/// to allocate enough space for the token, which is guaranteed to be at least 357/// Tok.getLength() bytes long. The actual length of the token is returned. 358/// 359/// Note that this method may do two possible things: it may either fill in 360/// the buffer specified with characters, or it may *change the input pointer* 361/// to point to a constant buffer with the data already in it (avoiding a 362/// copy). The caller is not allowed to modify the returned buffer pointer 363/// if an internal buffer is returned. 364unsigned Lexer::getSpelling(const Token &Tok, const char *&Buffer, 365 const SourceManager &SourceMgr, 366 const LangOptions &LangOpts, bool *Invalid) { 367 assert((int)Tok.getLength() >= 0 && "Token character range is bogus!"); 368 369 const char *TokStart = 0; 370 // NOTE: this has to be checked *before* testing for an IdentifierInfo. 371 if (Tok.is(tok::raw_identifier)) 372 TokStart = Tok.getRawIdentifierData(); 373 else if (!Tok.hasUCN()) { 374 if (const IdentifierInfo *II = Tok.getIdentifierInfo()) { 375 // Just return the string from the identifier table, which is very quick. 376 Buffer = II->getNameStart(); 377 return II->getLength(); 378 } 379 } 380 381 // NOTE: this can be checked even after testing for an IdentifierInfo. 382 if (Tok.isLiteral()) 383 TokStart = Tok.getLiteralData(); 384 385 if (TokStart == 0) { 386 // Compute the start of the token in the input lexer buffer. 387 bool CharDataInvalid = false; 388 TokStart = SourceMgr.getCharacterData(Tok.getLocation(), &CharDataInvalid); 389 if (Invalid) 390 *Invalid = CharDataInvalid; 391 if (CharDataInvalid) { 392 Buffer = ""; 393 return 0; 394 } 395 } 396 397 // If this token contains nothing interesting, return it directly. 398 if (!Tok.needsCleaning()) { 399 Buffer = TokStart; 400 return Tok.getLength(); 401 } 402 403 // Otherwise, hard case, relex the characters into the string. 404 return getSpellingSlow(Tok, TokStart, LangOpts, const_cast<char*>(Buffer)); 405} 406 407 408/// MeasureTokenLength - Relex the token at the specified location and return 409/// its length in bytes in the input file. If the token needs cleaning (e.g. 410/// includes a trigraph or an escaped newline) then this count includes bytes 411/// that are part of that. 412unsigned Lexer::MeasureTokenLength(SourceLocation Loc, 413 const SourceManager &SM, 414 const LangOptions &LangOpts) { 415 Token TheTok; 416 if (getRawToken(Loc, TheTok, SM, LangOpts)) 417 return 0; 418 return TheTok.getLength(); 419} 420 421/// \brief Relex the token at the specified location. 422/// \returns true if there was a failure, false on success. 423bool Lexer::getRawToken(SourceLocation Loc, Token &Result, 424 const SourceManager &SM, 425 const LangOptions &LangOpts) { 426 // TODO: this could be special cased for common tokens like identifiers, ')', 427 // etc to make this faster, if it mattered. Just look at StrData[0] to handle 428 // all obviously single-char tokens. This could use 429 // Lexer::isObviouslySimpleCharacter for example to handle identifiers or 430 // something. 431 432 // If this comes from a macro expansion, we really do want the macro name, not 433 // the token this macro expanded to. 434 Loc = SM.getExpansionLoc(Loc); 435 std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(Loc); 436 bool Invalid = false; 437 StringRef Buffer = SM.getBufferData(LocInfo.first, &Invalid); 438 if (Invalid) 439 return true; 440 441 const char *StrData = Buffer.data()+LocInfo.second; 442 443 if (isWhitespace(StrData[0])) 444 return true; 445 446 // Create a lexer starting at the beginning of this token. 447 Lexer TheLexer(SM.getLocForStartOfFile(LocInfo.first), LangOpts, 448 Buffer.begin(), StrData, Buffer.end()); 449 TheLexer.SetCommentRetentionState(true); 450 TheLexer.LexFromRawLexer(Result); 451 return false; 452} 453 454static SourceLocation getBeginningOfFileToken(SourceLocation Loc, 455 const SourceManager &SM, 456 const LangOptions &LangOpts) { 457 assert(Loc.isFileID()); 458 std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(Loc); 459 if (LocInfo.first.isInvalid()) 460 return Loc; 461 462 bool Invalid = false; 463 StringRef Buffer = SM.getBufferData(LocInfo.first, &Invalid); 464 if (Invalid) 465 return Loc; 466 467 // Back up from the current location until we hit the beginning of a line 468 // (or the buffer). We'll relex from that point. 469 const char *BufStart = Buffer.data(); 470 if (LocInfo.second >= Buffer.size()) 471 return Loc; 472 473 const char *StrData = BufStart+LocInfo.second; 474 if (StrData[0] == '\n' || StrData[0] == '\r') 475 return Loc; 476 477 const char *LexStart = StrData; 478 while (LexStart != BufStart) { 479 if (LexStart[0] == '\n' || LexStart[0] == '\r') { 480 ++LexStart; 481 break; 482 } 483 484 --LexStart; 485 } 486 487 // Create a lexer starting at the beginning of this token. 488 SourceLocation LexerStartLoc = Loc.getLocWithOffset(-LocInfo.second); 489 Lexer TheLexer(LexerStartLoc, LangOpts, BufStart, LexStart, Buffer.end()); 490 TheLexer.SetCommentRetentionState(true); 491 492 // Lex tokens until we find the token that contains the source location. 493 Token TheTok; 494 do { 495 TheLexer.LexFromRawLexer(TheTok); 496 497 if (TheLexer.getBufferLocation() > StrData) { 498 // Lexing this token has taken the lexer past the source location we're 499 // looking for. If the current token encompasses our source location, 500 // return the beginning of that token. 501 if (TheLexer.getBufferLocation() - TheTok.getLength() <= StrData) 502 return TheTok.getLocation(); 503 504 // We ended up skipping over the source location entirely, which means 505 // that it points into whitespace. We're done here. 506 break; 507 } 508 } while (TheTok.getKind() != tok::eof); 509 510 // We've passed our source location; just return the original source location. 511 return Loc; 512} 513 514SourceLocation Lexer::GetBeginningOfToken(SourceLocation Loc, 515 const SourceManager &SM, 516 const LangOptions &LangOpts) { 517 if (Loc.isFileID()) 518 return getBeginningOfFileToken(Loc, SM, LangOpts); 519 520 if (!SM.isMacroArgExpansion(Loc)) 521 return Loc; 522 523 SourceLocation FileLoc = SM.getSpellingLoc(Loc); 524 SourceLocation BeginFileLoc = getBeginningOfFileToken(FileLoc, SM, LangOpts); 525 std::pair<FileID, unsigned> FileLocInfo = SM.getDecomposedLoc(FileLoc); 526 std::pair<FileID, unsigned> BeginFileLocInfo 527 = SM.getDecomposedLoc(BeginFileLoc); 528 assert(FileLocInfo.first == BeginFileLocInfo.first && 529 FileLocInfo.second >= BeginFileLocInfo.second); 530 return Loc.getLocWithOffset(BeginFileLocInfo.second - FileLocInfo.second); 531} 532 533namespace { 534 enum PreambleDirectiveKind { 535 PDK_Skipped, 536 PDK_StartIf, 537 PDK_EndIf, 538 PDK_Unknown 539 }; 540} 541 542std::pair<unsigned, bool> 543Lexer::ComputePreamble(const llvm::MemoryBuffer *Buffer, 544 const LangOptions &LangOpts, unsigned MaxLines) { 545 // Create a lexer starting at the beginning of the file. Note that we use a 546 // "fake" file source location at offset 1 so that the lexer will track our 547 // position within the file. 548 const unsigned StartOffset = 1; 549 SourceLocation FileLoc = SourceLocation::getFromRawEncoding(StartOffset); 550 Lexer TheLexer(FileLoc, LangOpts, Buffer->getBufferStart(), 551 Buffer->getBufferStart(), Buffer->getBufferEnd()); 552 553 // StartLoc will differ from FileLoc if there is a BOM that was skipped. 554 SourceLocation StartLoc = TheLexer.getSourceLocation(); 555 556 bool InPreprocessorDirective = false; 557 Token TheTok; 558 Token IfStartTok; 559 unsigned IfCount = 0; 560 561 unsigned MaxLineOffset = 0; 562 if (MaxLines) { 563 const char *CurPtr = Buffer->getBufferStart(); 564 unsigned CurLine = 0; 565 while (CurPtr != Buffer->getBufferEnd()) { 566 char ch = *CurPtr++; 567 if (ch == '\n') { 568 ++CurLine; 569 if (CurLine == MaxLines) 570 break; 571 } 572 } 573 if (CurPtr != Buffer->getBufferEnd()) 574 MaxLineOffset = CurPtr - Buffer->getBufferStart(); 575 } 576 577 do { 578 TheLexer.LexFromRawLexer(TheTok); 579 580 if (InPreprocessorDirective) { 581 // If we've hit the end of the file, we're done. 582 if (TheTok.getKind() == tok::eof) { 583 break; 584 } 585 586 // If we haven't hit the end of the preprocessor directive, skip this 587 // token. 588 if (!TheTok.isAtStartOfLine()) 589 continue; 590 591 // We've passed the end of the preprocessor directive, and will look 592 // at this token again below. 593 InPreprocessorDirective = false; 594 } 595 596 // Keep track of the # of lines in the preamble. 597 if (TheTok.isAtStartOfLine()) { 598 unsigned TokOffset = TheTok.getLocation().getRawEncoding() - StartOffset; 599 600 // If we were asked to limit the number of lines in the preamble, 601 // and we're about to exceed that limit, we're done. 602 if (MaxLineOffset && TokOffset >= MaxLineOffset) 603 break; 604 } 605 606 // Comments are okay; skip over them. 607 if (TheTok.getKind() == tok::comment) 608 continue; 609 610 if (TheTok.isAtStartOfLine() && TheTok.getKind() == tok::hash) { 611 // This is the start of a preprocessor directive. 612 Token HashTok = TheTok; 613 InPreprocessorDirective = true; 614 615 // Figure out which directive this is. Since we're lexing raw tokens, 616 // we don't have an identifier table available. Instead, just look at 617 // the raw identifier to recognize and categorize preprocessor directives. 618 TheLexer.LexFromRawLexer(TheTok); 619 if (TheTok.getKind() == tok::raw_identifier && !TheTok.needsCleaning()) { 620 StringRef Keyword(TheTok.getRawIdentifierData(), 621 TheTok.getLength()); 622 PreambleDirectiveKind PDK 623 = llvm::StringSwitch<PreambleDirectiveKind>(Keyword) 624 .Case("include", PDK_Skipped) 625 .Case("__include_macros", PDK_Skipped) 626 .Case("define", PDK_Skipped) 627 .Case("undef", PDK_Skipped) 628 .Case("line", PDK_Skipped) 629 .Case("error", PDK_Skipped) 630 .Case("pragma", PDK_Skipped) 631 .Case("import", PDK_Skipped) 632 .Case("include_next", PDK_Skipped) 633 .Case("warning", PDK_Skipped) 634 .Case("ident", PDK_Skipped) 635 .Case("sccs", PDK_Skipped) 636 .Case("assert", PDK_Skipped) 637 .Case("unassert", PDK_Skipped) 638 .Case("if", PDK_StartIf) 639 .Case("ifdef", PDK_StartIf) 640 .Case("ifndef", PDK_StartIf) 641 .Case("elif", PDK_Skipped) 642 .Case("else", PDK_Skipped) 643 .Case("endif", PDK_EndIf) 644 .Default(PDK_Unknown); 645 646 switch (PDK) { 647 case PDK_Skipped: 648 continue; 649 650 case PDK_StartIf: 651 if (IfCount == 0) 652 IfStartTok = HashTok; 653 654 ++IfCount; 655 continue; 656 657 case PDK_EndIf: 658 // Mismatched #endif. The preamble ends here. 659 if (IfCount == 0) 660 break; 661 662 --IfCount; 663 continue; 664 665 case PDK_Unknown: 666 // We don't know what this directive is; stop at the '#'. 667 break; 668 } 669 } 670 671 // We only end up here if we didn't recognize the preprocessor 672 // directive or it was one that can't occur in the preamble at this 673 // point. Roll back the current token to the location of the '#'. 674 InPreprocessorDirective = false; 675 TheTok = HashTok; 676 } 677 678 // We hit a token that we don't recognize as being in the 679 // "preprocessing only" part of the file, so we're no longer in 680 // the preamble. 681 break; 682 } while (true); 683 684 SourceLocation End = IfCount? IfStartTok.getLocation() : TheTok.getLocation(); 685 return std::make_pair(End.getRawEncoding() - StartLoc.getRawEncoding(), 686 IfCount? IfStartTok.isAtStartOfLine() 687 : TheTok.isAtStartOfLine()); 688} 689 690 691/// AdvanceToTokenCharacter - Given a location that specifies the start of a 692/// token, return a new location that specifies a character within the token. 693SourceLocation Lexer::AdvanceToTokenCharacter(SourceLocation TokStart, 694 unsigned CharNo, 695 const SourceManager &SM, 696 const LangOptions &LangOpts) { 697 // Figure out how many physical characters away the specified expansion 698 // character is. This needs to take into consideration newlines and 699 // trigraphs. 700 bool Invalid = false; 701 const char *TokPtr = SM.getCharacterData(TokStart, &Invalid); 702 703 // If they request the first char of the token, we're trivially done. 704 if (Invalid || (CharNo == 0 && Lexer::isObviouslySimpleCharacter(*TokPtr))) 705 return TokStart; 706 707 unsigned PhysOffset = 0; 708 709 // The usual case is that tokens don't contain anything interesting. Skip 710 // over the uninteresting characters. If a token only consists of simple 711 // chars, this method is extremely fast. 712 while (Lexer::isObviouslySimpleCharacter(*TokPtr)) { 713 if (CharNo == 0) 714 return TokStart.getLocWithOffset(PhysOffset); 715 ++TokPtr, --CharNo, ++PhysOffset; 716 } 717 718 // If we have a character that may be a trigraph or escaped newline, use a 719 // lexer to parse it correctly. 720 for (; CharNo; --CharNo) { 721 unsigned Size; 722 Lexer::getCharAndSizeNoWarn(TokPtr, Size, LangOpts); 723 TokPtr += Size; 724 PhysOffset += Size; 725 } 726 727 // Final detail: if we end up on an escaped newline, we want to return the 728 // location of the actual byte of the token. For example foo\<newline>bar 729 // advanced by 3 should return the location of b, not of \\. One compounding 730 // detail of this is that the escape may be made by a trigraph. 731 if (!Lexer::isObviouslySimpleCharacter(*TokPtr)) 732 PhysOffset += Lexer::SkipEscapedNewLines(TokPtr)-TokPtr; 733 734 return TokStart.getLocWithOffset(PhysOffset); 735} 736 737/// \brief Computes the source location just past the end of the 738/// token at this source location. 739/// 740/// This routine can be used to produce a source location that 741/// points just past the end of the token referenced by \p Loc, and 742/// is generally used when a diagnostic needs to point just after a 743/// token where it expected something different that it received. If 744/// the returned source location would not be meaningful (e.g., if 745/// it points into a macro), this routine returns an invalid 746/// source location. 747/// 748/// \param Offset an offset from the end of the token, where the source 749/// location should refer to. The default offset (0) produces a source 750/// location pointing just past the end of the token; an offset of 1 produces 751/// a source location pointing to the last character in the token, etc. 752SourceLocation Lexer::getLocForEndOfToken(SourceLocation Loc, unsigned Offset, 753 const SourceManager &SM, 754 const LangOptions &LangOpts) { 755 if (Loc.isInvalid()) 756 return SourceLocation(); 757 758 if (Loc.isMacroID()) { 759 if (Offset > 0 || !isAtEndOfMacroExpansion(Loc, SM, LangOpts, &Loc)) 760 return SourceLocation(); // Points inside the macro expansion. 761 } 762 763 unsigned Len = Lexer::MeasureTokenLength(Loc, SM, LangOpts); 764 if (Len > Offset) 765 Len = Len - Offset; 766 else 767 return Loc; 768 769 return Loc.getLocWithOffset(Len); 770} 771 772/// \brief Returns true if the given MacroID location points at the first 773/// token of the macro expansion. 774bool Lexer::isAtStartOfMacroExpansion(SourceLocation loc, 775 const SourceManager &SM, 776 const LangOptions &LangOpts, 777 SourceLocation *MacroBegin) { 778 assert(loc.isValid() && loc.isMacroID() && "Expected a valid macro loc"); 779 780 std::pair<FileID, unsigned> infoLoc = SM.getDecomposedLoc(loc); 781 // FIXME: If the token comes from the macro token paste operator ('##') 782 // this function will always return false; 783 if (infoLoc.second > 0) 784 return false; // Does not point at the start of token. 785 786 SourceLocation expansionLoc = 787 SM.getSLocEntry(infoLoc.first).getExpansion().getExpansionLocStart(); 788 if (expansionLoc.isFileID()) { 789 // No other macro expansions, this is the first. 790 if (MacroBegin) 791 *MacroBegin = expansionLoc; 792 return true; 793 } 794 795 return isAtStartOfMacroExpansion(expansionLoc, SM, LangOpts, MacroBegin); 796} 797 798/// \brief Returns true if the given MacroID location points at the last 799/// token of the macro expansion. 800bool Lexer::isAtEndOfMacroExpansion(SourceLocation loc, 801 const SourceManager &SM, 802 const LangOptions &LangOpts, 803 SourceLocation *MacroEnd) { 804 assert(loc.isValid() && loc.isMacroID() && "Expected a valid macro loc"); 805 806 SourceLocation spellLoc = SM.getSpellingLoc(loc); 807 unsigned tokLen = MeasureTokenLength(spellLoc, SM, LangOpts); 808 if (tokLen == 0) 809 return false; 810 811 FileID FID = SM.getFileID(loc); 812 SourceLocation afterLoc = loc.getLocWithOffset(tokLen+1); 813 if (SM.isInFileID(afterLoc, FID)) 814 return false; // Still in the same FileID, does not point to the last token. 815 816 // FIXME: If the token comes from the macro token paste operator ('##') 817 // or the stringify operator ('#') this function will always return false; 818 819 SourceLocation expansionLoc = 820 SM.getSLocEntry(FID).getExpansion().getExpansionLocEnd(); 821 if (expansionLoc.isFileID()) { 822 // No other macro expansions. 823 if (MacroEnd) 824 *MacroEnd = expansionLoc; 825 return true; 826 } 827 828 return isAtEndOfMacroExpansion(expansionLoc, SM, LangOpts, MacroEnd); 829} 830 831static CharSourceRange makeRangeFromFileLocs(CharSourceRange Range, 832 const SourceManager &SM, 833 const LangOptions &LangOpts) { 834 SourceLocation Begin = Range.getBegin(); 835 SourceLocation End = Range.getEnd(); 836 assert(Begin.isFileID() && End.isFileID()); 837 if (Range.isTokenRange()) { 838 End = Lexer::getLocForEndOfToken(End, 0, SM,LangOpts); 839 if (End.isInvalid()) 840 return CharSourceRange(); 841 } 842 843 // Break down the source locations. 844 FileID FID; 845 unsigned BeginOffs; 846 llvm::tie(FID, BeginOffs) = SM.getDecomposedLoc(Begin); 847 if (FID.isInvalid()) 848 return CharSourceRange(); 849 850 unsigned EndOffs; 851 if (!SM.isInFileID(End, FID, &EndOffs) || 852 BeginOffs > EndOffs) 853 return CharSourceRange(); 854 855 return CharSourceRange::getCharRange(Begin, End); 856} 857 858CharSourceRange Lexer::makeFileCharRange(CharSourceRange Range, 859 const SourceManager &SM, 860 const LangOptions &LangOpts) { 861 SourceLocation Begin = Range.getBegin(); 862 SourceLocation End = Range.getEnd(); 863 if (Begin.isInvalid() || End.isInvalid()) 864 return CharSourceRange(); 865 866 if (Begin.isFileID() && End.isFileID()) 867 return makeRangeFromFileLocs(Range, SM, LangOpts); 868 869 if (Begin.isMacroID() && End.isFileID()) { 870 if (!isAtStartOfMacroExpansion(Begin, SM, LangOpts, &Begin)) 871 return CharSourceRange(); 872 Range.setBegin(Begin); 873 return makeRangeFromFileLocs(Range, SM, LangOpts); 874 } 875 876 if (Begin.isFileID() && End.isMacroID()) { 877 if ((Range.isTokenRange() && !isAtEndOfMacroExpansion(End, SM, LangOpts, 878 &End)) || 879 (Range.isCharRange() && !isAtStartOfMacroExpansion(End, SM, LangOpts, 880 &End))) 881 return CharSourceRange(); 882 Range.setEnd(End); 883 return makeRangeFromFileLocs(Range, SM, LangOpts); 884 } 885 886 assert(Begin.isMacroID() && End.isMacroID()); 887 SourceLocation MacroBegin, MacroEnd; 888 if (isAtStartOfMacroExpansion(Begin, SM, LangOpts, &MacroBegin) && 889 ((Range.isTokenRange() && isAtEndOfMacroExpansion(End, SM, LangOpts, 890 &MacroEnd)) || 891 (Range.isCharRange() && isAtStartOfMacroExpansion(End, SM, LangOpts, 892 &MacroEnd)))) { 893 Range.setBegin(MacroBegin); 894 Range.setEnd(MacroEnd); 895 return makeRangeFromFileLocs(Range, SM, LangOpts); 896 } 897 898 FileID FID; 899 unsigned BeginOffs; 900 llvm::tie(FID, BeginOffs) = SM.getDecomposedLoc(Begin); 901 if (FID.isInvalid()) 902 return CharSourceRange(); 903 904 unsigned EndOffs; 905 if (!SM.isInFileID(End, FID, &EndOffs) || 906 BeginOffs > EndOffs) 907 return CharSourceRange(); 908 909 const SrcMgr::SLocEntry *E = &SM.getSLocEntry(FID); 910 const SrcMgr::ExpansionInfo &Expansion = E->getExpansion(); 911 if (Expansion.isMacroArgExpansion() && 912 Expansion.getSpellingLoc().isFileID()) { 913 SourceLocation SpellLoc = Expansion.getSpellingLoc(); 914 Range.setBegin(SpellLoc.getLocWithOffset(BeginOffs)); 915 Range.setEnd(SpellLoc.getLocWithOffset(EndOffs)); 916 return makeRangeFromFileLocs(Range, SM, LangOpts); 917 } 918 919 return CharSourceRange(); 920} 921 922StringRef Lexer::getSourceText(CharSourceRange Range, 923 const SourceManager &SM, 924 const LangOptions &LangOpts, 925 bool *Invalid) { 926 Range = makeFileCharRange(Range, SM, LangOpts); 927 if (Range.isInvalid()) { 928 if (Invalid) *Invalid = true; 929 return StringRef(); 930 } 931 932 // Break down the source location. 933 std::pair<FileID, unsigned> beginInfo = SM.getDecomposedLoc(Range.getBegin()); 934 if (beginInfo.first.isInvalid()) { 935 if (Invalid) *Invalid = true; 936 return StringRef(); 937 } 938 939 unsigned EndOffs; 940 if (!SM.isInFileID(Range.getEnd(), beginInfo.first, &EndOffs) || 941 beginInfo.second > EndOffs) { 942 if (Invalid) *Invalid = true; 943 return StringRef(); 944 } 945 946 // Try to the load the file buffer. 947 bool invalidTemp = false; 948 StringRef file = SM.getBufferData(beginInfo.first, &invalidTemp); 949 if (invalidTemp) { 950 if (Invalid) *Invalid = true; 951 return StringRef(); 952 } 953 954 if (Invalid) *Invalid = false; 955 return file.substr(beginInfo.second, EndOffs - beginInfo.second); 956} 957 958StringRef Lexer::getImmediateMacroName(SourceLocation Loc, 959 const SourceManager &SM, 960 const LangOptions &LangOpts) { 961 assert(Loc.isMacroID() && "Only reasonble to call this on macros"); 962 963 // Find the location of the immediate macro expansion. 964 while (1) { 965 FileID FID = SM.getFileID(Loc); 966 const SrcMgr::SLocEntry *E = &SM.getSLocEntry(FID); 967 const SrcMgr::ExpansionInfo &Expansion = E->getExpansion(); 968 Loc = Expansion.getExpansionLocStart(); 969 if (!Expansion.isMacroArgExpansion()) 970 break; 971 972 // For macro arguments we need to check that the argument did not come 973 // from an inner macro, e.g: "MAC1( MAC2(foo) )" 974 975 // Loc points to the argument id of the macro definition, move to the 976 // macro expansion. 977 Loc = SM.getImmediateExpansionRange(Loc).first; 978 SourceLocation SpellLoc = Expansion.getSpellingLoc(); 979 if (SpellLoc.isFileID()) 980 break; // No inner macro. 981 982 // If spelling location resides in the same FileID as macro expansion 983 // location, it means there is no inner macro. 984 FileID MacroFID = SM.getFileID(Loc); 985 if (SM.isInFileID(SpellLoc, MacroFID)) 986 break; 987 988 // Argument came from inner macro. 989 Loc = SpellLoc; 990 } 991 992 // Find the spelling location of the start of the non-argument expansion 993 // range. This is where the macro name was spelled in order to begin 994 // expanding this macro. 995 Loc = SM.getSpellingLoc(Loc); 996 997 // Dig out the buffer where the macro name was spelled and the extents of the 998 // name so that we can render it into the expansion note. 999 std::pair<FileID, unsigned> ExpansionInfo = SM.getDecomposedLoc(Loc); 1000 unsigned MacroTokenLength = Lexer::MeasureTokenLength(Loc, SM, LangOpts); 1001 StringRef ExpansionBuffer = SM.getBufferData(ExpansionInfo.first); 1002 return ExpansionBuffer.substr(ExpansionInfo.second, MacroTokenLength); 1003} 1004 1005bool Lexer::isIdentifierBodyChar(char c, const LangOptions &LangOpts) { 1006 return isIdentifierBody(c, LangOpts.DollarIdents); 1007} 1008 1009 1010//===----------------------------------------------------------------------===// 1011// Diagnostics forwarding code. 1012//===----------------------------------------------------------------------===// 1013 1014/// GetMappedTokenLoc - If lexing out of a 'mapped buffer', where we pretend the 1015/// lexer buffer was all expanded at a single point, perform the mapping. 1016/// This is currently only used for _Pragma implementation, so it is the slow 1017/// path of the hot getSourceLocation method. Do not allow it to be inlined. 1018static LLVM_ATTRIBUTE_NOINLINE SourceLocation GetMappedTokenLoc( 1019 Preprocessor &PP, SourceLocation FileLoc, unsigned CharNo, unsigned TokLen); 1020static SourceLocation GetMappedTokenLoc(Preprocessor &PP, 1021 SourceLocation FileLoc, 1022 unsigned CharNo, unsigned TokLen) { 1023 assert(FileLoc.isMacroID() && "Must be a macro expansion"); 1024 1025 // Otherwise, we're lexing "mapped tokens". This is used for things like 1026 // _Pragma handling. Combine the expansion location of FileLoc with the 1027 // spelling location. 1028 SourceManager &SM = PP.getSourceManager(); 1029 1030 // Create a new SLoc which is expanded from Expansion(FileLoc) but whose 1031 // characters come from spelling(FileLoc)+Offset. 1032 SourceLocation SpellingLoc = SM.getSpellingLoc(FileLoc); 1033 SpellingLoc = SpellingLoc.getLocWithOffset(CharNo); 1034 1035 // Figure out the expansion loc range, which is the range covered by the 1036 // original _Pragma(...) sequence. 1037 std::pair<SourceLocation,SourceLocation> II = 1038 SM.getImmediateExpansionRange(FileLoc); 1039 1040 return SM.createExpansionLoc(SpellingLoc, II.first, II.second, TokLen); 1041} 1042 1043/// getSourceLocation - Return a source location identifier for the specified 1044/// offset in the current file. 1045SourceLocation Lexer::getSourceLocation(const char *Loc, 1046 unsigned TokLen) const { 1047 assert(Loc >= BufferStart && Loc <= BufferEnd && 1048 "Location out of range for this buffer!"); 1049 1050 // In the normal case, we're just lexing from a simple file buffer, return 1051 // the file id from FileLoc with the offset specified. 1052 unsigned CharNo = Loc-BufferStart; 1053 if (FileLoc.isFileID()) 1054 return FileLoc.getLocWithOffset(CharNo); 1055 1056 // Otherwise, this is the _Pragma lexer case, which pretends that all of the 1057 // tokens are lexed from where the _Pragma was defined. 1058 assert(PP && "This doesn't work on raw lexers"); 1059 return GetMappedTokenLoc(*PP, FileLoc, CharNo, TokLen); 1060} 1061 1062/// Diag - Forwarding function for diagnostics. This translate a source 1063/// position in the current buffer into a SourceLocation object for rendering. 1064DiagnosticBuilder Lexer::Diag(const char *Loc, unsigned DiagID) const { 1065 return PP->Diag(getSourceLocation(Loc), DiagID); 1066} 1067 1068//===----------------------------------------------------------------------===// 1069// Trigraph and Escaped Newline Handling Code. 1070//===----------------------------------------------------------------------===// 1071 1072/// GetTrigraphCharForLetter - Given a character that occurs after a ?? pair, 1073/// return the decoded trigraph letter it corresponds to, or '\0' if nothing. 1074static char GetTrigraphCharForLetter(char Letter) { 1075 switch (Letter) { 1076 default: return 0; 1077 case '=': return '#'; 1078 case ')': return ']'; 1079 case '(': return '['; 1080 case '!': return '|'; 1081 case '\'': return '^'; 1082 case '>': return '}'; 1083 case '/': return '\\'; 1084 case '<': return '{'; 1085 case '-': return '~'; 1086 } 1087} 1088 1089/// DecodeTrigraphChar - If the specified character is a legal trigraph when 1090/// prefixed with ??, emit a trigraph warning. If trigraphs are enabled, 1091/// return the result character. Finally, emit a warning about trigraph use 1092/// whether trigraphs are enabled or not. 1093static char DecodeTrigraphChar(const char *CP, Lexer *L) { 1094 char Res = GetTrigraphCharForLetter(*CP); 1095 if (!Res || !L) return Res; 1096 1097 if (!L->getLangOpts().Trigraphs) { 1098 if (!L->isLexingRawMode()) 1099 L->Diag(CP-2, diag::trigraph_ignored); 1100 return 0; 1101 } 1102 1103 if (!L->isLexingRawMode()) 1104 L->Diag(CP-2, diag::trigraph_converted) << StringRef(&Res, 1); 1105 return Res; 1106} 1107 1108/// getEscapedNewLineSize - Return the size of the specified escaped newline, 1109/// or 0 if it is not an escaped newline. P[-1] is known to be a "\" or a 1110/// trigraph equivalent on entry to this function. 1111unsigned Lexer::getEscapedNewLineSize(const char *Ptr) { 1112 unsigned Size = 0; 1113 while (isWhitespace(Ptr[Size])) { 1114 ++Size; 1115 1116 if (Ptr[Size-1] != '\n' && Ptr[Size-1] != '\r') 1117 continue; 1118 1119 // If this is a \r\n or \n\r, skip the other half. 1120 if ((Ptr[Size] == '\r' || Ptr[Size] == '\n') && 1121 Ptr[Size-1] != Ptr[Size]) 1122 ++Size; 1123 1124 return Size; 1125 } 1126 1127 // Not an escaped newline, must be a \t or something else. 1128 return 0; 1129} 1130 1131/// SkipEscapedNewLines - If P points to an escaped newline (or a series of 1132/// them), skip over them and return the first non-escaped-newline found, 1133/// otherwise return P. 1134const char *Lexer::SkipEscapedNewLines(const char *P) { 1135 while (1) { 1136 const char *AfterEscape; 1137 if (*P == '\\') { 1138 AfterEscape = P+1; 1139 } else if (*P == '?') { 1140 // If not a trigraph for escape, bail out. 1141 if (P[1] != '?' || P[2] != '/') 1142 return P; 1143 AfterEscape = P+3; 1144 } else { 1145 return P; 1146 } 1147 1148 unsigned NewLineSize = Lexer::getEscapedNewLineSize(AfterEscape); 1149 if (NewLineSize == 0) return P; 1150 P = AfterEscape+NewLineSize; 1151 } 1152} 1153 1154/// \brief Checks that the given token is the first token that occurs after the 1155/// given location (this excludes comments and whitespace). Returns the location 1156/// immediately after the specified token. If the token is not found or the 1157/// location is inside a macro, the returned source location will be invalid. 1158SourceLocation Lexer::findLocationAfterToken(SourceLocation Loc, 1159 tok::TokenKind TKind, 1160 const SourceManager &SM, 1161 const LangOptions &LangOpts, 1162 bool SkipTrailingWhitespaceAndNewLine) { 1163 if (Loc.isMacroID()) { 1164 if (!Lexer::isAtEndOfMacroExpansion(Loc, SM, LangOpts, &Loc)) 1165 return SourceLocation(); 1166 } 1167 Loc = Lexer::getLocForEndOfToken(Loc, 0, SM, LangOpts); 1168 1169 // Break down the source location. 1170 std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(Loc); 1171 1172 // Try to load the file buffer. 1173 bool InvalidTemp = false; 1174 StringRef File = SM.getBufferData(LocInfo.first, &InvalidTemp); 1175 if (InvalidTemp) 1176 return SourceLocation(); 1177 1178 const char *TokenBegin = File.data() + LocInfo.second; 1179 1180 // Lex from the start of the given location. 1181 Lexer lexer(SM.getLocForStartOfFile(LocInfo.first), LangOpts, File.begin(), 1182 TokenBegin, File.end()); 1183 // Find the token. 1184 Token Tok; 1185 lexer.LexFromRawLexer(Tok); 1186 if (Tok.isNot(TKind)) 1187 return SourceLocation(); 1188 SourceLocation TokenLoc = Tok.getLocation(); 1189 1190 // Calculate how much whitespace needs to be skipped if any. 1191 unsigned NumWhitespaceChars = 0; 1192 if (SkipTrailingWhitespaceAndNewLine) { 1193 const char *TokenEnd = SM.getCharacterData(TokenLoc) + 1194 Tok.getLength(); 1195 unsigned char C = *TokenEnd; 1196 while (isHorizontalWhitespace(C)) { 1197 C = *(++TokenEnd); 1198 NumWhitespaceChars++; 1199 } 1200 1201 // Skip \r, \n, \r\n, or \n\r 1202 if (C == '\n' || C == '\r') { 1203 char PrevC = C; 1204 C = *(++TokenEnd); 1205 NumWhitespaceChars++; 1206 if ((C == '\n' || C == '\r') && C != PrevC) 1207 NumWhitespaceChars++; 1208 } 1209 } 1210 1211 return TokenLoc.getLocWithOffset(Tok.getLength() + NumWhitespaceChars); 1212} 1213 1214/// getCharAndSizeSlow - Peek a single 'character' from the specified buffer, 1215/// get its size, and return it. This is tricky in several cases: 1216/// 1. If currently at the start of a trigraph, we warn about the trigraph, 1217/// then either return the trigraph (skipping 3 chars) or the '?', 1218/// depending on whether trigraphs are enabled or not. 1219/// 2. If this is an escaped newline (potentially with whitespace between 1220/// the backslash and newline), implicitly skip the newline and return 1221/// the char after it. 1222/// 1223/// This handles the slow/uncommon case of the getCharAndSize method. Here we 1224/// know that we can accumulate into Size, and that we have already incremented 1225/// Ptr by Size bytes. 1226/// 1227/// NOTE: When this method is updated, getCharAndSizeSlowNoWarn (below) should 1228/// be updated to match. 1229/// 1230char Lexer::getCharAndSizeSlow(const char *Ptr, unsigned &Size, 1231 Token *Tok) { 1232 // If we have a slash, look for an escaped newline. 1233 if (Ptr[0] == '\\') { 1234 ++Size; 1235 ++Ptr; 1236Slash: 1237 // Common case, backslash-char where the char is not whitespace. 1238 if (!isWhitespace(Ptr[0])) return '\\'; 1239 1240 // See if we have optional whitespace characters between the slash and 1241 // newline. 1242 if (unsigned EscapedNewLineSize = getEscapedNewLineSize(Ptr)) { 1243 // Remember that this token needs to be cleaned. 1244 if (Tok) Tok->setFlag(Token::NeedsCleaning); 1245 1246 // Warn if there was whitespace between the backslash and newline. 1247 if (Ptr[0] != '\n' && Ptr[0] != '\r' && Tok && !isLexingRawMode()) 1248 Diag(Ptr, diag::backslash_newline_space); 1249 1250 // Found backslash<whitespace><newline>. Parse the char after it. 1251 Size += EscapedNewLineSize; 1252 Ptr += EscapedNewLineSize; 1253 1254 // If the char that we finally got was a \n, then we must have had 1255 // something like \<newline><newline>. We don't want to consume the 1256 // second newline. 1257 if (*Ptr == '\n' || *Ptr == '\r' || *Ptr == '\0') 1258 return ' '; 1259 1260 // Use slow version to accumulate a correct size field. 1261 return getCharAndSizeSlow(Ptr, Size, Tok); 1262 } 1263 1264 // Otherwise, this is not an escaped newline, just return the slash. 1265 return '\\'; 1266 } 1267 1268 // If this is a trigraph, process it. 1269 if (Ptr[0] == '?' && Ptr[1] == '?') { 1270 // If this is actually a legal trigraph (not something like "??x"), emit 1271 // a trigraph warning. If so, and if trigraphs are enabled, return it. 1272 if (char C = DecodeTrigraphChar(Ptr+2, Tok ? this : 0)) { 1273 // Remember that this token needs to be cleaned. 1274 if (Tok) Tok->setFlag(Token::NeedsCleaning); 1275 1276 Ptr += 3; 1277 Size += 3; 1278 if (C == '\\') goto Slash; 1279 return C; 1280 } 1281 } 1282 1283 // If this is neither, return a single character. 1284 ++Size; 1285 return *Ptr; 1286} 1287 1288 1289/// getCharAndSizeSlowNoWarn - Handle the slow/uncommon case of the 1290/// getCharAndSizeNoWarn method. Here we know that we can accumulate into Size, 1291/// and that we have already incremented Ptr by Size bytes. 1292/// 1293/// NOTE: When this method is updated, getCharAndSizeSlow (above) should 1294/// be updated to match. 1295char Lexer::getCharAndSizeSlowNoWarn(const char *Ptr, unsigned &Size, 1296 const LangOptions &LangOpts) { 1297 // If we have a slash, look for an escaped newline. 1298 if (Ptr[0] == '\\') { 1299 ++Size; 1300 ++Ptr; 1301Slash: 1302 // Common case, backslash-char where the char is not whitespace. 1303 if (!isWhitespace(Ptr[0])) return '\\'; 1304 1305 // See if we have optional whitespace characters followed by a newline. 1306 if (unsigned EscapedNewLineSize = getEscapedNewLineSize(Ptr)) { 1307 // Found backslash<whitespace><newline>. Parse the char after it. 1308 Size += EscapedNewLineSize; 1309 Ptr += EscapedNewLineSize; 1310 1311 // If the char that we finally got was a \n, then we must have had 1312 // something like \<newline><newline>. We don't want to consume the 1313 // second newline. 1314 if (*Ptr == '\n' || *Ptr == '\r' || *Ptr == '\0') 1315 return ' '; 1316 1317 // Use slow version to accumulate a correct size field. 1318 return getCharAndSizeSlowNoWarn(Ptr, Size, LangOpts); 1319 } 1320 1321 // Otherwise, this is not an escaped newline, just return the slash. 1322 return '\\'; 1323 } 1324 1325 // If this is a trigraph, process it. 1326 if (LangOpts.Trigraphs && Ptr[0] == '?' && Ptr[1] == '?') { 1327 // If this is actually a legal trigraph (not something like "??x"), return 1328 // it. 1329 if (char C = GetTrigraphCharForLetter(Ptr[2])) { 1330 Ptr += 3; 1331 Size += 3; 1332 if (C == '\\') goto Slash; 1333 return C; 1334 } 1335 } 1336 1337 // If this is neither, return a single character. 1338 ++Size; 1339 return *Ptr; 1340} 1341 1342//===----------------------------------------------------------------------===// 1343// Helper methods for lexing. 1344//===----------------------------------------------------------------------===// 1345 1346/// \brief Routine that indiscriminately skips bytes in the source file. 1347void Lexer::SkipBytes(unsigned Bytes, bool StartOfLine) { 1348 BufferPtr += Bytes; 1349 if (BufferPtr > BufferEnd) 1350 BufferPtr = BufferEnd; 1351 IsAtStartOfLine = StartOfLine; 1352} 1353 1354namespace { 1355 struct UCNCharRange { 1356 uint32_t Lower; 1357 uint32_t Upper; 1358 }; 1359 1360 // C11 D.1, C++11 [charname.allowed] 1361 // FIXME: C99 and C++03 each have a different set of allowed UCNs. 1362 const UCNCharRange UCNAllowedCharRanges[] = { 1363 // 1 1364 { 0x00A8, 0x00A8 }, { 0x00AA, 0x00AA }, { 0x00AD, 0x00AD }, 1365 { 0x00AF, 0x00AF }, { 0x00B2, 0x00B5 }, { 0x00B7, 0x00BA }, 1366 { 0x00BC, 0x00BE }, { 0x00C0, 0x00D6 }, { 0x00D8, 0x00F6 }, 1367 { 0x00F8, 0x00FF }, 1368 // 2 1369 { 0x0100, 0x167F }, { 0x1681, 0x180D }, { 0x180F, 0x1FFF }, 1370 // 3 1371 { 0x200B, 0x200D }, { 0x202A, 0x202E }, { 0x203F, 0x2040 }, 1372 { 0x2054, 0x2054 }, { 0x2060, 0x206F }, 1373 // 4 1374 { 0x2070, 0x218F }, { 0x2460, 0x24FF }, { 0x2776, 0x2793 }, 1375 { 0x2C00, 0x2DFF }, { 0x2E80, 0x2FFF }, 1376 // 5 1377 { 0x3004, 0x3007 }, { 0x3021, 0x302F }, { 0x3031, 0x303F }, 1378 // 6 1379 { 0x3040, 0xD7FF }, 1380 // 7 1381 { 0xF900, 0xFD3D }, { 0xFD40, 0xFDCF }, { 0xFDF0, 0xFE44 }, 1382 { 0xFE47, 0xFFFD }, 1383 // 8 1384 { 0x10000, 0x1FFFD }, { 0x20000, 0x2FFFD }, { 0x30000, 0x3FFFD }, 1385 { 0x40000, 0x4FFFD }, { 0x50000, 0x5FFFD }, { 0x60000, 0x6FFFD }, 1386 { 0x70000, 0x7FFFD }, { 0x80000, 0x8FFFD }, { 0x90000, 0x9FFFD }, 1387 { 0xA0000, 0xAFFFD }, { 0xB0000, 0xBFFFD }, { 0xC0000, 0xCFFFD }, 1388 { 0xD0000, 0xDFFFD }, { 0xE0000, 0xEFFFD } 1389 }; 1390} 1391 1392static bool isAllowedIDChar(uint32_t c) { 1393 unsigned LowPoint = 0; 1394 unsigned HighPoint = llvm::array_lengthof(UCNAllowedCharRanges); 1395 1396 // Binary search the UCNAllowedCharRanges set. 1397 while (HighPoint != LowPoint) { 1398 unsigned MidPoint = (HighPoint + LowPoint) / 2; 1399 if (c < UCNAllowedCharRanges[MidPoint].Lower) 1400 HighPoint = MidPoint; 1401 else if (c > UCNAllowedCharRanges[MidPoint].Upper) 1402 LowPoint = MidPoint + 1; 1403 else 1404 return true; 1405 } 1406 1407 return false; 1408} 1409 1410static bool isAllowedInitiallyIDChar(uint32_t c) { 1411 // C11 D.2, C++11 [charname.disallowed] 1412 // FIXME: C99 only forbids "digits", presumably as described in C99 Annex D. 1413 // FIXME: C++03 does not forbid any initial characters. 1414 return !(0x0300 <= c && c <= 0x036F) && 1415 !(0x1DC0 <= c && c <= 0x1DFF) && 1416 !(0x20D0 <= c && c <= 0x20FF) && 1417 !(0xFE20 <= c && c <= 0xFE2F); 1418} 1419 1420 1421void Lexer::LexIdentifier(Token &Result, const char *CurPtr) { 1422 // Match [_A-Za-z0-9]*, we have already matched [_A-Za-z$] 1423 unsigned Size; 1424 unsigned char C = *CurPtr++; 1425 while (isIdentifierBody(C)) 1426 C = *CurPtr++; 1427 1428 --CurPtr; // Back up over the skipped character. 1429 1430 // Fast path, no $,\,? in identifier found. '\' might be an escaped newline 1431 // or UCN, and ? might be a trigraph for '\', an escaped newline or UCN. 1432 // 1433 // TODO: Could merge these checks into an InfoTable flag to make the 1434 // comparison cheaper 1435 if (isASCII(C) && C != '\\' && C != '?' && 1436 (C != '$' || !LangOpts.DollarIdents)) { 1437FinishIdentifier: 1438 const char *IdStart = BufferPtr; 1439 FormTokenWithChars(Result, CurPtr, tok::raw_identifier); 1440 Result.setRawIdentifierData(IdStart); 1441 1442 // If we are in raw mode, return this identifier raw. There is no need to 1443 // look up identifier information or attempt to macro expand it. 1444 if (LexingRawMode) 1445 return; 1446 1447 // Fill in Result.IdentifierInfo and update the token kind, 1448 // looking up the identifier in the identifier table. 1449 IdentifierInfo *II = PP->LookUpIdentifierInfo(Result); 1450 1451 // Finally, now that we know we have an identifier, pass this off to the 1452 // preprocessor, which may macro expand it or something. 1453 if (II->isHandleIdentifierCase()) 1454 PP->HandleIdentifier(Result); 1455 1456 return; 1457 } 1458 1459 // Otherwise, $,\,? in identifier found. Enter slower path. 1460 1461 C = getCharAndSize(CurPtr, Size); 1462 while (1) { 1463 if (C == '$') { 1464 // If we hit a $ and they are not supported in identifiers, we are done. 1465 if (!LangOpts.DollarIdents) goto FinishIdentifier; 1466 1467 // Otherwise, emit a diagnostic and continue. 1468 if (!isLexingRawMode()) 1469 Diag(CurPtr, diag::ext_dollar_in_identifier); 1470 CurPtr = ConsumeChar(CurPtr, Size, Result); 1471 C = getCharAndSize(CurPtr, Size); 1472 continue; 1473 1474 } else if (C == '\\') { 1475 const char *UCNPtr = CurPtr + Size; 1476 uint32_t CodePoint = tryReadUCN(UCNPtr, CurPtr, /*Token=*/0); 1477 if (CodePoint == 0 || !isAllowedIDChar(CodePoint)) 1478 goto FinishIdentifier; 1479 1480 Result.setFlag(Token::HasUCN); 1481 if ((UCNPtr - CurPtr == 6 && CurPtr[1] == 'u') || 1482 (UCNPtr - CurPtr == 10 && CurPtr[1] == 'U')) 1483 CurPtr = UCNPtr; 1484 else 1485 while (CurPtr != UCNPtr) 1486 (void)getAndAdvanceChar(CurPtr, Result); 1487 1488 C = getCharAndSize(CurPtr, Size); 1489 continue; 1490 } else if (!isASCII(C)) { 1491 const char *UnicodePtr = CurPtr; 1492 UTF32 CodePoint; 1493 ConversionResult Result = 1494 llvm::convertUTF8Sequence((const UTF8 **)&UnicodePtr, 1495 (const UTF8 *)BufferEnd, 1496 &CodePoint, 1497 strictConversion); 1498 if (Result != conversionOK || 1499 !isAllowedIDChar(static_cast<uint32_t>(CodePoint))) 1500 goto FinishIdentifier; 1501 1502 CurPtr = UnicodePtr; 1503 C = getCharAndSize(CurPtr, Size); 1504 continue; 1505 } else if (!isIdentifierBody(C)) { 1506 goto FinishIdentifier; 1507 } 1508 1509 // Otherwise, this character is good, consume it. 1510 CurPtr = ConsumeChar(CurPtr, Size, Result); 1511 1512 C = getCharAndSize(CurPtr, Size); 1513 while (isIdentifierBody(C)) { 1514 CurPtr = ConsumeChar(CurPtr, Size, Result); 1515 C = getCharAndSize(CurPtr, Size); 1516 } 1517 } 1518} 1519 1520/// isHexaLiteral - Return true if Start points to a hex constant. 1521/// in microsoft mode (where this is supposed to be several different tokens). 1522bool Lexer::isHexaLiteral(const char *Start, const LangOptions &LangOpts) { 1523 unsigned Size; 1524 char C1 = Lexer::getCharAndSizeNoWarn(Start, Size, LangOpts); 1525 if (C1 != '0') 1526 return false; 1527 char C2 = Lexer::getCharAndSizeNoWarn(Start + Size, Size, LangOpts); 1528 return (C2 == 'x' || C2 == 'X'); 1529} 1530 1531/// LexNumericConstant - Lex the remainder of a integer or floating point 1532/// constant. From[-1] is the first character lexed. Return the end of the 1533/// constant. 1534void Lexer::LexNumericConstant(Token &Result, const char *CurPtr) { 1535 unsigned Size; 1536 char C = getCharAndSize(CurPtr, Size); 1537 char PrevCh = 0; 1538 while (isPreprocessingNumberBody(C)) { // FIXME: UCNs in ud-suffix. 1539 CurPtr = ConsumeChar(CurPtr, Size, Result); 1540 PrevCh = C; 1541 C = getCharAndSize(CurPtr, Size); 1542 } 1543 1544 // If we fell out, check for a sign, due to 1e+12. If we have one, continue. 1545 if ((C == '-' || C == '+') && (PrevCh == 'E' || PrevCh == 'e')) { 1546 // If we are in Microsoft mode, don't continue if the constant is hex. 1547 // For example, MSVC will accept the following as 3 tokens: 0x1234567e+1 1548 if (!LangOpts.MicrosoftExt || !isHexaLiteral(BufferPtr, LangOpts)) 1549 return LexNumericConstant(Result, ConsumeChar(CurPtr, Size, Result)); 1550 } 1551 1552 // If we have a hex FP constant, continue. 1553 if ((C == '-' || C == '+') && (PrevCh == 'P' || PrevCh == 'p')) { 1554 // Outside C99, we accept hexadecimal floating point numbers as a 1555 // not-quite-conforming extension. Only do so if this looks like it's 1556 // actually meant to be a hexfloat, and not if it has a ud-suffix. 1557 bool IsHexFloat = true; 1558 if (!LangOpts.C99) { 1559 if (!isHexaLiteral(BufferPtr, LangOpts)) 1560 IsHexFloat = false; 1561 else if (std::find(BufferPtr, CurPtr, '_') != CurPtr) 1562 IsHexFloat = false; 1563 } 1564 if (IsHexFloat) 1565 return LexNumericConstant(Result, ConsumeChar(CurPtr, Size, Result)); 1566 } 1567 1568 // Update the location of token as well as BufferPtr. 1569 const char *TokStart = BufferPtr; 1570 FormTokenWithChars(Result, CurPtr, tok::numeric_constant); 1571 Result.setLiteralData(TokStart); 1572} 1573 1574/// LexUDSuffix - Lex the ud-suffix production for user-defined literal suffixes 1575/// in C++11, or warn on a ud-suffix in C++98. 1576const char *Lexer::LexUDSuffix(Token &Result, const char *CurPtr) { 1577 assert(getLangOpts().CPlusPlus); 1578 1579 // Maximally munch an identifier. FIXME: UCNs. 1580 unsigned Size; 1581 char C = getCharAndSize(CurPtr, Size); 1582 if (isIdentifierHead(C)) { 1583 if (!getLangOpts().CPlusPlus11) { 1584 if (!isLexingRawMode()) 1585 Diag(CurPtr, 1586 C == '_' ? diag::warn_cxx11_compat_user_defined_literal 1587 : diag::warn_cxx11_compat_reserved_user_defined_literal) 1588 << FixItHint::CreateInsertion(getSourceLocation(CurPtr), " "); 1589 return CurPtr; 1590 } 1591 1592 // C++11 [lex.ext]p10, [usrlit.suffix]p1: A program containing a ud-suffix 1593 // that does not start with an underscore is ill-formed. As a conforming 1594 // extension, we treat all such suffixes as if they had whitespace before 1595 // them. 1596 if (C != '_') { 1597 if (!isLexingRawMode()) 1598 Diag(CurPtr, getLangOpts().MicrosoftMode ? 1599 diag::ext_ms_reserved_user_defined_literal : 1600 diag::ext_reserved_user_defined_literal) 1601 << FixItHint::CreateInsertion(getSourceLocation(CurPtr), " "); 1602 return CurPtr; 1603 } 1604 1605 Result.setFlag(Token::HasUDSuffix); 1606 do { 1607 CurPtr = ConsumeChar(CurPtr, Size, Result); 1608 C = getCharAndSize(CurPtr, Size); 1609 } while (isIdentifierBody(C)); 1610 } 1611 return CurPtr; 1612} 1613 1614/// LexStringLiteral - Lex the remainder of a string literal, after having lexed 1615/// either " or L" or u8" or u" or U". 1616void Lexer::LexStringLiteral(Token &Result, const char *CurPtr, 1617 tok::TokenKind Kind) { 1618 const char *NulCharacter = 0; // Does this string contain the \0 character? 1619 1620 if (!isLexingRawMode() && 1621 (Kind == tok::utf8_string_literal || 1622 Kind == tok::utf16_string_literal || 1623 Kind == tok::utf32_string_literal)) 1624 Diag(BufferPtr, diag::warn_cxx98_compat_unicode_literal); 1625 1626 char C = getAndAdvanceChar(CurPtr, Result); 1627 while (C != '"') { 1628 // Skip escaped characters. Escaped newlines will already be processed by 1629 // getAndAdvanceChar. 1630 if (C == '\\') 1631 C = getAndAdvanceChar(CurPtr, Result); 1632 1633 if (C == '\n' || C == '\r' || // Newline. 1634 (C == 0 && CurPtr-1 == BufferEnd)) { // End of file. 1635 if (!isLexingRawMode() && !LangOpts.AsmPreprocessor) 1636 Diag(BufferPtr, diag::ext_unterminated_string); 1637 FormTokenWithChars(Result, CurPtr-1, tok::unknown); 1638 return; 1639 } 1640 1641 if (C == 0) { 1642 if (isCodeCompletionPoint(CurPtr-1)) { 1643 PP->CodeCompleteNaturalLanguage(); 1644 FormTokenWithChars(Result, CurPtr-1, tok::unknown); 1645 return cutOffLexing(); 1646 } 1647 1648 NulCharacter = CurPtr-1; 1649 } 1650 C = getAndAdvanceChar(CurPtr, Result); 1651 } 1652 1653 // If we are in C++11, lex the optional ud-suffix. 1654 if (getLangOpts().CPlusPlus) 1655 CurPtr = LexUDSuffix(Result, CurPtr); 1656 1657 // If a nul character existed in the string, warn about it. 1658 if (NulCharacter && !isLexingRawMode()) 1659 Diag(NulCharacter, diag::null_in_string); 1660 1661 // Update the location of the token as well as the BufferPtr instance var. 1662 const char *TokStart = BufferPtr; 1663 FormTokenWithChars(Result, CurPtr, Kind); 1664 Result.setLiteralData(TokStart); 1665} 1666 1667/// LexRawStringLiteral - Lex the remainder of a raw string literal, after 1668/// having lexed R", LR", u8R", uR", or UR". 1669void Lexer::LexRawStringLiteral(Token &Result, const char *CurPtr, 1670 tok::TokenKind Kind) { 1671 // This function doesn't use getAndAdvanceChar because C++0x [lex.pptoken]p3: 1672 // Between the initial and final double quote characters of the raw string, 1673 // any transformations performed in phases 1 and 2 (trigraphs, 1674 // universal-character-names, and line splicing) are reverted. 1675 1676 if (!isLexingRawMode()) 1677 Diag(BufferPtr, diag::warn_cxx98_compat_raw_string_literal); 1678 1679 unsigned PrefixLen = 0; 1680 1681 while (PrefixLen != 16 && isRawStringDelimBody(CurPtr[PrefixLen])) 1682 ++PrefixLen; 1683 1684 // If the last character was not a '(', then we didn't lex a valid delimiter. 1685 if (CurPtr[PrefixLen] != '(') { 1686 if (!isLexingRawMode()) { 1687 const char *PrefixEnd = &CurPtr[PrefixLen]; 1688 if (PrefixLen == 16) { 1689 Diag(PrefixEnd, diag::err_raw_delim_too_long); 1690 } else { 1691 Diag(PrefixEnd, diag::err_invalid_char_raw_delim) 1692 << StringRef(PrefixEnd, 1); 1693 } 1694 } 1695 1696 // Search for the next '"' in hopes of salvaging the lexer. Unfortunately, 1697 // it's possible the '"' was intended to be part of the raw string, but 1698 // there's not much we can do about that. 1699 while (1) { 1700 char C = *CurPtr++; 1701 1702 if (C == '"') 1703 break; 1704 if (C == 0 && CurPtr-1 == BufferEnd) { 1705 --CurPtr; 1706 break; 1707 } 1708 } 1709 1710 FormTokenWithChars(Result, CurPtr, tok::unknown); 1711 return; 1712 } 1713 1714 // Save prefix and move CurPtr past it 1715 const char *Prefix = CurPtr; 1716 CurPtr += PrefixLen + 1; // skip over prefix and '(' 1717 1718 while (1) { 1719 char C = *CurPtr++; 1720 1721 if (C == ')') { 1722 // Check for prefix match and closing quote. 1723 if (strncmp(CurPtr, Prefix, PrefixLen) == 0 && CurPtr[PrefixLen] == '"') { 1724 CurPtr += PrefixLen + 1; // skip over prefix and '"' 1725 break; 1726 } 1727 } else if (C == 0 && CurPtr-1 == BufferEnd) { // End of file. 1728 if (!isLexingRawMode()) 1729 Diag(BufferPtr, diag::err_unterminated_raw_string) 1730 << StringRef(Prefix, PrefixLen); 1731 FormTokenWithChars(Result, CurPtr-1, tok::unknown); 1732 return; 1733 } 1734 } 1735 1736 // If we are in C++11, lex the optional ud-suffix. 1737 if (getLangOpts().CPlusPlus) 1738 CurPtr = LexUDSuffix(Result, CurPtr); 1739 1740 // Update the location of token as well as BufferPtr. 1741 const char *TokStart = BufferPtr; 1742 FormTokenWithChars(Result, CurPtr, Kind); 1743 Result.setLiteralData(TokStart); 1744} 1745 1746/// LexAngledStringLiteral - Lex the remainder of an angled string literal, 1747/// after having lexed the '<' character. This is used for #include filenames. 1748void Lexer::LexAngledStringLiteral(Token &Result, const char *CurPtr) { 1749 const char *NulCharacter = 0; // Does this string contain the \0 character? 1750 const char *AfterLessPos = CurPtr; 1751 char C = getAndAdvanceChar(CurPtr, Result); 1752 while (C != '>') { 1753 // Skip escaped characters. 1754 if (C == '\\') { 1755 // Skip the escaped character. 1756 getAndAdvanceChar(CurPtr, Result); 1757 } else if (C == '\n' || C == '\r' || // Newline. 1758 (C == 0 && (CurPtr-1 == BufferEnd || // End of file. 1759 isCodeCompletionPoint(CurPtr-1)))) { 1760 // If the filename is unterminated, then it must just be a lone < 1761 // character. Return this as such. 1762 FormTokenWithChars(Result, AfterLessPos, tok::less); 1763 return; 1764 } else if (C == 0) { 1765 NulCharacter = CurPtr-1; 1766 } 1767 C = getAndAdvanceChar(CurPtr, Result); 1768 } 1769 1770 // If a nul character existed in the string, warn about it. 1771 if (NulCharacter && !isLexingRawMode()) 1772 Diag(NulCharacter, diag::null_in_string); 1773 1774 // Update the location of token as well as BufferPtr. 1775 const char *TokStart = BufferPtr; 1776 FormTokenWithChars(Result, CurPtr, tok::angle_string_literal); 1777 Result.setLiteralData(TokStart); 1778} 1779 1780 1781/// LexCharConstant - Lex the remainder of a character constant, after having 1782/// lexed either ' or L' or u' or U'. 1783void Lexer::LexCharConstant(Token &Result, const char *CurPtr, 1784 tok::TokenKind Kind) { 1785 const char *NulCharacter = 0; // Does this character contain the \0 character? 1786 1787 if (!isLexingRawMode() && 1788 (Kind == tok::utf16_char_constant || Kind == tok::utf32_char_constant)) 1789 Diag(BufferPtr, diag::warn_cxx98_compat_unicode_literal); 1790 1791 char C = getAndAdvanceChar(CurPtr, Result); 1792 if (C == '\'') { 1793 if (!isLexingRawMode() && !LangOpts.AsmPreprocessor) 1794 Diag(BufferPtr, diag::ext_empty_character); 1795 FormTokenWithChars(Result, CurPtr, tok::unknown); 1796 return; 1797 } 1798 1799 while (C != '\'') { 1800 // Skip escaped characters. 1801 if (C == '\\') 1802 C = getAndAdvanceChar(CurPtr, Result); 1803 1804 if (C == '\n' || C == '\r' || // Newline. 1805 (C == 0 && CurPtr-1 == BufferEnd)) { // End of file. 1806 if (!isLexingRawMode() && !LangOpts.AsmPreprocessor) 1807 Diag(BufferPtr, diag::ext_unterminated_char); 1808 FormTokenWithChars(Result, CurPtr-1, tok::unknown); 1809 return; 1810 } 1811 1812 if (C == 0) { 1813 if (isCodeCompletionPoint(CurPtr-1)) { 1814 PP->CodeCompleteNaturalLanguage(); 1815 FormTokenWithChars(Result, CurPtr-1, tok::unknown); 1816 return cutOffLexing(); 1817 } 1818 1819 NulCharacter = CurPtr-1; 1820 } 1821 C = getAndAdvanceChar(CurPtr, Result); 1822 } 1823 1824 // If we are in C++11, lex the optional ud-suffix. 1825 if (getLangOpts().CPlusPlus) 1826 CurPtr = LexUDSuffix(Result, CurPtr); 1827 1828 // If a nul character existed in the character, warn about it. 1829 if (NulCharacter && !isLexingRawMode()) 1830 Diag(NulCharacter, diag::null_in_char); 1831 1832 // Update the location of token as well as BufferPtr. 1833 const char *TokStart = BufferPtr; 1834 FormTokenWithChars(Result, CurPtr, Kind); 1835 Result.setLiteralData(TokStart); 1836} 1837 1838/// SkipWhitespace - Efficiently skip over a series of whitespace characters. 1839/// Update BufferPtr to point to the next non-whitespace character and return. 1840/// 1841/// This method forms a token and returns true if KeepWhitespaceMode is enabled. 1842/// 1843bool Lexer::SkipWhitespace(Token &Result, const char *CurPtr) { 1844 // Whitespace - Skip it, then return the token after the whitespace. 1845 unsigned char Char = *CurPtr; // Skip consequtive spaces efficiently. 1846 while (1) { 1847 // Skip horizontal whitespace very aggressively. 1848 while (isHorizontalWhitespace(Char)) 1849 Char = *++CurPtr; 1850 1851 // Otherwise if we have something other than whitespace, we're done. 1852 if (Char != '\n' && Char != '\r') 1853 break; 1854 1855 if (ParsingPreprocessorDirective) { 1856 // End of preprocessor directive line, let LexTokenInternal handle this. 1857 BufferPtr = CurPtr; 1858 return false; 1859 } 1860 1861 // ok, but handle newline. 1862 // The returned token is at the start of the line. 1863 Result.setFlag(Token::StartOfLine); 1864 // No leading whitespace seen so far. 1865 Result.clearFlag(Token::LeadingSpace); 1866 Char = *++CurPtr; 1867 } 1868 1869 // If this isn't immediately after a newline, there is leading space. 1870 char PrevChar = CurPtr[-1]; 1871 if (PrevChar != '\n' && PrevChar != '\r') 1872 Result.setFlag(Token::LeadingSpace); 1873 1874 // If the client wants us to return whitespace, return it now. 1875 if (isKeepWhitespaceMode()) { 1876 FormTokenWithChars(Result, CurPtr, tok::unknown); 1877 return true; 1878 } 1879 1880 BufferPtr = CurPtr; 1881 return false; 1882} 1883 1884/// We have just read the // characters from input. Skip until we find the 1885/// newline character thats terminate the comment. Then update BufferPtr and 1886/// return. 1887/// 1888/// If we're in KeepCommentMode or any CommentHandler has inserted 1889/// some tokens, this will store the first token and return true. 1890bool Lexer::SkipLineComment(Token &Result, const char *CurPtr) { 1891 // If Line comments aren't explicitly enabled for this language, emit an 1892 // extension warning. 1893 if (!LangOpts.LineComment && !isLexingRawMode()) { 1894 Diag(BufferPtr, diag::ext_line_comment); 1895 1896 // Mark them enabled so we only emit one warning for this translation 1897 // unit. 1898 LangOpts.LineComment = true; 1899 } 1900 1901 // Scan over the body of the comment. The common case, when scanning, is that 1902 // the comment contains normal ascii characters with nothing interesting in 1903 // them. As such, optimize for this case with the inner loop. 1904 char C; 1905 do { 1906 C = *CurPtr; 1907 // Skip over characters in the fast loop. 1908 while (C != 0 && // Potentially EOF. 1909 C != '\n' && C != '\r') // Newline or DOS-style newline. 1910 C = *++CurPtr; 1911 1912 const char *NextLine = CurPtr; 1913 if (C != 0) { 1914 // We found a newline, see if it's escaped. 1915 const char *EscapePtr = CurPtr-1; 1916 while (isHorizontalWhitespace(*EscapePtr)) // Skip whitespace. 1917 --EscapePtr; 1918 1919 if (*EscapePtr == '\\') // Escaped newline. 1920 CurPtr = EscapePtr; 1921 else if (EscapePtr[0] == '/' && EscapePtr[-1] == '?' && 1922 EscapePtr[-2] == '?') // Trigraph-escaped newline. 1923 CurPtr = EscapePtr-2; 1924 else 1925 break; // This is a newline, we're done. 1926 } 1927 1928 // Otherwise, this is a hard case. Fall back on getAndAdvanceChar to 1929 // properly decode the character. Read it in raw mode to avoid emitting 1930 // diagnostics about things like trigraphs. If we see an escaped newline, 1931 // we'll handle it below. 1932 const char *OldPtr = CurPtr; 1933 bool OldRawMode = isLexingRawMode(); 1934 LexingRawMode = true; 1935 C = getAndAdvanceChar(CurPtr, Result); 1936 LexingRawMode = OldRawMode; 1937 1938 // If we only read only one character, then no special handling is needed. 1939 // We're done and can skip forward to the newline. 1940 if (C != 0 && CurPtr == OldPtr+1) { 1941 CurPtr = NextLine; 1942 break; 1943 } 1944 1945 // If we read multiple characters, and one of those characters was a \r or 1946 // \n, then we had an escaped newline within the comment. Emit diagnostic 1947 // unless the next line is also a // comment. 1948 if (CurPtr != OldPtr+1 && C != '/' && CurPtr[0] != '/') { 1949 for (; OldPtr != CurPtr; ++OldPtr) 1950 if (OldPtr[0] == '\n' || OldPtr[0] == '\r') { 1951 // Okay, we found a // comment that ends in a newline, if the next 1952 // line is also a // comment, but has spaces, don't emit a diagnostic. 1953 if (isWhitespace(C)) { 1954 const char *ForwardPtr = CurPtr; 1955 while (isWhitespace(*ForwardPtr)) // Skip whitespace. 1956 ++ForwardPtr; 1957 if (ForwardPtr[0] == '/' && ForwardPtr[1] == '/') 1958 break; 1959 } 1960 1961 if (!isLexingRawMode()) 1962 Diag(OldPtr-1, diag::ext_multi_line_line_comment); 1963 break; 1964 } 1965 } 1966 1967 if (CurPtr == BufferEnd+1) { 1968 --CurPtr; 1969 break; 1970 } 1971 1972 if (C == '\0' && isCodeCompletionPoint(CurPtr-1)) { 1973 PP->CodeCompleteNaturalLanguage(); 1974 cutOffLexing(); 1975 return false; 1976 } 1977 1978 } while (C != '\n' && C != '\r'); 1979 1980 // Found but did not consume the newline. Notify comment handlers about the 1981 // comment unless we're in a #if 0 block. 1982 if (PP && !isLexingRawMode() && 1983 PP->HandleComment(Result, SourceRange(getSourceLocation(BufferPtr), 1984 getSourceLocation(CurPtr)))) { 1985 BufferPtr = CurPtr; 1986 return true; // A token has to be returned. 1987 } 1988 1989 // If we are returning comments as tokens, return this comment as a token. 1990 if (inKeepCommentMode()) 1991 return SaveLineComment(Result, CurPtr); 1992 1993 // If we are inside a preprocessor directive and we see the end of line, 1994 // return immediately, so that the lexer can return this as an EOD token. 1995 if (ParsingPreprocessorDirective || CurPtr == BufferEnd) { 1996 BufferPtr = CurPtr; 1997 return false; 1998 } 1999 2000 // Otherwise, eat the \n character. We don't care if this is a \n\r or 2001 // \r\n sequence. This is an efficiency hack (because we know the \n can't 2002 // contribute to another token), it isn't needed for correctness. Note that 2003 // this is ok even in KeepWhitespaceMode, because we would have returned the 2004 /// comment above in that mode. 2005 ++CurPtr; 2006 2007 // The next returned token is at the start of the line. 2008 Result.setFlag(Token::StartOfLine); 2009 // No leading whitespace seen so far. 2010 Result.clearFlag(Token::LeadingSpace); 2011 BufferPtr = CurPtr; 2012 return false; 2013} 2014 2015/// If in save-comment mode, package up this Line comment in an appropriate 2016/// way and return it. 2017bool Lexer::SaveLineComment(Token &Result, const char *CurPtr) { 2018 // If we're not in a preprocessor directive, just return the // comment 2019 // directly. 2020 FormTokenWithChars(Result, CurPtr, tok::comment); 2021 2022 if (!ParsingPreprocessorDirective || LexingRawMode) 2023 return true; 2024 2025 // If this Line-style comment is in a macro definition, transmogrify it into 2026 // a C-style block comment. 2027 bool Invalid = false; 2028 std::string Spelling = PP->getSpelling(Result, &Invalid); 2029 if (Invalid) 2030 return true; 2031 2032 assert(Spelling[0] == '/' && Spelling[1] == '/' && "Not line comment?"); 2033 Spelling[1] = '*'; // Change prefix to "/*". 2034 Spelling += "*/"; // add suffix. 2035 2036 Result.setKind(tok::comment); 2037 PP->CreateString(Spelling, Result, 2038 Result.getLocation(), Result.getLocation()); 2039 return true; 2040} 2041 2042/// isBlockCommentEndOfEscapedNewLine - Return true if the specified newline 2043/// character (either \\n or \\r) is part of an escaped newline sequence. Issue 2044/// a diagnostic if so. We know that the newline is inside of a block comment. 2045static bool isEndOfBlockCommentWithEscapedNewLine(const char *CurPtr, 2046 Lexer *L) { 2047 assert(CurPtr[0] == '\n' || CurPtr[0] == '\r'); 2048 2049 // Back up off the newline. 2050 --CurPtr; 2051 2052 // If this is a two-character newline sequence, skip the other character. 2053 if (CurPtr[0] == '\n' || CurPtr[0] == '\r') { 2054 // \n\n or \r\r -> not escaped newline. 2055 if (CurPtr[0] == CurPtr[1]) 2056 return false; 2057 // \n\r or \r\n -> skip the newline. 2058 --CurPtr; 2059 } 2060 2061 // If we have horizontal whitespace, skip over it. We allow whitespace 2062 // between the slash and newline. 2063 bool HasSpace = false; 2064 while (isHorizontalWhitespace(*CurPtr) || *CurPtr == 0) { 2065 --CurPtr; 2066 HasSpace = true; 2067 } 2068 2069 // If we have a slash, we know this is an escaped newline. 2070 if (*CurPtr == '\\') { 2071 if (CurPtr[-1] != '*') return false; 2072 } else { 2073 // It isn't a slash, is it the ?? / trigraph? 2074 if (CurPtr[0] != '/' || CurPtr[-1] != '?' || CurPtr[-2] != '?' || 2075 CurPtr[-3] != '*') 2076 return false; 2077 2078 // This is the trigraph ending the comment. Emit a stern warning! 2079 CurPtr -= 2; 2080 2081 // If no trigraphs are enabled, warn that we ignored this trigraph and 2082 // ignore this * character. 2083 if (!L->getLangOpts().Trigraphs) { 2084 if (!L->isLexingRawMode()) 2085 L->Diag(CurPtr, diag::trigraph_ignored_block_comment); 2086 return false; 2087 } 2088 if (!L->isLexingRawMode()) 2089 L->Diag(CurPtr, diag::trigraph_ends_block_comment); 2090 } 2091 2092 // Warn about having an escaped newline between the */ characters. 2093 if (!L->isLexingRawMode()) 2094 L->Diag(CurPtr, diag::escaped_newline_block_comment_end); 2095 2096 // If there was space between the backslash and newline, warn about it. 2097 if (HasSpace && !L->isLexingRawMode()) 2098 L->Diag(CurPtr, diag::backslash_newline_space); 2099 2100 return true; 2101} 2102 2103#ifdef __SSE2__ 2104#include <emmintrin.h> 2105#elif __ALTIVEC__ 2106#include <altivec.h> 2107#undef bool 2108#endif 2109 2110/// We have just read from input the / and * characters that started a comment. 2111/// Read until we find the * and / characters that terminate the comment. 2112/// Note that we don't bother decoding trigraphs or escaped newlines in block 2113/// comments, because they cannot cause the comment to end. The only thing 2114/// that can happen is the comment could end with an escaped newline between 2115/// the terminating * and /. 2116/// 2117/// If we're in KeepCommentMode or any CommentHandler has inserted 2118/// some tokens, this will store the first token and return true. 2119bool Lexer::SkipBlockComment(Token &Result, const char *CurPtr) { 2120 // Scan one character past where we should, looking for a '/' character. Once 2121 // we find it, check to see if it was preceded by a *. This common 2122 // optimization helps people who like to put a lot of * characters in their 2123 // comments. 2124 2125 // The first character we get with newlines and trigraphs skipped to handle 2126 // the degenerate /*/ case below correctly if the * has an escaped newline 2127 // after it. 2128 unsigned CharSize; 2129 unsigned char C = getCharAndSize(CurPtr, CharSize); 2130 CurPtr += CharSize; 2131 if (C == 0 && CurPtr == BufferEnd+1) { 2132 if (!isLexingRawMode()) 2133 Diag(BufferPtr, diag::err_unterminated_block_comment); 2134 --CurPtr; 2135 2136 // KeepWhitespaceMode should return this broken comment as a token. Since 2137 // it isn't a well formed comment, just return it as an 'unknown' token. 2138 if (isKeepWhitespaceMode()) { 2139 FormTokenWithChars(Result, CurPtr, tok::unknown); 2140 return true; 2141 } 2142 2143 BufferPtr = CurPtr; 2144 return false; 2145 } 2146 2147 // Check to see if the first character after the '/*' is another /. If so, 2148 // then this slash does not end the block comment, it is part of it. 2149 if (C == '/') 2150 C = *CurPtr++; 2151 2152 while (1) { 2153 // Skip over all non-interesting characters until we find end of buffer or a 2154 // (probably ending) '/' character. 2155 if (CurPtr + 24 < BufferEnd && 2156 // If there is a code-completion point avoid the fast scan because it 2157 // doesn't check for '\0'. 2158 !(PP && PP->getCodeCompletionFileLoc() == FileLoc)) { 2159 // While not aligned to a 16-byte boundary. 2160 while (C != '/' && ((intptr_t)CurPtr & 0x0F) != 0) 2161 C = *CurPtr++; 2162 2163 if (C == '/') goto FoundSlash; 2164 2165#ifdef __SSE2__ 2166 __m128i Slashes = _mm_set1_epi8('/'); 2167 while (CurPtr+16 <= BufferEnd) { 2168 int cmp = _mm_movemask_epi8(_mm_cmpeq_epi8(*(const __m128i*)CurPtr, 2169 Slashes)); 2170 if (cmp != 0) { 2171 // Adjust the pointer to point directly after the first slash. It's 2172 // not necessary to set C here, it will be overwritten at the end of 2173 // the outer loop. 2174 CurPtr += llvm::CountTrailingZeros_32(cmp) + 1; 2175 goto FoundSlash; 2176 } 2177 CurPtr += 16; 2178 } 2179#elif __ALTIVEC__ 2180 __vector unsigned char Slashes = { 2181 '/', '/', '/', '/', '/', '/', '/', '/', 2182 '/', '/', '/', '/', '/', '/', '/', '/' 2183 }; 2184 while (CurPtr+16 <= BufferEnd && 2185 !vec_any_eq(*(vector unsigned char*)CurPtr, Slashes)) 2186 CurPtr += 16; 2187#else 2188 // Scan for '/' quickly. Many block comments are very large. 2189 while (CurPtr[0] != '/' && 2190 CurPtr[1] != '/' && 2191 CurPtr[2] != '/' && 2192 CurPtr[3] != '/' && 2193 CurPtr+4 < BufferEnd) { 2194 CurPtr += 4; 2195 } 2196#endif 2197 2198 // It has to be one of the bytes scanned, increment to it and read one. 2199 C = *CurPtr++; 2200 } 2201 2202 // Loop to scan the remainder. 2203 while (C != '/' && C != '\0') 2204 C = *CurPtr++; 2205 2206 if (C == '/') { 2207 FoundSlash: 2208 if (CurPtr[-2] == '*') // We found the final */. We're done! 2209 break; 2210 2211 if ((CurPtr[-2] == '\n' || CurPtr[-2] == '\r')) { 2212 if (isEndOfBlockCommentWithEscapedNewLine(CurPtr-2, this)) { 2213 // We found the final */, though it had an escaped newline between the 2214 // * and /. We're done! 2215 break; 2216 } 2217 } 2218 if (CurPtr[0] == '*' && CurPtr[1] != '/') { 2219 // If this is a /* inside of the comment, emit a warning. Don't do this 2220 // if this is a /*/, which will end the comment. This misses cases with 2221 // embedded escaped newlines, but oh well. 2222 if (!isLexingRawMode()) 2223 Diag(CurPtr-1, diag::warn_nested_block_comment); 2224 } 2225 } else if (C == 0 && CurPtr == BufferEnd+1) { 2226 if (!isLexingRawMode()) 2227 Diag(BufferPtr, diag::err_unterminated_block_comment); 2228 // Note: the user probably forgot a */. We could continue immediately 2229 // after the /*, but this would involve lexing a lot of what really is the 2230 // comment, which surely would confuse the parser. 2231 --CurPtr; 2232 2233 // KeepWhitespaceMode should return this broken comment as a token. Since 2234 // it isn't a well formed comment, just return it as an 'unknown' token. 2235 if (isKeepWhitespaceMode()) { 2236 FormTokenWithChars(Result, CurPtr, tok::unknown); 2237 return true; 2238 } 2239 2240 BufferPtr = CurPtr; 2241 return false; 2242 } else if (C == '\0' && isCodeCompletionPoint(CurPtr-1)) { 2243 PP->CodeCompleteNaturalLanguage(); 2244 cutOffLexing(); 2245 return false; 2246 } 2247 2248 C = *CurPtr++; 2249 } 2250 2251 // Notify comment handlers about the comment unless we're in a #if 0 block. 2252 if (PP && !isLexingRawMode() && 2253 PP->HandleComment(Result, SourceRange(getSourceLocation(BufferPtr), 2254 getSourceLocation(CurPtr)))) { 2255 BufferPtr = CurPtr; 2256 return true; // A token has to be returned. 2257 } 2258 2259 // If we are returning comments as tokens, return this comment as a token. 2260 if (inKeepCommentMode()) { 2261 FormTokenWithChars(Result, CurPtr, tok::comment); 2262 return true; 2263 } 2264 2265 // It is common for the tokens immediately after a /**/ comment to be 2266 // whitespace. Instead of going through the big switch, handle it 2267 // efficiently now. This is safe even in KeepWhitespaceMode because we would 2268 // have already returned above with the comment as a token. 2269 if (isHorizontalWhitespace(*CurPtr)) { 2270 Result.setFlag(Token::LeadingSpace); 2271 SkipWhitespace(Result, CurPtr+1); 2272 return false; 2273 } 2274 2275 // Otherwise, just return so that the next character will be lexed as a token. 2276 BufferPtr = CurPtr; 2277 Result.setFlag(Token::LeadingSpace); 2278 return false; 2279} 2280 2281//===----------------------------------------------------------------------===// 2282// Primary Lexing Entry Points 2283//===----------------------------------------------------------------------===// 2284 2285/// ReadToEndOfLine - Read the rest of the current preprocessor line as an 2286/// uninterpreted string. This switches the lexer out of directive mode. 2287void Lexer::ReadToEndOfLine(SmallVectorImpl<char> *Result) { 2288 assert(ParsingPreprocessorDirective && ParsingFilename == false && 2289 "Must be in a preprocessing directive!"); 2290 Token Tmp; 2291 2292 // CurPtr - Cache BufferPtr in an automatic variable. 2293 const char *CurPtr = BufferPtr; 2294 while (1) { 2295 char Char = getAndAdvanceChar(CurPtr, Tmp); 2296 switch (Char) { 2297 default: 2298 if (Result) 2299 Result->push_back(Char); 2300 break; 2301 case 0: // Null. 2302 // Found end of file? 2303 if (CurPtr-1 != BufferEnd) { 2304 if (isCodeCompletionPoint(CurPtr-1)) { 2305 PP->CodeCompleteNaturalLanguage(); 2306 cutOffLexing(); 2307 return; 2308 } 2309 2310 // Nope, normal character, continue. 2311 if (Result) 2312 Result->push_back(Char); 2313 break; 2314 } 2315 // FALL THROUGH. 2316 case '\r': 2317 case '\n': 2318 // Okay, we found the end of the line. First, back up past the \0, \r, \n. 2319 assert(CurPtr[-1] == Char && "Trigraphs for newline?"); 2320 BufferPtr = CurPtr-1; 2321 2322 // Next, lex the character, which should handle the EOD transition. 2323 Lex(Tmp); 2324 if (Tmp.is(tok::code_completion)) { 2325 if (PP) 2326 PP->CodeCompleteNaturalLanguage(); 2327 Lex(Tmp); 2328 } 2329 assert(Tmp.is(tok::eod) && "Unexpected token!"); 2330 2331 // Finally, we're done; 2332 return; 2333 } 2334 } 2335} 2336 2337/// LexEndOfFile - CurPtr points to the end of this file. Handle this 2338/// condition, reporting diagnostics and handling other edge cases as required. 2339/// This returns true if Result contains a token, false if PP.Lex should be 2340/// called again. 2341bool Lexer::LexEndOfFile(Token &Result, const char *CurPtr) { 2342 // If we hit the end of the file while parsing a preprocessor directive, 2343 // end the preprocessor directive first. The next token returned will 2344 // then be the end of file. 2345 if (ParsingPreprocessorDirective) { 2346 // Done parsing the "line". 2347 ParsingPreprocessorDirective = false; 2348 // Update the location of token as well as BufferPtr. 2349 FormTokenWithChars(Result, CurPtr, tok::eod); 2350 2351 // Restore comment saving mode, in case it was disabled for directive. 2352 SetCommentRetentionState(PP->getCommentRetentionState()); 2353 return true; // Have a token. 2354 } 2355 2356 // If we are in raw mode, return this event as an EOF token. Let the caller 2357 // that put us in raw mode handle the event. 2358 if (isLexingRawMode()) { 2359 Result.startToken(); 2360 BufferPtr = BufferEnd; 2361 FormTokenWithChars(Result, BufferEnd, tok::eof); 2362 return true; 2363 } 2364 2365 // Issue diagnostics for unterminated #if and missing newline. 2366 2367 // If we are in a #if directive, emit an error. 2368 while (!ConditionalStack.empty()) { 2369 if (PP->getCodeCompletionFileLoc() != FileLoc) 2370 PP->Diag(ConditionalStack.back().IfLoc, 2371 diag::err_pp_unterminated_conditional); 2372 ConditionalStack.pop_back(); 2373 } 2374 2375 // C99 5.1.1.2p2: If the file is non-empty and didn't end in a newline, issue 2376 // a pedwarn. 2377 if (CurPtr != BufferStart && (CurPtr[-1] != '\n' && CurPtr[-1] != '\r')) 2378 Diag(BufferEnd, LangOpts.CPlusPlus11 ? // C++11 [lex.phases] 2.2 p2 2379 diag::warn_cxx98_compat_no_newline_eof : diag::ext_no_newline_eof) 2380 << FixItHint::CreateInsertion(getSourceLocation(BufferEnd), "\n"); 2381 2382 BufferPtr = CurPtr; 2383 2384 // Finally, let the preprocessor handle this. 2385 return PP->HandleEndOfFile(Result, isPragmaLexer()); 2386} 2387 2388/// isNextPPTokenLParen - Return 1 if the next unexpanded token lexed from 2389/// the specified lexer will return a tok::l_paren token, 0 if it is something 2390/// else and 2 if there are no more tokens in the buffer controlled by the 2391/// lexer. 2392unsigned Lexer::isNextPPTokenLParen() { 2393 assert(!LexingRawMode && "How can we expand a macro from a skipping buffer?"); 2394 2395 // Switch to 'skipping' mode. This will ensure that we can lex a token 2396 // without emitting diagnostics, disables macro expansion, and will cause EOF 2397 // to return an EOF token instead of popping the include stack. 2398 LexingRawMode = true; 2399 2400 // Save state that can be changed while lexing so that we can restore it. 2401 const char *TmpBufferPtr = BufferPtr; 2402 bool inPPDirectiveMode = ParsingPreprocessorDirective; 2403 2404 Token Tok; 2405 Tok.startToken(); 2406 LexTokenInternal(Tok); 2407 2408 // Restore state that may have changed. 2409 BufferPtr = TmpBufferPtr; 2410 ParsingPreprocessorDirective = inPPDirectiveMode; 2411 2412 // Restore the lexer back to non-skipping mode. 2413 LexingRawMode = false; 2414 2415 if (Tok.is(tok::eof)) 2416 return 2; 2417 return Tok.is(tok::l_paren); 2418} 2419 2420/// \brief Find the end of a version control conflict marker. 2421static const char *FindConflictEnd(const char *CurPtr, const char *BufferEnd, 2422 ConflictMarkerKind CMK) { 2423 const char *Terminator = CMK == CMK_Perforce ? "<<<<\n" : ">>>>>>>"; 2424 size_t TermLen = CMK == CMK_Perforce ? 5 : 7; 2425 StringRef RestOfBuffer(CurPtr+TermLen, BufferEnd-CurPtr-TermLen); 2426 size_t Pos = RestOfBuffer.find(Terminator); 2427 while (Pos != StringRef::npos) { 2428 // Must occur at start of line. 2429 if (RestOfBuffer[Pos-1] != '\r' && 2430 RestOfBuffer[Pos-1] != '\n') { 2431 RestOfBuffer = RestOfBuffer.substr(Pos+TermLen); 2432 Pos = RestOfBuffer.find(Terminator); 2433 continue; 2434 } 2435 return RestOfBuffer.data()+Pos; 2436 } 2437 return 0; 2438} 2439 2440/// IsStartOfConflictMarker - If the specified pointer is the start of a version 2441/// control conflict marker like '<<<<<<<', recognize it as such, emit an error 2442/// and recover nicely. This returns true if it is a conflict marker and false 2443/// if not. 2444bool Lexer::IsStartOfConflictMarker(const char *CurPtr) { 2445 // Only a conflict marker if it starts at the beginning of a line. 2446 if (CurPtr != BufferStart && 2447 CurPtr[-1] != '\n' && CurPtr[-1] != '\r') 2448 return false; 2449 2450 // Check to see if we have <<<<<<< or >>>>. 2451 if ((BufferEnd-CurPtr < 8 || StringRef(CurPtr, 7) != "<<<<<<<") && 2452 (BufferEnd-CurPtr < 6 || StringRef(CurPtr, 5) != ">>>> ")) 2453 return false; 2454 2455 // If we have a situation where we don't care about conflict markers, ignore 2456 // it. 2457 if (CurrentConflictMarkerState || isLexingRawMode()) 2458 return false; 2459 2460 ConflictMarkerKind Kind = *CurPtr == '<' ? CMK_Normal : CMK_Perforce; 2461 2462 // Check to see if there is an ending marker somewhere in the buffer at the 2463 // start of a line to terminate this conflict marker. 2464 if (FindConflictEnd(CurPtr, BufferEnd, Kind)) { 2465 // We found a match. We are really in a conflict marker. 2466 // Diagnose this, and ignore to the end of line. 2467 Diag(CurPtr, diag::err_conflict_marker); 2468 CurrentConflictMarkerState = Kind; 2469 2470 // Skip ahead to the end of line. We know this exists because the 2471 // end-of-conflict marker starts with \r or \n. 2472 while (*CurPtr != '\r' && *CurPtr != '\n') { 2473 assert(CurPtr != BufferEnd && "Didn't find end of line"); 2474 ++CurPtr; 2475 } 2476 BufferPtr = CurPtr; 2477 return true; 2478 } 2479 2480 // No end of conflict marker found. 2481 return false; 2482} 2483 2484 2485/// HandleEndOfConflictMarker - If this is a '====' or '||||' or '>>>>', or if 2486/// it is '<<<<' and the conflict marker started with a '>>>>' marker, then it 2487/// is the end of a conflict marker. Handle it by ignoring up until the end of 2488/// the line. This returns true if it is a conflict marker and false if not. 2489bool Lexer::HandleEndOfConflictMarker(const char *CurPtr) { 2490 // Only a conflict marker if it starts at the beginning of a line. 2491 if (CurPtr != BufferStart && 2492 CurPtr[-1] != '\n' && CurPtr[-1] != '\r') 2493 return false; 2494 2495 // If we have a situation where we don't care about conflict markers, ignore 2496 // it. 2497 if (!CurrentConflictMarkerState || isLexingRawMode()) 2498 return false; 2499 2500 // Check to see if we have the marker (4 characters in a row). 2501 for (unsigned i = 1; i != 4; ++i) 2502 if (CurPtr[i] != CurPtr[0]) 2503 return false; 2504 2505 // If we do have it, search for the end of the conflict marker. This could 2506 // fail if it got skipped with a '#if 0' or something. Note that CurPtr might 2507 // be the end of conflict marker. 2508 if (const char *End = FindConflictEnd(CurPtr, BufferEnd, 2509 CurrentConflictMarkerState)) { 2510 CurPtr = End; 2511 2512 // Skip ahead to the end of line. 2513 while (CurPtr != BufferEnd && *CurPtr != '\r' && *CurPtr != '\n') 2514 ++CurPtr; 2515 2516 BufferPtr = CurPtr; 2517 2518 // No longer in the conflict marker. 2519 CurrentConflictMarkerState = CMK_None; 2520 return true; 2521 } 2522 2523 return false; 2524} 2525 2526bool Lexer::isCodeCompletionPoint(const char *CurPtr) const { 2527 if (PP && PP->isCodeCompletionEnabled()) { 2528 SourceLocation Loc = FileLoc.getLocWithOffset(CurPtr-BufferStart); 2529 return Loc == PP->getCodeCompletionLoc(); 2530 } 2531 2532 return false; 2533} 2534 2535uint32_t Lexer::tryReadUCN(const char *&StartPtr, const char *SlashLoc, 2536 Token *Result) { 2537 unsigned CharSize; 2538 char Kind = getCharAndSize(StartPtr, CharSize); 2539 2540 unsigned NumHexDigits; 2541 if (Kind == 'u') 2542 NumHexDigits = 4; 2543 else if (Kind == 'U') 2544 NumHexDigits = 8; 2545 else 2546 return 0; 2547 2548 if (!LangOpts.CPlusPlus && !LangOpts.C99) { 2549 if (Result && !isLexingRawMode()) 2550 Diag(SlashLoc, diag::warn_ucn_not_valid_in_c89); 2551 return 0; 2552 } 2553 2554 const char *CurPtr = StartPtr + CharSize; 2555 const char *KindLoc = &CurPtr[-1]; 2556 2557 uint32_t CodePoint = 0; 2558 for (unsigned i = 0; i < NumHexDigits; ++i) { 2559 char C = getCharAndSize(CurPtr, CharSize); 2560 2561 unsigned Value = llvm::hexDigitValue(C); 2562 if (Value == -1U) { 2563 if (Result && !isLexingRawMode()) { 2564 if (i == 0) { 2565 Diag(BufferPtr, diag::warn_ucn_escape_no_digits) 2566 << StringRef(KindLoc, 1); 2567 } else { 2568 Diag(BufferPtr, diag::warn_ucn_escape_incomplete); 2569 2570 // If the user wrote \U1234, suggest a fixit to \u. 2571 if (i == 4 && NumHexDigits == 8) { 2572 CharSourceRange URange = 2573 CharSourceRange::getCharRange(getSourceLocation(KindLoc), 2574 getSourceLocation(KindLoc + 1)); 2575 Diag(KindLoc, diag::note_ucn_four_not_eight) 2576 << FixItHint::CreateReplacement(URange, "u"); 2577 } 2578 } 2579 } 2580 2581 return 0; 2582 } 2583 2584 CodePoint <<= 4; 2585 CodePoint += Value; 2586 2587 CurPtr += CharSize; 2588 } 2589 2590 if (Result) { 2591 Result->setFlag(Token::HasUCN); 2592 if (CurPtr - StartPtr == (ptrdiff_t)NumHexDigits + 2) 2593 StartPtr = CurPtr; 2594 else 2595 while (StartPtr != CurPtr) 2596 (void)getAndAdvanceChar(StartPtr, *Result); 2597 } else { 2598 StartPtr = CurPtr; 2599 } 2600 2601 // C99 6.4.3p2: A universal character name shall not specify a character whose 2602 // short identifier is less than 00A0 other than 0024 ($), 0040 (@), or 2603 // 0060 (`), nor one in the range D800 through DFFF inclusive.) 2604 // C++11 [lex.charset]p2: If the hexadecimal value for a 2605 // universal-character-name corresponds to a surrogate code point (in the 2606 // range 0xD800-0xDFFF, inclusive), the program is ill-formed. Additionally, 2607 // if the hexadecimal value for a universal-character-name outside the 2608 // c-char-sequence, s-char-sequence, or r-char-sequence of a character or 2609 // string literal corresponds to a control character (in either of the 2610 // ranges 0x00-0x1F or 0x7F-0x9F, both inclusive) or to a character in the 2611 // basic source character set, the program is ill-formed. 2612 if (CodePoint < 0xA0) { 2613 if (CodePoint == 0x24 || CodePoint == 0x40 || CodePoint == 0x60) 2614 return CodePoint; 2615 2616 // We don't use isLexingRawMode() here because we need to warn about bad 2617 // UCNs even when skipping preprocessing tokens in a #if block. 2618 if (Result && PP) { 2619 if (CodePoint < 0x20 || CodePoint >= 0x7F) 2620 Diag(BufferPtr, diag::err_ucn_control_character); 2621 else { 2622 char C = static_cast<char>(CodePoint); 2623 Diag(BufferPtr, diag::err_ucn_escape_basic_scs) << StringRef(&C, 1); 2624 } 2625 } 2626 2627 return 0; 2628 2629 } else if ((!LangOpts.CPlusPlus || LangOpts.CPlusPlus11) && 2630 (CodePoint >= 0xD800 && CodePoint <= 0xDFFF)) { 2631 // C++03 allows UCNs representing surrogate characters. C99 and C++11 don't. 2632 // We don't use isLexingRawMode() here because we need to warn about bad 2633 // UCNs even when skipping preprocessing tokens in a #if block. 2634 if (Result && PP) 2635 Diag(BufferPtr, diag::err_ucn_escape_invalid); 2636 return 0; 2637 } 2638 2639 return CodePoint; 2640} 2641 2642static bool isUnicodeWhitespace(uint32_t C) { 2643 return (C == 0x0085 || C == 0x00A0 || C == 0x1680 || 2644 C == 0x180E || (C >= 0x2000 && C <= 0x200A) || 2645 C == 0x2028 || C == 0x2029 || C == 0x202F || 2646 C == 0x205F || C == 0x3000); 2647} 2648 2649void Lexer::LexUnicode(Token &Result, uint32_t C, const char *CurPtr) { 2650 if (!isLexingRawMode() && !PP->isPreprocessedOutput() && 2651 isUnicodeWhitespace(C)) { 2652 CharSourceRange CharRange = 2653 CharSourceRange::getCharRange(getSourceLocation(), 2654 getSourceLocation(CurPtr)); 2655 Diag(BufferPtr, diag::ext_unicode_whitespace) 2656 << CharRange; 2657 2658 Result.setFlag(Token::LeadingSpace); 2659 if (SkipWhitespace(Result, CurPtr)) 2660 return; // KeepWhitespaceMode 2661 2662 return LexTokenInternal(Result); 2663 } 2664 2665 if (isAllowedIDChar(C) && isAllowedInitiallyIDChar(C)) { 2666 MIOpt.ReadToken(); 2667 return LexIdentifier(Result, CurPtr); 2668 } 2669 2670 if (!isLexingRawMode() && !ParsingPreprocessorDirective && 2671 !PP->isPreprocessedOutput() && 2672 !isASCII(*BufferPtr) && !isAllowedIDChar(C)) { 2673 // Non-ASCII characters tend to creep into source code unintentionally. 2674 // Instead of letting the parser complain about the unknown token, 2675 // just drop the character. 2676 // Note that we can /only/ do this when the non-ASCII character is actually 2677 // spelled as Unicode, not written as a UCN. The standard requires that 2678 // we not throw away any possible preprocessor tokens, but there's a 2679 // loophole in the mapping of Unicode characters to basic character set 2680 // characters that allows us to map these particular characters to, say, 2681 // whitespace. 2682 CharSourceRange CharRange = 2683 CharSourceRange::getCharRange(getSourceLocation(), 2684 getSourceLocation(CurPtr)); 2685 Diag(BufferPtr, diag::err_non_ascii) 2686 << FixItHint::CreateRemoval(CharRange); 2687 2688 BufferPtr = CurPtr; 2689 return LexTokenInternal(Result); 2690 } 2691 2692 // Otherwise, we have an explicit UCN or a character that's unlikely to show 2693 // up by accident. 2694 MIOpt.ReadToken(); 2695 FormTokenWithChars(Result, CurPtr, tok::unknown); 2696} 2697 2698 2699/// LexTokenInternal - This implements a simple C family lexer. It is an 2700/// extremely performance critical piece of code. This assumes that the buffer 2701/// has a null character at the end of the file. This returns a preprocessing 2702/// token, not a normal token, as such, it is an internal interface. It assumes 2703/// that the Flags of result have been cleared before calling this. 2704void Lexer::LexTokenInternal(Token &Result) { 2705LexNextToken: 2706 // New token, can't need cleaning yet. 2707 Result.clearFlag(Token::NeedsCleaning); 2708 Result.setIdentifierInfo(0); 2709 2710 // CurPtr - Cache BufferPtr in an automatic variable. 2711 const char *CurPtr = BufferPtr; 2712 2713 // Small amounts of horizontal whitespace is very common between tokens. 2714 if ((*CurPtr == ' ') || (*CurPtr == '\t')) { 2715 ++CurPtr; 2716 while ((*CurPtr == ' ') || (*CurPtr == '\t')) 2717 ++CurPtr; 2718 2719 // If we are keeping whitespace and other tokens, just return what we just 2720 // skipped. The next lexer invocation will return the token after the 2721 // whitespace. 2722 if (isKeepWhitespaceMode()) { 2723 FormTokenWithChars(Result, CurPtr, tok::unknown); 2724 return; 2725 } 2726 2727 BufferPtr = CurPtr; 2728 Result.setFlag(Token::LeadingSpace); 2729 } 2730 2731 unsigned SizeTmp, SizeTmp2; // Temporaries for use in cases below. 2732 2733 // Read a character, advancing over it. 2734 char Char = getAndAdvanceChar(CurPtr, Result); 2735 tok::TokenKind Kind; 2736 2737 switch (Char) { 2738 case 0: // Null. 2739 // Found end of file? 2740 if (CurPtr-1 == BufferEnd) { 2741 // Read the PP instance variable into an automatic variable, because 2742 // LexEndOfFile will often delete 'this'. 2743 Preprocessor *PPCache = PP; 2744 if (LexEndOfFile(Result, CurPtr-1)) // Retreat back into the file. 2745 return; // Got a token to return. 2746 assert(PPCache && "Raw buffer::LexEndOfFile should return a token"); 2747 return PPCache->Lex(Result); 2748 } 2749 2750 // Check if we are performing code completion. 2751 if (isCodeCompletionPoint(CurPtr-1)) { 2752 // Return the code-completion token. 2753 Result.startToken(); 2754 FormTokenWithChars(Result, CurPtr, tok::code_completion); 2755 return; 2756 } 2757 2758 if (!isLexingRawMode()) 2759 Diag(CurPtr-1, diag::null_in_file); 2760 Result.setFlag(Token::LeadingSpace); 2761 if (SkipWhitespace(Result, CurPtr)) 2762 return; // KeepWhitespaceMode 2763 2764 goto LexNextToken; // GCC isn't tail call eliminating. 2765 2766 case 26: // DOS & CP/M EOF: "^Z". 2767 // If we're in Microsoft extensions mode, treat this as end of file. 2768 if (LangOpts.MicrosoftExt) { 2769 // Read the PP instance variable into an automatic variable, because 2770 // LexEndOfFile will often delete 'this'. 2771 Preprocessor *PPCache = PP; 2772 if (LexEndOfFile(Result, CurPtr-1)) // Retreat back into the file. 2773 return; // Got a token to return. 2774 assert(PPCache && "Raw buffer::LexEndOfFile should return a token"); 2775 return PPCache->Lex(Result); 2776 } 2777 // If Microsoft extensions are disabled, this is just random garbage. 2778 Kind = tok::unknown; 2779 break; 2780 2781 case '\n': 2782 case '\r': 2783 // If we are inside a preprocessor directive and we see the end of line, 2784 // we know we are done with the directive, so return an EOD token. 2785 if (ParsingPreprocessorDirective) { 2786 // Done parsing the "line". 2787 ParsingPreprocessorDirective = false; 2788 2789 // Restore comment saving mode, in case it was disabled for directive. 2790 if (PP) 2791 SetCommentRetentionState(PP->getCommentRetentionState()); 2792 2793 // Since we consumed a newline, we are back at the start of a line. 2794 IsAtStartOfLine = true; 2795 2796 Kind = tok::eod; 2797 break; 2798 } 2799 // The returned token is at the start of the line. 2800 Result.setFlag(Token::StartOfLine); 2801 // No leading whitespace seen so far. 2802 Result.clearFlag(Token::LeadingSpace); 2803 2804 if (SkipWhitespace(Result, CurPtr)) 2805 return; // KeepWhitespaceMode 2806 goto LexNextToken; // GCC isn't tail call eliminating. 2807 case ' ': 2808 case '\t': 2809 case '\f': 2810 case '\v': 2811 SkipHorizontalWhitespace: 2812 Result.setFlag(Token::LeadingSpace); 2813 if (SkipWhitespace(Result, CurPtr)) 2814 return; // KeepWhitespaceMode 2815 2816 SkipIgnoredUnits: 2817 CurPtr = BufferPtr; 2818 2819 // If the next token is obviously a // or /* */ comment, skip it efficiently 2820 // too (without going through the big switch stmt). 2821 if (CurPtr[0] == '/' && CurPtr[1] == '/' && !inKeepCommentMode() && 2822 LangOpts.LineComment && !LangOpts.TraditionalCPP) { 2823 if (SkipLineComment(Result, CurPtr+2)) 2824 return; // There is a token to return. 2825 goto SkipIgnoredUnits; 2826 } else if (CurPtr[0] == '/' && CurPtr[1] == '*' && !inKeepCommentMode()) { 2827 if (SkipBlockComment(Result, CurPtr+2)) 2828 return; // There is a token to return. 2829 goto SkipIgnoredUnits; 2830 } else if (isHorizontalWhitespace(*CurPtr)) { 2831 goto SkipHorizontalWhitespace; 2832 } 2833 goto LexNextToken; // GCC isn't tail call eliminating. 2834 2835 // C99 6.4.4.1: Integer Constants. 2836 // C99 6.4.4.2: Floating Constants. 2837 case '0': case '1': case '2': case '3': case '4': 2838 case '5': case '6': case '7': case '8': case '9': 2839 // Notify MIOpt that we read a non-whitespace/non-comment token. 2840 MIOpt.ReadToken(); 2841 return LexNumericConstant(Result, CurPtr); 2842 2843 case 'u': // Identifier (uber) or C++0x UTF-8 or UTF-16 string literal 2844 // Notify MIOpt that we read a non-whitespace/non-comment token. 2845 MIOpt.ReadToken(); 2846 2847 if (LangOpts.CPlusPlus11) { 2848 Char = getCharAndSize(CurPtr, SizeTmp); 2849 2850 // UTF-16 string literal 2851 if (Char == '"') 2852 return LexStringLiteral(Result, ConsumeChar(CurPtr, SizeTmp, Result), 2853 tok::utf16_string_literal); 2854 2855 // UTF-16 character constant 2856 if (Char == '\'') 2857 return LexCharConstant(Result, ConsumeChar(CurPtr, SizeTmp, Result), 2858 tok::utf16_char_constant); 2859 2860 // UTF-16 raw string literal 2861 if (Char == 'R' && getCharAndSize(CurPtr + SizeTmp, SizeTmp2) == '"') 2862 return LexRawStringLiteral(Result, 2863 ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 2864 SizeTmp2, Result), 2865 tok::utf16_string_literal); 2866 2867 if (Char == '8') { 2868 char Char2 = getCharAndSize(CurPtr + SizeTmp, SizeTmp2); 2869 2870 // UTF-8 string literal 2871 if (Char2 == '"') 2872 return LexStringLiteral(Result, 2873 ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 2874 SizeTmp2, Result), 2875 tok::utf8_string_literal); 2876 2877 if (Char2 == 'R') { 2878 unsigned SizeTmp3; 2879 char Char3 = getCharAndSize(CurPtr + SizeTmp + SizeTmp2, SizeTmp3); 2880 // UTF-8 raw string literal 2881 if (Char3 == '"') { 2882 return LexRawStringLiteral(Result, 2883 ConsumeChar(ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 2884 SizeTmp2, Result), 2885 SizeTmp3, Result), 2886 tok::utf8_string_literal); 2887 } 2888 } 2889 } 2890 } 2891 2892 // treat u like the start of an identifier. 2893 return LexIdentifier(Result, CurPtr); 2894 2895 case 'U': // Identifier (Uber) or C++0x UTF-32 string literal 2896 // Notify MIOpt that we read a non-whitespace/non-comment token. 2897 MIOpt.ReadToken(); 2898 2899 if (LangOpts.CPlusPlus11) { 2900 Char = getCharAndSize(CurPtr, SizeTmp); 2901 2902 // UTF-32 string literal 2903 if (Char == '"') 2904 return LexStringLiteral(Result, ConsumeChar(CurPtr, SizeTmp, Result), 2905 tok::utf32_string_literal); 2906 2907 // UTF-32 character constant 2908 if (Char == '\'') 2909 return LexCharConstant(Result, ConsumeChar(CurPtr, SizeTmp, Result), 2910 tok::utf32_char_constant); 2911 2912 // UTF-32 raw string literal 2913 if (Char == 'R' && getCharAndSize(CurPtr + SizeTmp, SizeTmp2) == '"') 2914 return LexRawStringLiteral(Result, 2915 ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 2916 SizeTmp2, Result), 2917 tok::utf32_string_literal); 2918 } 2919 2920 // treat U like the start of an identifier. 2921 return LexIdentifier(Result, CurPtr); 2922 2923 case 'R': // Identifier or C++0x raw string literal 2924 // Notify MIOpt that we read a non-whitespace/non-comment token. 2925 MIOpt.ReadToken(); 2926 2927 if (LangOpts.CPlusPlus11) { 2928 Char = getCharAndSize(CurPtr, SizeTmp); 2929 2930 if (Char == '"') 2931 return LexRawStringLiteral(Result, 2932 ConsumeChar(CurPtr, SizeTmp, Result), 2933 tok::string_literal); 2934 } 2935 2936 // treat R like the start of an identifier. 2937 return LexIdentifier(Result, CurPtr); 2938 2939 case 'L': // Identifier (Loony) or wide literal (L'x' or L"xyz"). 2940 // Notify MIOpt that we read a non-whitespace/non-comment token. 2941 MIOpt.ReadToken(); 2942 Char = getCharAndSize(CurPtr, SizeTmp); 2943 2944 // Wide string literal. 2945 if (Char == '"') 2946 return LexStringLiteral(Result, ConsumeChar(CurPtr, SizeTmp, Result), 2947 tok::wide_string_literal); 2948 2949 // Wide raw string literal. 2950 if (LangOpts.CPlusPlus11 && Char == 'R' && 2951 getCharAndSize(CurPtr + SizeTmp, SizeTmp2) == '"') 2952 return LexRawStringLiteral(Result, 2953 ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 2954 SizeTmp2, Result), 2955 tok::wide_string_literal); 2956 2957 // Wide character constant. 2958 if (Char == '\'') 2959 return LexCharConstant(Result, ConsumeChar(CurPtr, SizeTmp, Result), 2960 tok::wide_char_constant); 2961 // FALL THROUGH, treating L like the start of an identifier. 2962 2963 // C99 6.4.2: Identifiers. 2964 case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': case 'G': 2965 case 'H': case 'I': case 'J': case 'K': /*'L'*/case 'M': case 'N': 2966 case 'O': case 'P': case 'Q': /*'R'*/case 'S': case 'T': /*'U'*/ 2967 case 'V': case 'W': case 'X': case 'Y': case 'Z': 2968 case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': case 'g': 2969 case 'h': case 'i': case 'j': case 'k': case 'l': case 'm': case 'n': 2970 case 'o': case 'p': case 'q': case 'r': case 's': case 't': /*'u'*/ 2971 case 'v': case 'w': case 'x': case 'y': case 'z': 2972 case '_': 2973 // Notify MIOpt that we read a non-whitespace/non-comment token. 2974 MIOpt.ReadToken(); 2975 return LexIdentifier(Result, CurPtr); 2976 2977 case '$': // $ in identifiers. 2978 if (LangOpts.DollarIdents) { 2979 if (!isLexingRawMode()) 2980 Diag(CurPtr-1, diag::ext_dollar_in_identifier); 2981 // Notify MIOpt that we read a non-whitespace/non-comment token. 2982 MIOpt.ReadToken(); 2983 return LexIdentifier(Result, CurPtr); 2984 } 2985 2986 Kind = tok::unknown; 2987 break; 2988 2989 // C99 6.4.4: Character Constants. 2990 case '\'': 2991 // Notify MIOpt that we read a non-whitespace/non-comment token. 2992 MIOpt.ReadToken(); 2993 return LexCharConstant(Result, CurPtr, tok::char_constant); 2994 2995 // C99 6.4.5: String Literals. 2996 case '"': 2997 // Notify MIOpt that we read a non-whitespace/non-comment token. 2998 MIOpt.ReadToken(); 2999 return LexStringLiteral(Result, CurPtr, tok::string_literal); 3000 3001 // C99 6.4.6: Punctuators. 3002 case '?': 3003 Kind = tok::question; 3004 break; 3005 case '[': 3006 Kind = tok::l_square; 3007 break; 3008 case ']': 3009 Kind = tok::r_square; 3010 break; 3011 case '(': 3012 Kind = tok::l_paren; 3013 break; 3014 case ')': 3015 Kind = tok::r_paren; 3016 break; 3017 case '{': 3018 Kind = tok::l_brace; 3019 break; 3020 case '}': 3021 Kind = tok::r_brace; 3022 break; 3023 case '.': 3024 Char = getCharAndSize(CurPtr, SizeTmp); 3025 if (Char >= '0' && Char <= '9') { 3026 // Notify MIOpt that we read a non-whitespace/non-comment token. 3027 MIOpt.ReadToken(); 3028 3029 return LexNumericConstant(Result, ConsumeChar(CurPtr, SizeTmp, Result)); 3030 } else if (LangOpts.CPlusPlus && Char == '*') { 3031 Kind = tok::periodstar; 3032 CurPtr += SizeTmp; 3033 } else if (Char == '.' && 3034 getCharAndSize(CurPtr+SizeTmp, SizeTmp2) == '.') { 3035 Kind = tok::ellipsis; 3036 CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 3037 SizeTmp2, Result); 3038 } else { 3039 Kind = tok::period; 3040 } 3041 break; 3042 case '&': 3043 Char = getCharAndSize(CurPtr, SizeTmp); 3044 if (Char == '&') { 3045 Kind = tok::ampamp; 3046 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3047 } else if (Char == '=') { 3048 Kind = tok::ampequal; 3049 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3050 } else { 3051 Kind = tok::amp; 3052 } 3053 break; 3054 case '*': 3055 if (getCharAndSize(CurPtr, SizeTmp) == '=') { 3056 Kind = tok::starequal; 3057 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3058 } else { 3059 Kind = tok::star; 3060 } 3061 break; 3062 case '+': 3063 Char = getCharAndSize(CurPtr, SizeTmp); 3064 if (Char == '+') { 3065 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3066 Kind = tok::plusplus; 3067 } else if (Char == '=') { 3068 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3069 Kind = tok::plusequal; 3070 } else { 3071 Kind = tok::plus; 3072 } 3073 break; 3074 case '-': 3075 Char = getCharAndSize(CurPtr, SizeTmp); 3076 if (Char == '-') { // -- 3077 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3078 Kind = tok::minusminus; 3079 } else if (Char == '>' && LangOpts.CPlusPlus && 3080 getCharAndSize(CurPtr+SizeTmp, SizeTmp2) == '*') { // C++ ->* 3081 CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 3082 SizeTmp2, Result); 3083 Kind = tok::arrowstar; 3084 } else if (Char == '>') { // -> 3085 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3086 Kind = tok::arrow; 3087 } else if (Char == '=') { // -= 3088 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3089 Kind = tok::minusequal; 3090 } else { 3091 Kind = tok::minus; 3092 } 3093 break; 3094 case '~': 3095 Kind = tok::tilde; 3096 break; 3097 case '!': 3098 if (getCharAndSize(CurPtr, SizeTmp) == '=') { 3099 Kind = tok::exclaimequal; 3100 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3101 } else { 3102 Kind = tok::exclaim; 3103 } 3104 break; 3105 case '/': 3106 // 6.4.9: Comments 3107 Char = getCharAndSize(CurPtr, SizeTmp); 3108 if (Char == '/') { // Line comment. 3109 // Even if Line comments are disabled (e.g. in C89 mode), we generally 3110 // want to lex this as a comment. There is one problem with this though, 3111 // that in one particular corner case, this can change the behavior of the 3112 // resultant program. For example, In "foo //**/ bar", C89 would lex 3113 // this as "foo / bar" and langauges with Line comments would lex it as 3114 // "foo". Check to see if the character after the second slash is a '*'. 3115 // If so, we will lex that as a "/" instead of the start of a comment. 3116 // However, we never do this in -traditional-cpp mode. 3117 if ((LangOpts.LineComment || 3118 getCharAndSize(CurPtr+SizeTmp, SizeTmp2) != '*') && 3119 !LangOpts.TraditionalCPP) { 3120 if (SkipLineComment(Result, ConsumeChar(CurPtr, SizeTmp, Result))) 3121 return; // There is a token to return. 3122 3123 // It is common for the tokens immediately after a // comment to be 3124 // whitespace (indentation for the next line). Instead of going through 3125 // the big switch, handle it efficiently now. 3126 goto SkipIgnoredUnits; 3127 } 3128 } 3129 3130 if (Char == '*') { // /**/ comment. 3131 if (SkipBlockComment(Result, ConsumeChar(CurPtr, SizeTmp, Result))) 3132 return; // There is a token to return. 3133 goto LexNextToken; // GCC isn't tail call eliminating. 3134 } 3135 3136 if (Char == '=') { 3137 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3138 Kind = tok::slashequal; 3139 } else { 3140 Kind = tok::slash; 3141 } 3142 break; 3143 case '%': 3144 Char = getCharAndSize(CurPtr, SizeTmp); 3145 if (Char == '=') { 3146 Kind = tok::percentequal; 3147 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3148 } else if (LangOpts.Digraphs && Char == '>') { 3149 Kind = tok::r_brace; // '%>' -> '}' 3150 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3151 } else if (LangOpts.Digraphs && Char == ':') { 3152 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3153 Char = getCharAndSize(CurPtr, SizeTmp); 3154 if (Char == '%' && getCharAndSize(CurPtr+SizeTmp, SizeTmp2) == ':') { 3155 Kind = tok::hashhash; // '%:%:' -> '##' 3156 CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 3157 SizeTmp2, Result); 3158 } else if (Char == '@' && LangOpts.MicrosoftExt) {// %:@ -> #@ -> Charize 3159 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3160 if (!isLexingRawMode()) 3161 Diag(BufferPtr, diag::ext_charize_microsoft); 3162 Kind = tok::hashat; 3163 } else { // '%:' -> '#' 3164 // We parsed a # character. If this occurs at the start of the line, 3165 // it's actually the start of a preprocessing directive. Callback to 3166 // the preprocessor to handle it. 3167 // FIXME: -fpreprocessed mode?? 3168 if (Result.isAtStartOfLine() && !LexingRawMode && !Is_PragmaLexer) 3169 goto HandleDirective; 3170 3171 Kind = tok::hash; 3172 } 3173 } else { 3174 Kind = tok::percent; 3175 } 3176 break; 3177 case '<': 3178 Char = getCharAndSize(CurPtr, SizeTmp); 3179 if (ParsingFilename) { 3180 return LexAngledStringLiteral(Result, CurPtr); 3181 } else if (Char == '<') { 3182 char After = getCharAndSize(CurPtr+SizeTmp, SizeTmp2); 3183 if (After == '=') { 3184 Kind = tok::lesslessequal; 3185 CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 3186 SizeTmp2, Result); 3187 } else if (After == '<' && IsStartOfConflictMarker(CurPtr-1)) { 3188 // If this is actually a '<<<<<<<' version control conflict marker, 3189 // recognize it as such and recover nicely. 3190 goto LexNextToken; 3191 } else if (After == '<' && HandleEndOfConflictMarker(CurPtr-1)) { 3192 // If this is '<<<<' and we're in a Perforce-style conflict marker, 3193 // ignore it. 3194 goto LexNextToken; 3195 } else if (LangOpts.CUDA && After == '<') { 3196 Kind = tok::lesslessless; 3197 CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 3198 SizeTmp2, Result); 3199 } else { 3200 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3201 Kind = tok::lessless; 3202 } 3203 } else if (Char == '=') { 3204 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3205 Kind = tok::lessequal; 3206 } else if (LangOpts.Digraphs && Char == ':') { // '<:' -> '[' 3207 if (LangOpts.CPlusPlus11 && 3208 getCharAndSize(CurPtr + SizeTmp, SizeTmp2) == ':') { 3209 // C++0x [lex.pptoken]p3: 3210 // Otherwise, if the next three characters are <:: and the subsequent 3211 // character is neither : nor >, the < is treated as a preprocessor 3212 // token by itself and not as the first character of the alternative 3213 // token <:. 3214 unsigned SizeTmp3; 3215 char After = getCharAndSize(CurPtr + SizeTmp + SizeTmp2, SizeTmp3); 3216 if (After != ':' && After != '>') { 3217 Kind = tok::less; 3218 if (!isLexingRawMode()) 3219 Diag(BufferPtr, diag::warn_cxx98_compat_less_colon_colon); 3220 break; 3221 } 3222 } 3223 3224 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3225 Kind = tok::l_square; 3226 } else if (LangOpts.Digraphs && Char == '%') { // '<%' -> '{' 3227 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3228 Kind = tok::l_brace; 3229 } else { 3230 Kind = tok::less; 3231 } 3232 break; 3233 case '>': 3234 Char = getCharAndSize(CurPtr, SizeTmp); 3235 if (Char == '=') { 3236 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3237 Kind = tok::greaterequal; 3238 } else if (Char == '>') { 3239 char After = getCharAndSize(CurPtr+SizeTmp, SizeTmp2); 3240 if (After == '=') { 3241 CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 3242 SizeTmp2, Result); 3243 Kind = tok::greatergreaterequal; 3244 } else if (After == '>' && IsStartOfConflictMarker(CurPtr-1)) { 3245 // If this is actually a '>>>>' conflict marker, recognize it as such 3246 // and recover nicely. 3247 goto LexNextToken; 3248 } else if (After == '>' && HandleEndOfConflictMarker(CurPtr-1)) { 3249 // If this is '>>>>>>>' and we're in a conflict marker, ignore it. 3250 goto LexNextToken; 3251 } else if (LangOpts.CUDA && After == '>') { 3252 Kind = tok::greatergreatergreater; 3253 CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 3254 SizeTmp2, Result); 3255 } else { 3256 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3257 Kind = tok::greatergreater; 3258 } 3259 3260 } else { 3261 Kind = tok::greater; 3262 } 3263 break; 3264 case '^': 3265 Char = getCharAndSize(CurPtr, SizeTmp); 3266 if (Char == '=') { 3267 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3268 Kind = tok::caretequal; 3269 } else { 3270 Kind = tok::caret; 3271 } 3272 break; 3273 case '|': 3274 Char = getCharAndSize(CurPtr, SizeTmp); 3275 if (Char == '=') { 3276 Kind = tok::pipeequal; 3277 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3278 } else if (Char == '|') { 3279 // If this is '|||||||' and we're in a conflict marker, ignore it. 3280 if (CurPtr[1] == '|' && HandleEndOfConflictMarker(CurPtr-1)) 3281 goto LexNextToken; 3282 Kind = tok::pipepipe; 3283 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3284 } else { 3285 Kind = tok::pipe; 3286 } 3287 break; 3288 case ':': 3289 Char = getCharAndSize(CurPtr, SizeTmp); 3290 if (LangOpts.Digraphs && Char == '>') { 3291 Kind = tok::r_square; // ':>' -> ']' 3292 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3293 } else if (LangOpts.CPlusPlus && Char == ':') { 3294 Kind = tok::coloncolon; 3295 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3296 } else { 3297 Kind = tok::colon; 3298 } 3299 break; 3300 case ';': 3301 Kind = tok::semi; 3302 break; 3303 case '=': 3304 Char = getCharAndSize(CurPtr, SizeTmp); 3305 if (Char == '=') { 3306 // If this is '====' and we're in a conflict marker, ignore it. 3307 if (CurPtr[1] == '=' && HandleEndOfConflictMarker(CurPtr-1)) 3308 goto LexNextToken; 3309 3310 Kind = tok::equalequal; 3311 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3312 } else { 3313 Kind = tok::equal; 3314 } 3315 break; 3316 case ',': 3317 Kind = tok::comma; 3318 break; 3319 case '#': 3320 Char = getCharAndSize(CurPtr, SizeTmp); 3321 if (Char == '#') { 3322 Kind = tok::hashhash; 3323 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3324 } else if (Char == '@' && LangOpts.MicrosoftExt) { // #@ -> Charize 3325 Kind = tok::hashat; 3326 if (!isLexingRawMode()) 3327 Diag(BufferPtr, diag::ext_charize_microsoft); 3328 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3329 } else { 3330 // We parsed a # character. If this occurs at the start of the line, 3331 // it's actually the start of a preprocessing directive. Callback to 3332 // the preprocessor to handle it. 3333 // FIXME: -fpreprocessed mode?? 3334 if (Result.isAtStartOfLine() && !LexingRawMode && !Is_PragmaLexer) 3335 goto HandleDirective; 3336 3337 Kind = tok::hash; 3338 } 3339 break; 3340 3341 case '@': 3342 // Objective C support. 3343 if (CurPtr[-1] == '@' && LangOpts.ObjC1) 3344 Kind = tok::at; 3345 else 3346 Kind = tok::unknown; 3347 break; 3348 3349 // UCNs (C99 6.4.3, C++11 [lex.charset]p2) 3350 case '\\': 3351 if (uint32_t CodePoint = tryReadUCN(CurPtr, BufferPtr, &Result)) 3352 return LexUnicode(Result, CodePoint, CurPtr); 3353 3354 Kind = tok::unknown; 3355 break; 3356 3357 default: { 3358 if (isASCII(Char)) { 3359 Kind = tok::unknown; 3360 break; 3361 } 3362 3363 UTF32 CodePoint; 3364 3365 // We can't just reset CurPtr to BufferPtr because BufferPtr may point to 3366 // an escaped newline. 3367 --CurPtr; 3368 ConversionResult Status = 3369 llvm::convertUTF8Sequence((const UTF8 **)&CurPtr, 3370 (const UTF8 *)BufferEnd, 3371 &CodePoint, 3372 strictConversion); 3373 if (Status == conversionOK) 3374 return LexUnicode(Result, CodePoint, CurPtr); 3375 3376 if (isLexingRawMode() || ParsingPreprocessorDirective || 3377 PP->isPreprocessedOutput()) { 3378 ++CurPtr; 3379 Kind = tok::unknown; 3380 break; 3381 } 3382 3383 // Non-ASCII characters tend to creep into source code unintentionally. 3384 // Instead of letting the parser complain about the unknown token, 3385 // just diagnose the invalid UTF-8, then drop the character. 3386 Diag(CurPtr, diag::err_invalid_utf8); 3387 3388 BufferPtr = CurPtr+1; 3389 goto LexNextToken; 3390 } 3391 } 3392 3393 // Notify MIOpt that we read a non-whitespace/non-comment token. 3394 MIOpt.ReadToken(); 3395 3396 // Update the location of token as well as BufferPtr. 3397 FormTokenWithChars(Result, CurPtr, Kind); 3398 return; 3399 3400HandleDirective: 3401 // We parsed a # character and it's the start of a preprocessing directive. 3402 3403 FormTokenWithChars(Result, CurPtr, tok::hash); 3404 PP->HandleDirective(Result); 3405 3406 // As an optimization, if the preprocessor didn't switch lexers, tail 3407 // recurse. 3408 if (PP->isCurrentLexer(this)) { 3409 // Start a new token. If this is a #include or something, the PP may 3410 // want us starting at the beginning of the line again. If so, set 3411 // the StartOfLine flag and clear LeadingSpace. 3412 if (IsAtStartOfLine) { 3413 Result.setFlag(Token::StartOfLine); 3414 Result.clearFlag(Token::LeadingSpace); 3415 IsAtStartOfLine = false; 3416 } 3417 goto LexNextToken; // GCC isn't tail call eliminating. 3418 } 3419 return PP->Lex(Result); 3420} 3421