Lexer.cpp revision a64ccefdf0ea4e03ec88805d71b0af74950c7472
15f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer//===--- Lexer.cpp - C Language Family Lexer ------------------------------===// 25f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// 35f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// The LLVM Compiler Infrastructure 45f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// 50bc735ffcfb223c0186419547abaa5c84482663eChris Lattner// This file is distributed under the University of Illinois Open Source 60bc735ffcfb223c0186419547abaa5c84482663eChris Lattner// License. See LICENSE.TXT for details. 75f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// 85f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer//===----------------------------------------------------------------------===// 95f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// 10d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner// This file implements the Lexer and Token interfaces. 115f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// 125f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer//===----------------------------------------------------------------------===// 135f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// 145f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// TODO: GCC Diagnostics emitted by the lexer: 155f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// PEDWARN: (form feed|vertical tab) in preprocessing directive 165f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// 175f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// Universal characters, unicode, char mapping: 185f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// WARNING: `%.*s' is not in NFKC 195f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// WARNING: `%.*s' is not in NFC 205f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// 215f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// Other: 225f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// TODO: Options to support: 235f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// -fexec-charset,-fwide-exec-charset 245f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// 255f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer//===----------------------------------------------------------------------===// 265f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 275f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer#include "clang/Lex/Lexer.h" 285f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer#include "clang/Lex/Preprocessor.h" 29500d3297d2a21edeac4d46cbcbe21bc2352c2a28Chris Lattner#include "clang/Lex/LexDiagnostic.h" 3055817afdf9d453a443262a733f6caf6692dca118Douglas Gregor#include "clang/Lex/CodeCompletionHandler.h" 319dc1f530c086d2c16f8cba758b0f59a5bf41323aChris Lattner#include "clang/Basic/SourceManager.h" 32f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor#include "llvm/ADT/StringSwitch.h" 33409a03671224d4e5bdab1594c43baf070148f830Chris Lattner#include "llvm/Support/Compiler.h" 345f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer#include "llvm/Support/MemoryBuffer.h" 352fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper#include <cstring> 365f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencerusing namespace clang; 375f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 38a2bf105739cef7aea8e4d219629c627fcd11742eChris Lattnerstatic void InitCharacterInfo(); 395f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 40dbf388b54bb1688ee4470c61235494cbc34b8f3cChris Lattner//===----------------------------------------------------------------------===// 41dbf388b54bb1688ee4470c61235494cbc34b8f3cChris Lattner// Token Class Implementation 42dbf388b54bb1688ee4470c61235494cbc34b8f3cChris Lattner//===----------------------------------------------------------------------===// 43dbf388b54bb1688ee4470c61235494cbc34b8f3cChris Lattner 441eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump/// isObjCAtKeyword - Return true if we have an ObjC keyword identifier. 45dbf388b54bb1688ee4470c61235494cbc34b8f3cChris Lattnerbool Token::isObjCAtKeyword(tok::ObjCKeywordKind objcKey) const { 46bec1c9d6f3feeec112cc8eeae90c1be29c6aaf13Douglas Gregor if (IdentifierInfo *II = getIdentifierInfo()) 47bec1c9d6f3feeec112cc8eeae90c1be29c6aaf13Douglas Gregor return II->getObjCKeywordID() == objcKey; 48bec1c9d6f3feeec112cc8eeae90c1be29c6aaf13Douglas Gregor return false; 49dbf388b54bb1688ee4470c61235494cbc34b8f3cChris Lattner} 50dbf388b54bb1688ee4470c61235494cbc34b8f3cChris Lattner 51dbf388b54bb1688ee4470c61235494cbc34b8f3cChris Lattner/// getObjCKeywordID - Return the ObjC keyword kind. 52dbf388b54bb1688ee4470c61235494cbc34b8f3cChris Lattnertok::ObjCKeywordKind Token::getObjCKeywordID() const { 53dbf388b54bb1688ee4470c61235494cbc34b8f3cChris Lattner IdentifierInfo *specId = getIdentifierInfo(); 54dbf388b54bb1688ee4470c61235494cbc34b8f3cChris Lattner return specId ? specId->getObjCKeywordID() : tok::objc_not_keyword; 55dbf388b54bb1688ee4470c61235494cbc34b8f3cChris Lattner} 56dbf388b54bb1688ee4470c61235494cbc34b8f3cChris Lattner 5753702cd401b8fdca985aede7732c2f6a82ad9b1cChris Lattner 58dbf388b54bb1688ee4470c61235494cbc34b8f3cChris Lattner//===----------------------------------------------------------------------===// 59dbf388b54bb1688ee4470c61235494cbc34b8f3cChris Lattner// Lexer Class Implementation 60dbf388b54bb1688ee4470c61235494cbc34b8f3cChris Lattner//===----------------------------------------------------------------------===// 61dbf388b54bb1688ee4470c61235494cbc34b8f3cChris Lattner 621eb4433ac451dc16f4133a88af2d002ac26c58efMike Stumpvoid Lexer::InitLexer(const char *BufStart, const char *BufPtr, 6322d91ca8d7c134eac5cc6a4869e6a84c461ad624Chris Lattner const char *BufEnd) { 64a2bf105739cef7aea8e4d219629c627fcd11742eChris Lattner InitCharacterInfo(); 651eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 6622d91ca8d7c134eac5cc6a4869e6a84c461ad624Chris Lattner BufferStart = BufStart; 6722d91ca8d7c134eac5cc6a4869e6a84c461ad624Chris Lattner BufferPtr = BufPtr; 6822d91ca8d7c134eac5cc6a4869e6a84c461ad624Chris Lattner BufferEnd = BufEnd; 691eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 7022d91ca8d7c134eac5cc6a4869e6a84c461ad624Chris Lattner assert(BufEnd[0] == 0 && 7122d91ca8d7c134eac5cc6a4869e6a84c461ad624Chris Lattner "We assume that the input buffer has a null character at the end" 7222d91ca8d7c134eac5cc6a4869e6a84c461ad624Chris Lattner " to simplify lexing!"); 731eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 74156119df1d076b63609618976281961283f871dbEric Christopher // Check whether we have a BOM in the beginning of the buffer. If yes - act 75156119df1d076b63609618976281961283f871dbEric Christopher // accordingly. Right now we support only UTF-8 with and without BOM, so, just 76156119df1d076b63609618976281961283f871dbEric Christopher // skip the UTF-8 BOM if it's present. 77156119df1d076b63609618976281961283f871dbEric Christopher if (BufferStart == BufferPtr) { 78156119df1d076b63609618976281961283f871dbEric Christopher // Determine the size of the BOM. 795f9e272e632e951b1efe824cd16acb4d96077930Chris Lattner StringRef Buf(BufferStart, BufferEnd - BufferStart); 80969f9d47338fc36ebb6d24ad3a51e45eda07fd58Eli Friedman size_t BOMLength = llvm::StringSwitch<size_t>(Buf) 81156119df1d076b63609618976281961283f871dbEric Christopher .StartsWith("\xEF\xBB\xBF", 3) // UTF-8 BOM 82156119df1d076b63609618976281961283f871dbEric Christopher .Default(0); 83156119df1d076b63609618976281961283f871dbEric Christopher 84156119df1d076b63609618976281961283f871dbEric Christopher // Skip the BOM. 85156119df1d076b63609618976281961283f871dbEric Christopher BufferPtr += BOMLength; 86156119df1d076b63609618976281961283f871dbEric Christopher } 87156119df1d076b63609618976281961283f871dbEric Christopher 8822d91ca8d7c134eac5cc6a4869e6a84c461ad624Chris Lattner Is_PragmaLexer = false; 8934f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner IsInConflictMarker = false; 90156119df1d076b63609618976281961283f871dbEric Christopher 9122d91ca8d7c134eac5cc6a4869e6a84c461ad624Chris Lattner // Start of the file is a start of line. 9222d91ca8d7c134eac5cc6a4869e6a84c461ad624Chris Lattner IsAtStartOfLine = true; 931eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 9422d91ca8d7c134eac5cc6a4869e6a84c461ad624Chris Lattner // We are not after parsing a #. 9522d91ca8d7c134eac5cc6a4869e6a84c461ad624Chris Lattner ParsingPreprocessorDirective = false; 961eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 9722d91ca8d7c134eac5cc6a4869e6a84c461ad624Chris Lattner // We are not after parsing #include. 9822d91ca8d7c134eac5cc6a4869e6a84c461ad624Chris Lattner ParsingFilename = false; 991eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 10022d91ca8d7c134eac5cc6a4869e6a84c461ad624Chris Lattner // We are not in raw mode. Raw mode disables diagnostics and interpretation 10122d91ca8d7c134eac5cc6a4869e6a84c461ad624Chris Lattner // of tokens (e.g. identifiers, thus disabling macro expansion). It is used 10222d91ca8d7c134eac5cc6a4869e6a84c461ad624Chris Lattner // to quickly lex the tokens of the buffer, e.g. when handling a "#if 0" block 10322d91ca8d7c134eac5cc6a4869e6a84c461ad624Chris Lattner // or otherwise skipping over tokens. 10422d91ca8d7c134eac5cc6a4869e6a84c461ad624Chris Lattner LexingRawMode = false; 1051eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 10622d91ca8d7c134eac5cc6a4869e6a84c461ad624Chris Lattner // Default to not keeping comments. 10722d91ca8d7c134eac5cc6a4869e6a84c461ad624Chris Lattner ExtendedTokenMode = 0; 10822d91ca8d7c134eac5cc6a4869e6a84c461ad624Chris Lattner} 10922d91ca8d7c134eac5cc6a4869e6a84c461ad624Chris Lattner 1100770dabb1ae81a2a9c2e7199262067103062a0b3Chris Lattner/// Lexer constructor - Create a new lexer object for the specified buffer 1110770dabb1ae81a2a9c2e7199262067103062a0b3Chris Lattner/// with the specified preprocessor managing the lexing process. This lexer 1120770dabb1ae81a2a9c2e7199262067103062a0b3Chris Lattner/// assumes that the associated file buffer and Preprocessor objects will 1130770dabb1ae81a2a9c2e7199262067103062a0b3Chris Lattner/// outlive it, so it doesn't take ownership of either of them. 1146e2901407bff59aeb4cc301cc58b034723d0eb49Chris LattnerLexer::Lexer(FileID FID, const llvm::MemoryBuffer *InputFile, Preprocessor &PP) 11588d3ac1341aa016cabd966c5b113a95ac05ea43fChris Lattner : PreprocessorLexer(&PP, FID), 11688d3ac1341aa016cabd966c5b113a95ac05ea43fChris Lattner FileLoc(PP.getSourceManager().getLocForStartOfFile(FID)), 11788d3ac1341aa016cabd966c5b113a95ac05ea43fChris Lattner Features(PP.getLangOptions()) { 1181eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 1190770dabb1ae81a2a9c2e7199262067103062a0b3Chris Lattner InitLexer(InputFile->getBufferStart(), InputFile->getBufferStart(), 1200770dabb1ae81a2a9c2e7199262067103062a0b3Chris Lattner InputFile->getBufferEnd()); 1211eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 1220770dabb1ae81a2a9c2e7199262067103062a0b3Chris Lattner // Default to keeping comments if the preprocessor wants them. 1230770dabb1ae81a2a9c2e7199262067103062a0b3Chris Lattner SetCommentRetentionState(PP.getCommentRetentionState()); 1240770dabb1ae81a2a9c2e7199262067103062a0b3Chris Lattner} 125dbf388b54bb1688ee4470c61235494cbc34b8f3cChris Lattner 126168ae2d44a443da75ea85db5f3b5081eb0bce113Chris Lattner/// Lexer constructor - Create a new raw lexer object. This object is only 127590f0cc643274267d4d41125b62557e1d87886c3Chris Lattner/// suitable for calls to 'LexRawToken'. This lexer assumes that the text 128590f0cc643274267d4d41125b62557e1d87886c3Chris Lattner/// range will outlive it, so it doesn't take ownership of it. 129168ae2d44a443da75ea85db5f3b5081eb0bce113Chris LattnerLexer::Lexer(SourceLocation fileloc, const LangOptions &features, 130de96c0f29c4cacabe6ea577c61db87c2a85aea6cChris Lattner const char *BufStart, const char *BufPtr, const char *BufEnd) 131c6fe32a91c7372caf09152ee31a24c4b5d24deedChris Lattner : FileLoc(fileloc), Features(features) { 13222d91ca8d7c134eac5cc6a4869e6a84c461ad624Chris Lattner 13322d91ca8d7c134eac5cc6a4869e6a84c461ad624Chris Lattner InitLexer(BufStart, BufPtr, BufEnd); 1341eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 135168ae2d44a443da75ea85db5f3b5081eb0bce113Chris Lattner // We *are* in raw mode. 136168ae2d44a443da75ea85db5f3b5081eb0bce113Chris Lattner LexingRawMode = true; 1375f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 1385f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 139025c3a66402fb713c2d9bf5dc174ff264765379aChris Lattner/// Lexer constructor - Create a new raw lexer object. This object is only 140025c3a66402fb713c2d9bf5dc174ff264765379aChris Lattner/// suitable for calls to 'LexRawToken'. This lexer assumes that the text 141025c3a66402fb713c2d9bf5dc174ff264765379aChris Lattner/// range will outlive it, so it doesn't take ownership of it. 1426e2901407bff59aeb4cc301cc58b034723d0eb49Chris LattnerLexer::Lexer(FileID FID, const llvm::MemoryBuffer *FromFile, 1436e2901407bff59aeb4cc301cc58b034723d0eb49Chris Lattner const SourceManager &SM, const LangOptions &features) 144025c3a66402fb713c2d9bf5dc174ff264765379aChris Lattner : FileLoc(SM.getLocForStartOfFile(FID)), Features(features) { 145025c3a66402fb713c2d9bf5dc174ff264765379aChris Lattner 1461eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump InitLexer(FromFile->getBufferStart(), FromFile->getBufferStart(), 147025c3a66402fb713c2d9bf5dc174ff264765379aChris Lattner FromFile->getBufferEnd()); 1481eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 149025c3a66402fb713c2d9bf5dc174ff264765379aChris Lattner // We *are* in raw mode. 150025c3a66402fb713c2d9bf5dc174ff264765379aChris Lattner LexingRawMode = true; 151025c3a66402fb713c2d9bf5dc174ff264765379aChris Lattner} 152025c3a66402fb713c2d9bf5dc174ff264765379aChris Lattner 15342e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner/// Create_PragmaLexer: Lexer constructor - Create a new lexer object for 15442e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner/// _Pragma expansion. This has a variety of magic semantics that this method 15542e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner/// sets up. It returns a new'd Lexer that must be delete'd when done. 15642e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner/// 15742e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner/// On entrance to this routine, TokStartLoc is a macro location which has a 15842e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner/// spelling loc that indicates the bytes to be lexed for the token and an 159433db06b614f26dc6829e86d6ff469e2cca7d4f9Chandler Carruth/// expansion location that indicates where all lexed tokens should be 16042e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner/// "expanded from". 16142e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner/// 16242e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner/// FIXME: It would really be nice to make _Pragma just be a wrapper around a 16342e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner/// normal lexer that remaps tokens as they fly by. This would require making 16442e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner/// Preprocessor::Lex virtual. Given that, we could just dump in a magic lexer 16542e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner/// interface that could handle this stuff. This would pull GetMappedTokenLoc 16642e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner/// out of the critical path of the lexer! 16742e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner/// 1681eb4433ac451dc16f4133a88af2d002ac26c58efMike StumpLexer *Lexer::Create_PragmaLexer(SourceLocation SpellingLoc, 169433db06b614f26dc6829e86d6ff469e2cca7d4f9Chandler Carruth SourceLocation ExpansionLocStart, 170433db06b614f26dc6829e86d6ff469e2cca7d4f9Chandler Carruth SourceLocation ExpansionLocEnd, 171bcc2a67e5180612417727cbdd8afd0f79fdf726dChris Lattner unsigned TokLen, Preprocessor &PP) { 17242e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner SourceManager &SM = PP.getSourceManager(); 17342e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner 17442e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner // Create the lexer as if we were going to lex the file normally. 175a11d61793341fea195c29a0dab3fbd74f2b39a8cChris Lattner FileID SpellingFID = SM.getFileID(SpellingLoc); 1766e2901407bff59aeb4cc301cc58b034723d0eb49Chris Lattner const llvm::MemoryBuffer *InputFile = SM.getBuffer(SpellingFID); 1776e2901407bff59aeb4cc301cc58b034723d0eb49Chris Lattner Lexer *L = new Lexer(SpellingFID, InputFile, PP); 1781eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 17942e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner // Now that the lexer is created, change the start/end locations so that we 18042e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner // just lex the subsection of the file that we want. This is lexing from a 18142e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner // scratch buffer. 18242e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner const char *StrData = SM.getCharacterData(SpellingLoc); 1831eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 18442e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner L->BufferPtr = StrData; 18542e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner L->BufferEnd = StrData+TokLen; 1861fa495304c81e03f07f278a47b5efe9317104aabChris Lattner assert(L->BufferEnd[0] == 0 && "Buffer is not nul terminated!"); 18742e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner 18842e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner // Set the SourceLocation with the remapping information. This ensures that 18942e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner // GetMappedTokenLoc will remap the tokens as they are lexed. 190bf340e452339e374ea6eef78c1f0a2abdd16c5a3Chandler Carruth L->FileLoc = SM.createExpansionLoc(SM.getLocForStartOfFile(SpellingFID), 191bf340e452339e374ea6eef78c1f0a2abdd16c5a3Chandler Carruth ExpansionLocStart, 192bf340e452339e374ea6eef78c1f0a2abdd16c5a3Chandler Carruth ExpansionLocEnd, TokLen); 1931eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 19442e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner // Ensure that the lexer thinks it is inside a directive, so that end \n will 19584021556baceb76eedf7d44be8ba71d9b8cfaccePeter Collingbourne // return an EOD token. 19642e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner L->ParsingPreprocessorDirective = true; 1971eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 19842e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner // This lexer really is for _Pragma. 19942e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner L->Is_PragmaLexer = true; 20042e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner return L; 20142e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner} 20242e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner 203168ae2d44a443da75ea85db5f3b5081eb0bce113Chris Lattner 2045f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// Stringify - Convert the specified string into a C string, with surrounding 2055f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// ""'s, and with escaped \ and " characters. 2065f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencerstd::string Lexer::Stringify(const std::string &Str, bool Charify) { 2075f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer std::string Result = Str; 2085f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer char Quote = Charify ? '\'' : '"'; 2095f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer for (unsigned i = 0, e = Result.size(); i != e; ++i) { 2105f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (Result[i] == '\\' || Result[i] == Quote) { 2115f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Result.insert(Result.begin()+i, '\\'); 2125f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer ++i; ++e; 2135f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 2145f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 2155f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer return Result; 2165f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 2175f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 218d8e3083840fef752d11ca183f42786470ed061e3Chris Lattner/// Stringify - Convert the specified string into a C string by escaping '\' 219d8e3083840fef752d11ca183f42786470ed061e3Chris Lattner/// and " characters. This does not add surrounding ""'s to the string. 2205f9e272e632e951b1efe824cd16acb4d96077930Chris Lattnervoid Lexer::Stringify(SmallVectorImpl<char> &Str) { 221d8e3083840fef752d11ca183f42786470ed061e3Chris Lattner for (unsigned i = 0, e = Str.size(); i != e; ++i) { 222d8e3083840fef752d11ca183f42786470ed061e3Chris Lattner if (Str[i] == '\\' || Str[i] == '"') { 223d8e3083840fef752d11ca183f42786470ed061e3Chris Lattner Str.insert(Str.begin()+i, '\\'); 224d8e3083840fef752d11ca183f42786470ed061e3Chris Lattner ++i; ++e; 225d8e3083840fef752d11ca183f42786470ed061e3Chris Lattner } 226d8e3083840fef752d11ca183f42786470ed061e3Chris Lattner } 227d8e3083840fef752d11ca183f42786470ed061e3Chris Lattner} 228d8e3083840fef752d11ca183f42786470ed061e3Chris Lattner 229b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner//===----------------------------------------------------------------------===// 230b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner// Token Spelling 231b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner//===----------------------------------------------------------------------===// 232b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner 233b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner/// getSpelling() - Return the 'spelling' of this token. The spelling of a 234b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner/// token are the characters used to represent the token in the source file 235b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner/// after trigraph expansion and escaped-newline folding. In particular, this 236b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner/// wants to get the true, uncanonicalized, spelling of things like digraphs 237b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner/// UCNs, etc. 2385f9e272e632e951b1efe824cd16acb4d96077930Chris LattnerStringRef Lexer::getSpelling(SourceLocation loc, 2395f9e272e632e951b1efe824cd16acb4d96077930Chris Lattner SmallVectorImpl<char> &buffer, 240834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall const SourceManager &SM, 241834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall const LangOptions &options, 242834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall bool *invalid) { 243834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall // Break down the source location. 244834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall std::pair<FileID, unsigned> locInfo = SM.getDecomposedLoc(loc); 245834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall 246834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall // Try to the load the file buffer. 247834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall bool invalidTemp = false; 2485f9e272e632e951b1efe824cd16acb4d96077930Chris Lattner StringRef file = SM.getBufferData(locInfo.first, &invalidTemp); 249834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall if (invalidTemp) { 250834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall if (invalid) *invalid = true; 2515f9e272e632e951b1efe824cd16acb4d96077930Chris Lattner return StringRef(); 252834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall } 253834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall 254834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall const char *tokenBegin = file.data() + locInfo.second; 255834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall 256834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall // Lex from the start of the given location. 257834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall Lexer lexer(SM.getLocForStartOfFile(locInfo.first), options, 258834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall file.begin(), tokenBegin, file.end()); 259834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall Token token; 260834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall lexer.LexFromRawLexer(token); 261834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall 262834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall unsigned length = token.getLength(); 263834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall 264834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall // Common case: no need for cleaning. 265834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall if (!token.needsCleaning()) 2665f9e272e632e951b1efe824cd16acb4d96077930Chris Lattner return StringRef(tokenBegin, length); 267834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall 268834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall // Hard case, we need to relex the characters into the string. 269834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall buffer.clear(); 270834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall buffer.reserve(length); 271834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall 272834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall for (const char *ti = tokenBegin, *te = ti + length; ti != te; ) { 273834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall unsigned charSize; 274834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall buffer.push_back(Lexer::getCharAndSizeNoWarn(ti, charSize, options)); 275834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall ti += charSize; 276834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall } 277834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall 2785f9e272e632e951b1efe824cd16acb4d96077930Chris Lattner return StringRef(buffer.data(), buffer.size()); 279834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall} 280834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall 281834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall/// getSpelling() - Return the 'spelling' of this token. The spelling of a 282834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall/// token are the characters used to represent the token in the source file 283834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall/// after trigraph expansion and escaped-newline folding. In particular, this 284834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall/// wants to get the true, uncanonicalized, spelling of things like digraphs 285834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall/// UCNs, etc. 286b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattnerstd::string Lexer::getSpelling(const Token &Tok, const SourceManager &SourceMgr, 287b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner const LangOptions &Features, bool *Invalid) { 288b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner assert((int)Tok.getLength() >= 0 && "Token character range is bogus!"); 289b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner 290b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner // If this token contains nothing interesting, return it directly. 291b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner bool CharDataInvalid = false; 292b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner const char* TokStart = SourceMgr.getCharacterData(Tok.getLocation(), 293b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner &CharDataInvalid); 294b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner if (Invalid) 295b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner *Invalid = CharDataInvalid; 296b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner if (CharDataInvalid) 297b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner return std::string(); 298b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner 299b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner if (!Tok.needsCleaning()) 300b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner return std::string(TokStart, TokStart+Tok.getLength()); 301b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner 302b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner std::string Result; 303b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner Result.reserve(Tok.getLength()); 304b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner 305b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner // Otherwise, hard case, relex the characters into the string. 306b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner for (const char *Ptr = TokStart, *End = TokStart+Tok.getLength(); 307b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner Ptr != End; ) { 308b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner unsigned CharSize; 309b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner Result.push_back(Lexer::getCharAndSizeNoWarn(Ptr, CharSize, Features)); 310b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner Ptr += CharSize; 311b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner } 312b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner assert(Result.size() != unsigned(Tok.getLength()) && 313b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner "NeedsCleaning flag set on something that didn't need cleaning!"); 314b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner return Result; 315b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner} 316b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner 317b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner/// getSpelling - This method is used to get the spelling of a token into a 318b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner/// preallocated buffer, instead of as an std::string. The caller is required 319b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner/// to allocate enough space for the token, which is guaranteed to be at least 320b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner/// Tok.getLength() bytes long. The actual length of the token is returned. 321b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner/// 322b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner/// Note that this method may do two possible things: it may either fill in 323b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner/// the buffer specified with characters, or it may *change the input pointer* 324b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner/// to point to a constant buffer with the data already in it (avoiding a 325b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner/// copy). The caller is not allowed to modify the returned buffer pointer 326b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner/// if an internal buffer is returned. 327b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattnerunsigned Lexer::getSpelling(const Token &Tok, const char *&Buffer, 328b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner const SourceManager &SourceMgr, 329b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner const LangOptions &Features, bool *Invalid) { 330b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner assert((int)Tok.getLength() >= 0 && "Token character range is bogus!"); 331c4bf2b9afb7d47445a9dc6bc848657098a4e3851Abramo Bagnara 332c4bf2b9afb7d47445a9dc6bc848657098a4e3851Abramo Bagnara const char *TokStart = 0; 333c4bf2b9afb7d47445a9dc6bc848657098a4e3851Abramo Bagnara // NOTE: this has to be checked *before* testing for an IdentifierInfo. 334c4bf2b9afb7d47445a9dc6bc848657098a4e3851Abramo Bagnara if (Tok.is(tok::raw_identifier)) 335c4bf2b9afb7d47445a9dc6bc848657098a4e3851Abramo Bagnara TokStart = Tok.getRawIdentifierData(); 336c4bf2b9afb7d47445a9dc6bc848657098a4e3851Abramo Bagnara else if (const IdentifierInfo *II = Tok.getIdentifierInfo()) { 337c4bf2b9afb7d47445a9dc6bc848657098a4e3851Abramo Bagnara // Just return the string from the identifier table, which is very quick. 338b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner Buffer = II->getNameStart(); 339b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner return II->getLength(); 340b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner } 341c4bf2b9afb7d47445a9dc6bc848657098a4e3851Abramo Bagnara 342c4bf2b9afb7d47445a9dc6bc848657098a4e3851Abramo Bagnara // NOTE: this can be checked even after testing for an IdentifierInfo. 343b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner if (Tok.isLiteral()) 344b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner TokStart = Tok.getLiteralData(); 345c4bf2b9afb7d47445a9dc6bc848657098a4e3851Abramo Bagnara 346b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner if (TokStart == 0) { 347c4bf2b9afb7d47445a9dc6bc848657098a4e3851Abramo Bagnara // Compute the start of the token in the input lexer buffer. 348b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner bool CharDataInvalid = false; 349b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner TokStart = SourceMgr.getCharacterData(Tok.getLocation(), &CharDataInvalid); 350b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner if (Invalid) 351b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner *Invalid = CharDataInvalid; 352b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner if (CharDataInvalid) { 353b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner Buffer = ""; 354b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner return 0; 355b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner } 356b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner } 357c4bf2b9afb7d47445a9dc6bc848657098a4e3851Abramo Bagnara 358b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner // If this token contains nothing interesting, return it directly. 359b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner if (!Tok.needsCleaning()) { 360b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner Buffer = TokStart; 361b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner return Tok.getLength(); 362b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner } 363c4bf2b9afb7d47445a9dc6bc848657098a4e3851Abramo Bagnara 364b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner // Otherwise, hard case, relex the characters into the string. 365b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner char *OutBuf = const_cast<char*>(Buffer); 366b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner for (const char *Ptr = TokStart, *End = TokStart+Tok.getLength(); 367b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner Ptr != End; ) { 368b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner unsigned CharSize; 369b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner *OutBuf++ = Lexer::getCharAndSizeNoWarn(Ptr, CharSize, Features); 370b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner Ptr += CharSize; 371b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner } 372b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner assert(unsigned(OutBuf-Buffer) != Tok.getLength() && 373b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner "NeedsCleaning flag set on something that didn't need cleaning!"); 374c4bf2b9afb7d47445a9dc6bc848657098a4e3851Abramo Bagnara 375b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner return OutBuf-Buffer; 376b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner} 377b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner 378b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner 379b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner 38033e9abd21083a0191a7676a04b497006d2da184dDouglas Gregorstatic bool isWhitespace(unsigned char c); 3815f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 3829a6119437672f42be5f50c3fe89fe843b1bfa5b5Chris Lattner/// MeasureTokenLength - Relex the token at the specified location and return 3839a6119437672f42be5f50c3fe89fe843b1bfa5b5Chris Lattner/// its length in bytes in the input file. If the token needs cleaning (e.g. 3849a6119437672f42be5f50c3fe89fe843b1bfa5b5Chris Lattner/// includes a trigraph or an escaped newline) then this count includes bytes 3859a6119437672f42be5f50c3fe89fe843b1bfa5b5Chris Lattner/// that are part of that. 3869a6119437672f42be5f50c3fe89fe843b1bfa5b5Chris Lattnerunsigned Lexer::MeasureTokenLength(SourceLocation Loc, 3872c78b873f4f3823ae859c15674cb3d76c8554113Chris Lattner const SourceManager &SM, 3882c78b873f4f3823ae859c15674cb3d76c8554113Chris Lattner const LangOptions &LangOpts) { 3899a6119437672f42be5f50c3fe89fe843b1bfa5b5Chris Lattner // TODO: this could be special cased for common tokens like identifiers, ')', 3909a6119437672f42be5f50c3fe89fe843b1bfa5b5Chris Lattner // etc to make this faster, if it mattered. Just look at StrData[0] to handle 3911eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump // all obviously single-char tokens. This could use 3929a6119437672f42be5f50c3fe89fe843b1bfa5b5Chris Lattner // Lexer::isObviouslySimpleCharacter for example to handle identifiers or 3939a6119437672f42be5f50c3fe89fe843b1bfa5b5Chris Lattner // something. 394de7aeefc5573d669ed476d7bda7a8940d3bcadb7Chris Lattner 395de7aeefc5573d669ed476d7bda7a8940d3bcadb7Chris Lattner // If this comes from a macro expansion, we really do want the macro name, not 396de7aeefc5573d669ed476d7bda7a8940d3bcadb7Chris Lattner // the token this macro expanded to. 397402785357ab053dd53f4fdd858b9630a5e0f8badChandler Carruth Loc = SM.getExpansionLoc(Loc); 398363fdc29656cc03c1817268888f95e6343470aa8Chris Lattner std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(Loc); 399f715ca12bfc9fddfde75f98a197424434428b821Douglas Gregor bool Invalid = false; 4005f9e272e632e951b1efe824cd16acb4d96077930Chris Lattner StringRef Buffer = SM.getBufferData(LocInfo.first, &Invalid); 401f715ca12bfc9fddfde75f98a197424434428b821Douglas Gregor if (Invalid) 402aea67dbd653a2dd6dd5cc2159279e81e855b2482Douglas Gregor return 0; 403f6ac97b101c8840efa92bf29166077ce4049e293Benjamin Kramer 404f6ac97b101c8840efa92bf29166077ce4049e293Benjamin Kramer const char *StrData = Buffer.data()+LocInfo.second; 4058350394c65b81bba3986dfe44ae17423873741deChris Lattner 40633e9abd21083a0191a7676a04b497006d2da184dDouglas Gregor if (isWhitespace(StrData[0])) 40733e9abd21083a0191a7676a04b497006d2da184dDouglas Gregor return 0; 40833e9abd21083a0191a7676a04b497006d2da184dDouglas Gregor 4099a6119437672f42be5f50c3fe89fe843b1bfa5b5Chris Lattner // Create a lexer starting at the beginning of this token. 410c3526d89ef9c31639ec8b25180cfb22354344241Sebastian Redl Lexer TheLexer(SM.getLocForStartOfFile(LocInfo.first), LangOpts, 411c3526d89ef9c31639ec8b25180cfb22354344241Sebastian Redl Buffer.begin(), StrData, Buffer.end()); 41239de7409bffb6b725a8aa64f0ba77ab51e8c9eb3Chris Lattner TheLexer.SetCommentRetentionState(true); 4139a6119437672f42be5f50c3fe89fe843b1bfa5b5Chris Lattner Token TheTok; 414590f0cc643274267d4d41125b62557e1d87886c3Chris Lattner TheLexer.LexFromRawLexer(TheTok); 4159a6119437672f42be5f50c3fe89fe843b1bfa5b5Chris Lattner return TheTok.getLength(); 4169a6119437672f42be5f50c3fe89fe843b1bfa5b5Chris Lattner} 4179a6119437672f42be5f50c3fe89fe843b1bfa5b5Chris Lattner 4180e870622e4d4b2ecb7bc6ffd2c97f74fd14220b6Argyrios Kyrtzidisstatic SourceLocation getBeginningOfFileToken(SourceLocation Loc, 4190e870622e4d4b2ecb7bc6ffd2c97f74fd14220b6Argyrios Kyrtzidis const SourceManager &SM, 4200e870622e4d4b2ecb7bc6ffd2c97f74fd14220b6Argyrios Kyrtzidis const LangOptions &LangOpts) { 4210e870622e4d4b2ecb7bc6ffd2c97f74fd14220b6Argyrios Kyrtzidis assert(Loc.isFileID()); 422a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(Loc); 4233de84241d90f3dd280126fdf2c4651667151c967Douglas Gregor if (LocInfo.first.isInvalid()) 4243de84241d90f3dd280126fdf2c4651667151c967Douglas Gregor return Loc; 4253de84241d90f3dd280126fdf2c4651667151c967Douglas Gregor 426a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor bool Invalid = false; 4275f9e272e632e951b1efe824cd16acb4d96077930Chris Lattner StringRef Buffer = SM.getBufferData(LocInfo.first, &Invalid); 428a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor if (Invalid) 429a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor return Loc; 430a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor 431a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor // Back up from the current location until we hit the beginning of a line 432a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor // (or the buffer). We'll relex from that point. 433a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor const char *BufStart = Buffer.data(); 4343de84241d90f3dd280126fdf2c4651667151c967Douglas Gregor if (LocInfo.second >= Buffer.size()) 4353de84241d90f3dd280126fdf2c4651667151c967Douglas Gregor return Loc; 4363de84241d90f3dd280126fdf2c4651667151c967Douglas Gregor 437a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor const char *StrData = BufStart+LocInfo.second; 438a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor if (StrData[0] == '\n' || StrData[0] == '\r') 439a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor return Loc; 440a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor 441a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor const char *LexStart = StrData; 442a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor while (LexStart != BufStart) { 443a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor if (LexStart[0] == '\n' || LexStart[0] == '\r') { 444a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor ++LexStart; 445a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor break; 446a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor } 447a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor 448a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor --LexStart; 449a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor } 450a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor 451a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor // Create a lexer starting at the beginning of this token. 452a64ccefdf0ea4e03ec88805d71b0af74950c7472Argyrios Kyrtzidis SourceLocation LexerStartLoc = Loc.getLocWithOffset(-LocInfo.second); 453a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor Lexer TheLexer(LexerStartLoc, LangOpts, BufStart, LexStart, Buffer.end()); 454a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor TheLexer.SetCommentRetentionState(true); 455a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor 456a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor // Lex tokens until we find the token that contains the source location. 457a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor Token TheTok; 458a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor do { 459a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor TheLexer.LexFromRawLexer(TheTok); 460a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor 461a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor if (TheLexer.getBufferLocation() > StrData) { 462a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor // Lexing this token has taken the lexer past the source location we're 463a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor // looking for. If the current token encompasses our source location, 464a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor // return the beginning of that token. 465a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor if (TheLexer.getBufferLocation() - TheTok.getLength() <= StrData) 466a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor return TheTok.getLocation(); 467a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor 468a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor // We ended up skipping over the source location entirely, which means 469a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor // that it points into whitespace. We're done here. 470a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor break; 471a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor } 472a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor } while (TheTok.getKind() != tok::eof); 473a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor 474a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor // We've passed our source location; just return the original source location. 475a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor return Loc; 476a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor} 477a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor 4780e870622e4d4b2ecb7bc6ffd2c97f74fd14220b6Argyrios KyrtzidisSourceLocation Lexer::GetBeginningOfToken(SourceLocation Loc, 4790e870622e4d4b2ecb7bc6ffd2c97f74fd14220b6Argyrios Kyrtzidis const SourceManager &SM, 4800e870622e4d4b2ecb7bc6ffd2c97f74fd14220b6Argyrios Kyrtzidis const LangOptions &LangOpts) { 4810e870622e4d4b2ecb7bc6ffd2c97f74fd14220b6Argyrios Kyrtzidis if (Loc.isFileID()) 4820e870622e4d4b2ecb7bc6ffd2c97f74fd14220b6Argyrios Kyrtzidis return getBeginningOfFileToken(Loc, SM, LangOpts); 4830e870622e4d4b2ecb7bc6ffd2c97f74fd14220b6Argyrios Kyrtzidis 4840e870622e4d4b2ecb7bc6ffd2c97f74fd14220b6Argyrios Kyrtzidis if (!SM.isMacroArgExpansion(Loc)) 4850e870622e4d4b2ecb7bc6ffd2c97f74fd14220b6Argyrios Kyrtzidis return Loc; 4860e870622e4d4b2ecb7bc6ffd2c97f74fd14220b6Argyrios Kyrtzidis 4870e870622e4d4b2ecb7bc6ffd2c97f74fd14220b6Argyrios Kyrtzidis SourceLocation FileLoc = SM.getSpellingLoc(Loc); 4880e870622e4d4b2ecb7bc6ffd2c97f74fd14220b6Argyrios Kyrtzidis SourceLocation BeginFileLoc = getBeginningOfFileToken(FileLoc, SM, LangOpts); 4890e870622e4d4b2ecb7bc6ffd2c97f74fd14220b6Argyrios Kyrtzidis std::pair<FileID, unsigned> FileLocInfo = SM.getDecomposedLoc(FileLoc); 4900e870622e4d4b2ecb7bc6ffd2c97f74fd14220b6Argyrios Kyrtzidis std::pair<FileID, unsigned> BeginFileLocInfo= SM.getDecomposedLoc(BeginFileLoc); 4910e870622e4d4b2ecb7bc6ffd2c97f74fd14220b6Argyrios Kyrtzidis assert(FileLocInfo.first == BeginFileLocInfo.first && 4920e870622e4d4b2ecb7bc6ffd2c97f74fd14220b6Argyrios Kyrtzidis FileLocInfo.second >= BeginFileLocInfo.second); 493a64ccefdf0ea4e03ec88805d71b0af74950c7472Argyrios Kyrtzidis return Loc.getLocWithOffset(SM.getDecomposedLoc(BeginFileLoc).second - 4940e870622e4d4b2ecb7bc6ffd2c97f74fd14220b6Argyrios Kyrtzidis SM.getDecomposedLoc(FileLoc).second); 4950e870622e4d4b2ecb7bc6ffd2c97f74fd14220b6Argyrios Kyrtzidis} 4960e870622e4d4b2ecb7bc6ffd2c97f74fd14220b6Argyrios Kyrtzidis 497f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregornamespace { 498f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor enum PreambleDirectiveKind { 499f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor PDK_Skipped, 500f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor PDK_StartIf, 501f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor PDK_EndIf, 502f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor PDK_Unknown 503f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor }; 504f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor} 505f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor 506f4f6c9db68465b886ec2e596feaa6ecc782395a4Douglas Gregorstd::pair<unsigned, bool> 50703c107a42fae79e89d0016999a1a04c07d65591aArgyrios KyrtzidisLexer::ComputePreamble(const llvm::MemoryBuffer *Buffer, 50803c107a42fae79e89d0016999a1a04c07d65591aArgyrios Kyrtzidis const LangOptions &Features, unsigned MaxLines) { 509f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor // Create a lexer starting at the beginning of the file. Note that we use a 510f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor // "fake" file source location at offset 1 so that the lexer will track our 511f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor // position within the file. 512f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor const unsigned StartOffset = 1; 513f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor SourceLocation StartLoc = SourceLocation::getFromRawEncoding(StartOffset); 51403c107a42fae79e89d0016999a1a04c07d65591aArgyrios Kyrtzidis Lexer TheLexer(StartLoc, Features, Buffer->getBufferStart(), 515f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor Buffer->getBufferStart(), Buffer->getBufferEnd()); 516f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor 517f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor bool InPreprocessorDirective = false; 518f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor Token TheTok; 519f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor Token IfStartTok; 520f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor unsigned IfCount = 0; 521c8c97a03eb0fdeb4f5fc9c4dea308ebbf46c2c93Argyrios Kyrtzidis 522c8c97a03eb0fdeb4f5fc9c4dea308ebbf46c2c93Argyrios Kyrtzidis unsigned MaxLineOffset = 0; 523c8c97a03eb0fdeb4f5fc9c4dea308ebbf46c2c93Argyrios Kyrtzidis if (MaxLines) { 524c8c97a03eb0fdeb4f5fc9c4dea308ebbf46c2c93Argyrios Kyrtzidis const char *CurPtr = Buffer->getBufferStart(); 525c8c97a03eb0fdeb4f5fc9c4dea308ebbf46c2c93Argyrios Kyrtzidis unsigned CurLine = 0; 526c8c97a03eb0fdeb4f5fc9c4dea308ebbf46c2c93Argyrios Kyrtzidis while (CurPtr != Buffer->getBufferEnd()) { 527c8c97a03eb0fdeb4f5fc9c4dea308ebbf46c2c93Argyrios Kyrtzidis char ch = *CurPtr++; 528c8c97a03eb0fdeb4f5fc9c4dea308ebbf46c2c93Argyrios Kyrtzidis if (ch == '\n') { 529c8c97a03eb0fdeb4f5fc9c4dea308ebbf46c2c93Argyrios Kyrtzidis ++CurLine; 530c8c97a03eb0fdeb4f5fc9c4dea308ebbf46c2c93Argyrios Kyrtzidis if (CurLine == MaxLines) 531c8c97a03eb0fdeb4f5fc9c4dea308ebbf46c2c93Argyrios Kyrtzidis break; 532c8c97a03eb0fdeb4f5fc9c4dea308ebbf46c2c93Argyrios Kyrtzidis } 533c8c97a03eb0fdeb4f5fc9c4dea308ebbf46c2c93Argyrios Kyrtzidis } 534c8c97a03eb0fdeb4f5fc9c4dea308ebbf46c2c93Argyrios Kyrtzidis if (CurPtr != Buffer->getBufferEnd()) 535c8c97a03eb0fdeb4f5fc9c4dea308ebbf46c2c93Argyrios Kyrtzidis MaxLineOffset = CurPtr - Buffer->getBufferStart(); 536c8c97a03eb0fdeb4f5fc9c4dea308ebbf46c2c93Argyrios Kyrtzidis } 537df95a13ec73d2cdaea79555cb412d767f4963120Douglas Gregor 538f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor do { 539f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor TheLexer.LexFromRawLexer(TheTok); 540f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor 541f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor if (InPreprocessorDirective) { 542f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor // If we've hit the end of the file, we're done. 543f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor if (TheTok.getKind() == tok::eof) { 544f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor InPreprocessorDirective = false; 545f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor break; 546f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor } 547f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor 548f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor // If we haven't hit the end of the preprocessor directive, skip this 549f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor // token. 550f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor if (!TheTok.isAtStartOfLine()) 551f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor continue; 552f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor 553f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor // We've passed the end of the preprocessor directive, and will look 554f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor // at this token again below. 555f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor InPreprocessorDirective = false; 556f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor } 557f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor 558df95a13ec73d2cdaea79555cb412d767f4963120Douglas Gregor // Keep track of the # of lines in the preamble. 559df95a13ec73d2cdaea79555cb412d767f4963120Douglas Gregor if (TheTok.isAtStartOfLine()) { 560c8c97a03eb0fdeb4f5fc9c4dea308ebbf46c2c93Argyrios Kyrtzidis unsigned TokOffset = TheTok.getLocation().getRawEncoding() - StartOffset; 561df95a13ec73d2cdaea79555cb412d767f4963120Douglas Gregor 562df95a13ec73d2cdaea79555cb412d767f4963120Douglas Gregor // If we were asked to limit the number of lines in the preamble, 563df95a13ec73d2cdaea79555cb412d767f4963120Douglas Gregor // and we're about to exceed that limit, we're done. 564c8c97a03eb0fdeb4f5fc9c4dea308ebbf46c2c93Argyrios Kyrtzidis if (MaxLineOffset && TokOffset >= MaxLineOffset) 565df95a13ec73d2cdaea79555cb412d767f4963120Douglas Gregor break; 566df95a13ec73d2cdaea79555cb412d767f4963120Douglas Gregor } 567df95a13ec73d2cdaea79555cb412d767f4963120Douglas Gregor 568f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor // Comments are okay; skip over them. 569f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor if (TheTok.getKind() == tok::comment) 570f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor continue; 571f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor 572f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor if (TheTok.isAtStartOfLine() && TheTok.getKind() == tok::hash) { 573f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor // This is the start of a preprocessor directive. 574f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor Token HashTok = TheTok; 575f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor InPreprocessorDirective = true; 576f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor 57719207f1e5f51261a33492602501fb7ada50ea546Joerg Sonnenberger // Figure out which directive this is. Since we're lexing raw tokens, 578f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor // we don't have an identifier table available. Instead, just look at 579f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor // the raw identifier to recognize and categorize preprocessor directives. 580f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor TheLexer.LexFromRawLexer(TheTok); 581c4bf2b9afb7d47445a9dc6bc848657098a4e3851Abramo Bagnara if (TheTok.getKind() == tok::raw_identifier && !TheTok.needsCleaning()) { 5825f9e272e632e951b1efe824cd16acb4d96077930Chris Lattner StringRef Keyword(TheTok.getRawIdentifierData(), 583c4bf2b9afb7d47445a9dc6bc848657098a4e3851Abramo Bagnara TheTok.getLength()); 584f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor PreambleDirectiveKind PDK 585f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor = llvm::StringSwitch<PreambleDirectiveKind>(Keyword) 586f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor .Case("include", PDK_Skipped) 587f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor .Case("__include_macros", PDK_Skipped) 588f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor .Case("define", PDK_Skipped) 589f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor .Case("undef", PDK_Skipped) 590f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor .Case("line", PDK_Skipped) 591f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor .Case("error", PDK_Skipped) 592f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor .Case("pragma", PDK_Skipped) 593f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor .Case("import", PDK_Skipped) 594f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor .Case("include_next", PDK_Skipped) 595f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor .Case("warning", PDK_Skipped) 596f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor .Case("ident", PDK_Skipped) 597f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor .Case("sccs", PDK_Skipped) 598f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor .Case("assert", PDK_Skipped) 599f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor .Case("unassert", PDK_Skipped) 600f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor .Case("if", PDK_StartIf) 601f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor .Case("ifdef", PDK_StartIf) 602f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor .Case("ifndef", PDK_StartIf) 603f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor .Case("elif", PDK_Skipped) 604f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor .Case("else", PDK_Skipped) 605f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor .Case("endif", PDK_EndIf) 606f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor .Default(PDK_Unknown); 607f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor 608f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor switch (PDK) { 609f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor case PDK_Skipped: 610f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor continue; 611f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor 612f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor case PDK_StartIf: 613f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor if (IfCount == 0) 614f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor IfStartTok = HashTok; 615f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor 616f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor ++IfCount; 617f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor continue; 618f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor 619f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor case PDK_EndIf: 620f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor // Mismatched #endif. The preamble ends here. 621f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor if (IfCount == 0) 622f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor break; 623f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor 624f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor --IfCount; 625f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor continue; 626f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor 627f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor case PDK_Unknown: 628f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor // We don't know what this directive is; stop at the '#'. 629f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor break; 630f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor } 631f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor } 632f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor 633f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor // We only end up here if we didn't recognize the preprocessor 634f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor // directive or it was one that can't occur in the preamble at this 635f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor // point. Roll back the current token to the location of the '#'. 636f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor InPreprocessorDirective = false; 637f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor TheTok = HashTok; 638f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor } 639f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor 640df95a13ec73d2cdaea79555cb412d767f4963120Douglas Gregor // We hit a token that we don't recognize as being in the 641df95a13ec73d2cdaea79555cb412d767f4963120Douglas Gregor // "preprocessing only" part of the file, so we're no longer in 642df95a13ec73d2cdaea79555cb412d767f4963120Douglas Gregor // the preamble. 643f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor break; 644f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor } while (true); 645f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor 646f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor SourceLocation End = IfCount? IfStartTok.getLocation() : TheTok.getLocation(); 647f4f6c9db68465b886ec2e596feaa6ecc782395a4Douglas Gregor return std::make_pair(End.getRawEncoding() - StartLoc.getRawEncoding(), 648f4f6c9db68465b886ec2e596feaa6ecc782395a4Douglas Gregor IfCount? IfStartTok.isAtStartOfLine() 649f4f6c9db68465b886ec2e596feaa6ecc782395a4Douglas Gregor : TheTok.isAtStartOfLine()); 650f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor} 651f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor 6527ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner 6537ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner/// AdvanceToTokenCharacter - Given a location that specifies the start of a 6547ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner/// token, return a new location that specifies a character within the token. 6557ef5c27eb6e8ebe58b52013246c06753c3613263Chris LattnerSourceLocation Lexer::AdvanceToTokenCharacter(SourceLocation TokStart, 6567ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner unsigned CharNo, 6577ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner const SourceManager &SM, 6587ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner const LangOptions &Features) { 659433db06b614f26dc6829e86d6ff469e2cca7d4f9Chandler Carruth // Figure out how many physical characters away the specified expansion 6607ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner // character is. This needs to take into consideration newlines and 6617ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner // trigraphs. 6627ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner bool Invalid = false; 6637ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner const char *TokPtr = SM.getCharacterData(TokStart, &Invalid); 6647ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner 6657ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner // If they request the first char of the token, we're trivially done. 6667ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner if (Invalid || (CharNo == 0 && Lexer::isObviouslySimpleCharacter(*TokPtr))) 6677ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner return TokStart; 6687ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner 6697ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner unsigned PhysOffset = 0; 6707ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner 6717ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner // The usual case is that tokens don't contain anything interesting. Skip 6727ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner // over the uninteresting characters. If a token only consists of simple 6737ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner // chars, this method is extremely fast. 6747ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner while (Lexer::isObviouslySimpleCharacter(*TokPtr)) { 6757ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner if (CharNo == 0) 676a64ccefdf0ea4e03ec88805d71b0af74950c7472Argyrios Kyrtzidis return TokStart.getLocWithOffset(PhysOffset); 6777ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner ++TokPtr, --CharNo, ++PhysOffset; 6787ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner } 6797ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner 6807ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner // If we have a character that may be a trigraph or escaped newline, use a 6817ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner // lexer to parse it correctly. 6827ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner for (; CharNo; --CharNo) { 6837ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner unsigned Size; 6847ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner Lexer::getCharAndSizeNoWarn(TokPtr, Size, Features); 6857ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner TokPtr += Size; 6867ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner PhysOffset += Size; 6877ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner } 6887ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner 6897ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner // Final detail: if we end up on an escaped newline, we want to return the 6907ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner // location of the actual byte of the token. For example foo\<newline>bar 6917ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner // advanced by 3 should return the location of b, not of \\. One compounding 6927ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner // detail of this is that the escape may be made by a trigraph. 6937ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner if (!Lexer::isObviouslySimpleCharacter(*TokPtr)) 6947ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner PhysOffset += Lexer::SkipEscapedNewLines(TokPtr)-TokPtr; 6957ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner 696a64ccefdf0ea4e03ec88805d71b0af74950c7472Argyrios Kyrtzidis return TokStart.getLocWithOffset(PhysOffset); 6977ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner} 6987ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner 6997ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner/// \brief Computes the source location just past the end of the 7007ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner/// token at this source location. 7017ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner/// 7027ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner/// This routine can be used to produce a source location that 7037ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner/// points just past the end of the token referenced by \p Loc, and 7047ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner/// is generally used when a diagnostic needs to point just after a 7057ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner/// token where it expected something different that it received. If 7067ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner/// the returned source location would not be meaningful (e.g., if 7077ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner/// it points into a macro), this routine returns an invalid 7087ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner/// source location. 7097ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner/// 7107ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner/// \param Offset an offset from the end of the token, where the source 7117ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner/// location should refer to. The default offset (0) produces a source 7127ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner/// location pointing just past the end of the token; an offset of 1 produces 7137ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner/// a source location pointing to the last character in the token, etc. 7147ef5c27eb6e8ebe58b52013246c06753c3613263Chris LattnerSourceLocation Lexer::getLocForEndOfToken(SourceLocation Loc, unsigned Offset, 7157ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner const SourceManager &SM, 7167ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner const LangOptions &Features) { 7177ddf6b2d77ac4cb27f78d817d7884e6ce17afd0cArgyrios Kyrtzidis if (Loc.isInvalid()) 7187ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner return SourceLocation(); 7197ddf6b2d77ac4cb27f78d817d7884e6ce17afd0cArgyrios Kyrtzidis 7207ddf6b2d77ac4cb27f78d817d7884e6ce17afd0cArgyrios Kyrtzidis if (Loc.isMacroID()) { 721433db06b614f26dc6829e86d6ff469e2cca7d4f9Chandler Carruth if (Offset > 0 || !isAtEndOfMacroExpansion(Loc, SM, Features)) 722433db06b614f26dc6829e86d6ff469e2cca7d4f9Chandler Carruth return SourceLocation(); // Points inside the macro expansion. 7237ddf6b2d77ac4cb27f78d817d7884e6ce17afd0cArgyrios Kyrtzidis 724433db06b614f26dc6829e86d6ff469e2cca7d4f9Chandler Carruth // Continue and find the location just after the macro expansion. 725edc3dccece244a584f8ebdb81da6c962c08e79beChandler Carruth Loc = SM.getExpansionRange(Loc).second; 7267ddf6b2d77ac4cb27f78d817d7884e6ce17afd0cArgyrios Kyrtzidis } 7277ddf6b2d77ac4cb27f78d817d7884e6ce17afd0cArgyrios Kyrtzidis 7287ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner unsigned Len = Lexer::MeasureTokenLength(Loc, SM, Features); 7297ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner if (Len > Offset) 7307ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner Len = Len - Offset; 7317ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner else 7327ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner return Loc; 7337ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner 734a64ccefdf0ea4e03ec88805d71b0af74950c7472Argyrios Kyrtzidis return Loc.getLocWithOffset(Len); 7357ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner} 7367ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner 7377a759606d93975866051f67104ae58446e55f404Argyrios Kyrtzidis/// \brief Returns true if the given MacroID location points at the first 738433db06b614f26dc6829e86d6ff469e2cca7d4f9Chandler Carruth/// token of the macro expansion. 739433db06b614f26dc6829e86d6ff469e2cca7d4f9Chandler Carruthbool Lexer::isAtStartOfMacroExpansion(SourceLocation loc, 740f62d43d2afe1960755a1b5813cae1e5983bcac1bDouglas Gregor const SourceManager &SM, 741f62d43d2afe1960755a1b5813cae1e5983bcac1bDouglas Gregor const LangOptions &LangOpts) { 7427a759606d93975866051f67104ae58446e55f404Argyrios Kyrtzidis assert(loc.isValid() && loc.isMacroID() && "Expected a valid macro loc"); 7437a759606d93975866051f67104ae58446e55f404Argyrios Kyrtzidis 7447a759606d93975866051f67104ae58446e55f404Argyrios Kyrtzidis std::pair<FileID, unsigned> infoLoc = SM.getDecomposedLoc(loc); 7457a759606d93975866051f67104ae58446e55f404Argyrios Kyrtzidis // FIXME: If the token comes from the macro token paste operator ('##') 7467a759606d93975866051f67104ae58446e55f404Argyrios Kyrtzidis // this function will always return false; 7477a759606d93975866051f67104ae58446e55f404Argyrios Kyrtzidis if (infoLoc.second > 0) 7487a759606d93975866051f67104ae58446e55f404Argyrios Kyrtzidis return false; // Does not point at the start of token. 7497a759606d93975866051f67104ae58446e55f404Argyrios Kyrtzidis 750433db06b614f26dc6829e86d6ff469e2cca7d4f9Chandler Carruth SourceLocation expansionLoc = 7511728762d5a8cfaf8d64385f47b311e84de1ae7a2Chandler Carruth SM.getSLocEntry(infoLoc.first).getExpansion().getExpansionLocStart(); 752433db06b614f26dc6829e86d6ff469e2cca7d4f9Chandler Carruth if (expansionLoc.isFileID()) 753433db06b614f26dc6829e86d6ff469e2cca7d4f9Chandler Carruth return true; // No other macro expansions, this is the first. 7547a759606d93975866051f67104ae58446e55f404Argyrios Kyrtzidis 755433db06b614f26dc6829e86d6ff469e2cca7d4f9Chandler Carruth return isAtStartOfMacroExpansion(expansionLoc, SM, LangOpts); 7567a759606d93975866051f67104ae58446e55f404Argyrios Kyrtzidis} 7577a759606d93975866051f67104ae58446e55f404Argyrios Kyrtzidis 7587a759606d93975866051f67104ae58446e55f404Argyrios Kyrtzidis/// \brief Returns true if the given MacroID location points at the last 759433db06b614f26dc6829e86d6ff469e2cca7d4f9Chandler Carruth/// token of the macro expansion. 760433db06b614f26dc6829e86d6ff469e2cca7d4f9Chandler Carruthbool Lexer::isAtEndOfMacroExpansion(SourceLocation loc, 7617a759606d93975866051f67104ae58446e55f404Argyrios Kyrtzidis const SourceManager &SM, 7627a759606d93975866051f67104ae58446e55f404Argyrios Kyrtzidis const LangOptions &LangOpts) { 7637a759606d93975866051f67104ae58446e55f404Argyrios Kyrtzidis assert(loc.isValid() && loc.isMacroID() && "Expected a valid macro loc"); 7647a759606d93975866051f67104ae58446e55f404Argyrios Kyrtzidis 7657a759606d93975866051f67104ae58446e55f404Argyrios Kyrtzidis SourceLocation spellLoc = SM.getSpellingLoc(loc); 7667a759606d93975866051f67104ae58446e55f404Argyrios Kyrtzidis unsigned tokLen = MeasureTokenLength(spellLoc, SM, LangOpts); 7677a759606d93975866051f67104ae58446e55f404Argyrios Kyrtzidis if (tokLen == 0) 7687a759606d93975866051f67104ae58446e55f404Argyrios Kyrtzidis return false; 7697a759606d93975866051f67104ae58446e55f404Argyrios Kyrtzidis 7707a759606d93975866051f67104ae58446e55f404Argyrios Kyrtzidis FileID FID = SM.getFileID(loc); 771a64ccefdf0ea4e03ec88805d71b0af74950c7472Argyrios Kyrtzidis SourceLocation afterLoc = loc.getLocWithOffset(tokLen+1); 772f8c50652f7b224e66b0b6098d1fba07e036019b4Argyrios Kyrtzidis if (SM.isInFileID(afterLoc, FID)) 773f8c50652f7b224e66b0b6098d1fba07e036019b4Argyrios Kyrtzidis return false; // Still in the same FileID, does not point to the last token. 7747a759606d93975866051f67104ae58446e55f404Argyrios Kyrtzidis 7757a759606d93975866051f67104ae58446e55f404Argyrios Kyrtzidis // FIXME: If the token comes from the macro token paste operator ('##') 7767a759606d93975866051f67104ae58446e55f404Argyrios Kyrtzidis // or the stringify operator ('#') this function will always return false; 777f8c50652f7b224e66b0b6098d1fba07e036019b4Argyrios Kyrtzidis 778433db06b614f26dc6829e86d6ff469e2cca7d4f9Chandler Carruth SourceLocation expansionLoc = 7791728762d5a8cfaf8d64385f47b311e84de1ae7a2Chandler Carruth SM.getSLocEntry(FID).getExpansion().getExpansionLocEnd(); 780433db06b614f26dc6829e86d6ff469e2cca7d4f9Chandler Carruth if (expansionLoc.isFileID()) 781433db06b614f26dc6829e86d6ff469e2cca7d4f9Chandler Carruth return true; // No other macro expansions. 7827a759606d93975866051f67104ae58446e55f404Argyrios Kyrtzidis 783433db06b614f26dc6829e86d6ff469e2cca7d4f9Chandler Carruth return isAtEndOfMacroExpansion(expansionLoc, SM, LangOpts); 7847a759606d93975866051f67104ae58446e55f404Argyrios Kyrtzidis} 7857a759606d93975866051f67104ae58446e55f404Argyrios Kyrtzidis 7865f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer//===----------------------------------------------------------------------===// 7875f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// Character information. 7885f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer//===----------------------------------------------------------------------===// 7895f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 7905f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencerenum { 7915f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CHAR_HORZ_WS = 0x01, // ' ', '\t', '\f', '\v'. Note, no '\0' 7925f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CHAR_VERT_WS = 0x02, // '\r', '\n' 7935f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CHAR_LETTER = 0x04, // a-z,A-Z 7945f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CHAR_NUMBER = 0x08, // 0-9 7955f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CHAR_UNDER = 0x10, // _ 7962fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper CHAR_PERIOD = 0x20, // . 7972fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper CHAR_RAWDEL = 0x40 // {}[]#<>%:;?*+-/^&|~!=,"' 7985f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer}; 7995f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 80003b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner// Statically initialize CharInfo table based on ASCII character set 80103b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner// Reference: FreeBSD 7.2 /usr/share/misc/ascii 802a2bf105739cef7aea8e4d219629c627fcd11742eChris Lattnerstatic const unsigned char CharInfo[256] = 80303b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner{ 80403b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner// 0 NUL 1 SOH 2 STX 3 ETX 80503b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner// 4 EOT 5 ENQ 6 ACK 7 BEL 80603b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner 0 , 0 , 0 , 0 , 80703b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner 0 , 0 , 0 , 0 , 80803b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner// 8 BS 9 HT 10 NL 11 VT 80903b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner//12 NP 13 CR 14 SO 15 SI 81003b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner 0 , CHAR_HORZ_WS, CHAR_VERT_WS, CHAR_HORZ_WS, 81103b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner CHAR_HORZ_WS, CHAR_VERT_WS, 0 , 0 , 81203b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner//16 DLE 17 DC1 18 DC2 19 DC3 81303b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner//20 DC4 21 NAK 22 SYN 23 ETB 81403b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner 0 , 0 , 0 , 0 , 81503b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner 0 , 0 , 0 , 0 , 81603b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner//24 CAN 25 EM 26 SUB 27 ESC 81703b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner//28 FS 29 GS 30 RS 31 US 81803b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner 0 , 0 , 0 , 0 , 81903b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner 0 , 0 , 0 , 0 , 82003b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner//32 SP 33 ! 34 " 35 # 82103b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner//36 $ 37 % 38 & 39 ' 8222fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper CHAR_HORZ_WS, CHAR_RAWDEL , CHAR_RAWDEL , CHAR_RAWDEL , 8232fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper 0 , CHAR_RAWDEL , CHAR_RAWDEL , CHAR_RAWDEL , 82403b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner//40 ( 41 ) 42 * 43 + 82503b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner//44 , 45 - 46 . 47 / 8262fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper 0 , 0 , CHAR_RAWDEL , CHAR_RAWDEL , 8272fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper CHAR_RAWDEL , CHAR_RAWDEL , CHAR_PERIOD , CHAR_RAWDEL , 82803b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner//48 0 49 1 50 2 51 3 82903b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner//52 4 53 5 54 6 55 7 83003b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner CHAR_NUMBER , CHAR_NUMBER , CHAR_NUMBER , CHAR_NUMBER , 83103b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner CHAR_NUMBER , CHAR_NUMBER , CHAR_NUMBER , CHAR_NUMBER , 83203b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner//56 8 57 9 58 : 59 ; 83303b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner//60 < 61 = 62 > 63 ? 8342fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper CHAR_NUMBER , CHAR_NUMBER , CHAR_RAWDEL , CHAR_RAWDEL , 8352fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper CHAR_RAWDEL , CHAR_RAWDEL , CHAR_RAWDEL , CHAR_RAWDEL , 83603b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner//64 @ 65 A 66 B 67 C 83703b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner//68 D 69 E 70 F 71 G 83803b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner 0 , CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , 83903b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , 84003b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner//72 H 73 I 74 J 75 K 84103b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner//76 L 77 M 78 N 79 O 84203b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , 84303b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , 84403b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner//80 P 81 Q 82 R 83 S 84503b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner//84 T 85 U 86 V 87 W 84603b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , 84703b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , 84803b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner//88 X 89 Y 90 Z 91 [ 84903b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner//92 \ 93 ] 94 ^ 95 _ 8502fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , CHAR_RAWDEL , 8512fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper 0 , CHAR_RAWDEL , CHAR_RAWDEL , CHAR_UNDER , 85203b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner//96 ` 97 a 98 b 99 c 85303b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner//100 d 101 e 102 f 103 g 85403b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner 0 , CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , 85503b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , 85603b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner//104 h 105 i 106 j 107 k 85703b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner//108 l 109 m 110 n 111 o 85803b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , 85903b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , 86003b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner//112 p 113 q 114 r 115 s 86103b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner//116 t 117 u 118 v 119 w 86203b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , 86303b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , 86403b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner//120 x 121 y 122 z 123 { 8652fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper//124 | 125 } 126 ~ 127 DEL 8662fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , CHAR_RAWDEL , 8672fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper CHAR_RAWDEL , CHAR_RAWDEL , CHAR_RAWDEL , 0 86803b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner}; 86903b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner 870a2bf105739cef7aea8e4d219629c627fcd11742eChris Lattnerstatic void InitCharacterInfo() { 8715f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer static bool isInited = false; 8725f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (isInited) return; 87303b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner // check the statically-initialized CharInfo table 87403b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner assert(CHAR_HORZ_WS == CharInfo[(int)' ']); 87503b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner assert(CHAR_HORZ_WS == CharInfo[(int)'\t']); 87603b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner assert(CHAR_HORZ_WS == CharInfo[(int)'\f']); 87703b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner assert(CHAR_HORZ_WS == CharInfo[(int)'\v']); 87803b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner assert(CHAR_VERT_WS == CharInfo[(int)'\n']); 87903b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner assert(CHAR_VERT_WS == CharInfo[(int)'\r']); 88003b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner assert(CHAR_UNDER == CharInfo[(int)'_']); 88103b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner assert(CHAR_PERIOD == CharInfo[(int)'.']); 88203b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner for (unsigned i = 'a'; i <= 'z'; ++i) { 88303b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner assert(CHAR_LETTER == CharInfo[i]); 88403b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner assert(CHAR_LETTER == CharInfo[i+'A'-'a']); 88503b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner } 8865f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer for (unsigned i = '0'; i <= '9'; ++i) 88703b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner assert(CHAR_NUMBER == CharInfo[i]); 8887b682656310b0d7e94a98d982444bc7ddedc653fSteve Naroff 88903b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner isInited = true; 8905f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 8915f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 89203b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner 8935f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// isIdentifierBody - Return true if this is the body character of an 8945f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// identifier, which is [a-zA-Z0-9_]. 8955f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencerstatic inline bool isIdentifierBody(unsigned char c) { 89695c062b94e2af5380617cf5dc115497ae7a5f817Hartmut Kaiser return (CharInfo[c] & (CHAR_LETTER|CHAR_NUMBER|CHAR_UNDER)) ? true : false; 8975f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 8985f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 8995f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// isHorizontalWhitespace - Return true if this character is horizontal 9005f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// whitespace: ' ', '\t', '\f', '\v'. Note that this returns false for '\0'. 9015f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencerstatic inline bool isHorizontalWhitespace(unsigned char c) { 90295c062b94e2af5380617cf5dc115497ae7a5f817Hartmut Kaiser return (CharInfo[c] & CHAR_HORZ_WS) ? true : false; 9035f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 9045f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 905aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks/// isVerticalWhitespace - Return true if this character is vertical 906aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks/// whitespace: '\n', '\r'. Note that this returns false for '\0'. 907aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaksstatic inline bool isVerticalWhitespace(unsigned char c) { 908aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks return (CharInfo[c] & CHAR_VERT_WS) ? true : false; 909aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks} 910aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks 9115f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// isWhitespace - Return true if this character is horizontal or vertical 9125f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// whitespace: ' ', '\t', '\f', '\v', '\n', '\r'. Note that this returns false 9135f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// for '\0'. 9145f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencerstatic inline bool isWhitespace(unsigned char c) { 91595c062b94e2af5380617cf5dc115497ae7a5f817Hartmut Kaiser return (CharInfo[c] & (CHAR_HORZ_WS|CHAR_VERT_WS)) ? true : false; 9165f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 9175f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 9185f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// isNumberBody - Return true if this is the body character of an 9195f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// preprocessing number, which is [a-zA-Z0-9_.]. 9205f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencerstatic inline bool isNumberBody(unsigned char c) { 9211eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump return (CharInfo[c] & (CHAR_LETTER|CHAR_NUMBER|CHAR_UNDER|CHAR_PERIOD)) ? 92295c062b94e2af5380617cf5dc115497ae7a5f817Hartmut Kaiser true : false; 9235f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 9245f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 9252fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper/// isRawStringDelimBody - Return true if this is the body character of a 9262fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper/// raw string delimiter. 9272fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topperstatic inline bool isRawStringDelimBody(unsigned char c) { 9282fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper return (CharInfo[c] & 9292fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper (CHAR_LETTER|CHAR_NUMBER|CHAR_UNDER|CHAR_PERIOD|CHAR_RAWDEL)) ? 9302fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper true : false; 9312fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper} 9322fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper 9335f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 9345f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer//===----------------------------------------------------------------------===// 9355f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// Diagnostics forwarding code. 9365f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer//===----------------------------------------------------------------------===// 9375f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 938409a03671224d4e5bdab1594c43baf070148f830Chris Lattner/// GetMappedTokenLoc - If lexing out of a 'mapped buffer', where we pretend the 939433db06b614f26dc6829e86d6ff469e2cca7d4f9Chandler Carruth/// lexer buffer was all expanded at a single point, perform the mapping. 940409a03671224d4e5bdab1594c43baf070148f830Chris Lattner/// This is currently only used for _Pragma implementation, so it is the slow 941409a03671224d4e5bdab1594c43baf070148f830Chris Lattner/// path of the hot getSourceLocation method. Do not allow it to be inlined. 94214bd96571ef6f0e97dc79ec4d01b547d60e8fa68Chandler Carruthstatic LLVM_ATTRIBUTE_NOINLINE SourceLocation GetMappedTokenLoc( 94314bd96571ef6f0e97dc79ec4d01b547d60e8fa68Chandler Carruth Preprocessor &PP, SourceLocation FileLoc, unsigned CharNo, unsigned TokLen); 944409a03671224d4e5bdab1594c43baf070148f830Chris Lattnerstatic SourceLocation GetMappedTokenLoc(Preprocessor &PP, 945409a03671224d4e5bdab1594c43baf070148f830Chris Lattner SourceLocation FileLoc, 946de7aeefc5573d669ed476d7bda7a8940d3bcadb7Chris Lattner unsigned CharNo, unsigned TokLen) { 947433db06b614f26dc6829e86d6ff469e2cca7d4f9Chandler Carruth assert(FileLoc.isMacroID() && "Must be a macro expansion"); 9481eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 949409a03671224d4e5bdab1594c43baf070148f830Chris Lattner // Otherwise, we're lexing "mapped tokens". This is used for things like 950433db06b614f26dc6829e86d6ff469e2cca7d4f9Chandler Carruth // _Pragma handling. Combine the expansion location of FileLoc with the 951df7c17a8d02fe09a3466786bae3e40fc3252687aChris Lattner // spelling location. 952e7fb48466afcbf2c4ccdfa658824282fdc3c512cChris Lattner SourceManager &SM = PP.getSourceManager(); 9531eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 954433db06b614f26dc6829e86d6ff469e2cca7d4f9Chandler Carruth // Create a new SLoc which is expanded from Expansion(FileLoc) but whose 955df7c17a8d02fe09a3466786bae3e40fc3252687aChris Lattner // characters come from spelling(FileLoc)+Offset. 956e7fb48466afcbf2c4ccdfa658824282fdc3c512cChris Lattner SourceLocation SpellingLoc = SM.getSpellingLoc(FileLoc); 957a64ccefdf0ea4e03ec88805d71b0af74950c7472Argyrios Kyrtzidis SpellingLoc = SpellingLoc.getLocWithOffset(CharNo); 9581eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 959e7fb48466afcbf2c4ccdfa658824282fdc3c512cChris Lattner // Figure out the expansion loc range, which is the range covered by the 960e7fb48466afcbf2c4ccdfa658824282fdc3c512cChris Lattner // original _Pragma(...) sequence. 961e7fb48466afcbf2c4ccdfa658824282fdc3c512cChris Lattner std::pair<SourceLocation,SourceLocation> II = 962999f739404edf2078cf9f9c28b4dc45c19765842Chandler Carruth SM.getImmediateExpansionRange(FileLoc); 9631eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 964bf340e452339e374ea6eef78c1f0a2abdd16c5a3Chandler Carruth return SM.createExpansionLoc(SpellingLoc, II.first, II.second, TokLen); 965409a03671224d4e5bdab1594c43baf070148f830Chris Lattner} 966409a03671224d4e5bdab1594c43baf070148f830Chris Lattner 9675f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// getSourceLocation - Return a source location identifier for the specified 9685f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// offset in the current file. 969de7aeefc5573d669ed476d7bda7a8940d3bcadb7Chris LattnerSourceLocation Lexer::getSourceLocation(const char *Loc, 970de7aeefc5573d669ed476d7bda7a8940d3bcadb7Chris Lattner unsigned TokLen) const { 971448cec4c1c3705f6f49ffdefb58a7329942a2dd8Chris Lattner assert(Loc >= BufferStart && Loc <= BufferEnd && 9725f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer "Location out of range for this buffer!"); 9739dc1f530c086d2c16f8cba758b0f59a5bf41323aChris Lattner 9749dc1f530c086d2c16f8cba758b0f59a5bf41323aChris Lattner // In the normal case, we're just lexing from a simple file buffer, return 9759dc1f530c086d2c16f8cba758b0f59a5bf41323aChris Lattner // the file id from FileLoc with the offset specified. 976448cec4c1c3705f6f49ffdefb58a7329942a2dd8Chris Lattner unsigned CharNo = Loc-BufferStart; 9779dc1f530c086d2c16f8cba758b0f59a5bf41323aChris Lattner if (FileLoc.isFileID()) 978a64ccefdf0ea4e03ec88805d71b0af74950c7472Argyrios Kyrtzidis return FileLoc.getLocWithOffset(CharNo); 9791eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 9802b2453a7d8fe732561795431f39ceb2b2a832d84Chris Lattner // Otherwise, this is the _Pragma lexer case, which pretends that all of the 9812b2453a7d8fe732561795431f39ceb2b2a832d84Chris Lattner // tokens are lexed from where the _Pragma was defined. 982168ae2d44a443da75ea85db5f3b5081eb0bce113Chris Lattner assert(PP && "This doesn't work on raw lexers"); 983de7aeefc5573d669ed476d7bda7a8940d3bcadb7Chris Lattner return GetMappedTokenLoc(*PP, FileLoc, CharNo, TokLen); 9845f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 9855f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 9865f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// Diag - Forwarding function for diagnostics. This translate a source 9875f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// position in the current buffer into a SourceLocation object for rendering. 9883cbfe2c4159e0a219ae660d50625c013aa4afbd0Chris LattnerDiagnosticBuilder Lexer::Diag(const char *Loc, unsigned DiagID) const { 9893692b09faa9fe346f39bc922db6dce48cdcc3f63Chris Lattner return PP->Diag(getSourceLocation(Loc), DiagID); 9905f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 9915f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 9925f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer//===----------------------------------------------------------------------===// 9935f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// Trigraph and Escaped Newline Handling Code. 9945f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer//===----------------------------------------------------------------------===// 9955f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 9965f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// GetTrigraphCharForLetter - Given a character that occurs after a ?? pair, 9975f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// return the decoded trigraph letter it corresponds to, or '\0' if nothing. 9985f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencerstatic char GetTrigraphCharForLetter(char Letter) { 9995f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer switch (Letter) { 10005f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer default: return 0; 10015f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '=': return '#'; 10025f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case ')': return ']'; 10035f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '(': return '['; 10045f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '!': return '|'; 10055f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '\'': return '^'; 10065f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '>': return '}'; 10075f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '/': return '\\'; 10085f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '<': return '{'; 10095f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '-': return '~'; 10105f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 10115f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 10125f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 10135f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// DecodeTrigraphChar - If the specified character is a legal trigraph when 10145f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// prefixed with ??, emit a trigraph warning. If trigraphs are enabled, 10155f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// return the result character. Finally, emit a warning about trigraph use 10165f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// whether trigraphs are enabled or not. 10175f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencerstatic char DecodeTrigraphChar(const char *CP, Lexer *L) { 10185f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer char Res = GetTrigraphCharForLetter(*CP); 10193692b09faa9fe346f39bc922db6dce48cdcc3f63Chris Lattner if (!Res || !L) return Res; 10201eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 10213692b09faa9fe346f39bc922db6dce48cdcc3f63Chris Lattner if (!L->getFeatures().Trigraphs) { 102274d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner if (!L->isLexingRawMode()) 102374d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner L->Diag(CP-2, diag::trigraph_ignored); 10243692b09faa9fe346f39bc922db6dce48cdcc3f63Chris Lattner return 0; 10255f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 10261eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 102774d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner if (!L->isLexingRawMode()) 10285f9e272e632e951b1efe824cd16acb4d96077930Chris Lattner L->Diag(CP-2, diag::trigraph_converted) << StringRef(&Res, 1); 10295f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer return Res; 10305f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 10315f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 103224f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner/// getEscapedNewLineSize - Return the size of the specified escaped newline, 103324f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner/// or 0 if it is not an escaped newline. P[-1] is known to be a "\" or a 10341eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump/// trigraph equivalent on entry to this function. 103524f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattnerunsigned Lexer::getEscapedNewLineSize(const char *Ptr) { 103624f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner unsigned Size = 0; 103724f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner while (isWhitespace(Ptr[Size])) { 103824f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner ++Size; 10391eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 104024f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner if (Ptr[Size-1] != '\n' && Ptr[Size-1] != '\r') 104124f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner continue; 104224f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner 104324f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner // If this is a \r\n or \n\r, skip the other half. 104424f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner if ((Ptr[Size] == '\r' || Ptr[Size] == '\n') && 104524f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner Ptr[Size-1] != Ptr[Size]) 104624f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner ++Size; 10471eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 104824f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner return Size; 10491eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump } 10501eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 105124f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner // Not an escaped newline, must be a \t or something else. 105224f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner return 0; 105324f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner} 105424f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner 1055033749571f8d4c804eeb357c70b06424aa24503bChris Lattner/// SkipEscapedNewLines - If P points to an escaped newline (or a series of 1056033749571f8d4c804eeb357c70b06424aa24503bChris Lattner/// them), skip over them and return the first non-escaped-newline found, 1057033749571f8d4c804eeb357c70b06424aa24503bChris Lattner/// otherwise return P. 1058033749571f8d4c804eeb357c70b06424aa24503bChris Lattnerconst char *Lexer::SkipEscapedNewLines(const char *P) { 1059033749571f8d4c804eeb357c70b06424aa24503bChris Lattner while (1) { 1060033749571f8d4c804eeb357c70b06424aa24503bChris Lattner const char *AfterEscape; 1061033749571f8d4c804eeb357c70b06424aa24503bChris Lattner if (*P == '\\') { 1062033749571f8d4c804eeb357c70b06424aa24503bChris Lattner AfterEscape = P+1; 1063033749571f8d4c804eeb357c70b06424aa24503bChris Lattner } else if (*P == '?') { 1064033749571f8d4c804eeb357c70b06424aa24503bChris Lattner // If not a trigraph for escape, bail out. 1065033749571f8d4c804eeb357c70b06424aa24503bChris Lattner if (P[1] != '?' || P[2] != '/') 1066033749571f8d4c804eeb357c70b06424aa24503bChris Lattner return P; 1067033749571f8d4c804eeb357c70b06424aa24503bChris Lattner AfterEscape = P+3; 1068033749571f8d4c804eeb357c70b06424aa24503bChris Lattner } else { 1069033749571f8d4c804eeb357c70b06424aa24503bChris Lattner return P; 1070033749571f8d4c804eeb357c70b06424aa24503bChris Lattner } 10711eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 1072033749571f8d4c804eeb357c70b06424aa24503bChris Lattner unsigned NewLineSize = Lexer::getEscapedNewLineSize(AfterEscape); 1073033749571f8d4c804eeb357c70b06424aa24503bChris Lattner if (NewLineSize == 0) return P; 1074033749571f8d4c804eeb357c70b06424aa24503bChris Lattner P = AfterEscape+NewLineSize; 1075033749571f8d4c804eeb357c70b06424aa24503bChris Lattner } 1076033749571f8d4c804eeb357c70b06424aa24503bChris Lattner} 1077033749571f8d4c804eeb357c70b06424aa24503bChris Lattner 1078aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks/// \brief Checks that the given token is the first token that occurs after the 1079aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks/// given location (this excludes comments and whitespace). Returns the location 1080aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks/// immediately after the specified token. If the token is not found or the 1081aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks/// location is inside a macro, the returned source location will be invalid. 1082aca25bccefe56121b686706afc84c8cb5d46e65bAnna ZaksSourceLocation Lexer::findLocationAfterToken(SourceLocation Loc, 1083aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks tok::TokenKind TKind, 1084aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks const SourceManager &SM, 1085aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks const LangOptions &LangOpts, 1086aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks bool SkipTrailingWhitespaceAndNewLine) { 1087aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks if (Loc.isMacroID()) { 1088aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks if (!Lexer::isAtEndOfMacroExpansion(Loc, SM, LangOpts)) 1089aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks return SourceLocation(); 1090aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks Loc = SM.getExpansionRange(Loc).second; 1091aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks } 1092aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks Loc = Lexer::getLocForEndOfToken(Loc, 0, SM, LangOpts); 1093aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks 1094aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks // Break down the source location. 1095aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(Loc); 1096aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks 1097aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks // Try to load the file buffer. 1098aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks bool InvalidTemp = false; 1099aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks llvm::StringRef File = SM.getBufferData(LocInfo.first, &InvalidTemp); 1100aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks if (InvalidTemp) 1101aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks return SourceLocation(); 1102aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks 1103aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks const char *TokenBegin = File.data() + LocInfo.second; 1104aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks 1105aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks // Lex from the start of the given location. 1106aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks Lexer lexer(SM.getLocForStartOfFile(LocInfo.first), LangOpts, File.begin(), 1107aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks TokenBegin, File.end()); 1108aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks // Find the token. 1109aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks Token Tok; 1110aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks lexer.LexFromRawLexer(Tok); 1111aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks if (Tok.isNot(TKind)) 1112aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks return SourceLocation(); 1113aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks SourceLocation TokenLoc = Tok.getLocation(); 1114aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks 1115aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks // Calculate how much whitespace needs to be skipped if any. 1116aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks unsigned NumWhitespaceChars = 0; 1117aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks if (SkipTrailingWhitespaceAndNewLine) { 1118aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks const char *TokenEnd = SM.getCharacterData(TokenLoc) + 1119aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks Tok.getLength(); 1120aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks unsigned char C = *TokenEnd; 1121aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks while (isHorizontalWhitespace(C)) { 1122aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks C = *(++TokenEnd); 1123aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks NumWhitespaceChars++; 1124aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks } 1125aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks if (isVerticalWhitespace(C)) 1126aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks NumWhitespaceChars++; 1127aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks } 1128aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks 1129a64ccefdf0ea4e03ec88805d71b0af74950c7472Argyrios Kyrtzidis return TokenLoc.getLocWithOffset(Tok.getLength() + NumWhitespaceChars); 1130aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks} 113124f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner 11325f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// getCharAndSizeSlow - Peek a single 'character' from the specified buffer, 11335f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// get its size, and return it. This is tricky in several cases: 11345f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// 1. If currently at the start of a trigraph, we warn about the trigraph, 11355f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// then either return the trigraph (skipping 3 chars) or the '?', 11365f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// depending on whether trigraphs are enabled or not. 11375f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// 2. If this is an escaped newline (potentially with whitespace between 11385f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// the backslash and newline), implicitly skip the newline and return 11395f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// the char after it. 11405f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// 3. If this is a UCN, return it. FIXME: C++ UCN's? 11415f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// 11425f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// This handles the slow/uncommon case of the getCharAndSize method. Here we 11435f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// know that we can accumulate into Size, and that we have already incremented 11445f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// Ptr by Size bytes. 11455f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// 11465f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// NOTE: When this method is updated, getCharAndSizeSlowNoWarn (below) should 11475f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// be updated to match. 11485f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// 11495f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencerchar Lexer::getCharAndSizeSlow(const char *Ptr, unsigned &Size, 1150d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner Token *Tok) { 11515f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If we have a slash, look for an escaped newline. 11525f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (Ptr[0] == '\\') { 11535f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer ++Size; 11545f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer ++Ptr; 11555f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid SpencerSlash: 11565f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Common case, backslash-char where the char is not whitespace. 11575f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (!isWhitespace(Ptr[0])) return '\\'; 11581eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 11595636a3b6ece2c1f413464b72545e08eb0b7f06e4Chris Lattner // See if we have optional whitespace characters between the slash and 11605636a3b6ece2c1f413464b72545e08eb0b7f06e4Chris Lattner // newline. 116124f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner if (unsigned EscapedNewLineSize = getEscapedNewLineSize(Ptr)) { 116224f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner // Remember that this token needs to be cleaned. 116324f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner if (Tok) Tok->setFlag(Token::NeedsCleaning); 116424f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner 116524f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner // Warn if there was whitespace between the backslash and newline. 11665636a3b6ece2c1f413464b72545e08eb0b7f06e4Chris Lattner if (Ptr[0] != '\n' && Ptr[0] != '\r' && Tok && !isLexingRawMode()) 116724f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner Diag(Ptr, diag::backslash_newline_space); 11681eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 116924f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner // Found backslash<whitespace><newline>. Parse the char after it. 117024f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner Size += EscapedNewLineSize; 117124f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner Ptr += EscapedNewLineSize; 117224f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner // Use slow version to accumulate a correct size field. 117324f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner return getCharAndSizeSlow(Ptr, Size, Tok); 117424f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner } 11751eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 11765f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Otherwise, this is not an escaped newline, just return the slash. 11775f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer return '\\'; 11785f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 11791eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 11805f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If this is a trigraph, process it. 11815f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (Ptr[0] == '?' && Ptr[1] == '?') { 11825f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If this is actually a legal trigraph (not something like "??x"), emit 11835f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // a trigraph warning. If so, and if trigraphs are enabled, return it. 11845f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (char C = DecodeTrigraphChar(Ptr+2, Tok ? this : 0)) { 11855f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Remember that this token needs to be cleaned. 1186d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner if (Tok) Tok->setFlag(Token::NeedsCleaning); 11875f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 11885f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Ptr += 3; 11895f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Size += 3; 11905f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (C == '\\') goto Slash; 11915f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer return C; 11925f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 11935f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 11941eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 11955f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If this is neither, return a single character. 11965f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer ++Size; 11975f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer return *Ptr; 11985f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 11995f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 12005f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 12015f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// getCharAndSizeSlowNoWarn - Handle the slow/uncommon case of the 12025f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// getCharAndSizeNoWarn method. Here we know that we can accumulate into Size, 12035f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// and that we have already incremented Ptr by Size bytes. 12045f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// 12055f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// NOTE: When this method is updated, getCharAndSizeSlow (above) should 12065f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// be updated to match. 12075f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencerchar Lexer::getCharAndSizeSlowNoWarn(const char *Ptr, unsigned &Size, 12085f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer const LangOptions &Features) { 12095f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If we have a slash, look for an escaped newline. 12105f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (Ptr[0] == '\\') { 12115f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer ++Size; 12125f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer ++Ptr; 12135f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid SpencerSlash: 12145f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Common case, backslash-char where the char is not whitespace. 12155f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (!isWhitespace(Ptr[0])) return '\\'; 12161eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 12175f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // See if we have optional whitespace characters followed by a newline. 121824f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner if (unsigned EscapedNewLineSize = getEscapedNewLineSize(Ptr)) { 121924f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner // Found backslash<whitespace><newline>. Parse the char after it. 122024f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner Size += EscapedNewLineSize; 122124f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner Ptr += EscapedNewLineSize; 12221eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 122324f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner // Use slow version to accumulate a correct size field. 122424f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner return getCharAndSizeSlowNoWarn(Ptr, Size, Features); 122524f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner } 12261eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 12275f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Otherwise, this is not an escaped newline, just return the slash. 12285f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer return '\\'; 12295f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 12301eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 12315f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If this is a trigraph, process it. 12325f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (Features.Trigraphs && Ptr[0] == '?' && Ptr[1] == '?') { 12335f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If this is actually a legal trigraph (not something like "??x"), return 12345f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // it. 12355f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (char C = GetTrigraphCharForLetter(Ptr[2])) { 12365f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Ptr += 3; 12375f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Size += 3; 12385f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (C == '\\') goto Slash; 12395f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer return C; 12405f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 12415f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 12421eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 12435f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If this is neither, return a single character. 12445f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer ++Size; 12455f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer return *Ptr; 12465f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 12475f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 12485f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer//===----------------------------------------------------------------------===// 12495f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// Helper methods for lexing. 12505f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer//===----------------------------------------------------------------------===// 12515f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 1252f4f6c9db68465b886ec2e596feaa6ecc782395a4Douglas Gregor/// \brief Routine that indiscriminately skips bytes in the source file. 1253f4f6c9db68465b886ec2e596feaa6ecc782395a4Douglas Gregorvoid Lexer::SkipBytes(unsigned Bytes, bool StartOfLine) { 1254f4f6c9db68465b886ec2e596feaa6ecc782395a4Douglas Gregor BufferPtr += Bytes; 1255f4f6c9db68465b886ec2e596feaa6ecc782395a4Douglas Gregor if (BufferPtr > BufferEnd) 1256f4f6c9db68465b886ec2e596feaa6ecc782395a4Douglas Gregor BufferPtr = BufferEnd; 1257f4f6c9db68465b886ec2e596feaa6ecc782395a4Douglas Gregor IsAtStartOfLine = StartOfLine; 1258f4f6c9db68465b886ec2e596feaa6ecc782395a4Douglas Gregor} 1259f4f6c9db68465b886ec2e596feaa6ecc782395a4Douglas Gregor 1260d217773f106856a11879ec79dc468efefaf2ee75Chris Lattnervoid Lexer::LexIdentifier(Token &Result, const char *CurPtr) { 12615f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Match [_A-Za-z0-9]*, we have already matched [_A-Za-z$] 12625f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer unsigned Size; 12635f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer unsigned char C = *CurPtr++; 1264cd991dbb12f24368753cef086c7ad3ec203c9ea6Chris Lattner while (isIdentifierBody(C)) 12655f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer C = *CurPtr++; 1266cd991dbb12f24368753cef086c7ad3ec203c9ea6Chris Lattner 12675f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer --CurPtr; // Back up over the skipped character. 12685f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 12695f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Fast path, no $,\,? in identifier found. '\' might be an escaped newline 12705f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // or UCN, and ? might be a trigraph for '\', an escaped newline or UCN. 12715f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // FIXME: UCNs. 1272cd991dbb12f24368753cef086c7ad3ec203c9ea6Chris Lattner // 1273cd991dbb12f24368753cef086c7ad3ec203c9ea6Chris Lattner // TODO: Could merge these checks into a CharInfo flag to make the comparison 1274cd991dbb12f24368753cef086c7ad3ec203c9ea6Chris Lattner // cheaper 12755f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (C != '\\' && C != '?' && (C != '$' || !Features.DollarIdents)) { 12765f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid SpencerFinishIdentifier: 12775f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer const char *IdStart = BufferPtr; 1278c4bf2b9afb7d47445a9dc6bc848657098a4e3851Abramo Bagnara FormTokenWithChars(Result, CurPtr, tok::raw_identifier); 1279c4bf2b9afb7d47445a9dc6bc848657098a4e3851Abramo Bagnara Result.setRawIdentifierData(IdStart); 12801eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 12815f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If we are in raw mode, return this identifier raw. There is no need to 12825f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // look up identifier information or attempt to macro expand it. 1283c4bf2b9afb7d47445a9dc6bc848657098a4e3851Abramo Bagnara if (LexingRawMode) 1284c4bf2b9afb7d47445a9dc6bc848657098a4e3851Abramo Bagnara return; 12851eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 1286c4bf2b9afb7d47445a9dc6bc848657098a4e3851Abramo Bagnara // Fill in Result.IdentifierInfo and update the token kind, 1287c4bf2b9afb7d47445a9dc6bc848657098a4e3851Abramo Bagnara // looking up the identifier in the identifier table. 1288c4bf2b9afb7d47445a9dc6bc848657098a4e3851Abramo Bagnara IdentifierInfo *II = PP->LookUpIdentifierInfo(Result); 12891eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 12905f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Finally, now that we know we have an identifier, pass this off to the 12915f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // preprocessor, which may macro expand it or something. 1292d1186fa38166a581b51975f0382a45fc3a0733d0Chris Lattner if (II->isHandleIdentifierCase()) 12936a170eb3ea6d6319277becabef68eb1a26bf8766Chris Lattner PP->HandleIdentifier(Result); 12946aa52ec6b969faabf3764baf79d89810b8249a7eDouglas Gregor 12956a170eb3ea6d6319277becabef68eb1a26bf8766Chris Lattner return; 12965f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 12971eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 12985f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Otherwise, $,\,? in identifier found. Enter slower path. 12991eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 13005f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer C = getCharAndSize(CurPtr, Size); 13015f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer while (1) { 13025f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (C == '$') { 13035f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If we hit a $ and they are not supported in identifiers, we are done. 13045f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (!Features.DollarIdents) goto FinishIdentifier; 13051eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 13065f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Otherwise, emit a diagnostic and continue. 130774d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner if (!isLexingRawMode()) 130874d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner Diag(CurPtr, diag::ext_dollar_in_identifier); 13095f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(CurPtr, Size, Result); 13105f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer C = getCharAndSize(CurPtr, Size); 13115f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer continue; 13125f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } else if (!isIdentifierBody(C)) { // FIXME: UCNs. 13135f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Found end of identifier. 13145f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer goto FinishIdentifier; 13155f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 13165f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 13175f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Otherwise, this character is good, consume it. 13185f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(CurPtr, Size, Result); 13195f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 13205f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer C = getCharAndSize(CurPtr, Size); 13215f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer while (isIdentifierBody(C)) { // FIXME: UCNs. 13225f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(CurPtr, Size, Result); 13235f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer C = getCharAndSize(CurPtr, Size); 13245f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 13255f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 13265f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 13275f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 1328a75ec43d625753b4439b0d6f70bd988444c74617Douglas Gregor/// isHexaLiteral - Return true if Start points to a hex constant. 13294a551000bee716ac8b1bbe16134a53f0ad221a5aChris Lattner/// in microsoft mode (where this is supposed to be several different tokens). 13306ab55ebab20086f725c4017b48ef8d7691ef870fChris Lattnerstatic bool isHexaLiteral(const char *Start, const LangOptions &Features) { 13316ab55ebab20086f725c4017b48ef8d7691ef870fChris Lattner unsigned Size; 13326ab55ebab20086f725c4017b48ef8d7691ef870fChris Lattner char C1 = Lexer::getCharAndSizeNoWarn(Start, Size, Features); 13336ab55ebab20086f725c4017b48ef8d7691ef870fChris Lattner if (C1 != '0') 13346ab55ebab20086f725c4017b48ef8d7691ef870fChris Lattner return false; 13356ab55ebab20086f725c4017b48ef8d7691ef870fChris Lattner char C2 = Lexer::getCharAndSizeNoWarn(Start + Size, Size, Features); 13366ab55ebab20086f725c4017b48ef8d7691ef870fChris Lattner return (C2 == 'x' || C2 == 'X'); 1337a75ec43d625753b4439b0d6f70bd988444c74617Douglas Gregor} 13385f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 13395253c7ff266ae79308050c9f43d60dd1a67c5fb9Nate Begeman/// LexNumericConstant - Lex the remainder of a integer or floating point 13405f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// constant. From[-1] is the first character lexed. Return the end of the 13415f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// constant. 1342d217773f106856a11879ec79dc468efefaf2ee75Chris Lattnervoid Lexer::LexNumericConstant(Token &Result, const char *CurPtr) { 13435f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer unsigned Size; 13445f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer char C = getCharAndSize(CurPtr, Size); 13455f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer char PrevCh = 0; 13465f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer while (isNumberBody(C)) { // FIXME: UCNs? 13475f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(CurPtr, Size, Result); 13485f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer PrevCh = C; 13495f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer C = getCharAndSize(CurPtr, Size); 13505f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 13511eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 13525f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If we fell out, check for a sign, due to 1e+12. If we have one, continue. 1353b2f4a20ddc281b194caa00e850ad74759e0d50baChris Lattner if ((C == '-' || C == '+') && (PrevCh == 'E' || PrevCh == 'e')) { 1354b2f4a20ddc281b194caa00e850ad74759e0d50baChris Lattner // If we are in Microsoft mode, don't continue if the constant is hex. 1355b2f4a20ddc281b194caa00e850ad74759e0d50baChris Lattner // For example, MSVC will accept the following as 3 tokens: 0x1234567e+1 135662ec1f2fd7368542bb926c04797fb07023547694Francois Pichet if (!Features.MicrosoftExt || !isHexaLiteral(BufferPtr, Features)) 1357b2f4a20ddc281b194caa00e850ad74759e0d50baChris Lattner return LexNumericConstant(Result, ConsumeChar(CurPtr, Size, Result)); 1358b2f4a20ddc281b194caa00e850ad74759e0d50baChris Lattner } 13595f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 13605f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If we have a hex FP constant, continue. 13618c723404c34e9347fae473489194e8c200c4ac83Sean Hunt if ((C == '-' || C == '+') && (PrevCh == 'P' || PrevCh == 'p') && 1362b2f4a20ddc281b194caa00e850ad74759e0d50baChris Lattner !Features.CPlusPlus0x) 13635f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer return LexNumericConstant(Result, ConsumeChar(CurPtr, Size, Result)); 13641eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 13655f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Update the location of token as well as BufferPtr. 136647246be8ac5b0ddde6c402b8fc6946b6135487b5Chris Lattner const char *TokStart = BufferPtr; 13679e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner FormTokenWithChars(Result, CurPtr, tok::numeric_constant); 136847246be8ac5b0ddde6c402b8fc6946b6135487b5Chris Lattner Result.setLiteralData(TokStart); 13695f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 13705f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 13715f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// LexStringLiteral - Lex the remainder of a string literal, after having lexed 13725cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor/// either " or L" or u8" or u" or U". 13735cee1195584fa8672253139c86e922daeda69b9eDouglas Gregorvoid Lexer::LexStringLiteral(Token &Result, const char *CurPtr, 13745cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor tok::TokenKind Kind) { 13755f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer const char *NulCharacter = 0; // Does this string contain the \0 character? 13761eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 13775f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer char C = getAndAdvanceChar(CurPtr, Result); 13785f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer while (C != '"') { 1379571339c215dc3fa44acccb89c7c6b1d5b962b7f2Chris Lattner // Skip escaped characters. Escaped newlines will already be processed by 1380571339c215dc3fa44acccb89c7c6b1d5b962b7f2Chris Lattner // getAndAdvanceChar. 1381571339c215dc3fa44acccb89c7c6b1d5b962b7f2Chris Lattner if (C == '\\') 13825f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer C = getAndAdvanceChar(CurPtr, Result); 138333611e0d5ab1372608a7649b1877cd4300621c71Douglas Gregor 1384571339c215dc3fa44acccb89c7c6b1d5b962b7f2Chris Lattner if (C == '\n' || C == '\r' || // Newline. 138533611e0d5ab1372608a7649b1877cd4300621c71Douglas Gregor (C == 0 && CurPtr-1 == BufferEnd)) { // End of file. 13867d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis if (!isLexingRawMode() && !Features.AsmPreprocessor) 1387ff1ed985ede24b9a45e52d726db95767e35fe21eArgyrios Kyrtzidis Diag(BufferPtr, diag::warn_unterminated_string); 13889e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner FormTokenWithChars(Result, CurPtr-1, tok::unknown); 13895f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer return; 13905f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 1391571339c215dc3fa44acccb89c7c6b1d5b962b7f2Chris Lattner 13927d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis if (C == 0) { 13937d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis if (isCodeCompletionPoint(CurPtr-1)) { 13947d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis PP->CodeCompleteNaturalLanguage(); 13957d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis FormTokenWithChars(Result, CurPtr-1, tok::unknown); 13967d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis return cutOffLexing(); 13977d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis } 13987d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis 1399571339c215dc3fa44acccb89c7c6b1d5b962b7f2Chris Lattner NulCharacter = CurPtr-1; 14007d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis } 14015f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer C = getAndAdvanceChar(CurPtr, Result); 14025f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 14031eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 14045f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If a nul character existed in the string, warn about it. 140574d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner if (NulCharacter && !isLexingRawMode()) 140674d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner Diag(NulCharacter, diag::null_in_string); 14075f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 14085f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Update the location of the token as well as the BufferPtr instance var. 140947246be8ac5b0ddde6c402b8fc6946b6135487b5Chris Lattner const char *TokStart = BufferPtr; 14105cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor FormTokenWithChars(Result, CurPtr, Kind); 141147246be8ac5b0ddde6c402b8fc6946b6135487b5Chris Lattner Result.setLiteralData(TokStart); 14125f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 14135f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 14142fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper/// LexRawStringLiteral - Lex the remainder of a raw string literal, after 14152fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper/// having lexed R", LR", u8R", uR", or UR". 14162fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Toppervoid Lexer::LexRawStringLiteral(Token &Result, const char *CurPtr, 14172fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper tok::TokenKind Kind) { 14182fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper // This function doesn't use getAndAdvanceChar because C++0x [lex.pptoken]p3: 14192fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper // Between the initial and final double quote characters of the raw string, 14202fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper // any transformations performed in phases 1 and 2 (trigraphs, 14212fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper // universal-character-names, and line splicing) are reverted. 14222fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper 14232fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper unsigned PrefixLen = 0; 14242fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper 14252fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper while (PrefixLen != 16 && isRawStringDelimBody(CurPtr[PrefixLen])) 14262fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper ++PrefixLen; 14272fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper 14282fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper // If the last character was not a '(', then we didn't lex a valid delimiter. 14292fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper if (CurPtr[PrefixLen] != '(') { 14302fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper if (!isLexingRawMode()) { 14312fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper const char *PrefixEnd = &CurPtr[PrefixLen]; 14322fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper if (PrefixLen == 16) { 14332fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper Diag(PrefixEnd, diag::err_raw_delim_too_long); 14342fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper } else { 14352fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper Diag(PrefixEnd, diag::err_invalid_char_raw_delim) 14362fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper << StringRef(PrefixEnd, 1); 14372fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper } 14382fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper } 14392fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper 14402fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper // Search for the next '"' in hopes of salvaging the lexer. Unfortunately, 14412fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper // it's possible the '"' was intended to be part of the raw string, but 14422fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper // there's not much we can do about that. 14432fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper while (1) { 14442fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper char C = *CurPtr++; 14452fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper 14462fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper if (C == '"') 14472fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper break; 14482fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper if (C == 0 && CurPtr-1 == BufferEnd) { 14492fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper --CurPtr; 14502fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper break; 14512fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper } 14522fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper } 14532fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper 14542fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper FormTokenWithChars(Result, CurPtr, tok::unknown); 14552fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper return; 14562fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper } 14572fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper 14582fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper // Save prefix and move CurPtr past it 14592fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper const char *Prefix = CurPtr; 14602fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper CurPtr += PrefixLen + 1; // skip over prefix and '(' 14612fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper 14622fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper while (1) { 14632fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper char C = *CurPtr++; 14642fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper 14652fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper if (C == ')') { 14662fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper // Check for prefix match and closing quote. 14672fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper if (strncmp(CurPtr, Prefix, PrefixLen) == 0 && CurPtr[PrefixLen] == '"') { 14682fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper CurPtr += PrefixLen + 1; // skip over prefix and '"' 14692fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper break; 14702fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper } 14712fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper } else if (C == 0 && CurPtr-1 == BufferEnd) { // End of file. 14722fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper if (!isLexingRawMode()) 14732fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper Diag(BufferPtr, diag::err_unterminated_raw_string) 14742fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper << StringRef(Prefix, PrefixLen); 14752fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper FormTokenWithChars(Result, CurPtr-1, tok::unknown); 14762fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper return; 14772fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper } 14782fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper } 14792fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper 14802fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper // Update the location of token as well as BufferPtr. 14812fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper const char *TokStart = BufferPtr; 14822fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper FormTokenWithChars(Result, CurPtr, Kind); 14832fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper Result.setLiteralData(TokStart); 14842fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper} 14852fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper 14865f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// LexAngledStringLiteral - Lex the remainder of an angled string literal, 14875f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// after having lexed the '<' character. This is used for #include filenames. 1488d217773f106856a11879ec79dc468efefaf2ee75Chris Lattnervoid Lexer::LexAngledStringLiteral(Token &Result, const char *CurPtr) { 14895f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer const char *NulCharacter = 0; // Does this string contain the \0 character? 14909cb51ce03caf0755759f7e67aea7d4be684ff738Chris Lattner const char *AfterLessPos = CurPtr; 14915f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer char C = getAndAdvanceChar(CurPtr, Result); 14925f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer while (C != '>') { 14935f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Skip escaped characters. 14945f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (C == '\\') { 14955f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Skip the escaped character. 14965f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer C = getAndAdvanceChar(CurPtr, Result); 14975f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } else if (C == '\n' || C == '\r' || // Newline. 14987d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis (C == 0 && (CurPtr-1 == BufferEnd || // End of file. 14997d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis isCodeCompletionPoint(CurPtr-1)))) { 15009cb51ce03caf0755759f7e67aea7d4be684ff738Chris Lattner // If the filename is unterminated, then it must just be a lone < 15019cb51ce03caf0755759f7e67aea7d4be684ff738Chris Lattner // character. Return this as such. 15029cb51ce03caf0755759f7e67aea7d4be684ff738Chris Lattner FormTokenWithChars(Result, AfterLessPos, tok::less); 15035f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer return; 15045f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } else if (C == 0) { 15055f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer NulCharacter = CurPtr-1; 15065f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 15075f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer C = getAndAdvanceChar(CurPtr, Result); 15085f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 15091eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 15105f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If a nul character existed in the string, warn about it. 151174d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner if (NulCharacter && !isLexingRawMode()) 151274d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner Diag(NulCharacter, diag::null_in_string); 15131eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 15145f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Update the location of token as well as BufferPtr. 151547246be8ac5b0ddde6c402b8fc6946b6135487b5Chris Lattner const char *TokStart = BufferPtr; 15169e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner FormTokenWithChars(Result, CurPtr, tok::angle_string_literal); 151747246be8ac5b0ddde6c402b8fc6946b6135487b5Chris Lattner Result.setLiteralData(TokStart); 15185f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 15195f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 15205f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 15215f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// LexCharConstant - Lex the remainder of a character constant, after having 15225cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor/// lexed either ' or L' or u' or U'. 15235cee1195584fa8672253139c86e922daeda69b9eDouglas Gregorvoid Lexer::LexCharConstant(Token &Result, const char *CurPtr, 15245cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor tok::TokenKind Kind) { 15255f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer const char *NulCharacter = 0; // Does this character contain the \0 character? 15265f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 15275f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer char C = getAndAdvanceChar(CurPtr, Result); 15285f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (C == '\'') { 152933ab3f6c7e3065550de202088f8f335ecfa16ae1Chris Lattner if (!isLexingRawMode() && !Features.AsmPreprocessor) 153074d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner Diag(BufferPtr, diag::err_empty_character); 15319e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner FormTokenWithChars(Result, CurPtr, tok::unknown); 15325f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer return; 15335f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 15341eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 1535d80f786689d608e5c22d6e1045884de7aff76c40Chris Lattner while (C != '\'') { 1536d80f786689d608e5c22d6e1045884de7aff76c40Chris Lattner // Skip escaped characters. 1537d80f786689d608e5c22d6e1045884de7aff76c40Chris Lattner if (C == '\\') { 1538d80f786689d608e5c22d6e1045884de7aff76c40Chris Lattner // Skip the escaped character. 1539d80f786689d608e5c22d6e1045884de7aff76c40Chris Lattner // FIXME: UCN's 15405f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer C = getAndAdvanceChar(CurPtr, Result); 1541d80f786689d608e5c22d6e1045884de7aff76c40Chris Lattner } else if (C == '\n' || C == '\r' || // Newline. 1542d80f786689d608e5c22d6e1045884de7aff76c40Chris Lattner (C == 0 && CurPtr-1 == BufferEnd)) { // End of file. 15437d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis if (!isLexingRawMode() && !Features.AsmPreprocessor) 1544ff1ed985ede24b9a45e52d726db95767e35fe21eArgyrios Kyrtzidis Diag(BufferPtr, diag::warn_unterminated_char); 1545d80f786689d608e5c22d6e1045884de7aff76c40Chris Lattner FormTokenWithChars(Result, CurPtr-1, tok::unknown); 1546d80f786689d608e5c22d6e1045884de7aff76c40Chris Lattner return; 1547d80f786689d608e5c22d6e1045884de7aff76c40Chris Lattner } else if (C == 0) { 15487d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis if (isCodeCompletionPoint(CurPtr-1)) { 15497d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis PP->CodeCompleteNaturalLanguage(); 15507d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis FormTokenWithChars(Result, CurPtr-1, tok::unknown); 15517d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis return cutOffLexing(); 15527d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis } 15537d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis 1554d80f786689d608e5c22d6e1045884de7aff76c40Chris Lattner NulCharacter = CurPtr-1; 1555d80f786689d608e5c22d6e1045884de7aff76c40Chris Lattner } 1556d80f786689d608e5c22d6e1045884de7aff76c40Chris Lattner C = getAndAdvanceChar(CurPtr, Result); 15575f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 15581eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 1559d80f786689d608e5c22d6e1045884de7aff76c40Chris Lattner // If a nul character existed in the character, warn about it. 156074d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner if (NulCharacter && !isLexingRawMode()) 156174d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner Diag(NulCharacter, diag::null_in_char); 15625f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 15635f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Update the location of token as well as BufferPtr. 156447246be8ac5b0ddde6c402b8fc6946b6135487b5Chris Lattner const char *TokStart = BufferPtr; 15655cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor FormTokenWithChars(Result, CurPtr, Kind); 156647246be8ac5b0ddde6c402b8fc6946b6135487b5Chris Lattner Result.setLiteralData(TokStart); 15675f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 15685f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 15695f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// SkipWhitespace - Efficiently skip over a series of whitespace characters. 15705f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// Update BufferPtr to point to the next non-whitespace character and return. 1571d88dc48e33d71732708960170e57a3d1bdc8f847Chris Lattner/// 1572d88dc48e33d71732708960170e57a3d1bdc8f847Chris Lattner/// This method forms a token and returns true if KeepWhitespaceMode is enabled. 1573d88dc48e33d71732708960170e57a3d1bdc8f847Chris Lattner/// 1574d88dc48e33d71732708960170e57a3d1bdc8f847Chris Lattnerbool Lexer::SkipWhitespace(Token &Result, const char *CurPtr) { 15755f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Whitespace - Skip it, then return the token after the whitespace. 15765f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer unsigned char Char = *CurPtr; // Skip consequtive spaces efficiently. 15775f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer while (1) { 15785f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Skip horizontal whitespace very aggressively. 15795f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer while (isHorizontalWhitespace(Char)) 15805f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Char = *++CurPtr; 15811eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 1582ddd3e8b90a92c39f620bda7a0945320c0b9e60dbDaniel Dunbar // Otherwise if we have something other than whitespace, we're done. 15835f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (Char != '\n' && Char != '\r') 15845f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 15851eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 15865f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (ParsingPreprocessorDirective) { 15875f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // End of preprocessor directive line, let LexTokenInternal handle this. 15885f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer BufferPtr = CurPtr; 1589d88dc48e33d71732708960170e57a3d1bdc8f847Chris Lattner return false; 15905f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 15911eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 15925f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // ok, but handle newline. 15935f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // The returned token is at the start of the line. 1594d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner Result.setFlag(Token::StartOfLine); 15955f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // No leading whitespace seen so far. 1596d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner Result.clearFlag(Token::LeadingSpace); 15975f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Char = *++CurPtr; 15985f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 15995f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 16005f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If this isn't immediately after a newline, there is leading space. 16015f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer char PrevChar = CurPtr[-1]; 16025f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (PrevChar != '\n' && PrevChar != '\r') 1603d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner Result.setFlag(Token::LeadingSpace); 16045f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 1605d88dc48e33d71732708960170e57a3d1bdc8f847Chris Lattner // If the client wants us to return whitespace, return it now. 1606d88dc48e33d71732708960170e57a3d1bdc8f847Chris Lattner if (isKeepWhitespaceMode()) { 16079e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner FormTokenWithChars(Result, CurPtr, tok::unknown); 1608d88dc48e33d71732708960170e57a3d1bdc8f847Chris Lattner return true; 1609d88dc48e33d71732708960170e57a3d1bdc8f847Chris Lattner } 16101eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 16115f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer BufferPtr = CurPtr; 1612d88dc48e33d71732708960170e57a3d1bdc8f847Chris Lattner return false; 16135f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 16145f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 16155f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// SkipBCPLComment - We have just read the // characters from input. Skip until 16165f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// we find the newline character thats terminate the comment. Then update 1617046c2277dcbcc8eb89dbb5b1b8c5226b7cb81635Chris Lattner/// BufferPtr and return. 1618046c2277dcbcc8eb89dbb5b1b8c5226b7cb81635Chris Lattner/// 1619046c2277dcbcc8eb89dbb5b1b8c5226b7cb81635Chris Lattner/// If we're in KeepCommentMode or any CommentHandler has inserted 1620046c2277dcbcc8eb89dbb5b1b8c5226b7cb81635Chris Lattner/// some tokens, this will store the first token and return true. 1621d217773f106856a11879ec79dc468efefaf2ee75Chris Lattnerbool Lexer::SkipBCPLComment(Token &Result, const char *CurPtr) { 16225f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If BCPL comments aren't explicitly enabled for this language, emit an 16235f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // extension warning. 162474d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner if (!Features.BCPLComment && !isLexingRawMode()) { 16255f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Diag(BufferPtr, diag::ext_bcpl_comment); 16261eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 16275f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Mark them enabled so we only emit one warning for this translation 16285f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // unit. 16295f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Features.BCPLComment = true; 16305f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 16311eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 16325f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Scan over the body of the comment. The common case, when scanning, is that 16335f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // the comment contains normal ascii characters with nothing interesting in 16345f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // them. As such, optimize for this case with the inner loop. 16355f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer char C; 16365f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer do { 16375f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer C = *CurPtr; 16385f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Skip over characters in the fast loop. 16395f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer while (C != 0 && // Potentially EOF. 16405f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer C != '\n' && C != '\r') // Newline or DOS-style newline. 16415f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer C = *++CurPtr; 16425f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 16431daa58ea8973854b62e79f0939fa37cb6fed00e6Benjamin Kramer const char *NextLine = CurPtr; 16441daa58ea8973854b62e79f0939fa37cb6fed00e6Benjamin Kramer if (C != 0) { 16451daa58ea8973854b62e79f0939fa37cb6fed00e6Benjamin Kramer // We found a newline, see if it's escaped. 16461daa58ea8973854b62e79f0939fa37cb6fed00e6Benjamin Kramer const char *EscapePtr = CurPtr-1; 16471daa58ea8973854b62e79f0939fa37cb6fed00e6Benjamin Kramer while (isHorizontalWhitespace(*EscapePtr)) // Skip whitespace. 16481daa58ea8973854b62e79f0939fa37cb6fed00e6Benjamin Kramer --EscapePtr; 16491daa58ea8973854b62e79f0939fa37cb6fed00e6Benjamin Kramer 16501daa58ea8973854b62e79f0939fa37cb6fed00e6Benjamin Kramer if (*EscapePtr == '\\') // Escaped newline. 16511daa58ea8973854b62e79f0939fa37cb6fed00e6Benjamin Kramer CurPtr = EscapePtr; 16521daa58ea8973854b62e79f0939fa37cb6fed00e6Benjamin Kramer else if (EscapePtr[0] == '/' && EscapePtr[-1] == '?' && 16531daa58ea8973854b62e79f0939fa37cb6fed00e6Benjamin Kramer EscapePtr[-2] == '?') // Trigraph-escaped newline. 16541daa58ea8973854b62e79f0939fa37cb6fed00e6Benjamin Kramer CurPtr = EscapePtr-2; 16551daa58ea8973854b62e79f0939fa37cb6fed00e6Benjamin Kramer else 16561daa58ea8973854b62e79f0939fa37cb6fed00e6Benjamin Kramer break; // This is a newline, we're done. 16571daa58ea8973854b62e79f0939fa37cb6fed00e6Benjamin Kramer 16581daa58ea8973854b62e79f0939fa37cb6fed00e6Benjamin Kramer C = *CurPtr; 16591daa58ea8973854b62e79f0939fa37cb6fed00e6Benjamin Kramer } 16601eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 16615f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Otherwise, this is a hard case. Fall back on getAndAdvanceChar to 1662bc3e9842212b78bcf49682641307e4f3419ecc5eChris Lattner // properly decode the character. Read it in raw mode to avoid emitting 1663bc3e9842212b78bcf49682641307e4f3419ecc5eChris Lattner // diagnostics about things like trigraphs. If we see an escaped newline, 1664bc3e9842212b78bcf49682641307e4f3419ecc5eChris Lattner // we'll handle it below. 16655f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer const char *OldPtr = CurPtr; 1666bc3e9842212b78bcf49682641307e4f3419ecc5eChris Lattner bool OldRawMode = isLexingRawMode(); 1667bc3e9842212b78bcf49682641307e4f3419ecc5eChris Lattner LexingRawMode = true; 16685f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer C = getAndAdvanceChar(CurPtr, Result); 1669bc3e9842212b78bcf49682641307e4f3419ecc5eChris Lattner LexingRawMode = OldRawMode; 1670ead616c5d8681a41b11273327813e61bda01907aChris Lattner 16711daa58ea8973854b62e79f0939fa37cb6fed00e6Benjamin Kramer // If we only read only one character, then no special handling is needed. 16721daa58ea8973854b62e79f0939fa37cb6fed00e6Benjamin Kramer // We're done and can skip forward to the newline. 16731daa58ea8973854b62e79f0939fa37cb6fed00e6Benjamin Kramer if (C != 0 && CurPtr == OldPtr+1) { 16741daa58ea8973854b62e79f0939fa37cb6fed00e6Benjamin Kramer CurPtr = NextLine; 16751daa58ea8973854b62e79f0939fa37cb6fed00e6Benjamin Kramer break; 16761daa58ea8973854b62e79f0939fa37cb6fed00e6Benjamin Kramer } 16771daa58ea8973854b62e79f0939fa37cb6fed00e6Benjamin Kramer 1678ead616c5d8681a41b11273327813e61bda01907aChris Lattner // If the char that we finally got was a \n, then we must have had something 1679ead616c5d8681a41b11273327813e61bda01907aChris Lattner // like \<newline><newline>. We don't want to have consumed the second 1680ead616c5d8681a41b11273327813e61bda01907aChris Lattner // newline, we want CurPtr, to end up pointing to it down below. 1681ead616c5d8681a41b11273327813e61bda01907aChris Lattner if (C == '\n' || C == '\r') { 1682ead616c5d8681a41b11273327813e61bda01907aChris Lattner --CurPtr; 1683ead616c5d8681a41b11273327813e61bda01907aChris Lattner C = 'x'; // doesn't matter what this is. 1684ead616c5d8681a41b11273327813e61bda01907aChris Lattner } 16851eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 16865f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If we read multiple characters, and one of those characters was a \r or 16875f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // \n, then we had an escaped newline within the comment. Emit diagnostic 16885f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // unless the next line is also a // comment. 16895f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (CurPtr != OldPtr+1 && C != '/' && CurPtr[0] != '/') { 16905f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer for (; OldPtr != CurPtr; ++OldPtr) 16915f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (OldPtr[0] == '\n' || OldPtr[0] == '\r') { 16925f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Okay, we found a // comment that ends in a newline, if the next 16935f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // line is also a // comment, but has spaces, don't emit a diagnostic. 16945d6ae288bc661572e52ea71fc14b249eb5e2d9e9Benjamin Kramer if (isWhitespace(C)) { 16955f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer const char *ForwardPtr = CurPtr; 16965d6ae288bc661572e52ea71fc14b249eb5e2d9e9Benjamin Kramer while (isWhitespace(*ForwardPtr)) // Skip whitespace. 16975f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer ++ForwardPtr; 16985f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (ForwardPtr[0] == '/' && ForwardPtr[1] == '/') 16995f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 17005f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 17011eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 170274d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner if (!isLexingRawMode()) 170374d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner Diag(OldPtr-1, diag::ext_multi_line_bcpl_comment); 17045f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 17055f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 17065f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 17071eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 170855817afdf9d453a443262a733f6caf6692dca118Douglas Gregor if (CurPtr == BufferEnd+1) { 170955817afdf9d453a443262a733f6caf6692dca118Douglas Gregor --CurPtr; 171055817afdf9d453a443262a733f6caf6692dca118Douglas Gregor break; 171155817afdf9d453a443262a733f6caf6692dca118Douglas Gregor } 17127d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis 17137d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis if (C == '\0' && isCodeCompletionPoint(CurPtr-1)) { 17147d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis PP->CodeCompleteNaturalLanguage(); 17157d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis cutOffLexing(); 17167d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis return false; 17177d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis } 17187d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis 17195f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } while (C != '\n' && C != '\r'); 17205f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 17213d0ad58b28e0d50fca7f21c6a078b05370510288Chris Lattner // Found but did not consume the newline. Notify comment handlers about the 17223d0ad58b28e0d50fca7f21c6a078b05370510288Chris Lattner // comment unless we're in a #if 0 block. 17233d0ad58b28e0d50fca7f21c6a078b05370510288Chris Lattner if (PP && !isLexingRawMode() && 17243d0ad58b28e0d50fca7f21c6a078b05370510288Chris Lattner PP->HandleComment(Result, SourceRange(getSourceLocation(BufferPtr), 17253d0ad58b28e0d50fca7f21c6a078b05370510288Chris Lattner getSourceLocation(CurPtr)))) { 1726046c2277dcbcc8eb89dbb5b1b8c5226b7cb81635Chris Lattner BufferPtr = CurPtr; 1727046c2277dcbcc8eb89dbb5b1b8c5226b7cb81635Chris Lattner return true; // A token has to be returned. 1728046c2277dcbcc8eb89dbb5b1b8c5226b7cb81635Chris Lattner } 17291eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 17305f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If we are returning comments as tokens, return this comment as a token. 1731fa95a019da00b926d64ff83358ba73bbc6ae1e37Chris Lattner if (inKeepCommentMode()) 17325f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer return SaveBCPLComment(Result, CurPtr); 17335f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 17345f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If we are inside a preprocessor directive and we see the end of line, 173584021556baceb76eedf7d44be8ba71d9b8cfaccePeter Collingbourne // return immediately, so that the lexer can return this as an EOD token. 17365f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (ParsingPreprocessorDirective || CurPtr == BufferEnd) { 17375f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer BufferPtr = CurPtr; 17382d3818959df7b71bfcf8c2f2b573856bfc4c3a21Chris Lattner return false; 17395f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 17401eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 17415f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Otherwise, eat the \n character. We don't care if this is a \n\r or 17427a4f004a77421af876152281b92c4f8ea24afb2dChris Lattner // \r\n sequence. This is an efficiency hack (because we know the \n can't 1743d88dc48e33d71732708960170e57a3d1bdc8f847Chris Lattner // contribute to another token), it isn't needed for correctness. Note that 1744d88dc48e33d71732708960170e57a3d1bdc8f847Chris Lattner // this is ok even in KeepWhitespaceMode, because we would have returned the 1745d88dc48e33d71732708960170e57a3d1bdc8f847Chris Lattner /// comment above in that mode. 17465f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer ++CurPtr; 17471eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 17485f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // The next returned token is at the start of the line. 1749d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner Result.setFlag(Token::StartOfLine); 17505f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // No leading whitespace seen so far. 1751d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner Result.clearFlag(Token::LeadingSpace); 17525f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer BufferPtr = CurPtr; 17532d3818959df7b71bfcf8c2f2b573856bfc4c3a21Chris Lattner return false; 17545f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 17555f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 17565f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// SaveBCPLComment - If in save-comment mode, package up this BCPL comment in 17575f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// an appropriate way and return it. 1758d217773f106856a11879ec79dc468efefaf2ee75Chris Lattnerbool Lexer::SaveBCPLComment(Token &Result, const char *CurPtr) { 17599e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner // If we're not in a preprocessor directive, just return the // comment 17609e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner // directly. 17619e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner FormTokenWithChars(Result, CurPtr, tok::comment); 17621eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 17639e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner if (!ParsingPreprocessorDirective) 17649e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner return true; 17651eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 17669e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner // If this BCPL-style comment is in a macro definition, transmogrify it into 17679e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner // a C-style block comment. 1768453091cc2082e207ea2c2dda645a9bc01b37fb0cDouglas Gregor bool Invalid = false; 1769453091cc2082e207ea2c2dda645a9bc01b37fb0cDouglas Gregor std::string Spelling = PP->getSpelling(Result, &Invalid); 1770453091cc2082e207ea2c2dda645a9bc01b37fb0cDouglas Gregor if (Invalid) 1771453091cc2082e207ea2c2dda645a9bc01b37fb0cDouglas Gregor return true; 1772453091cc2082e207ea2c2dda645a9bc01b37fb0cDouglas Gregor 17739e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner assert(Spelling[0] == '/' && Spelling[1] == '/' && "Not bcpl comment?"); 17749e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Spelling[1] = '*'; // Change prefix to "/*". 17759e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Spelling += "*/"; // add suffix. 17761eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 17779e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Result.setKind(tok::comment); 177847246be8ac5b0ddde6c402b8fc6946b6135487b5Chris Lattner PP->CreateString(&Spelling[0], Spelling.size(), Result, 177947246be8ac5b0ddde6c402b8fc6946b6135487b5Chris Lattner Result.getLocation()); 17802d3818959df7b71bfcf8c2f2b573856bfc4c3a21Chris Lattner return true; 17815f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 17825f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 17835f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// isBlockCommentEndOfEscapedNewLine - Return true if the specified newline 17845f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// character (either \n or \r) is part of an escaped newline sequence. Issue a 178547a2b40b7a3d9c89d6c3d12e5a81e2498c207750Chris Lattner/// diagnostic if so. We know that the newline is inside of a block comment. 17861eb4433ac451dc16f4133a88af2d002ac26c58efMike Stumpstatic bool isEndOfBlockCommentWithEscapedNewLine(const char *CurPtr, 17875f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Lexer *L) { 17885f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer assert(CurPtr[0] == '\n' || CurPtr[0] == '\r'); 17891eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 17905f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Back up off the newline. 17915f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer --CurPtr; 17921eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 17935f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If this is a two-character newline sequence, skip the other character. 17945f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (CurPtr[0] == '\n' || CurPtr[0] == '\r') { 17955f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // \n\n or \r\r -> not escaped newline. 17965f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (CurPtr[0] == CurPtr[1]) 17975f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer return false; 17985f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // \n\r or \r\n -> skip the newline. 17995f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer --CurPtr; 18005f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 18011eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 18025f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If we have horizontal whitespace, skip over it. We allow whitespace 18035f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // between the slash and newline. 18045f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer bool HasSpace = false; 18055f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer while (isHorizontalWhitespace(*CurPtr) || *CurPtr == 0) { 18065f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer --CurPtr; 18075f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer HasSpace = true; 18085f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 18091eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 18105f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If we have a slash, we know this is an escaped newline. 18115f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (*CurPtr == '\\') { 18125f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (CurPtr[-1] != '*') return false; 18135f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } else { 18145f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // It isn't a slash, is it the ?? / trigraph? 18155f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (CurPtr[0] != '/' || CurPtr[-1] != '?' || CurPtr[-2] != '?' || 18165f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr[-3] != '*') 18175f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer return false; 18181eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 18195f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // This is the trigraph ending the comment. Emit a stern warning! 18205f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr -= 2; 18215f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 18225f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If no trigraphs are enabled, warn that we ignored this trigraph and 18235f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // ignore this * character. 18245f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (!L->getFeatures().Trigraphs) { 182574d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner if (!L->isLexingRawMode()) 182674d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner L->Diag(CurPtr, diag::trigraph_ignored_block_comment); 18275f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer return false; 18285f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 182974d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner if (!L->isLexingRawMode()) 183074d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner L->Diag(CurPtr, diag::trigraph_ends_block_comment); 18315f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 18321eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 18335f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Warn about having an escaped newline between the */ characters. 183474d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner if (!L->isLexingRawMode()) 183574d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner L->Diag(CurPtr, diag::escaped_newline_block_comment_end); 18361eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 18375f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If there was space between the backslash and newline, warn about it. 183874d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner if (HasSpace && !L->isLexingRawMode()) 183974d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner L->Diag(CurPtr, diag::backslash_newline_space); 18401eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 18415f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer return true; 18425f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 18435f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 18445f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer#ifdef __SSE2__ 18455f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer#include <emmintrin.h> 18465f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer#elif __ALTIVEC__ 18475f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer#include <altivec.h> 18485f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer#undef bool 18495f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer#endif 18505f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 18515f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// SkipBlockComment - We have just read the /* characters from input. Read 18525f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// until we find the */ characters that terminate the comment. Note that we 18535f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// don't bother decoding trigraphs or escaped newlines in block comments, 18545f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// because they cannot cause the comment to end. The only thing that can 18555f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// happen is the comment could end with an escaped newline between the */ end 18565f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// of comment. 18572d3818959df7b71bfcf8c2f2b573856bfc4c3a21Chris Lattner/// 1858046c2277dcbcc8eb89dbb5b1b8c5226b7cb81635Chris Lattner/// If we're in KeepCommentMode or any CommentHandler has inserted 1859046c2277dcbcc8eb89dbb5b1b8c5226b7cb81635Chris Lattner/// some tokens, this will store the first token and return true. 1860d217773f106856a11879ec79dc468efefaf2ee75Chris Lattnerbool Lexer::SkipBlockComment(Token &Result, const char *CurPtr) { 18615f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Scan one character past where we should, looking for a '/' character. Once 1862fc8f0e14ad142ed811e90fbd9a30e419e301c717Chris Lattner // we find it, check to see if it was preceded by a *. This common 18635f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // optimization helps people who like to put a lot of * characters in their 18645f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // comments. 18658146b6851e873eab71341f1b1d3198894bc4c0acChris Lattner 18668146b6851e873eab71341f1b1d3198894bc4c0acChris Lattner // The first character we get with newlines and trigraphs skipped to handle 18678146b6851e873eab71341f1b1d3198894bc4c0acChris Lattner // the degenerate /*/ case below correctly if the * has an escaped newline 18688146b6851e873eab71341f1b1d3198894bc4c0acChris Lattner // after it. 18698146b6851e873eab71341f1b1d3198894bc4c0acChris Lattner unsigned CharSize; 18708146b6851e873eab71341f1b1d3198894bc4c0acChris Lattner unsigned char C = getCharAndSize(CurPtr, CharSize); 18718146b6851e873eab71341f1b1d3198894bc4c0acChris Lattner CurPtr += CharSize; 18725f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (C == 0 && CurPtr == BufferEnd+1) { 18737d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis if (!isLexingRawMode()) 18740af574270d3be2b0e73a3379dfaa633746f8fc6fChris Lattner Diag(BufferPtr, diag::err_unterminated_block_comment); 187531f0ecae4b2d4d268d04c562a43d9fbfee96f189Chris Lattner --CurPtr; 18761eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 187731f0ecae4b2d4d268d04c562a43d9fbfee96f189Chris Lattner // KeepWhitespaceMode should return this broken comment as a token. Since 187831f0ecae4b2d4d268d04c562a43d9fbfee96f189Chris Lattner // it isn't a well formed comment, just return it as an 'unknown' token. 187931f0ecae4b2d4d268d04c562a43d9fbfee96f189Chris Lattner if (isKeepWhitespaceMode()) { 18809e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner FormTokenWithChars(Result, CurPtr, tok::unknown); 188131f0ecae4b2d4d268d04c562a43d9fbfee96f189Chris Lattner return true; 188231f0ecae4b2d4d268d04c562a43d9fbfee96f189Chris Lattner } 18831eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 188431f0ecae4b2d4d268d04c562a43d9fbfee96f189Chris Lattner BufferPtr = CurPtr; 18852d3818959df7b71bfcf8c2f2b573856bfc4c3a21Chris Lattner return false; 18865f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 18871eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 18888146b6851e873eab71341f1b1d3198894bc4c0acChris Lattner // Check to see if the first character after the '/*' is another /. If so, 18898146b6851e873eab71341f1b1d3198894bc4c0acChris Lattner // then this slash does not end the block comment, it is part of it. 18908146b6851e873eab71341f1b1d3198894bc4c0acChris Lattner if (C == '/') 18918146b6851e873eab71341f1b1d3198894bc4c0acChris Lattner C = *CurPtr++; 18921eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 18935f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer while (1) { 18945f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Skip over all non-interesting characters until we find end of buffer or a 18955f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // (probably ending) '/' character. 18967d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis if (CurPtr + 24 < BufferEnd && 18977d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis // If there is a code-completion point avoid the fast scan because it 18987d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis // doesn't check for '\0'. 18997d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis !(PP && PP->getCodeCompletionFileLoc() == FileLoc)) { 19005f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // While not aligned to a 16-byte boundary. 19015f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer while (C != '/' && ((intptr_t)CurPtr & 0x0F) != 0) 19025f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer C = *CurPtr++; 19031eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 19045f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (C == '/') goto FoundSlash; 19055f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 19065f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer#ifdef __SSE2__ 19075f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer __m128i Slashes = _mm_set_epi8('/', '/', '/', '/', '/', '/', '/', '/', 19085f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer '/', '/', '/', '/', '/', '/', '/', '/'); 19095f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer while (CurPtr+16 <= BufferEnd && 19105f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer _mm_movemask_epi8(_mm_cmpeq_epi8(*(__m128i*)CurPtr, Slashes)) == 0) 19115f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr += 16; 19125f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer#elif __ALTIVEC__ 19135f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer __vector unsigned char Slashes = { 19141eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump '/', '/', '/', '/', '/', '/', '/', '/', 19155f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer '/', '/', '/', '/', '/', '/', '/', '/' 19165f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer }; 19175f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer while (CurPtr+16 <= BufferEnd && 19185f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer !vec_any_eq(*(vector unsigned char*)CurPtr, Slashes)) 19195f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr += 16; 19201eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump#else 19215f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Scan for '/' quickly. Many block comments are very large. 19225f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer while (CurPtr[0] != '/' && 19235f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr[1] != '/' && 19245f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr[2] != '/' && 19255f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr[3] != '/' && 19265f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr+4 < BufferEnd) { 19275f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr += 4; 19285f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 19295f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer#endif 19301eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 19315f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // It has to be one of the bytes scanned, increment to it and read one. 19325f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer C = *CurPtr++; 19335f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 19341eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 19355f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Loop to scan the remainder. 19365f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer while (C != '/' && C != '\0') 19375f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer C = *CurPtr++; 19381eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 19395f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer FoundSlash: 19405f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (C == '/') { 19415f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (CurPtr[-2] == '*') // We found the final */. We're done! 19425f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 19431eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 19445f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if ((CurPtr[-2] == '\n' || CurPtr[-2] == '\r')) { 19455f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (isEndOfBlockCommentWithEscapedNewLine(CurPtr-2, this)) { 19465f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // We found the final */, though it had an escaped newline between the 19475f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // * and /. We're done! 19485f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 19495f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 19505f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 19515f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (CurPtr[0] == '*' && CurPtr[1] != '/') { 19525f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If this is a /* inside of the comment, emit a warning. Don't do this 19535f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // if this is a /*/, which will end the comment. This misses cases with 19545f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // embedded escaped newlines, but oh well. 195574d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner if (!isLexingRawMode()) 195674d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner Diag(CurPtr-1, diag::warn_nested_block_comment); 19575f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 19585f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } else if (C == 0 && CurPtr == BufferEnd+1) { 19597d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis if (!isLexingRawMode()) 196074d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner Diag(BufferPtr, diag::err_unterminated_block_comment); 19615f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Note: the user probably forgot a */. We could continue immediately 19625f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // after the /*, but this would involve lexing a lot of what really is the 19635f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // comment, which surely would confuse the parser. 196431f0ecae4b2d4d268d04c562a43d9fbfee96f189Chris Lattner --CurPtr; 19651eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 196631f0ecae4b2d4d268d04c562a43d9fbfee96f189Chris Lattner // KeepWhitespaceMode should return this broken comment as a token. Since 196731f0ecae4b2d4d268d04c562a43d9fbfee96f189Chris Lattner // it isn't a well formed comment, just return it as an 'unknown' token. 196831f0ecae4b2d4d268d04c562a43d9fbfee96f189Chris Lattner if (isKeepWhitespaceMode()) { 19699e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner FormTokenWithChars(Result, CurPtr, tok::unknown); 197031f0ecae4b2d4d268d04c562a43d9fbfee96f189Chris Lattner return true; 197131f0ecae4b2d4d268d04c562a43d9fbfee96f189Chris Lattner } 19721eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 197331f0ecae4b2d4d268d04c562a43d9fbfee96f189Chris Lattner BufferPtr = CurPtr; 19742d3818959df7b71bfcf8c2f2b573856bfc4c3a21Chris Lattner return false; 19757d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis } else if (C == '\0' && isCodeCompletionPoint(CurPtr-1)) { 19767d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis PP->CodeCompleteNaturalLanguage(); 19777d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis cutOffLexing(); 19787d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis return false; 19795f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 19807d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis 19815f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer C = *CurPtr++; 19825f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 19831eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 19843d0ad58b28e0d50fca7f21c6a078b05370510288Chris Lattner // Notify comment handlers about the comment unless we're in a #if 0 block. 19853d0ad58b28e0d50fca7f21c6a078b05370510288Chris Lattner if (PP && !isLexingRawMode() && 19863d0ad58b28e0d50fca7f21c6a078b05370510288Chris Lattner PP->HandleComment(Result, SourceRange(getSourceLocation(BufferPtr), 19873d0ad58b28e0d50fca7f21c6a078b05370510288Chris Lattner getSourceLocation(CurPtr)))) { 1988046c2277dcbcc8eb89dbb5b1b8c5226b7cb81635Chris Lattner BufferPtr = CurPtr; 1989046c2277dcbcc8eb89dbb5b1b8c5226b7cb81635Chris Lattner return true; // A token has to be returned. 1990046c2277dcbcc8eb89dbb5b1b8c5226b7cb81635Chris Lattner } 19912e22253e03e175144aeb9d13350a12fd83f858beDouglas Gregor 19925f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If we are returning comments as tokens, return this comment as a token. 1993fa95a019da00b926d64ff83358ba73bbc6ae1e37Chris Lattner if (inKeepCommentMode()) { 19949e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner FormTokenWithChars(Result, CurPtr, tok::comment); 19952d3818959df7b71bfcf8c2f2b573856bfc4c3a21Chris Lattner return true; 19965f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 19975f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 19985f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // It is common for the tokens immediately after a /**/ comment to be 19995f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // whitespace. Instead of going through the big switch, handle it 2000d88dc48e33d71732708960170e57a3d1bdc8f847Chris Lattner // efficiently now. This is safe even in KeepWhitespaceMode because we would 2001d88dc48e33d71732708960170e57a3d1bdc8f847Chris Lattner // have already returned above with the comment as a token. 20025f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (isHorizontalWhitespace(*CurPtr)) { 2003d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner Result.setFlag(Token::LeadingSpace); 20045f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer SkipWhitespace(Result, CurPtr+1); 20052d3818959df7b71bfcf8c2f2b573856bfc4c3a21Chris Lattner return false; 20065f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 20075f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 20085f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Otherwise, just return so that the next character will be lexed as a token. 20095f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer BufferPtr = CurPtr; 2010d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner Result.setFlag(Token::LeadingSpace); 20112d3818959df7b71bfcf8c2f2b573856bfc4c3a21Chris Lattner return false; 20125f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 20135f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 20145f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer//===----------------------------------------------------------------------===// 20155f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// Primary Lexing Entry Points 20165f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer//===----------------------------------------------------------------------===// 20175f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 20185f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// ReadToEndOfLine - Read the rest of the current preprocessor line as an 20195f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// uninterpreted string. This switches the lexer out of directive mode. 20205f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencerstd::string Lexer::ReadToEndOfLine() { 20215f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer assert(ParsingPreprocessorDirective && ParsingFilename == false && 20225f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer "Must be in a preprocessing directive!"); 20235f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer std::string Result; 2024d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner Token Tmp; 20255f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 20265f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // CurPtr - Cache BufferPtr in an automatic variable. 20275f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer const char *CurPtr = BufferPtr; 20285f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer while (1) { 20295f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer char Char = getAndAdvanceChar(CurPtr, Tmp); 20305f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer switch (Char) { 20315f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer default: 20325f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Result += Char; 20335f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 20345f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case 0: // Null. 20355f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Found end of file? 20365f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (CurPtr-1 != BufferEnd) { 20377d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis if (isCodeCompletionPoint(CurPtr-1)) { 20387d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis PP->CodeCompleteNaturalLanguage(); 20397d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis cutOffLexing(); 20407d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis return Result; 20417d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis } 20427d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis 20435f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Nope, normal character, continue. 20445f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Result += Char; 20455f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 20465f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 20475f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // FALL THROUGH. 20485f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '\r': 20495f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '\n': 20505f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Okay, we found the end of the line. First, back up past the \0, \r, \n. 20515f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer assert(CurPtr[-1] == Char && "Trigraphs for newline?"); 20525f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer BufferPtr = CurPtr-1; 20531eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 205484021556baceb76eedf7d44be8ba71d9b8cfaccePeter Collingbourne // Next, lex the character, which should handle the EOD transition. 20555f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Lex(Tmp); 205655817afdf9d453a443262a733f6caf6692dca118Douglas Gregor if (Tmp.is(tok::code_completion)) { 20577d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis if (PP) 20587d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis PP->CodeCompleteNaturalLanguage(); 205955817afdf9d453a443262a733f6caf6692dca118Douglas Gregor Lex(Tmp); 206055817afdf9d453a443262a733f6caf6692dca118Douglas Gregor } 206184021556baceb76eedf7d44be8ba71d9b8cfaccePeter Collingbourne assert(Tmp.is(tok::eod) && "Unexpected token!"); 20621eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 20635f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Finally, we're done, return the string we found. 20645f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer return Result; 20655f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 20665f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 20675f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 20685f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 20695f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// LexEndOfFile - CurPtr points to the end of this file. Handle this 20705f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// condition, reporting diagnostics and handling other edge cases as required. 20715f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// This returns true if Result contains a token, false if PP.Lex should be 20725f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// called again. 2073d217773f106856a11879ec79dc468efefaf2ee75Chris Lattnerbool Lexer::LexEndOfFile(Token &Result, const char *CurPtr) { 20745f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If we hit the end of the file while parsing a preprocessor directive, 20755f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // end the preprocessor directive first. The next token returned will 20765f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // then be the end of file. 20775f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (ParsingPreprocessorDirective) { 20785f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Done parsing the "line". 20795f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer ParsingPreprocessorDirective = false; 20805f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Update the location of token as well as BufferPtr. 208184021556baceb76eedf7d44be8ba71d9b8cfaccePeter Collingbourne FormTokenWithChars(Result, CurPtr, tok::eod); 20821eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 20835f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Restore comment saving mode, in case it was disabled for directive. 2084f744d133e8d36eec65828d046ee695bda238125cChris Lattner SetCommentRetentionState(PP->getCommentRetentionState()); 20855f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer return true; // Have a token. 20861eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump } 208786d9a52c24d390631a888d4ff812e1b15445e0a0Douglas Gregor 208886d9a52c24d390631a888d4ff812e1b15445e0a0Douglas Gregor // If we are in raw mode, return this event as an EOF token. Let the caller 208986d9a52c24d390631a888d4ff812e1b15445e0a0Douglas Gregor // that put us in raw mode handle the event. 209086d9a52c24d390631a888d4ff812e1b15445e0a0Douglas Gregor if (isLexingRawMode()) { 209186d9a52c24d390631a888d4ff812e1b15445e0a0Douglas Gregor Result.startToken(); 209286d9a52c24d390631a888d4ff812e1b15445e0a0Douglas Gregor BufferPtr = BufferEnd; 209386d9a52c24d390631a888d4ff812e1b15445e0a0Douglas Gregor FormTokenWithChars(Result, BufferEnd, tok::eof); 209486d9a52c24d390631a888d4ff812e1b15445e0a0Douglas Gregor return true; 209586d9a52c24d390631a888d4ff812e1b15445e0a0Douglas Gregor } 209681b747b7fcc91c2fba9a3183d8fac80adbfc1d3eDouglas Gregor 2097f44e854ed1e3aa86d2ed6d615ccd109d50ddcff9Douglas Gregor // Issue diagnostics for unterminated #if and missing newline. 2098f44e854ed1e3aa86d2ed6d615ccd109d50ddcff9Douglas Gregor 20995f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If we are in a #if directive, emit an error. 21005f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer while (!ConditionalStack.empty()) { 21017d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis if (PP->getCodeCompletionFileLoc() != FileLoc) 21022d474ba9e8ae43a1a5a9f72718c0d79092b9453fDouglas Gregor PP->Diag(ConditionalStack.back().IfLoc, 21032d474ba9e8ae43a1a5a9f72718c0d79092b9453fDouglas Gregor diag::err_pp_unterminated_conditional); 21045f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer ConditionalStack.pop_back(); 21055f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 21061eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 2107b25e5d79d9d1967df058a242e96a62d0d0ace074Chris Lattner // C99 5.1.1.2p2: If the file is non-empty and didn't end in a newline, issue 2108b25e5d79d9d1967df058a242e96a62d0d0ace074Chris Lattner // a pedwarn. 2109b25e5d79d9d1967df058a242e96a62d0d0ace074Chris Lattner if (CurPtr != BufferStart && (CurPtr[-1] != '\n' && CurPtr[-1] != '\r')) 211020d0ee5d23c0f0e879ce1711d11df3118b3fb033Mike Stump Diag(BufferEnd, diag::ext_no_newline_eof) 2111849b243d4065f56742a4677d6dc8277609a151f8Douglas Gregor << FixItHint::CreateInsertion(getSourceLocation(BufferEnd), "\n"); 21121eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 21135f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer BufferPtr = CurPtr; 21145f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 21155f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Finally, let the preprocessor handle this. 2116168ae2d44a443da75ea85db5f3b5081eb0bce113Chris Lattner return PP->HandleEndOfFile(Result); 21175f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 21185f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 21195f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// isNextPPTokenLParen - Return 1 if the next unexpanded token lexed from 21205f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// the specified lexer will return a tok::l_paren token, 0 if it is something 21215f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// else and 2 if there are no more tokens in the buffer controlled by the 21225f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// lexer. 21235f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencerunsigned Lexer::isNextPPTokenLParen() { 21245f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer assert(!LexingRawMode && "How can we expand a macro from a skipping buffer?"); 21251eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 21265f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Switch to 'skipping' mode. This will ensure that we can lex a token 21275f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // without emitting diagnostics, disables macro expansion, and will cause EOF 21285f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // to return an EOF token instead of popping the include stack. 21295f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer LexingRawMode = true; 21301eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 21315f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Save state that can be changed while lexing so that we can restore it. 21325f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer const char *TmpBufferPtr = BufferPtr; 2133a864cf7c1d774a0f790bfc46befc87d9dbf1f65cChris Lattner bool inPPDirectiveMode = ParsingPreprocessorDirective; 21341eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 2135d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner Token Tok; 21365f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Tok.startToken(); 21375f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer LexTokenInternal(Tok); 21381eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 21395f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Restore state that may have changed. 21405f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer BufferPtr = TmpBufferPtr; 2141a864cf7c1d774a0f790bfc46befc87d9dbf1f65cChris Lattner ParsingPreprocessorDirective = inPPDirectiveMode; 21421eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 21435f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Restore the lexer back to non-skipping mode. 21445f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer LexingRawMode = false; 21451eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 214622f6bbcafa8871f4f20c4402d9cbc5c024fee99aChris Lattner if (Tok.is(tok::eof)) 21475f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer return 2; 214822f6bbcafa8871f4f20c4402d9cbc5c024fee99aChris Lattner return Tok.is(tok::l_paren); 21495f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 21505f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 215134f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner/// FindConflictEnd - Find the end of a version control conflict marker. 215234f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattnerstatic const char *FindConflictEnd(const char *CurPtr, const char *BufferEnd) { 21535f9e272e632e951b1efe824cd16acb4d96077930Chris Lattner StringRef RestOfBuffer(CurPtr+7, BufferEnd-CurPtr-7); 215434f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner size_t Pos = RestOfBuffer.find(">>>>>>>"); 21555f9e272e632e951b1efe824cd16acb4d96077930Chris Lattner while (Pos != StringRef::npos) { 215634f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner // Must occur at start of line. 215734f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner if (RestOfBuffer[Pos-1] != '\r' && 215834f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner RestOfBuffer[Pos-1] != '\n') { 215934f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner RestOfBuffer = RestOfBuffer.substr(Pos+7); 21603d488990e13835cb35538b2f0cd53dbda92e36beChris Lattner Pos = RestOfBuffer.find(">>>>>>>"); 216134f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner continue; 216234f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner } 216334f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner return RestOfBuffer.data()+Pos; 216434f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner } 216534f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner return 0; 216634f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner} 216734f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner 216834f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner/// IsStartOfConflictMarker - If the specified pointer is the start of a version 216934f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner/// control conflict marker like '<<<<<<<', recognize it as such, emit an error 217034f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner/// and recover nicely. This returns true if it is a conflict marker and false 217134f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner/// if not. 217234f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattnerbool Lexer::IsStartOfConflictMarker(const char *CurPtr) { 217334f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner // Only a conflict marker if it starts at the beginning of a line. 217434f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner if (CurPtr != BufferStart && 217534f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner CurPtr[-1] != '\n' && CurPtr[-1] != '\r') 217634f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner return false; 217734f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner 217834f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner // Check to see if we have <<<<<<<. 217934f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner if (BufferEnd-CurPtr < 8 || 21805f9e272e632e951b1efe824cd16acb4d96077930Chris Lattner StringRef(CurPtr, 7) != "<<<<<<<") 218134f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner return false; 218234f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner 218334f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner // If we have a situation where we don't care about conflict markers, ignore 218434f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner // it. 218534f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner if (IsInConflictMarker || isLexingRawMode()) 218634f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner return false; 218734f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner 218834f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner // Check to see if there is a >>>>>>> somewhere in the buffer at the start of 218934f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner // a line to terminate this conflict marker. 21903d488990e13835cb35538b2f0cd53dbda92e36beChris Lattner if (FindConflictEnd(CurPtr, BufferEnd)) { 219134f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner // We found a match. We are really in a conflict marker. 219234f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner // Diagnose this, and ignore to the end of line. 219334f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner Diag(CurPtr, diag::err_conflict_marker); 219434f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner IsInConflictMarker = true; 219534f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner 219634f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner // Skip ahead to the end of line. We know this exists because the 219734f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner // end-of-conflict marker starts with \r or \n. 219834f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner while (*CurPtr != '\r' && *CurPtr != '\n') { 219934f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner assert(CurPtr != BufferEnd && "Didn't find end of line"); 220034f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner ++CurPtr; 220134f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner } 220234f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner BufferPtr = CurPtr; 220334f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner return true; 220434f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner } 220534f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner 220634f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner // No end of conflict marker found. 220734f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner return false; 220834f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner} 220934f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner 221034f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner 221134f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner/// HandleEndOfConflictMarker - If this is a '=======' or '|||||||' or '>>>>>>>' 221234f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner/// marker, then it is the end of a conflict marker. Handle it by ignoring up 221334f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner/// until the end of the line. This returns true if it is a conflict marker and 221434f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner/// false if not. 221534f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattnerbool Lexer::HandleEndOfConflictMarker(const char *CurPtr) { 221634f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner // Only a conflict marker if it starts at the beginning of a line. 221734f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner if (CurPtr != BufferStart && 221834f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner CurPtr[-1] != '\n' && CurPtr[-1] != '\r') 221934f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner return false; 222034f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner 222134f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner // If we have a situation where we don't care about conflict markers, ignore 222234f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner // it. 222334f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner if (!IsInConflictMarker || isLexingRawMode()) 222434f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner return false; 222534f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner 222634f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner // Check to see if we have the marker (7 characters in a row). 222734f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner for (unsigned i = 1; i != 7; ++i) 222834f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner if (CurPtr[i] != CurPtr[0]) 222934f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner return false; 223034f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner 223134f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner // If we do have it, search for the end of the conflict marker. This could 223234f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner // fail if it got skipped with a '#if 0' or something. Note that CurPtr might 223334f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner // be the end of conflict marker. 223434f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner if (const char *End = FindConflictEnd(CurPtr, BufferEnd)) { 223534f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner CurPtr = End; 223634f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner 223734f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner // Skip ahead to the end of line. 223834f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner while (CurPtr != BufferEnd && *CurPtr != '\r' && *CurPtr != '\n') 223934f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner ++CurPtr; 224034f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner 224134f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner BufferPtr = CurPtr; 224234f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner 224334f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner // No longer in the conflict marker. 224434f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner IsInConflictMarker = false; 224534f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner return true; 224634f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner } 224734f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner 224834f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner return false; 224934f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner} 225034f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner 22517d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidisbool Lexer::isCodeCompletionPoint(const char *CurPtr) const { 22527d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis if (PP && PP->isCodeCompletionEnabled()) { 2253a64ccefdf0ea4e03ec88805d71b0af74950c7472Argyrios Kyrtzidis SourceLocation Loc = FileLoc.getLocWithOffset(CurPtr-BufferStart); 22547d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis return Loc == PP->getCodeCompletionLoc(); 22557d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis } 22567d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis 22577d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis return false; 22587d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis} 22597d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis 22605f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 22615f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// LexTokenInternal - This implements a simple C family lexer. It is an 22625f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// extremely performance critical piece of code. This assumes that the buffer 2263efb173ddd95325c7fd3c15070392b27c07a49a85Chris Lattner/// has a null character at the end of the file. This returns a preprocessing 2264efb173ddd95325c7fd3c15070392b27c07a49a85Chris Lattner/// token, not a normal token, as such, it is an internal interface. It assumes 2265efb173ddd95325c7fd3c15070392b27c07a49a85Chris Lattner/// that the Flags of result have been cleared before calling this. 2266d217773f106856a11879ec79dc468efefaf2ee75Chris Lattnervoid Lexer::LexTokenInternal(Token &Result) { 22675f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid SpencerLexNextToken: 22685f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // New token, can't need cleaning yet. 2269d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner Result.clearFlag(Token::NeedsCleaning); 22705f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Result.setIdentifierInfo(0); 22711eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 22725f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // CurPtr - Cache BufferPtr in an automatic variable. 22735f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer const char *CurPtr = BufferPtr; 22745f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 22755f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Small amounts of horizontal whitespace is very common between tokens. 22765f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if ((*CurPtr == ' ') || (*CurPtr == '\t')) { 22775f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer ++CurPtr; 22785f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer while ((*CurPtr == ' ') || (*CurPtr == '\t')) 22795f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer ++CurPtr; 22801eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 2281d88dc48e33d71732708960170e57a3d1bdc8f847Chris Lattner // If we are keeping whitespace and other tokens, just return what we just 2282d88dc48e33d71732708960170e57a3d1bdc8f847Chris Lattner // skipped. The next lexer invocation will return the token after the 2283d88dc48e33d71732708960170e57a3d1bdc8f847Chris Lattner // whitespace. 2284d88dc48e33d71732708960170e57a3d1bdc8f847Chris Lattner if (isKeepWhitespaceMode()) { 22859e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner FormTokenWithChars(Result, CurPtr, tok::unknown); 2286d88dc48e33d71732708960170e57a3d1bdc8f847Chris Lattner return; 2287d88dc48e33d71732708960170e57a3d1bdc8f847Chris Lattner } 22881eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 22895f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer BufferPtr = CurPtr; 2290d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner Result.setFlag(Token::LeadingSpace); 22915f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 22921eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 22935f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer unsigned SizeTmp, SizeTmp2; // Temporaries for use in cases below. 22941eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 22955f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Read a character, advancing over it. 22965f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer char Char = getAndAdvanceChar(CurPtr, Result); 22979e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner tok::TokenKind Kind; 22981eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 22995f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer switch (Char) { 23005f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case 0: // Null. 23015f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Found end of file? 23025f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (CurPtr-1 == BufferEnd) { 23035f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Read the PP instance variable into an automatic variable, because 23045f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // LexEndOfFile will often delete 'this'. 2305168ae2d44a443da75ea85db5f3b5081eb0bce113Chris Lattner Preprocessor *PPCache = PP; 23065f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (LexEndOfFile(Result, CurPtr-1)) // Retreat back into the file. 23075f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer return; // Got a token to return. 2308168ae2d44a443da75ea85db5f3b5081eb0bce113Chris Lattner assert(PPCache && "Raw buffer::LexEndOfFile should return a token"); 2309168ae2d44a443da75ea85db5f3b5081eb0bce113Chris Lattner return PPCache->Lex(Result); 23105f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 23111eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 23127d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis // Check if we are performing code completion. 23137d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis if (isCodeCompletionPoint(CurPtr-1)) { 23147d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis // Return the code-completion token. 23157d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis Result.startToken(); 23167d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis FormTokenWithChars(Result, CurPtr, tok::code_completion); 23177d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis return; 23187d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis } 23197d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis 232074d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner if (!isLexingRawMode()) 232174d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner Diag(CurPtr-1, diag::null_in_file); 2322d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner Result.setFlag(Token::LeadingSpace); 2323d88dc48e33d71732708960170e57a3d1bdc8f847Chris Lattner if (SkipWhitespace(Result, CurPtr)) 2324d88dc48e33d71732708960170e57a3d1bdc8f847Chris Lattner return; // KeepWhitespaceMode 23251eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 23265f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer goto LexNextToken; // GCC isn't tail call eliminating. 2327a2bf105739cef7aea8e4d219629c627fcd11742eChris Lattner 2328a2bf105739cef7aea8e4d219629c627fcd11742eChris Lattner case 26: // DOS & CP/M EOF: "^Z". 2329a2bf105739cef7aea8e4d219629c627fcd11742eChris Lattner // If we're in Microsoft extensions mode, treat this as end of file. 233062ec1f2fd7368542bb926c04797fb07023547694Francois Pichet if (Features.MicrosoftExt) { 2331a2bf105739cef7aea8e4d219629c627fcd11742eChris Lattner // Read the PP instance variable into an automatic variable, because 2332a2bf105739cef7aea8e4d219629c627fcd11742eChris Lattner // LexEndOfFile will often delete 'this'. 2333a2bf105739cef7aea8e4d219629c627fcd11742eChris Lattner Preprocessor *PPCache = PP; 2334a2bf105739cef7aea8e4d219629c627fcd11742eChris Lattner if (LexEndOfFile(Result, CurPtr-1)) // Retreat back into the file. 2335a2bf105739cef7aea8e4d219629c627fcd11742eChris Lattner return; // Got a token to return. 2336a2bf105739cef7aea8e4d219629c627fcd11742eChris Lattner assert(PPCache && "Raw buffer::LexEndOfFile should return a token"); 2337a2bf105739cef7aea8e4d219629c627fcd11742eChris Lattner return PPCache->Lex(Result); 2338a2bf105739cef7aea8e4d219629c627fcd11742eChris Lattner } 2339a2bf105739cef7aea8e4d219629c627fcd11742eChris Lattner // If Microsoft extensions are disabled, this is just random garbage. 2340a2bf105739cef7aea8e4d219629c627fcd11742eChris Lattner Kind = tok::unknown; 2341a2bf105739cef7aea8e4d219629c627fcd11742eChris Lattner break; 2342a2bf105739cef7aea8e4d219629c627fcd11742eChris Lattner 23435f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '\n': 23445f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '\r': 23455f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If we are inside a preprocessor directive and we see the end of line, 234684021556baceb76eedf7d44be8ba71d9b8cfaccePeter Collingbourne // we know we are done with the directive, so return an EOD token. 23475f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (ParsingPreprocessorDirective) { 23485f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Done parsing the "line". 23495f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer ParsingPreprocessorDirective = false; 23501eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 23515f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Restore comment saving mode, in case it was disabled for directive. 2352f744d133e8d36eec65828d046ee695bda238125cChris Lattner SetCommentRetentionState(PP->getCommentRetentionState()); 23531eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 23545f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Since we consumed a newline, we are back at the start of a line. 23555f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer IsAtStartOfLine = true; 23561eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 235784021556baceb76eedf7d44be8ba71d9b8cfaccePeter Collingbourne Kind = tok::eod; 23585f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 23595f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 23605f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // The returned token is at the start of the line. 2361d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner Result.setFlag(Token::StartOfLine); 23625f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // No leading whitespace seen so far. 2363d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner Result.clearFlag(Token::LeadingSpace); 23641eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 2365d88dc48e33d71732708960170e57a3d1bdc8f847Chris Lattner if (SkipWhitespace(Result, CurPtr)) 2366d88dc48e33d71732708960170e57a3d1bdc8f847Chris Lattner return; // KeepWhitespaceMode 23675f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer goto LexNextToken; // GCC isn't tail call eliminating. 23685f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case ' ': 23695f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '\t': 23705f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '\f': 23715f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '\v': 23728133cfcc0b672cc86d1ef73ce0c61eccecdb3690Chris Lattner SkipHorizontalWhitespace: 2373d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner Result.setFlag(Token::LeadingSpace); 2374d88dc48e33d71732708960170e57a3d1bdc8f847Chris Lattner if (SkipWhitespace(Result, CurPtr)) 2375d88dc48e33d71732708960170e57a3d1bdc8f847Chris Lattner return; // KeepWhitespaceMode 23768133cfcc0b672cc86d1ef73ce0c61eccecdb3690Chris Lattner 23778133cfcc0b672cc86d1ef73ce0c61eccecdb3690Chris Lattner SkipIgnoredUnits: 23788133cfcc0b672cc86d1ef73ce0c61eccecdb3690Chris Lattner CurPtr = BufferPtr; 23791eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 23808133cfcc0b672cc86d1ef73ce0c61eccecdb3690Chris Lattner // If the next token is obviously a // or /* */ comment, skip it efficiently 23818133cfcc0b672cc86d1ef73ce0c61eccecdb3690Chris Lattner // too (without going through the big switch stmt). 23828402c73dd880e8af46c826d873681820aebe32ebChris Lattner if (CurPtr[0] == '/' && CurPtr[1] == '/' && !inKeepCommentMode() && 23832ed42287b9b88a0bb08ab10e95e306da3f294ba9Daniel Dunbar Features.BCPLComment && !Features.TraditionalCPP) { 2384046c2277dcbcc8eb89dbb5b1b8c5226b7cb81635Chris Lattner if (SkipBCPLComment(Result, CurPtr+2)) 2385046c2277dcbcc8eb89dbb5b1b8c5226b7cb81635Chris Lattner return; // There is a token to return. 23868133cfcc0b672cc86d1ef73ce0c61eccecdb3690Chris Lattner goto SkipIgnoredUnits; 2387fa95a019da00b926d64ff83358ba73bbc6ae1e37Chris Lattner } else if (CurPtr[0] == '/' && CurPtr[1] == '*' && !inKeepCommentMode()) { 2388046c2277dcbcc8eb89dbb5b1b8c5226b7cb81635Chris Lattner if (SkipBlockComment(Result, CurPtr+2)) 2389046c2277dcbcc8eb89dbb5b1b8c5226b7cb81635Chris Lattner return; // There is a token to return. 23908133cfcc0b672cc86d1ef73ce0c61eccecdb3690Chris Lattner goto SkipIgnoredUnits; 23918133cfcc0b672cc86d1ef73ce0c61eccecdb3690Chris Lattner } else if (isHorizontalWhitespace(*CurPtr)) { 23928133cfcc0b672cc86d1ef73ce0c61eccecdb3690Chris Lattner goto SkipHorizontalWhitespace; 23938133cfcc0b672cc86d1ef73ce0c61eccecdb3690Chris Lattner } 23945f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer goto LexNextToken; // GCC isn't tail call eliminating. 2395a2bf105739cef7aea8e4d219629c627fcd11742eChris Lattner 23963a5707766850f9ee9daa35299794328b5caf96dcChris Lattner // C99 6.4.4.1: Integer Constants. 23973a5707766850f9ee9daa35299794328b5caf96dcChris Lattner // C99 6.4.4.2: Floating Constants. 23983a5707766850f9ee9daa35299794328b5caf96dcChris Lattner case '0': case '1': case '2': case '3': case '4': 23993a5707766850f9ee9daa35299794328b5caf96dcChris Lattner case '5': case '6': case '7': case '8': case '9': 24003a5707766850f9ee9daa35299794328b5caf96dcChris Lattner // Notify MIOpt that we read a non-whitespace/non-comment token. 24013a5707766850f9ee9daa35299794328b5caf96dcChris Lattner MIOpt.ReadToken(); 24023a5707766850f9ee9daa35299794328b5caf96dcChris Lattner return LexNumericConstant(Result, CurPtr); 24031eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 24045cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor case 'u': // Identifier (uber) or C++0x UTF-8 or UTF-16 string literal 24055cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor // Notify MIOpt that we read a non-whitespace/non-comment token. 24065cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor MIOpt.ReadToken(); 24075cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor 24085cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor if (Features.CPlusPlus0x) { 24095cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor Char = getCharAndSize(CurPtr, SizeTmp); 24105cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor 24115cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor // UTF-16 string literal 24125cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor if (Char == '"') 24135cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor return LexStringLiteral(Result, ConsumeChar(CurPtr, SizeTmp, Result), 24145cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor tok::utf16_string_literal); 24155cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor 24165cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor // UTF-16 character constant 24175cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor if (Char == '\'') 24185cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor return LexCharConstant(Result, ConsumeChar(CurPtr, SizeTmp, Result), 24195cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor tok::utf16_char_constant); 24205cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor 24212fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper // UTF-16 raw string literal 24222fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper if (Char == 'R' && getCharAndSize(CurPtr + SizeTmp, SizeTmp2) == '"') 24232fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper return LexRawStringLiteral(Result, 24242fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 24252fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper SizeTmp2, Result), 24262fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper tok::utf16_string_literal); 24272fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper 24282fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper if (Char == '8') { 24292fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper char Char2 = getCharAndSize(CurPtr + SizeTmp, SizeTmp2); 24302fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper 24312fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper // UTF-8 string literal 24322fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper if (Char2 == '"') 24332fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper return LexStringLiteral(Result, 24342fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 24352fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper SizeTmp2, Result), 24362fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper tok::utf8_string_literal); 24372fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper 24382fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper if (Char2 == 'R') { 24392fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper unsigned SizeTmp3; 24402fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper char Char3 = getCharAndSize(CurPtr + SizeTmp + SizeTmp2, SizeTmp3); 24412fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper // UTF-8 raw string literal 24422fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper if (Char3 == '"') { 24432fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper return LexRawStringLiteral(Result, 24442fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper ConsumeChar(ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 24452fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper SizeTmp2, Result), 24462fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper SizeTmp3, Result), 24472fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper tok::utf8_string_literal); 24482fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper } 24492fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper } 24502fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper } 24515cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor } 24525cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor 24535cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor // treat u like the start of an identifier. 24545cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor return LexIdentifier(Result, CurPtr); 24555cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor 24565cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor case 'U': // Identifier (Uber) or C++0x UTF-32 string literal 24575cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor // Notify MIOpt that we read a non-whitespace/non-comment token. 24585cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor MIOpt.ReadToken(); 24595cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor 24605cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor if (Features.CPlusPlus0x) { 24615cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor Char = getCharAndSize(CurPtr, SizeTmp); 24625cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor 24635cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor // UTF-32 string literal 24645cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor if (Char == '"') 24655cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor return LexStringLiteral(Result, ConsumeChar(CurPtr, SizeTmp, Result), 24665cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor tok::utf32_string_literal); 24675cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor 24685cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor // UTF-32 character constant 24695cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor if (Char == '\'') 24705cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor return LexCharConstant(Result, ConsumeChar(CurPtr, SizeTmp, Result), 24715cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor tok::utf32_char_constant); 24722fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper 24732fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper // UTF-32 raw string literal 24742fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper if (Char == 'R' && getCharAndSize(CurPtr + SizeTmp, SizeTmp2) == '"') 24752fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper return LexRawStringLiteral(Result, 24762fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 24772fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper SizeTmp2, Result), 24782fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper tok::utf32_string_literal); 24795cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor } 24805cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor 24815cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor // treat U like the start of an identifier. 24825cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor return LexIdentifier(Result, CurPtr); 24835cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor 24842fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper case 'R': // Identifier or C++0x raw string literal 24852fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper // Notify MIOpt that we read a non-whitespace/non-comment token. 24862fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper MIOpt.ReadToken(); 24872fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper 24882fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper if (Features.CPlusPlus0x) { 24892fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper Char = getCharAndSize(CurPtr, SizeTmp); 24902fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper 24912fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper if (Char == '"') 24922fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper return LexRawStringLiteral(Result, 24932fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper ConsumeChar(CurPtr, SizeTmp, Result), 24942fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper tok::string_literal); 24952fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper } 24962fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper 24972fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper // treat R like the start of an identifier. 24982fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper return LexIdentifier(Result, CurPtr); 24992fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper 25003a5707766850f9ee9daa35299794328b5caf96dcChris Lattner case 'L': // Identifier (Loony) or wide literal (L'x' or L"xyz"). 25015f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Notify MIOpt that we read a non-whitespace/non-comment token. 25025f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer MIOpt.ReadToken(); 25035f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Char = getCharAndSize(CurPtr, SizeTmp); 25045f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 25055f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Wide string literal. 25065f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (Char == '"') 25075f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer return LexStringLiteral(Result, ConsumeChar(CurPtr, SizeTmp, Result), 25085cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor tok::wide_string_literal); 25095f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 25102fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper // Wide raw string literal. 25112fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper if (Features.CPlusPlus0x && Char == 'R' && 25122fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper getCharAndSize(CurPtr + SizeTmp, SizeTmp2) == '"') 25132fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper return LexRawStringLiteral(Result, 25142fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 25152fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper SizeTmp2, Result), 25162fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper tok::wide_string_literal); 25172fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper 25185f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Wide character constant. 25195f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (Char == '\'') 25205cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor return LexCharConstant(Result, ConsumeChar(CurPtr, SizeTmp, Result), 25215cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor tok::wide_char_constant); 25225f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // FALL THROUGH, treating L like the start of an identifier. 25231eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 25245f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // C99 6.4.2: Identifiers. 25255f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': case 'G': 25265f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case 'H': case 'I': case 'J': case 'K': /*'L'*/case 'M': case 'N': 25272fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper case 'O': case 'P': case 'Q': /*'R'*/case 'S': case 'T': /*'U'*/ 25285f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case 'V': case 'W': case 'X': case 'Y': case 'Z': 25295f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': case 'g': 25305f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case 'h': case 'i': case 'j': case 'k': case 'l': case 'm': case 'n': 25315cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor case 'o': case 'p': case 'q': case 'r': case 's': case 't': /*'u'*/ 25325f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case 'v': case 'w': case 'x': case 'y': case 'z': 25335f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '_': 25345f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Notify MIOpt that we read a non-whitespace/non-comment token. 25355f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer MIOpt.ReadToken(); 25365f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer return LexIdentifier(Result, CurPtr); 25373a5707766850f9ee9daa35299794328b5caf96dcChris Lattner 25383a5707766850f9ee9daa35299794328b5caf96dcChris Lattner case '$': // $ in identifiers. 25393a5707766850f9ee9daa35299794328b5caf96dcChris Lattner if (Features.DollarIdents) { 254074d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner if (!isLexingRawMode()) 254174d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner Diag(CurPtr-1, diag::ext_dollar_in_identifier); 25423a5707766850f9ee9daa35299794328b5caf96dcChris Lattner // Notify MIOpt that we read a non-whitespace/non-comment token. 25433a5707766850f9ee9daa35299794328b5caf96dcChris Lattner MIOpt.ReadToken(); 25443a5707766850f9ee9daa35299794328b5caf96dcChris Lattner return LexIdentifier(Result, CurPtr); 25453a5707766850f9ee9daa35299794328b5caf96dcChris Lattner } 25461eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 25479e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::unknown; 25483a5707766850f9ee9daa35299794328b5caf96dcChris Lattner break; 25491eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 25505f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // C99 6.4.4: Character Constants. 25515f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '\'': 25525f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Notify MIOpt that we read a non-whitespace/non-comment token. 25535f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer MIOpt.ReadToken(); 25545cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor return LexCharConstant(Result, CurPtr, tok::char_constant); 25555f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 25565f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // C99 6.4.5: String Literals. 25575f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '"': 25585f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Notify MIOpt that we read a non-whitespace/non-comment token. 25595f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer MIOpt.ReadToken(); 25605cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor return LexStringLiteral(Result, CurPtr, tok::string_literal); 25615f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 25625f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // C99 6.4.6: Punctuators. 25635f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '?': 25649e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::question; 25655f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 25665f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '[': 25679e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::l_square; 25685f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 25695f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case ']': 25709e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::r_square; 25715f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 25725f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '(': 25739e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::l_paren; 25745f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 25755f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case ')': 25769e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::r_paren; 25775f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 25785f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '{': 25799e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::l_brace; 25805f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 25815f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '}': 25829e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::r_brace; 25835f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 25845f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '.': 25855f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Char = getCharAndSize(CurPtr, SizeTmp); 25865f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (Char >= '0' && Char <= '9') { 25875f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Notify MIOpt that we read a non-whitespace/non-comment token. 25885f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer MIOpt.ReadToken(); 25895f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 25905f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer return LexNumericConstant(Result, ConsumeChar(CurPtr, SizeTmp, Result)); 25915f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } else if (Features.CPlusPlus && Char == '*') { 25929e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::periodstar; 25935f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr += SizeTmp; 25945f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } else if (Char == '.' && 25955f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer getCharAndSize(CurPtr+SizeTmp, SizeTmp2) == '.') { 25969e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::ellipsis; 25975f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 25985f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer SizeTmp2, Result); 25995f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } else { 26009e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::period; 26015f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 26025f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 26035f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '&': 26045f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Char = getCharAndSize(CurPtr, SizeTmp); 26055f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (Char == '&') { 26069e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::ampamp; 26075f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 26085f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } else if (Char == '=') { 26099e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::ampequal; 26105f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 26115f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } else { 26129e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::amp; 26135f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 26145f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 26151eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump case '*': 26165f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (getCharAndSize(CurPtr, SizeTmp) == '=') { 26179e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::starequal; 26185f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 26195f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } else { 26209e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::star; 26215f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 26225f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 26235f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '+': 26245f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Char = getCharAndSize(CurPtr, SizeTmp); 26255f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (Char == '+') { 26265f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 26279e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::plusplus; 26285f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } else if (Char == '=') { 26295f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 26309e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::plusequal; 26315f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } else { 26329e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::plus; 26335f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 26345f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 26355f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '-': 26365f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Char = getCharAndSize(CurPtr, SizeTmp); 26379e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner if (Char == '-') { // -- 26385f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 26399e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::minusminus; 26401eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump } else if (Char == '>' && Features.CPlusPlus && 26419e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner getCharAndSize(CurPtr+SizeTmp, SizeTmp2) == '*') { // C++ ->* 26425f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 26435f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer SizeTmp2, Result); 26449e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::arrowstar; 26459e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner } else if (Char == '>') { // -> 26465f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 26479e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::arrow; 26489e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner } else if (Char == '=') { // -= 26495f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 26509e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::minusequal; 26515f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } else { 26529e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::minus; 26535f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 26545f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 26555f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '~': 26569e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::tilde; 26575f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 26585f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '!': 26595f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (getCharAndSize(CurPtr, SizeTmp) == '=') { 26609e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::exclaimequal; 26615f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 26625f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } else { 26639e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::exclaim; 26645f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 26655f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 26665f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '/': 26675f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // 6.4.9: Comments 26685f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Char = getCharAndSize(CurPtr, SizeTmp); 26695f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (Char == '/') { // BCPL comment. 26708402c73dd880e8af46c826d873681820aebe32ebChris Lattner // Even if BCPL comments are disabled (e.g. in C89 mode), we generally 26718402c73dd880e8af46c826d873681820aebe32ebChris Lattner // want to lex this as a comment. There is one problem with this though, 26728402c73dd880e8af46c826d873681820aebe32ebChris Lattner // that in one particular corner case, this can change the behavior of the 26738402c73dd880e8af46c826d873681820aebe32ebChris Lattner // resultant program. For example, In "foo //**/ bar", C89 would lex 26748402c73dd880e8af46c826d873681820aebe32ebChris Lattner // this as "foo / bar" and langauges with BCPL comments would lex it as 26758402c73dd880e8af46c826d873681820aebe32ebChris Lattner // "foo". Check to see if the character after the second slash is a '*'. 26768402c73dd880e8af46c826d873681820aebe32ebChris Lattner // If so, we will lex that as a "/" instead of the start of a comment. 26772ed42287b9b88a0bb08ab10e95e306da3f294ba9Daniel Dunbar // However, we never do this in -traditional-cpp mode. 26782ed42287b9b88a0bb08ab10e95e306da3f294ba9Daniel Dunbar if ((Features.BCPLComment || 26792ed42287b9b88a0bb08ab10e95e306da3f294ba9Daniel Dunbar getCharAndSize(CurPtr+SizeTmp, SizeTmp2) != '*') && 26802ed42287b9b88a0bb08ab10e95e306da3f294ba9Daniel Dunbar !Features.TraditionalCPP) { 26818402c73dd880e8af46c826d873681820aebe32ebChris Lattner if (SkipBCPLComment(Result, ConsumeChar(CurPtr, SizeTmp, Result))) 2682046c2277dcbcc8eb89dbb5b1b8c5226b7cb81635Chris Lattner return; // There is a token to return. 26831eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 26848402c73dd880e8af46c826d873681820aebe32ebChris Lattner // It is common for the tokens immediately after a // comment to be 26858402c73dd880e8af46c826d873681820aebe32ebChris Lattner // whitespace (indentation for the next line). Instead of going through 26868402c73dd880e8af46c826d873681820aebe32ebChris Lattner // the big switch, handle it efficiently now. 26878402c73dd880e8af46c826d873681820aebe32ebChris Lattner goto SkipIgnoredUnits; 26888402c73dd880e8af46c826d873681820aebe32ebChris Lattner } 26898402c73dd880e8af46c826d873681820aebe32ebChris Lattner } 26901eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 26918402c73dd880e8af46c826d873681820aebe32ebChris Lattner if (Char == '*') { // /**/ comment. 26925f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (SkipBlockComment(Result, ConsumeChar(CurPtr, SizeTmp, Result))) 2693046c2277dcbcc8eb89dbb5b1b8c5226b7cb81635Chris Lattner return; // There is a token to return. 26942d3818959df7b71bfcf8c2f2b573856bfc4c3a21Chris Lattner goto LexNextToken; // GCC isn't tail call eliminating. 26958402c73dd880e8af46c826d873681820aebe32ebChris Lattner } 26961eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 26978402c73dd880e8af46c826d873681820aebe32ebChris Lattner if (Char == '=') { 26985f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 26999e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::slashequal; 27005f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } else { 27019e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::slash; 27025f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 27035f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 27045f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '%': 27055f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Char = getCharAndSize(CurPtr, SizeTmp); 27065f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (Char == '=') { 27079e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::percentequal; 27085f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 27095f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } else if (Features.Digraphs && Char == '>') { 27109e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::r_brace; // '%>' -> '}' 27115f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 27125f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } else if (Features.Digraphs && Char == ':') { 27135f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 27145f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Char = getCharAndSize(CurPtr, SizeTmp); 27155f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (Char == '%' && getCharAndSize(CurPtr+SizeTmp, SizeTmp2) == ':') { 27169e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::hashhash; // '%:%:' -> '##' 27175f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 27185f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer SizeTmp2, Result); 271962ec1f2fd7368542bb926c04797fb07023547694Francois Pichet } else if (Char == '@' && Features.MicrosoftExt) {// %:@ -> #@ -> Charize 27205f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 272174d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner if (!isLexingRawMode()) 272274d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner Diag(BufferPtr, diag::charize_microsoft_ext); 27239e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::hashat; 2724e91e93225db2e66906878513c6ef4dd6a7ee2b6aChris Lattner } else { // '%:' -> '#' 27255f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // We parsed a # character. If this occurs at the start of the line, 27265f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // it's actually the start of a preprocessing directive. Callback to 27275f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // the preprocessor to handle it. 27285f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // FIXME: -fpreprocessed mode?? 2729766703b76dc1eb0c49d11cafc2a17c7407876ebcChris Lattner if (Result.isAtStartOfLine() && !LexingRawMode && !Is_PragmaLexer) { 2730e91e93225db2e66906878513c6ef4dd6a7ee2b6aChris Lattner FormTokenWithChars(Result, CurPtr, tok::hash); 2731168ae2d44a443da75ea85db5f3b5081eb0bce113Chris Lattner PP->HandleDirective(Result); 27321eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 27335f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // As an optimization, if the preprocessor didn't switch lexers, tail 27345f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // recurse. 2735168ae2d44a443da75ea85db5f3b5081eb0bce113Chris Lattner if (PP->isCurrentLexer(this)) { 27365f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Start a new token. If this is a #include or something, the PP may 27375f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // want us starting at the beginning of the line again. If so, set 2738515f43f9f23de50d155b481b8774ec40bdfd7ff2Chris Lattner // the StartOfLine flag and clear LeadingSpace. 27395f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (IsAtStartOfLine) { 2740d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner Result.setFlag(Token::StartOfLine); 2741515f43f9f23de50d155b481b8774ec40bdfd7ff2Chris Lattner Result.clearFlag(Token::LeadingSpace); 27425f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer IsAtStartOfLine = false; 27435f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 27445f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer goto LexNextToken; // GCC isn't tail call eliminating. 27455f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 27461eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 2747168ae2d44a443da75ea85db5f3b5081eb0bce113Chris Lattner return PP->Lex(Result); 27485f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 27491eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 2750e91e93225db2e66906878513c6ef4dd6a7ee2b6aChris Lattner Kind = tok::hash; 27515f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 27525f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } else { 27539e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::percent; 27545f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 27555f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 27565f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '<': 27575f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Char = getCharAndSize(CurPtr, SizeTmp); 27585f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (ParsingFilename) { 27599cb51ce03caf0755759f7e67aea7d4be684ff738Chris Lattner return LexAngledStringLiteral(Result, CurPtr); 27605f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } else if (Char == '<') { 276134f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner char After = getCharAndSize(CurPtr+SizeTmp, SizeTmp2); 276234f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner if (After == '=') { 276334f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner Kind = tok::lesslessequal; 276434f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 276534f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner SizeTmp2, Result); 276634f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner } else if (After == '<' && IsStartOfConflictMarker(CurPtr-1)) { 276734f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner // If this is actually a '<<<<<<<' version control conflict marker, 276834f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner // recognize it as such and recover nicely. 276934f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner goto LexNextToken; 27701b791d6465d42a9763927be1dd8af229efcbbf5ePeter Collingbourne } else if (Features.CUDA && After == '<') { 27711b791d6465d42a9763927be1dd8af229efcbbf5ePeter Collingbourne Kind = tok::lesslessless; 27721b791d6465d42a9763927be1dd8af229efcbbf5ePeter Collingbourne CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 27731b791d6465d42a9763927be1dd8af229efcbbf5ePeter Collingbourne SizeTmp2, Result); 277434f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner } else { 277534f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 277634f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner Kind = tok::lessless; 277734f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner } 27785f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } else if (Char == '=') { 27795f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 27809e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::lessequal; 27819e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner } else if (Features.Digraphs && Char == ':') { // '<:' -> '[' 278287a1e19735a27d84edc2ac1331c040e2fb4c3b1aRichard Smith if (Features.CPlusPlus0x && 278387a1e19735a27d84edc2ac1331c040e2fb4c3b1aRichard Smith getCharAndSize(CurPtr + SizeTmp, SizeTmp2) == ':') { 278487a1e19735a27d84edc2ac1331c040e2fb4c3b1aRichard Smith // C++0x [lex.pptoken]p3: 278587a1e19735a27d84edc2ac1331c040e2fb4c3b1aRichard Smith // Otherwise, if the next three characters are <:: and the subsequent 278687a1e19735a27d84edc2ac1331c040e2fb4c3b1aRichard Smith // character is neither : nor >, the < is treated as a preprocessor 278787a1e19735a27d84edc2ac1331c040e2fb4c3b1aRichard Smith // token by itself and not as the first character of the alternative 278887a1e19735a27d84edc2ac1331c040e2fb4c3b1aRichard Smith // token <:. 278987a1e19735a27d84edc2ac1331c040e2fb4c3b1aRichard Smith unsigned SizeTmp3; 279087a1e19735a27d84edc2ac1331c040e2fb4c3b1aRichard Smith char After = getCharAndSize(CurPtr + SizeTmp + SizeTmp2, SizeTmp3); 279187a1e19735a27d84edc2ac1331c040e2fb4c3b1aRichard Smith if (After != ':' && After != '>') { 279287a1e19735a27d84edc2ac1331c040e2fb4c3b1aRichard Smith Kind = tok::less; 279387a1e19735a27d84edc2ac1331c040e2fb4c3b1aRichard Smith break; 279487a1e19735a27d84edc2ac1331c040e2fb4c3b1aRichard Smith } 279587a1e19735a27d84edc2ac1331c040e2fb4c3b1aRichard Smith } 279687a1e19735a27d84edc2ac1331c040e2fb4c3b1aRichard Smith 27975f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 27989e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::l_square; 27999e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner } else if (Features.Digraphs && Char == '%') { // '<%' -> '{' 28005f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 28019e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::l_brace; 28025f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } else { 28039e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::less; 28045f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 28055f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 28065f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '>': 28075f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Char = getCharAndSize(CurPtr, SizeTmp); 28085f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (Char == '=') { 28095f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 28109e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::greaterequal; 28115f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } else if (Char == '>') { 281234f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner char After = getCharAndSize(CurPtr+SizeTmp, SizeTmp2); 281334f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner if (After == '=') { 281434f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 281534f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner SizeTmp2, Result); 281634f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner Kind = tok::greatergreaterequal; 281734f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner } else if (After == '>' && HandleEndOfConflictMarker(CurPtr-1)) { 281834f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner // If this is '>>>>>>>' and we're in a conflict marker, ignore it. 281934f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner goto LexNextToken; 28201b791d6465d42a9763927be1dd8af229efcbbf5ePeter Collingbourne } else if (Features.CUDA && After == '>') { 28211b791d6465d42a9763927be1dd8af229efcbbf5ePeter Collingbourne Kind = tok::greatergreatergreater; 28221b791d6465d42a9763927be1dd8af229efcbbf5ePeter Collingbourne CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 28231b791d6465d42a9763927be1dd8af229efcbbf5ePeter Collingbourne SizeTmp2, Result); 282434f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner } else { 282534f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 282634f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner Kind = tok::greatergreater; 282734f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner } 282834f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner 28295f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } else { 28309e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::greater; 28315f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 28325f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 28335f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '^': 28345f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Char = getCharAndSize(CurPtr, SizeTmp); 28355f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (Char == '=') { 28365f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 28379e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::caretequal; 28385f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } else { 28399e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::caret; 28405f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 28415f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 28425f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '|': 28435f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Char = getCharAndSize(CurPtr, SizeTmp); 28445f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (Char == '=') { 28459e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::pipeequal; 28465f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 28475f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } else if (Char == '|') { 284834f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner // If this is '|||||||' and we're in a conflict marker, ignore it. 284934f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner if (CurPtr[1] == '|' && HandleEndOfConflictMarker(CurPtr-1)) 285034f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner goto LexNextToken; 28519e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::pipepipe; 28525f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 28535f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } else { 28549e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::pipe; 28555f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 28565f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 28575f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case ':': 28585f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Char = getCharAndSize(CurPtr, SizeTmp); 28595f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (Features.Digraphs && Char == '>') { 28609e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::r_square; // ':>' -> ']' 28615f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 28625f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } else if (Features.CPlusPlus && Char == ':') { 28639e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::coloncolon; 28645f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 28651eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump } else { 28669e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::colon; 28675f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 28685f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 28695f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case ';': 28709e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::semi; 28715f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 28725f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '=': 28735f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Char = getCharAndSize(CurPtr, SizeTmp); 28745f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (Char == '=') { 287534f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner // If this is '=======' and we're in a conflict marker, ignore it. 287634f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner if (CurPtr[1] == '=' && HandleEndOfConflictMarker(CurPtr-1)) 287734f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner goto LexNextToken; 287834f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner 28799e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::equalequal; 28805f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 28811eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump } else { 28829e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::equal; 28835f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 28845f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 28855f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case ',': 28869e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::comma; 28875f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 28885f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '#': 28895f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Char = getCharAndSize(CurPtr, SizeTmp); 28905f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (Char == '#') { 28919e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::hashhash; 28925f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 289362ec1f2fd7368542bb926c04797fb07023547694Francois Pichet } else if (Char == '@' && Features.MicrosoftExt) { // #@ -> Charize 28949e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::hashat; 289574d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner if (!isLexingRawMode()) 289674d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner Diag(BufferPtr, diag::charize_microsoft_ext); 28975f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 28985f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } else { 28995f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // We parsed a # character. If this occurs at the start of the line, 29005f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // it's actually the start of a preprocessing directive. Callback to 29015f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // the preprocessor to handle it. 29025f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // FIXME: -fpreprocessed mode?? 2903766703b76dc1eb0c49d11cafc2a17c7407876ebcChris Lattner if (Result.isAtStartOfLine() && !LexingRawMode && !Is_PragmaLexer) { 2904e91e93225db2e66906878513c6ef4dd6a7ee2b6aChris Lattner FormTokenWithChars(Result, CurPtr, tok::hash); 2905168ae2d44a443da75ea85db5f3b5081eb0bce113Chris Lattner PP->HandleDirective(Result); 29061eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 29075f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // As an optimization, if the preprocessor didn't switch lexers, tail 29085f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // recurse. 2909168ae2d44a443da75ea85db5f3b5081eb0bce113Chris Lattner if (PP->isCurrentLexer(this)) { 29105f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Start a new token. If this is a #include or something, the PP may 29115f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // want us starting at the beginning of the line again. If so, set 2912515f43f9f23de50d155b481b8774ec40bdfd7ff2Chris Lattner // the StartOfLine flag and clear LeadingSpace. 29135f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (IsAtStartOfLine) { 2914d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner Result.setFlag(Token::StartOfLine); 2915515f43f9f23de50d155b481b8774ec40bdfd7ff2Chris Lattner Result.clearFlag(Token::LeadingSpace); 29165f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer IsAtStartOfLine = false; 29175f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 29185f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer goto LexNextToken; // GCC isn't tail call eliminating. 29195f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 2920168ae2d44a443da75ea85db5f3b5081eb0bce113Chris Lattner return PP->Lex(Result); 29215f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 29221eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 2923e91e93225db2e66906878513c6ef4dd6a7ee2b6aChris Lattner Kind = tok::hash; 29245f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 29255f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 29265f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 29273a5707766850f9ee9daa35299794328b5caf96dcChris Lattner case '@': 29283a5707766850f9ee9daa35299794328b5caf96dcChris Lattner // Objective C support. 29293a5707766850f9ee9daa35299794328b5caf96dcChris Lattner if (CurPtr[-1] == '@' && Features.ObjC1) 29309e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::at; 29313a5707766850f9ee9daa35299794328b5caf96dcChris Lattner else 29329e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::unknown; 29333a5707766850f9ee9daa35299794328b5caf96dcChris Lattner break; 29341eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 29355f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '\\': 29365f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // FIXME: UCN's. 29375f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // FALL THROUGH. 29385f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer default: 29399e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::unknown; 29405f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 29415f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 29421eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 29435f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Notify MIOpt that we read a non-whitespace/non-comment token. 29445f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer MIOpt.ReadToken(); 29455f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 29465f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Update the location of token as well as BufferPtr. 29479e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner FormTokenWithChars(Result, CurPtr, Kind); 29485f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 2949