Lexer.cpp revision 7d100872341f233c81e1d7b72b40457e62c36862
15f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer//===--- Lexer.cpp - C Language Family Lexer ------------------------------===// 25f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// 35f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// The LLVM Compiler Infrastructure 45f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// 50bc735ffcfb223c0186419547abaa5c84482663eChris Lattner// This file is distributed under the University of Illinois Open Source 60bc735ffcfb223c0186419547abaa5c84482663eChris Lattner// License. See LICENSE.TXT for details. 75f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// 85f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer//===----------------------------------------------------------------------===// 95f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// 10d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner// This file implements the Lexer and Token interfaces. 115f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// 125f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer//===----------------------------------------------------------------------===// 135f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// 145f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// TODO: GCC Diagnostics emitted by the lexer: 155f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// PEDWARN: (form feed|vertical tab) in preprocessing directive 165f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// 175f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// Universal characters, unicode, char mapping: 185f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// WARNING: `%.*s' is not in NFKC 195f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// WARNING: `%.*s' is not in NFC 205f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// 215f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// Other: 225f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// TODO: Options to support: 235f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// -fexec-charset,-fwide-exec-charset 245f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// 255f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer//===----------------------------------------------------------------------===// 265f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 275f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer#include "clang/Lex/Lexer.h" 285f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer#include "clang/Lex/Preprocessor.h" 29500d3297d2a21edeac4d46cbcbe21bc2352c2a28Chris Lattner#include "clang/Lex/LexDiagnostic.h" 3055817afdf9d453a443262a733f6caf6692dca118Douglas Gregor#include "clang/Lex/CodeCompletionHandler.h" 319dc1f530c086d2c16f8cba758b0f59a5bf41323aChris Lattner#include "clang/Basic/SourceManager.h" 32f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor#include "llvm/ADT/StringSwitch.h" 33409a03671224d4e5bdab1594c43baf070148f830Chris Lattner#include "llvm/Support/Compiler.h" 345f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer#include "llvm/Support/MemoryBuffer.h" 355f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer#include <cctype> 362fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper#include <cstring> 375f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencerusing namespace clang; 385f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 39a2bf105739cef7aea8e4d219629c627fcd11742eChris Lattnerstatic void InitCharacterInfo(); 405f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 41dbf388b54bb1688ee4470c61235494cbc34b8f3cChris Lattner//===----------------------------------------------------------------------===// 42dbf388b54bb1688ee4470c61235494cbc34b8f3cChris Lattner// Token Class Implementation 43dbf388b54bb1688ee4470c61235494cbc34b8f3cChris Lattner//===----------------------------------------------------------------------===// 44dbf388b54bb1688ee4470c61235494cbc34b8f3cChris Lattner 451eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump/// isObjCAtKeyword - Return true if we have an ObjC keyword identifier. 46dbf388b54bb1688ee4470c61235494cbc34b8f3cChris Lattnerbool Token::isObjCAtKeyword(tok::ObjCKeywordKind objcKey) const { 47bec1c9d6f3feeec112cc8eeae90c1be29c6aaf13Douglas Gregor if (IdentifierInfo *II = getIdentifierInfo()) 48bec1c9d6f3feeec112cc8eeae90c1be29c6aaf13Douglas Gregor return II->getObjCKeywordID() == objcKey; 49bec1c9d6f3feeec112cc8eeae90c1be29c6aaf13Douglas Gregor return false; 50dbf388b54bb1688ee4470c61235494cbc34b8f3cChris Lattner} 51dbf388b54bb1688ee4470c61235494cbc34b8f3cChris Lattner 52dbf388b54bb1688ee4470c61235494cbc34b8f3cChris Lattner/// getObjCKeywordID - Return the ObjC keyword kind. 53dbf388b54bb1688ee4470c61235494cbc34b8f3cChris Lattnertok::ObjCKeywordKind Token::getObjCKeywordID() const { 54dbf388b54bb1688ee4470c61235494cbc34b8f3cChris Lattner IdentifierInfo *specId = getIdentifierInfo(); 55dbf388b54bb1688ee4470c61235494cbc34b8f3cChris Lattner return specId ? specId->getObjCKeywordID() : tok::objc_not_keyword; 56dbf388b54bb1688ee4470c61235494cbc34b8f3cChris Lattner} 57dbf388b54bb1688ee4470c61235494cbc34b8f3cChris Lattner 5853702cd401b8fdca985aede7732c2f6a82ad9b1cChris Lattner 59dbf388b54bb1688ee4470c61235494cbc34b8f3cChris Lattner//===----------------------------------------------------------------------===// 60dbf388b54bb1688ee4470c61235494cbc34b8f3cChris Lattner// Lexer Class Implementation 61dbf388b54bb1688ee4470c61235494cbc34b8f3cChris Lattner//===----------------------------------------------------------------------===// 62dbf388b54bb1688ee4470c61235494cbc34b8f3cChris Lattner 631eb4433ac451dc16f4133a88af2d002ac26c58efMike Stumpvoid Lexer::InitLexer(const char *BufStart, const char *BufPtr, 6422d91ca8d7c134eac5cc6a4869e6a84c461ad624Chris Lattner const char *BufEnd) { 65a2bf105739cef7aea8e4d219629c627fcd11742eChris Lattner InitCharacterInfo(); 661eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 6722d91ca8d7c134eac5cc6a4869e6a84c461ad624Chris Lattner BufferStart = BufStart; 6822d91ca8d7c134eac5cc6a4869e6a84c461ad624Chris Lattner BufferPtr = BufPtr; 6922d91ca8d7c134eac5cc6a4869e6a84c461ad624Chris Lattner BufferEnd = BufEnd; 701eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 7122d91ca8d7c134eac5cc6a4869e6a84c461ad624Chris Lattner assert(BufEnd[0] == 0 && 7222d91ca8d7c134eac5cc6a4869e6a84c461ad624Chris Lattner "We assume that the input buffer has a null character at the end" 7322d91ca8d7c134eac5cc6a4869e6a84c461ad624Chris Lattner " to simplify lexing!"); 741eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 75156119df1d076b63609618976281961283f871dbEric Christopher // Check whether we have a BOM in the beginning of the buffer. If yes - act 76156119df1d076b63609618976281961283f871dbEric Christopher // accordingly. Right now we support only UTF-8 with and without BOM, so, just 77156119df1d076b63609618976281961283f871dbEric Christopher // skip the UTF-8 BOM if it's present. 78156119df1d076b63609618976281961283f871dbEric Christopher if (BufferStart == BufferPtr) { 79156119df1d076b63609618976281961283f871dbEric Christopher // Determine the size of the BOM. 805f9e272e632e951b1efe824cd16acb4d96077930Chris Lattner StringRef Buf(BufferStart, BufferEnd - BufferStart); 81969f9d47338fc36ebb6d24ad3a51e45eda07fd58Eli Friedman size_t BOMLength = llvm::StringSwitch<size_t>(Buf) 82156119df1d076b63609618976281961283f871dbEric Christopher .StartsWith("\xEF\xBB\xBF", 3) // UTF-8 BOM 83156119df1d076b63609618976281961283f871dbEric Christopher .Default(0); 84156119df1d076b63609618976281961283f871dbEric Christopher 85156119df1d076b63609618976281961283f871dbEric Christopher // Skip the BOM. 86156119df1d076b63609618976281961283f871dbEric Christopher BufferPtr += BOMLength; 87156119df1d076b63609618976281961283f871dbEric Christopher } 88156119df1d076b63609618976281961283f871dbEric Christopher 8922d91ca8d7c134eac5cc6a4869e6a84c461ad624Chris Lattner Is_PragmaLexer = false; 9034f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner IsInConflictMarker = false; 91156119df1d076b63609618976281961283f871dbEric Christopher 9222d91ca8d7c134eac5cc6a4869e6a84c461ad624Chris Lattner // Start of the file is a start of line. 9322d91ca8d7c134eac5cc6a4869e6a84c461ad624Chris Lattner IsAtStartOfLine = true; 941eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 9522d91ca8d7c134eac5cc6a4869e6a84c461ad624Chris Lattner // We are not after parsing a #. 9622d91ca8d7c134eac5cc6a4869e6a84c461ad624Chris Lattner ParsingPreprocessorDirective = false; 971eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 9822d91ca8d7c134eac5cc6a4869e6a84c461ad624Chris Lattner // We are not after parsing #include. 9922d91ca8d7c134eac5cc6a4869e6a84c461ad624Chris Lattner ParsingFilename = false; 1001eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 10122d91ca8d7c134eac5cc6a4869e6a84c461ad624Chris Lattner // We are not in raw mode. Raw mode disables diagnostics and interpretation 10222d91ca8d7c134eac5cc6a4869e6a84c461ad624Chris Lattner // of tokens (e.g. identifiers, thus disabling macro expansion). It is used 10322d91ca8d7c134eac5cc6a4869e6a84c461ad624Chris Lattner // to quickly lex the tokens of the buffer, e.g. when handling a "#if 0" block 10422d91ca8d7c134eac5cc6a4869e6a84c461ad624Chris Lattner // or otherwise skipping over tokens. 10522d91ca8d7c134eac5cc6a4869e6a84c461ad624Chris Lattner LexingRawMode = false; 1061eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 10722d91ca8d7c134eac5cc6a4869e6a84c461ad624Chris Lattner // Default to not keeping comments. 10822d91ca8d7c134eac5cc6a4869e6a84c461ad624Chris Lattner ExtendedTokenMode = 0; 10922d91ca8d7c134eac5cc6a4869e6a84c461ad624Chris Lattner} 11022d91ca8d7c134eac5cc6a4869e6a84c461ad624Chris Lattner 1110770dabb1ae81a2a9c2e7199262067103062a0b3Chris Lattner/// Lexer constructor - Create a new lexer object for the specified buffer 1120770dabb1ae81a2a9c2e7199262067103062a0b3Chris Lattner/// with the specified preprocessor managing the lexing process. This lexer 1130770dabb1ae81a2a9c2e7199262067103062a0b3Chris Lattner/// assumes that the associated file buffer and Preprocessor objects will 1140770dabb1ae81a2a9c2e7199262067103062a0b3Chris Lattner/// outlive it, so it doesn't take ownership of either of them. 1156e2901407bff59aeb4cc301cc58b034723d0eb49Chris LattnerLexer::Lexer(FileID FID, const llvm::MemoryBuffer *InputFile, Preprocessor &PP) 11688d3ac1341aa016cabd966c5b113a95ac05ea43fChris Lattner : PreprocessorLexer(&PP, FID), 11788d3ac1341aa016cabd966c5b113a95ac05ea43fChris Lattner FileLoc(PP.getSourceManager().getLocForStartOfFile(FID)), 11888d3ac1341aa016cabd966c5b113a95ac05ea43fChris Lattner Features(PP.getLangOptions()) { 1191eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 1200770dabb1ae81a2a9c2e7199262067103062a0b3Chris Lattner InitLexer(InputFile->getBufferStart(), InputFile->getBufferStart(), 1210770dabb1ae81a2a9c2e7199262067103062a0b3Chris Lattner InputFile->getBufferEnd()); 1221eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 1230770dabb1ae81a2a9c2e7199262067103062a0b3Chris Lattner // Default to keeping comments if the preprocessor wants them. 1240770dabb1ae81a2a9c2e7199262067103062a0b3Chris Lattner SetCommentRetentionState(PP.getCommentRetentionState()); 1250770dabb1ae81a2a9c2e7199262067103062a0b3Chris Lattner} 126dbf388b54bb1688ee4470c61235494cbc34b8f3cChris Lattner 127168ae2d44a443da75ea85db5f3b5081eb0bce113Chris Lattner/// Lexer constructor - Create a new raw lexer object. This object is only 128590f0cc643274267d4d41125b62557e1d87886c3Chris Lattner/// suitable for calls to 'LexRawToken'. This lexer assumes that the text 129590f0cc643274267d4d41125b62557e1d87886c3Chris Lattner/// range will outlive it, so it doesn't take ownership of it. 130168ae2d44a443da75ea85db5f3b5081eb0bce113Chris LattnerLexer::Lexer(SourceLocation fileloc, const LangOptions &features, 131de96c0f29c4cacabe6ea577c61db87c2a85aea6cChris Lattner const char *BufStart, const char *BufPtr, const char *BufEnd) 132c6fe32a91c7372caf09152ee31a24c4b5d24deedChris Lattner : FileLoc(fileloc), Features(features) { 13322d91ca8d7c134eac5cc6a4869e6a84c461ad624Chris Lattner 13422d91ca8d7c134eac5cc6a4869e6a84c461ad624Chris Lattner InitLexer(BufStart, BufPtr, BufEnd); 1351eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 136168ae2d44a443da75ea85db5f3b5081eb0bce113Chris Lattner // We *are* in raw mode. 137168ae2d44a443da75ea85db5f3b5081eb0bce113Chris Lattner LexingRawMode = true; 1385f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 1395f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 140025c3a66402fb713c2d9bf5dc174ff264765379aChris Lattner/// Lexer constructor - Create a new raw lexer object. This object is only 141025c3a66402fb713c2d9bf5dc174ff264765379aChris Lattner/// suitable for calls to 'LexRawToken'. This lexer assumes that the text 142025c3a66402fb713c2d9bf5dc174ff264765379aChris Lattner/// range will outlive it, so it doesn't take ownership of it. 1436e2901407bff59aeb4cc301cc58b034723d0eb49Chris LattnerLexer::Lexer(FileID FID, const llvm::MemoryBuffer *FromFile, 1446e2901407bff59aeb4cc301cc58b034723d0eb49Chris Lattner const SourceManager &SM, const LangOptions &features) 145025c3a66402fb713c2d9bf5dc174ff264765379aChris Lattner : FileLoc(SM.getLocForStartOfFile(FID)), Features(features) { 146025c3a66402fb713c2d9bf5dc174ff264765379aChris Lattner 1471eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump InitLexer(FromFile->getBufferStart(), FromFile->getBufferStart(), 148025c3a66402fb713c2d9bf5dc174ff264765379aChris Lattner FromFile->getBufferEnd()); 1491eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 150025c3a66402fb713c2d9bf5dc174ff264765379aChris Lattner // We *are* in raw mode. 151025c3a66402fb713c2d9bf5dc174ff264765379aChris Lattner LexingRawMode = true; 152025c3a66402fb713c2d9bf5dc174ff264765379aChris Lattner} 153025c3a66402fb713c2d9bf5dc174ff264765379aChris Lattner 15442e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner/// Create_PragmaLexer: Lexer constructor - Create a new lexer object for 15542e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner/// _Pragma expansion. This has a variety of magic semantics that this method 15642e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner/// sets up. It returns a new'd Lexer that must be delete'd when done. 15742e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner/// 15842e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner/// On entrance to this routine, TokStartLoc is a macro location which has a 15942e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner/// spelling loc that indicates the bytes to be lexed for the token and an 160433db06b614f26dc6829e86d6ff469e2cca7d4f9Chandler Carruth/// expansion location that indicates where all lexed tokens should be 16142e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner/// "expanded from". 16242e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner/// 16342e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner/// FIXME: It would really be nice to make _Pragma just be a wrapper around a 16442e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner/// normal lexer that remaps tokens as they fly by. This would require making 16542e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner/// Preprocessor::Lex virtual. Given that, we could just dump in a magic lexer 16642e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner/// interface that could handle this stuff. This would pull GetMappedTokenLoc 16742e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner/// out of the critical path of the lexer! 16842e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner/// 1691eb4433ac451dc16f4133a88af2d002ac26c58efMike StumpLexer *Lexer::Create_PragmaLexer(SourceLocation SpellingLoc, 170433db06b614f26dc6829e86d6ff469e2cca7d4f9Chandler Carruth SourceLocation ExpansionLocStart, 171433db06b614f26dc6829e86d6ff469e2cca7d4f9Chandler Carruth SourceLocation ExpansionLocEnd, 172bcc2a67e5180612417727cbdd8afd0f79fdf726dChris Lattner unsigned TokLen, Preprocessor &PP) { 17342e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner SourceManager &SM = PP.getSourceManager(); 17442e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner 17542e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner // Create the lexer as if we were going to lex the file normally. 176a11d61793341fea195c29a0dab3fbd74f2b39a8cChris Lattner FileID SpellingFID = SM.getFileID(SpellingLoc); 1776e2901407bff59aeb4cc301cc58b034723d0eb49Chris Lattner const llvm::MemoryBuffer *InputFile = SM.getBuffer(SpellingFID); 1786e2901407bff59aeb4cc301cc58b034723d0eb49Chris Lattner Lexer *L = new Lexer(SpellingFID, InputFile, PP); 1791eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 18042e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner // Now that the lexer is created, change the start/end locations so that we 18142e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner // just lex the subsection of the file that we want. This is lexing from a 18242e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner // scratch buffer. 18342e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner const char *StrData = SM.getCharacterData(SpellingLoc); 1841eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 18542e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner L->BufferPtr = StrData; 18642e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner L->BufferEnd = StrData+TokLen; 1871fa495304c81e03f07f278a47b5efe9317104aabChris Lattner assert(L->BufferEnd[0] == 0 && "Buffer is not nul terminated!"); 18842e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner 18942e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner // Set the SourceLocation with the remapping information. This ensures that 19042e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner // GetMappedTokenLoc will remap the tokens as they are lexed. 191bf340e452339e374ea6eef78c1f0a2abdd16c5a3Chandler Carruth L->FileLoc = SM.createExpansionLoc(SM.getLocForStartOfFile(SpellingFID), 192bf340e452339e374ea6eef78c1f0a2abdd16c5a3Chandler Carruth ExpansionLocStart, 193bf340e452339e374ea6eef78c1f0a2abdd16c5a3Chandler Carruth ExpansionLocEnd, TokLen); 1941eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 19542e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner // Ensure that the lexer thinks it is inside a directive, so that end \n will 19684021556baceb76eedf7d44be8ba71d9b8cfaccePeter Collingbourne // return an EOD token. 19742e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner L->ParsingPreprocessorDirective = true; 1981eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 19942e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner // This lexer really is for _Pragma. 20042e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner L->Is_PragmaLexer = true; 20142e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner return L; 20242e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner} 20342e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner 204168ae2d44a443da75ea85db5f3b5081eb0bce113Chris Lattner 2055f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// Stringify - Convert the specified string into a C string, with surrounding 2065f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// ""'s, and with escaped \ and " characters. 2075f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencerstd::string Lexer::Stringify(const std::string &Str, bool Charify) { 2085f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer std::string Result = Str; 2095f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer char Quote = Charify ? '\'' : '"'; 2105f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer for (unsigned i = 0, e = Result.size(); i != e; ++i) { 2115f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (Result[i] == '\\' || Result[i] == Quote) { 2125f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Result.insert(Result.begin()+i, '\\'); 2135f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer ++i; ++e; 2145f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 2155f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 2165f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer return Result; 2175f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 2185f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 219d8e3083840fef752d11ca183f42786470ed061e3Chris Lattner/// Stringify - Convert the specified string into a C string by escaping '\' 220d8e3083840fef752d11ca183f42786470ed061e3Chris Lattner/// and " characters. This does not add surrounding ""'s to the string. 2215f9e272e632e951b1efe824cd16acb4d96077930Chris Lattnervoid Lexer::Stringify(SmallVectorImpl<char> &Str) { 222d8e3083840fef752d11ca183f42786470ed061e3Chris Lattner for (unsigned i = 0, e = Str.size(); i != e; ++i) { 223d8e3083840fef752d11ca183f42786470ed061e3Chris Lattner if (Str[i] == '\\' || Str[i] == '"') { 224d8e3083840fef752d11ca183f42786470ed061e3Chris Lattner Str.insert(Str.begin()+i, '\\'); 225d8e3083840fef752d11ca183f42786470ed061e3Chris Lattner ++i; ++e; 226d8e3083840fef752d11ca183f42786470ed061e3Chris Lattner } 227d8e3083840fef752d11ca183f42786470ed061e3Chris Lattner } 228d8e3083840fef752d11ca183f42786470ed061e3Chris Lattner} 229d8e3083840fef752d11ca183f42786470ed061e3Chris Lattner 230b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner//===----------------------------------------------------------------------===// 231b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner// Token Spelling 232b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner//===----------------------------------------------------------------------===// 233b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner 234b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner/// getSpelling() - Return the 'spelling' of this token. The spelling of a 235b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner/// token are the characters used to represent the token in the source file 236b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner/// after trigraph expansion and escaped-newline folding. In particular, this 237b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner/// wants to get the true, uncanonicalized, spelling of things like digraphs 238b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner/// UCNs, etc. 2395f9e272e632e951b1efe824cd16acb4d96077930Chris LattnerStringRef Lexer::getSpelling(SourceLocation loc, 2405f9e272e632e951b1efe824cd16acb4d96077930Chris Lattner SmallVectorImpl<char> &buffer, 241834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall const SourceManager &SM, 242834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall const LangOptions &options, 243834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall bool *invalid) { 244834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall // Break down the source location. 245834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall std::pair<FileID, unsigned> locInfo = SM.getDecomposedLoc(loc); 246834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall 247834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall // Try to the load the file buffer. 248834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall bool invalidTemp = false; 2495f9e272e632e951b1efe824cd16acb4d96077930Chris Lattner StringRef file = SM.getBufferData(locInfo.first, &invalidTemp); 250834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall if (invalidTemp) { 251834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall if (invalid) *invalid = true; 2525f9e272e632e951b1efe824cd16acb4d96077930Chris Lattner return StringRef(); 253834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall } 254834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall 255834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall const char *tokenBegin = file.data() + locInfo.second; 256834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall 257834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall // Lex from the start of the given location. 258834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall Lexer lexer(SM.getLocForStartOfFile(locInfo.first), options, 259834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall file.begin(), tokenBegin, file.end()); 260834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall Token token; 261834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall lexer.LexFromRawLexer(token); 262834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall 263834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall unsigned length = token.getLength(); 264834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall 265834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall // Common case: no need for cleaning. 266834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall if (!token.needsCleaning()) 2675f9e272e632e951b1efe824cd16acb4d96077930Chris Lattner return StringRef(tokenBegin, length); 268834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall 269834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall // Hard case, we need to relex the characters into the string. 270834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall buffer.clear(); 271834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall buffer.reserve(length); 272834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall 273834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall for (const char *ti = tokenBegin, *te = ti + length; ti != te; ) { 274834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall unsigned charSize; 275834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall buffer.push_back(Lexer::getCharAndSizeNoWarn(ti, charSize, options)); 276834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall ti += charSize; 277834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall } 278834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall 2795f9e272e632e951b1efe824cd16acb4d96077930Chris Lattner return StringRef(buffer.data(), buffer.size()); 280834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall} 281834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall 282834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall/// getSpelling() - Return the 'spelling' of this token. The spelling of a 283834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall/// token are the characters used to represent the token in the source file 284834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall/// after trigraph expansion and escaped-newline folding. In particular, this 285834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall/// wants to get the true, uncanonicalized, spelling of things like digraphs 286834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall/// UCNs, etc. 287b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattnerstd::string Lexer::getSpelling(const Token &Tok, const SourceManager &SourceMgr, 288b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner const LangOptions &Features, bool *Invalid) { 289b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner assert((int)Tok.getLength() >= 0 && "Token character range is bogus!"); 290b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner 291b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner // If this token contains nothing interesting, return it directly. 292b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner bool CharDataInvalid = false; 293b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner const char* TokStart = SourceMgr.getCharacterData(Tok.getLocation(), 294b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner &CharDataInvalid); 295b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner if (Invalid) 296b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner *Invalid = CharDataInvalid; 297b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner if (CharDataInvalid) 298b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner return std::string(); 299b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner 300b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner if (!Tok.needsCleaning()) 301b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner return std::string(TokStart, TokStart+Tok.getLength()); 302b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner 303b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner std::string Result; 304b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner Result.reserve(Tok.getLength()); 305b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner 306b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner // Otherwise, hard case, relex the characters into the string. 307b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner for (const char *Ptr = TokStart, *End = TokStart+Tok.getLength(); 308b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner Ptr != End; ) { 309b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner unsigned CharSize; 310b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner Result.push_back(Lexer::getCharAndSizeNoWarn(Ptr, CharSize, Features)); 311b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner Ptr += CharSize; 312b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner } 313b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner assert(Result.size() != unsigned(Tok.getLength()) && 314b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner "NeedsCleaning flag set on something that didn't need cleaning!"); 315b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner return Result; 316b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner} 317b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner 318b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner/// getSpelling - This method is used to get the spelling of a token into a 319b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner/// preallocated buffer, instead of as an std::string. The caller is required 320b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner/// to allocate enough space for the token, which is guaranteed to be at least 321b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner/// Tok.getLength() bytes long. The actual length of the token is returned. 322b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner/// 323b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner/// Note that this method may do two possible things: it may either fill in 324b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner/// the buffer specified with characters, or it may *change the input pointer* 325b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner/// to point to a constant buffer with the data already in it (avoiding a 326b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner/// copy). The caller is not allowed to modify the returned buffer pointer 327b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner/// if an internal buffer is returned. 328b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattnerunsigned Lexer::getSpelling(const Token &Tok, const char *&Buffer, 329b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner const SourceManager &SourceMgr, 330b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner const LangOptions &Features, bool *Invalid) { 331b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner assert((int)Tok.getLength() >= 0 && "Token character range is bogus!"); 332c4bf2b9afb7d47445a9dc6bc848657098a4e3851Abramo Bagnara 333c4bf2b9afb7d47445a9dc6bc848657098a4e3851Abramo Bagnara const char *TokStart = 0; 334c4bf2b9afb7d47445a9dc6bc848657098a4e3851Abramo Bagnara // NOTE: this has to be checked *before* testing for an IdentifierInfo. 335c4bf2b9afb7d47445a9dc6bc848657098a4e3851Abramo Bagnara if (Tok.is(tok::raw_identifier)) 336c4bf2b9afb7d47445a9dc6bc848657098a4e3851Abramo Bagnara TokStart = Tok.getRawIdentifierData(); 337c4bf2b9afb7d47445a9dc6bc848657098a4e3851Abramo Bagnara else if (const IdentifierInfo *II = Tok.getIdentifierInfo()) { 338c4bf2b9afb7d47445a9dc6bc848657098a4e3851Abramo Bagnara // Just return the string from the identifier table, which is very quick. 339b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner Buffer = II->getNameStart(); 340b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner return II->getLength(); 341b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner } 342c4bf2b9afb7d47445a9dc6bc848657098a4e3851Abramo Bagnara 343c4bf2b9afb7d47445a9dc6bc848657098a4e3851Abramo Bagnara // NOTE: this can be checked even after testing for an IdentifierInfo. 344b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner if (Tok.isLiteral()) 345b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner TokStart = Tok.getLiteralData(); 346c4bf2b9afb7d47445a9dc6bc848657098a4e3851Abramo Bagnara 347b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner if (TokStart == 0) { 348c4bf2b9afb7d47445a9dc6bc848657098a4e3851Abramo Bagnara // Compute the start of the token in the input lexer buffer. 349b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner bool CharDataInvalid = false; 350b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner TokStart = SourceMgr.getCharacterData(Tok.getLocation(), &CharDataInvalid); 351b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner if (Invalid) 352b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner *Invalid = CharDataInvalid; 353b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner if (CharDataInvalid) { 354b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner Buffer = ""; 355b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner return 0; 356b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner } 357b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner } 358c4bf2b9afb7d47445a9dc6bc848657098a4e3851Abramo Bagnara 359b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner // If this token contains nothing interesting, return it directly. 360b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner if (!Tok.needsCleaning()) { 361b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner Buffer = TokStart; 362b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner return Tok.getLength(); 363b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner } 364c4bf2b9afb7d47445a9dc6bc848657098a4e3851Abramo Bagnara 365b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner // Otherwise, hard case, relex the characters into the string. 366b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner char *OutBuf = const_cast<char*>(Buffer); 367b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner for (const char *Ptr = TokStart, *End = TokStart+Tok.getLength(); 368b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner Ptr != End; ) { 369b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner unsigned CharSize; 370b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner *OutBuf++ = Lexer::getCharAndSizeNoWarn(Ptr, CharSize, Features); 371b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner Ptr += CharSize; 372b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner } 373b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner assert(unsigned(OutBuf-Buffer) != Tok.getLength() && 374b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner "NeedsCleaning flag set on something that didn't need cleaning!"); 375c4bf2b9afb7d47445a9dc6bc848657098a4e3851Abramo Bagnara 376b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner return OutBuf-Buffer; 377b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner} 378b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner 379b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner 380b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner 38133e9abd21083a0191a7676a04b497006d2da184dDouglas Gregorstatic bool isWhitespace(unsigned char c); 3825f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 3839a6119437672f42be5f50c3fe89fe843b1bfa5b5Chris Lattner/// MeasureTokenLength - Relex the token at the specified location and return 3849a6119437672f42be5f50c3fe89fe843b1bfa5b5Chris Lattner/// its length in bytes in the input file. If the token needs cleaning (e.g. 3859a6119437672f42be5f50c3fe89fe843b1bfa5b5Chris Lattner/// includes a trigraph or an escaped newline) then this count includes bytes 3869a6119437672f42be5f50c3fe89fe843b1bfa5b5Chris Lattner/// that are part of that. 3879a6119437672f42be5f50c3fe89fe843b1bfa5b5Chris Lattnerunsigned Lexer::MeasureTokenLength(SourceLocation Loc, 3882c78b873f4f3823ae859c15674cb3d76c8554113Chris Lattner const SourceManager &SM, 3892c78b873f4f3823ae859c15674cb3d76c8554113Chris Lattner const LangOptions &LangOpts) { 3909a6119437672f42be5f50c3fe89fe843b1bfa5b5Chris Lattner // TODO: this could be special cased for common tokens like identifiers, ')', 3919a6119437672f42be5f50c3fe89fe843b1bfa5b5Chris Lattner // etc to make this faster, if it mattered. Just look at StrData[0] to handle 3921eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump // all obviously single-char tokens. This could use 3939a6119437672f42be5f50c3fe89fe843b1bfa5b5Chris Lattner // Lexer::isObviouslySimpleCharacter for example to handle identifiers or 3949a6119437672f42be5f50c3fe89fe843b1bfa5b5Chris Lattner // something. 395de7aeefc5573d669ed476d7bda7a8940d3bcadb7Chris Lattner 396de7aeefc5573d669ed476d7bda7a8940d3bcadb7Chris Lattner // If this comes from a macro expansion, we really do want the macro name, not 397de7aeefc5573d669ed476d7bda7a8940d3bcadb7Chris Lattner // the token this macro expanded to. 398402785357ab053dd53f4fdd858b9630a5e0f8badChandler Carruth Loc = SM.getExpansionLoc(Loc); 399363fdc29656cc03c1817268888f95e6343470aa8Chris Lattner std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(Loc); 400f715ca12bfc9fddfde75f98a197424434428b821Douglas Gregor bool Invalid = false; 4015f9e272e632e951b1efe824cd16acb4d96077930Chris Lattner StringRef Buffer = SM.getBufferData(LocInfo.first, &Invalid); 402f715ca12bfc9fddfde75f98a197424434428b821Douglas Gregor if (Invalid) 403aea67dbd653a2dd6dd5cc2159279e81e855b2482Douglas Gregor return 0; 404f6ac97b101c8840efa92bf29166077ce4049e293Benjamin Kramer 405f6ac97b101c8840efa92bf29166077ce4049e293Benjamin Kramer const char *StrData = Buffer.data()+LocInfo.second; 4068350394c65b81bba3986dfe44ae17423873741deChris Lattner 40733e9abd21083a0191a7676a04b497006d2da184dDouglas Gregor if (isWhitespace(StrData[0])) 40833e9abd21083a0191a7676a04b497006d2da184dDouglas Gregor return 0; 40933e9abd21083a0191a7676a04b497006d2da184dDouglas Gregor 4109a6119437672f42be5f50c3fe89fe843b1bfa5b5Chris Lattner // Create a lexer starting at the beginning of this token. 411c3526d89ef9c31639ec8b25180cfb22354344241Sebastian Redl Lexer TheLexer(SM.getLocForStartOfFile(LocInfo.first), LangOpts, 412c3526d89ef9c31639ec8b25180cfb22354344241Sebastian Redl Buffer.begin(), StrData, Buffer.end()); 41339de7409bffb6b725a8aa64f0ba77ab51e8c9eb3Chris Lattner TheLexer.SetCommentRetentionState(true); 4149a6119437672f42be5f50c3fe89fe843b1bfa5b5Chris Lattner Token TheTok; 415590f0cc643274267d4d41125b62557e1d87886c3Chris Lattner TheLexer.LexFromRawLexer(TheTok); 4169a6119437672f42be5f50c3fe89fe843b1bfa5b5Chris Lattner return TheTok.getLength(); 4179a6119437672f42be5f50c3fe89fe843b1bfa5b5Chris Lattner} 4189a6119437672f42be5f50c3fe89fe843b1bfa5b5Chris Lattner 4190e870622e4d4b2ecb7bc6ffd2c97f74fd14220b6Argyrios Kyrtzidisstatic SourceLocation getBeginningOfFileToken(SourceLocation Loc, 4200e870622e4d4b2ecb7bc6ffd2c97f74fd14220b6Argyrios Kyrtzidis const SourceManager &SM, 4210e870622e4d4b2ecb7bc6ffd2c97f74fd14220b6Argyrios Kyrtzidis const LangOptions &LangOpts) { 4220e870622e4d4b2ecb7bc6ffd2c97f74fd14220b6Argyrios Kyrtzidis assert(Loc.isFileID()); 423a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(Loc); 4243de84241d90f3dd280126fdf2c4651667151c967Douglas Gregor if (LocInfo.first.isInvalid()) 4253de84241d90f3dd280126fdf2c4651667151c967Douglas Gregor return Loc; 4263de84241d90f3dd280126fdf2c4651667151c967Douglas Gregor 427a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor bool Invalid = false; 4285f9e272e632e951b1efe824cd16acb4d96077930Chris Lattner StringRef Buffer = SM.getBufferData(LocInfo.first, &Invalid); 429a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor if (Invalid) 430a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor return Loc; 431a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor 432a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor // Back up from the current location until we hit the beginning of a line 433a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor // (or the buffer). We'll relex from that point. 434a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor const char *BufStart = Buffer.data(); 4353de84241d90f3dd280126fdf2c4651667151c967Douglas Gregor if (LocInfo.second >= Buffer.size()) 4363de84241d90f3dd280126fdf2c4651667151c967Douglas Gregor return Loc; 4373de84241d90f3dd280126fdf2c4651667151c967Douglas Gregor 438a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor const char *StrData = BufStart+LocInfo.second; 439a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor if (StrData[0] == '\n' || StrData[0] == '\r') 440a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor return Loc; 441a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor 442a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor const char *LexStart = StrData; 443a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor while (LexStart != BufStart) { 444a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor if (LexStart[0] == '\n' || LexStart[0] == '\r') { 445a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor ++LexStart; 446a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor break; 447a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor } 448a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor 449a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor --LexStart; 450a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor } 451a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor 452a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor // Create a lexer starting at the beginning of this token. 453a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor SourceLocation LexerStartLoc = Loc.getFileLocWithOffset(-LocInfo.second); 454a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor Lexer TheLexer(LexerStartLoc, LangOpts, BufStart, LexStart, Buffer.end()); 455a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor TheLexer.SetCommentRetentionState(true); 456a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor 457a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor // Lex tokens until we find the token that contains the source location. 458a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor Token TheTok; 459a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor do { 460a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor TheLexer.LexFromRawLexer(TheTok); 461a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor 462a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor if (TheLexer.getBufferLocation() > StrData) { 463a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor // Lexing this token has taken the lexer past the source location we're 464a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor // looking for. If the current token encompasses our source location, 465a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor // return the beginning of that token. 466a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor if (TheLexer.getBufferLocation() - TheTok.getLength() <= StrData) 467a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor return TheTok.getLocation(); 468a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor 469a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor // We ended up skipping over the source location entirely, which means 470a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor // that it points into whitespace. We're done here. 471a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor break; 472a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor } 473a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor } while (TheTok.getKind() != tok::eof); 474a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor 475a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor // We've passed our source location; just return the original source location. 476a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor return Loc; 477a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor} 478a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor 4790e870622e4d4b2ecb7bc6ffd2c97f74fd14220b6Argyrios KyrtzidisSourceLocation Lexer::GetBeginningOfToken(SourceLocation Loc, 4800e870622e4d4b2ecb7bc6ffd2c97f74fd14220b6Argyrios Kyrtzidis const SourceManager &SM, 4810e870622e4d4b2ecb7bc6ffd2c97f74fd14220b6Argyrios Kyrtzidis const LangOptions &LangOpts) { 4820e870622e4d4b2ecb7bc6ffd2c97f74fd14220b6Argyrios Kyrtzidis if (Loc.isFileID()) 4830e870622e4d4b2ecb7bc6ffd2c97f74fd14220b6Argyrios Kyrtzidis return getBeginningOfFileToken(Loc, SM, LangOpts); 4840e870622e4d4b2ecb7bc6ffd2c97f74fd14220b6Argyrios Kyrtzidis 4850e870622e4d4b2ecb7bc6ffd2c97f74fd14220b6Argyrios Kyrtzidis if (!SM.isMacroArgExpansion(Loc)) 4860e870622e4d4b2ecb7bc6ffd2c97f74fd14220b6Argyrios Kyrtzidis return Loc; 4870e870622e4d4b2ecb7bc6ffd2c97f74fd14220b6Argyrios Kyrtzidis 4880e870622e4d4b2ecb7bc6ffd2c97f74fd14220b6Argyrios Kyrtzidis SourceLocation FileLoc = SM.getSpellingLoc(Loc); 4890e870622e4d4b2ecb7bc6ffd2c97f74fd14220b6Argyrios Kyrtzidis SourceLocation BeginFileLoc = getBeginningOfFileToken(FileLoc, SM, LangOpts); 4900e870622e4d4b2ecb7bc6ffd2c97f74fd14220b6Argyrios Kyrtzidis std::pair<FileID, unsigned> FileLocInfo = SM.getDecomposedLoc(FileLoc); 4910e870622e4d4b2ecb7bc6ffd2c97f74fd14220b6Argyrios Kyrtzidis std::pair<FileID, unsigned> BeginFileLocInfo= SM.getDecomposedLoc(BeginFileLoc); 4920e870622e4d4b2ecb7bc6ffd2c97f74fd14220b6Argyrios Kyrtzidis assert(FileLocInfo.first == BeginFileLocInfo.first && 4930e870622e4d4b2ecb7bc6ffd2c97f74fd14220b6Argyrios Kyrtzidis FileLocInfo.second >= BeginFileLocInfo.second); 4940e870622e4d4b2ecb7bc6ffd2c97f74fd14220b6Argyrios Kyrtzidis return Loc.getFileLocWithOffset(SM.getDecomposedLoc(BeginFileLoc).second - 4950e870622e4d4b2ecb7bc6ffd2c97f74fd14220b6Argyrios Kyrtzidis SM.getDecomposedLoc(FileLoc).second); 4960e870622e4d4b2ecb7bc6ffd2c97f74fd14220b6Argyrios Kyrtzidis} 4970e870622e4d4b2ecb7bc6ffd2c97f74fd14220b6Argyrios Kyrtzidis 498f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregornamespace { 499f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor enum PreambleDirectiveKind { 500f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor PDK_Skipped, 501f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor PDK_StartIf, 502f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor PDK_EndIf, 503f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor PDK_Unknown 504f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor }; 505f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor} 506f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor 507f4f6c9db68465b886ec2e596feaa6ecc782395a4Douglas Gregorstd::pair<unsigned, bool> 50803c107a42fae79e89d0016999a1a04c07d65591aArgyrios KyrtzidisLexer::ComputePreamble(const llvm::MemoryBuffer *Buffer, 50903c107a42fae79e89d0016999a1a04c07d65591aArgyrios Kyrtzidis const LangOptions &Features, unsigned MaxLines) { 510f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor // Create a lexer starting at the beginning of the file. Note that we use a 511f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor // "fake" file source location at offset 1 so that the lexer will track our 512f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor // position within the file. 513f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor const unsigned StartOffset = 1; 514f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor SourceLocation StartLoc = SourceLocation::getFromRawEncoding(StartOffset); 51503c107a42fae79e89d0016999a1a04c07d65591aArgyrios Kyrtzidis Lexer TheLexer(StartLoc, Features, Buffer->getBufferStart(), 516f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor Buffer->getBufferStart(), Buffer->getBufferEnd()); 517f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor 518f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor bool InPreprocessorDirective = false; 519f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor Token TheTok; 520f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor Token IfStartTok; 521f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor unsigned IfCount = 0; 522c8c97a03eb0fdeb4f5fc9c4dea308ebbf46c2c93Argyrios Kyrtzidis 523c8c97a03eb0fdeb4f5fc9c4dea308ebbf46c2c93Argyrios Kyrtzidis unsigned MaxLineOffset = 0; 524c8c97a03eb0fdeb4f5fc9c4dea308ebbf46c2c93Argyrios Kyrtzidis if (MaxLines) { 525c8c97a03eb0fdeb4f5fc9c4dea308ebbf46c2c93Argyrios Kyrtzidis const char *CurPtr = Buffer->getBufferStart(); 526c8c97a03eb0fdeb4f5fc9c4dea308ebbf46c2c93Argyrios Kyrtzidis unsigned CurLine = 0; 527c8c97a03eb0fdeb4f5fc9c4dea308ebbf46c2c93Argyrios Kyrtzidis while (CurPtr != Buffer->getBufferEnd()) { 528c8c97a03eb0fdeb4f5fc9c4dea308ebbf46c2c93Argyrios Kyrtzidis char ch = *CurPtr++; 529c8c97a03eb0fdeb4f5fc9c4dea308ebbf46c2c93Argyrios Kyrtzidis if (ch == '\n') { 530c8c97a03eb0fdeb4f5fc9c4dea308ebbf46c2c93Argyrios Kyrtzidis ++CurLine; 531c8c97a03eb0fdeb4f5fc9c4dea308ebbf46c2c93Argyrios Kyrtzidis if (CurLine == MaxLines) 532c8c97a03eb0fdeb4f5fc9c4dea308ebbf46c2c93Argyrios Kyrtzidis break; 533c8c97a03eb0fdeb4f5fc9c4dea308ebbf46c2c93Argyrios Kyrtzidis } 534c8c97a03eb0fdeb4f5fc9c4dea308ebbf46c2c93Argyrios Kyrtzidis } 535c8c97a03eb0fdeb4f5fc9c4dea308ebbf46c2c93Argyrios Kyrtzidis if (CurPtr != Buffer->getBufferEnd()) 536c8c97a03eb0fdeb4f5fc9c4dea308ebbf46c2c93Argyrios Kyrtzidis MaxLineOffset = CurPtr - Buffer->getBufferStart(); 537c8c97a03eb0fdeb4f5fc9c4dea308ebbf46c2c93Argyrios Kyrtzidis } 538df95a13ec73d2cdaea79555cb412d767f4963120Douglas Gregor 539f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor do { 540f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor TheLexer.LexFromRawLexer(TheTok); 541f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor 542f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor if (InPreprocessorDirective) { 543f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor // If we've hit the end of the file, we're done. 544f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor if (TheTok.getKind() == tok::eof) { 545f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor InPreprocessorDirective = false; 546f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor break; 547f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor } 548f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor 549f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor // If we haven't hit the end of the preprocessor directive, skip this 550f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor // token. 551f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor if (!TheTok.isAtStartOfLine()) 552f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor continue; 553f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor 554f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor // We've passed the end of the preprocessor directive, and will look 555f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor // at this token again below. 556f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor InPreprocessorDirective = false; 557f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor } 558f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor 559df95a13ec73d2cdaea79555cb412d767f4963120Douglas Gregor // Keep track of the # of lines in the preamble. 560df95a13ec73d2cdaea79555cb412d767f4963120Douglas Gregor if (TheTok.isAtStartOfLine()) { 561c8c97a03eb0fdeb4f5fc9c4dea308ebbf46c2c93Argyrios Kyrtzidis unsigned TokOffset = TheTok.getLocation().getRawEncoding() - StartOffset; 562df95a13ec73d2cdaea79555cb412d767f4963120Douglas Gregor 563df95a13ec73d2cdaea79555cb412d767f4963120Douglas Gregor // If we were asked to limit the number of lines in the preamble, 564df95a13ec73d2cdaea79555cb412d767f4963120Douglas Gregor // and we're about to exceed that limit, we're done. 565c8c97a03eb0fdeb4f5fc9c4dea308ebbf46c2c93Argyrios Kyrtzidis if (MaxLineOffset && TokOffset >= MaxLineOffset) 566df95a13ec73d2cdaea79555cb412d767f4963120Douglas Gregor break; 567df95a13ec73d2cdaea79555cb412d767f4963120Douglas Gregor } 568df95a13ec73d2cdaea79555cb412d767f4963120Douglas Gregor 569f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor // Comments are okay; skip over them. 570f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor if (TheTok.getKind() == tok::comment) 571f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor continue; 572f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor 573f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor if (TheTok.isAtStartOfLine() && TheTok.getKind() == tok::hash) { 574f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor // This is the start of a preprocessor directive. 575f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor Token HashTok = TheTok; 576f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor InPreprocessorDirective = true; 577f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor 57819207f1e5f51261a33492602501fb7ada50ea546Joerg Sonnenberger // Figure out which directive this is. Since we're lexing raw tokens, 579f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor // we don't have an identifier table available. Instead, just look at 580f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor // the raw identifier to recognize and categorize preprocessor directives. 581f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor TheLexer.LexFromRawLexer(TheTok); 582c4bf2b9afb7d47445a9dc6bc848657098a4e3851Abramo Bagnara if (TheTok.getKind() == tok::raw_identifier && !TheTok.needsCleaning()) { 5835f9e272e632e951b1efe824cd16acb4d96077930Chris Lattner StringRef Keyword(TheTok.getRawIdentifierData(), 584c4bf2b9afb7d47445a9dc6bc848657098a4e3851Abramo Bagnara TheTok.getLength()); 585f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor PreambleDirectiveKind PDK 586f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor = llvm::StringSwitch<PreambleDirectiveKind>(Keyword) 587f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor .Case("include", PDK_Skipped) 588f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor .Case("__include_macros", PDK_Skipped) 589f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor .Case("define", PDK_Skipped) 590f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor .Case("undef", PDK_Skipped) 591f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor .Case("line", PDK_Skipped) 592f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor .Case("error", PDK_Skipped) 593f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor .Case("pragma", PDK_Skipped) 594f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor .Case("import", PDK_Skipped) 595f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor .Case("include_next", PDK_Skipped) 596f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor .Case("warning", PDK_Skipped) 597f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor .Case("ident", PDK_Skipped) 598f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor .Case("sccs", PDK_Skipped) 599f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor .Case("assert", PDK_Skipped) 600f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor .Case("unassert", PDK_Skipped) 601f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor .Case("if", PDK_StartIf) 602f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor .Case("ifdef", PDK_StartIf) 603f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor .Case("ifndef", PDK_StartIf) 604f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor .Case("elif", PDK_Skipped) 605f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor .Case("else", PDK_Skipped) 606f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor .Case("endif", PDK_EndIf) 607f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor .Default(PDK_Unknown); 608f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor 609f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor switch (PDK) { 610f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor case PDK_Skipped: 611f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor continue; 612f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor 613f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor case PDK_StartIf: 614f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor if (IfCount == 0) 615f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor IfStartTok = HashTok; 616f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor 617f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor ++IfCount; 618f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor continue; 619f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor 620f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor case PDK_EndIf: 621f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor // Mismatched #endif. The preamble ends here. 622f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor if (IfCount == 0) 623f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor break; 624f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor 625f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor --IfCount; 626f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor continue; 627f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor 628f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor case PDK_Unknown: 629f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor // We don't know what this directive is; stop at the '#'. 630f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor break; 631f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor } 632f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor } 633f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor 634f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor // We only end up here if we didn't recognize the preprocessor 635f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor // directive or it was one that can't occur in the preamble at this 636f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor // point. Roll back the current token to the location of the '#'. 637f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor InPreprocessorDirective = false; 638f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor TheTok = HashTok; 639f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor } 640f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor 641df95a13ec73d2cdaea79555cb412d767f4963120Douglas Gregor // We hit a token that we don't recognize as being in the 642df95a13ec73d2cdaea79555cb412d767f4963120Douglas Gregor // "preprocessing only" part of the file, so we're no longer in 643df95a13ec73d2cdaea79555cb412d767f4963120Douglas Gregor // the preamble. 644f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor break; 645f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor } while (true); 646f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor 647f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor SourceLocation End = IfCount? IfStartTok.getLocation() : TheTok.getLocation(); 648f4f6c9db68465b886ec2e596feaa6ecc782395a4Douglas Gregor return std::make_pair(End.getRawEncoding() - StartLoc.getRawEncoding(), 649f4f6c9db68465b886ec2e596feaa6ecc782395a4Douglas Gregor IfCount? IfStartTok.isAtStartOfLine() 650f4f6c9db68465b886ec2e596feaa6ecc782395a4Douglas Gregor : TheTok.isAtStartOfLine()); 651f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor} 652f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor 6537ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner 6547ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner/// AdvanceToTokenCharacter - Given a location that specifies the start of a 6557ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner/// token, return a new location that specifies a character within the token. 6567ef5c27eb6e8ebe58b52013246c06753c3613263Chris LattnerSourceLocation Lexer::AdvanceToTokenCharacter(SourceLocation TokStart, 6577ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner unsigned CharNo, 6587ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner const SourceManager &SM, 6597ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner const LangOptions &Features) { 660433db06b614f26dc6829e86d6ff469e2cca7d4f9Chandler Carruth // Figure out how many physical characters away the specified expansion 6617ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner // character is. This needs to take into consideration newlines and 6627ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner // trigraphs. 6637ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner bool Invalid = false; 6647ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner const char *TokPtr = SM.getCharacterData(TokStart, &Invalid); 6657ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner 6667ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner // If they request the first char of the token, we're trivially done. 6677ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner if (Invalid || (CharNo == 0 && Lexer::isObviouslySimpleCharacter(*TokPtr))) 6687ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner return TokStart; 6697ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner 6707ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner unsigned PhysOffset = 0; 6717ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner 6727ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner // The usual case is that tokens don't contain anything interesting. Skip 6737ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner // over the uninteresting characters. If a token only consists of simple 6747ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner // chars, this method is extremely fast. 6757ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner while (Lexer::isObviouslySimpleCharacter(*TokPtr)) { 6767ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner if (CharNo == 0) 6777ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner return TokStart.getFileLocWithOffset(PhysOffset); 6787ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner ++TokPtr, --CharNo, ++PhysOffset; 6797ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner } 6807ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner 6817ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner // If we have a character that may be a trigraph or escaped newline, use a 6827ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner // lexer to parse it correctly. 6837ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner for (; CharNo; --CharNo) { 6847ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner unsigned Size; 6857ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner Lexer::getCharAndSizeNoWarn(TokPtr, Size, Features); 6867ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner TokPtr += Size; 6877ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner PhysOffset += Size; 6887ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner } 6897ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner 6907ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner // Final detail: if we end up on an escaped newline, we want to return the 6917ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner // location of the actual byte of the token. For example foo\<newline>bar 6927ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner // advanced by 3 should return the location of b, not of \\. One compounding 6937ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner // detail of this is that the escape may be made by a trigraph. 6947ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner if (!Lexer::isObviouslySimpleCharacter(*TokPtr)) 6957ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner PhysOffset += Lexer::SkipEscapedNewLines(TokPtr)-TokPtr; 6967ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner 6977ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner return TokStart.getFileLocWithOffset(PhysOffset); 6987ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner} 6997ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner 7007ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner/// \brief Computes the source location just past the end of the 7017ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner/// token at this source location. 7027ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner/// 7037ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner/// This routine can be used to produce a source location that 7047ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner/// points just past the end of the token referenced by \p Loc, and 7057ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner/// is generally used when a diagnostic needs to point just after a 7067ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner/// token where it expected something different that it received. If 7077ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner/// the returned source location would not be meaningful (e.g., if 7087ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner/// it points into a macro), this routine returns an invalid 7097ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner/// source location. 7107ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner/// 7117ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner/// \param Offset an offset from the end of the token, where the source 7127ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner/// location should refer to. The default offset (0) produces a source 7137ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner/// location pointing just past the end of the token; an offset of 1 produces 7147ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner/// a source location pointing to the last character in the token, etc. 7157ef5c27eb6e8ebe58b52013246c06753c3613263Chris LattnerSourceLocation Lexer::getLocForEndOfToken(SourceLocation Loc, unsigned Offset, 7167ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner const SourceManager &SM, 7177ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner const LangOptions &Features) { 7187ddf6b2d77ac4cb27f78d817d7884e6ce17afd0cArgyrios Kyrtzidis if (Loc.isInvalid()) 7197ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner return SourceLocation(); 7207ddf6b2d77ac4cb27f78d817d7884e6ce17afd0cArgyrios Kyrtzidis 7217ddf6b2d77ac4cb27f78d817d7884e6ce17afd0cArgyrios Kyrtzidis if (Loc.isMacroID()) { 722433db06b614f26dc6829e86d6ff469e2cca7d4f9Chandler Carruth if (Offset > 0 || !isAtEndOfMacroExpansion(Loc, SM, Features)) 723433db06b614f26dc6829e86d6ff469e2cca7d4f9Chandler Carruth return SourceLocation(); // Points inside the macro expansion. 7247ddf6b2d77ac4cb27f78d817d7884e6ce17afd0cArgyrios Kyrtzidis 725433db06b614f26dc6829e86d6ff469e2cca7d4f9Chandler Carruth // Continue and find the location just after the macro expansion. 726edc3dccece244a584f8ebdb81da6c962c08e79beChandler Carruth Loc = SM.getExpansionRange(Loc).second; 7277ddf6b2d77ac4cb27f78d817d7884e6ce17afd0cArgyrios Kyrtzidis } 7287ddf6b2d77ac4cb27f78d817d7884e6ce17afd0cArgyrios Kyrtzidis 7297ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner unsigned Len = Lexer::MeasureTokenLength(Loc, SM, Features); 7307ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner if (Len > Offset) 7317ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner Len = Len - Offset; 7327ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner else 7337ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner return Loc; 7347ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner 73577ebb3829f2abb9be2d40927b793b72347626aadJohn McCall return Loc.getFileLocWithOffset(Len); 7367ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner} 7377ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner 7387a759606d93975866051f67104ae58446e55f404Argyrios Kyrtzidis/// \brief Returns true if the given MacroID location points at the first 739433db06b614f26dc6829e86d6ff469e2cca7d4f9Chandler Carruth/// token of the macro expansion. 740433db06b614f26dc6829e86d6ff469e2cca7d4f9Chandler Carruthbool Lexer::isAtStartOfMacroExpansion(SourceLocation loc, 741f62d43d2afe1960755a1b5813cae1e5983bcac1bDouglas Gregor const SourceManager &SM, 742f62d43d2afe1960755a1b5813cae1e5983bcac1bDouglas Gregor const LangOptions &LangOpts) { 7437a759606d93975866051f67104ae58446e55f404Argyrios Kyrtzidis assert(loc.isValid() && loc.isMacroID() && "Expected a valid macro loc"); 7447a759606d93975866051f67104ae58446e55f404Argyrios Kyrtzidis 7457a759606d93975866051f67104ae58446e55f404Argyrios Kyrtzidis std::pair<FileID, unsigned> infoLoc = SM.getDecomposedLoc(loc); 7467a759606d93975866051f67104ae58446e55f404Argyrios Kyrtzidis // FIXME: If the token comes from the macro token paste operator ('##') 7477a759606d93975866051f67104ae58446e55f404Argyrios Kyrtzidis // this function will always return false; 7487a759606d93975866051f67104ae58446e55f404Argyrios Kyrtzidis if (infoLoc.second > 0) 7497a759606d93975866051f67104ae58446e55f404Argyrios Kyrtzidis return false; // Does not point at the start of token. 7507a759606d93975866051f67104ae58446e55f404Argyrios Kyrtzidis 751433db06b614f26dc6829e86d6ff469e2cca7d4f9Chandler Carruth SourceLocation expansionLoc = 7521728762d5a8cfaf8d64385f47b311e84de1ae7a2Chandler Carruth SM.getSLocEntry(infoLoc.first).getExpansion().getExpansionLocStart(); 753433db06b614f26dc6829e86d6ff469e2cca7d4f9Chandler Carruth if (expansionLoc.isFileID()) 754433db06b614f26dc6829e86d6ff469e2cca7d4f9Chandler Carruth return true; // No other macro expansions, this is the first. 7557a759606d93975866051f67104ae58446e55f404Argyrios Kyrtzidis 756433db06b614f26dc6829e86d6ff469e2cca7d4f9Chandler Carruth return isAtStartOfMacroExpansion(expansionLoc, SM, LangOpts); 7577a759606d93975866051f67104ae58446e55f404Argyrios Kyrtzidis} 7587a759606d93975866051f67104ae58446e55f404Argyrios Kyrtzidis 7597a759606d93975866051f67104ae58446e55f404Argyrios Kyrtzidis/// \brief Returns true if the given MacroID location points at the last 760433db06b614f26dc6829e86d6ff469e2cca7d4f9Chandler Carruth/// token of the macro expansion. 761433db06b614f26dc6829e86d6ff469e2cca7d4f9Chandler Carruthbool Lexer::isAtEndOfMacroExpansion(SourceLocation loc, 7627a759606d93975866051f67104ae58446e55f404Argyrios Kyrtzidis const SourceManager &SM, 7637a759606d93975866051f67104ae58446e55f404Argyrios Kyrtzidis const LangOptions &LangOpts) { 7647a759606d93975866051f67104ae58446e55f404Argyrios Kyrtzidis assert(loc.isValid() && loc.isMacroID() && "Expected a valid macro loc"); 7657a759606d93975866051f67104ae58446e55f404Argyrios Kyrtzidis 7667a759606d93975866051f67104ae58446e55f404Argyrios Kyrtzidis SourceLocation spellLoc = SM.getSpellingLoc(loc); 7677a759606d93975866051f67104ae58446e55f404Argyrios Kyrtzidis unsigned tokLen = MeasureTokenLength(spellLoc, SM, LangOpts); 7687a759606d93975866051f67104ae58446e55f404Argyrios Kyrtzidis if (tokLen == 0) 7697a759606d93975866051f67104ae58446e55f404Argyrios Kyrtzidis return false; 7707a759606d93975866051f67104ae58446e55f404Argyrios Kyrtzidis 7717a759606d93975866051f67104ae58446e55f404Argyrios Kyrtzidis FileID FID = SM.getFileID(loc); 7727a759606d93975866051f67104ae58446e55f404Argyrios Kyrtzidis SourceLocation afterLoc = loc.getFileLocWithOffset(tokLen+1); 773f8c50652f7b224e66b0b6098d1fba07e036019b4Argyrios Kyrtzidis if (SM.isInFileID(afterLoc, FID)) 774f8c50652f7b224e66b0b6098d1fba07e036019b4Argyrios Kyrtzidis return false; // Still in the same FileID, does not point to the last token. 7757a759606d93975866051f67104ae58446e55f404Argyrios Kyrtzidis 7767a759606d93975866051f67104ae58446e55f404Argyrios Kyrtzidis // FIXME: If the token comes from the macro token paste operator ('##') 7777a759606d93975866051f67104ae58446e55f404Argyrios Kyrtzidis // or the stringify operator ('#') this function will always return false; 778f8c50652f7b224e66b0b6098d1fba07e036019b4Argyrios Kyrtzidis 779433db06b614f26dc6829e86d6ff469e2cca7d4f9Chandler Carruth SourceLocation expansionLoc = 7801728762d5a8cfaf8d64385f47b311e84de1ae7a2Chandler Carruth SM.getSLocEntry(FID).getExpansion().getExpansionLocEnd(); 781433db06b614f26dc6829e86d6ff469e2cca7d4f9Chandler Carruth if (expansionLoc.isFileID()) 782433db06b614f26dc6829e86d6ff469e2cca7d4f9Chandler Carruth return true; // No other macro expansions. 7837a759606d93975866051f67104ae58446e55f404Argyrios Kyrtzidis 784433db06b614f26dc6829e86d6ff469e2cca7d4f9Chandler Carruth return isAtEndOfMacroExpansion(expansionLoc, SM, LangOpts); 7857a759606d93975866051f67104ae58446e55f404Argyrios Kyrtzidis} 7867a759606d93975866051f67104ae58446e55f404Argyrios Kyrtzidis 7875f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer//===----------------------------------------------------------------------===// 7885f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// Character information. 7895f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer//===----------------------------------------------------------------------===// 7905f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 7915f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencerenum { 7925f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CHAR_HORZ_WS = 0x01, // ' ', '\t', '\f', '\v'. Note, no '\0' 7935f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CHAR_VERT_WS = 0x02, // '\r', '\n' 7945f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CHAR_LETTER = 0x04, // a-z,A-Z 7955f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CHAR_NUMBER = 0x08, // 0-9 7965f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CHAR_UNDER = 0x10, // _ 7972fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper CHAR_PERIOD = 0x20, // . 7982fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper CHAR_RAWDEL = 0x40 // {}[]#<>%:;?*+-/^&|~!=,"' 7995f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer}; 8005f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 80103b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner// Statically initialize CharInfo table based on ASCII character set 80203b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner// Reference: FreeBSD 7.2 /usr/share/misc/ascii 803a2bf105739cef7aea8e4d219629c627fcd11742eChris Lattnerstatic const unsigned char CharInfo[256] = 80403b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner{ 80503b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner// 0 NUL 1 SOH 2 STX 3 ETX 80603b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner// 4 EOT 5 ENQ 6 ACK 7 BEL 80703b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner 0 , 0 , 0 , 0 , 80803b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner 0 , 0 , 0 , 0 , 80903b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner// 8 BS 9 HT 10 NL 11 VT 81003b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner//12 NP 13 CR 14 SO 15 SI 81103b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner 0 , CHAR_HORZ_WS, CHAR_VERT_WS, CHAR_HORZ_WS, 81203b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner CHAR_HORZ_WS, CHAR_VERT_WS, 0 , 0 , 81303b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner//16 DLE 17 DC1 18 DC2 19 DC3 81403b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner//20 DC4 21 NAK 22 SYN 23 ETB 81503b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner 0 , 0 , 0 , 0 , 81603b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner 0 , 0 , 0 , 0 , 81703b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner//24 CAN 25 EM 26 SUB 27 ESC 81803b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner//28 FS 29 GS 30 RS 31 US 81903b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner 0 , 0 , 0 , 0 , 82003b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner 0 , 0 , 0 , 0 , 82103b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner//32 SP 33 ! 34 " 35 # 82203b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner//36 $ 37 % 38 & 39 ' 8232fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper CHAR_HORZ_WS, CHAR_RAWDEL , CHAR_RAWDEL , CHAR_RAWDEL , 8242fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper 0 , CHAR_RAWDEL , CHAR_RAWDEL , CHAR_RAWDEL , 82503b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner//40 ( 41 ) 42 * 43 + 82603b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner//44 , 45 - 46 . 47 / 8272fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper 0 , 0 , CHAR_RAWDEL , CHAR_RAWDEL , 8282fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper CHAR_RAWDEL , CHAR_RAWDEL , CHAR_PERIOD , CHAR_RAWDEL , 82903b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner//48 0 49 1 50 2 51 3 83003b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner//52 4 53 5 54 6 55 7 83103b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner CHAR_NUMBER , CHAR_NUMBER , CHAR_NUMBER , CHAR_NUMBER , 83203b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner CHAR_NUMBER , CHAR_NUMBER , CHAR_NUMBER , CHAR_NUMBER , 83303b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner//56 8 57 9 58 : 59 ; 83403b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner//60 < 61 = 62 > 63 ? 8352fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper CHAR_NUMBER , CHAR_NUMBER , CHAR_RAWDEL , CHAR_RAWDEL , 8362fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper CHAR_RAWDEL , CHAR_RAWDEL , CHAR_RAWDEL , CHAR_RAWDEL , 83703b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner//64 @ 65 A 66 B 67 C 83803b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner//68 D 69 E 70 F 71 G 83903b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner 0 , CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , 84003b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , 84103b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner//72 H 73 I 74 J 75 K 84203b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner//76 L 77 M 78 N 79 O 84303b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , 84403b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , 84503b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner//80 P 81 Q 82 R 83 S 84603b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner//84 T 85 U 86 V 87 W 84703b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , 84803b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , 84903b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner//88 X 89 Y 90 Z 91 [ 85003b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner//92 \ 93 ] 94 ^ 95 _ 8512fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , CHAR_RAWDEL , 8522fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper 0 , CHAR_RAWDEL , CHAR_RAWDEL , CHAR_UNDER , 85303b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner//96 ` 97 a 98 b 99 c 85403b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner//100 d 101 e 102 f 103 g 85503b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner 0 , CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , 85603b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , 85703b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner//104 h 105 i 106 j 107 k 85803b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner//108 l 109 m 110 n 111 o 85903b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , 86003b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , 86103b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner//112 p 113 q 114 r 115 s 86203b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner//116 t 117 u 118 v 119 w 86303b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , 86403b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , 86503b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner//120 x 121 y 122 z 123 { 8662fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper//124 | 125 } 126 ~ 127 DEL 8672fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , CHAR_RAWDEL , 8682fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper CHAR_RAWDEL , CHAR_RAWDEL , CHAR_RAWDEL , 0 86903b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner}; 87003b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner 871a2bf105739cef7aea8e4d219629c627fcd11742eChris Lattnerstatic void InitCharacterInfo() { 8725f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer static bool isInited = false; 8735f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (isInited) return; 87403b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner // check the statically-initialized CharInfo table 87503b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner assert(CHAR_HORZ_WS == CharInfo[(int)' ']); 87603b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner assert(CHAR_HORZ_WS == CharInfo[(int)'\t']); 87703b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner assert(CHAR_HORZ_WS == CharInfo[(int)'\f']); 87803b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner assert(CHAR_HORZ_WS == CharInfo[(int)'\v']); 87903b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner assert(CHAR_VERT_WS == CharInfo[(int)'\n']); 88003b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner assert(CHAR_VERT_WS == CharInfo[(int)'\r']); 88103b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner assert(CHAR_UNDER == CharInfo[(int)'_']); 88203b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner assert(CHAR_PERIOD == CharInfo[(int)'.']); 88303b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner for (unsigned i = 'a'; i <= 'z'; ++i) { 88403b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner assert(CHAR_LETTER == CharInfo[i]); 88503b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner assert(CHAR_LETTER == CharInfo[i+'A'-'a']); 88603b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner } 8875f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer for (unsigned i = '0'; i <= '9'; ++i) 88803b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner assert(CHAR_NUMBER == CharInfo[i]); 8897b682656310b0d7e94a98d982444bc7ddedc653fSteve Naroff 89003b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner isInited = true; 8915f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 8925f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 89303b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner 8945f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// isIdentifierBody - Return true if this is the body character of an 8955f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// identifier, which is [a-zA-Z0-9_]. 8965f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencerstatic inline bool isIdentifierBody(unsigned char c) { 89795c062b94e2af5380617cf5dc115497ae7a5f817Hartmut Kaiser return (CharInfo[c] & (CHAR_LETTER|CHAR_NUMBER|CHAR_UNDER)) ? true : false; 8985f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 8995f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 9005f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// isHorizontalWhitespace - Return true if this character is horizontal 9015f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// whitespace: ' ', '\t', '\f', '\v'. Note that this returns false for '\0'. 9025f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencerstatic inline bool isHorizontalWhitespace(unsigned char c) { 90395c062b94e2af5380617cf5dc115497ae7a5f817Hartmut Kaiser return (CharInfo[c] & CHAR_HORZ_WS) ? true : false; 9045f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 9055f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 906aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks/// isVerticalWhitespace - Return true if this character is vertical 907aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks/// whitespace: '\n', '\r'. Note that this returns false for '\0'. 908aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaksstatic inline bool isVerticalWhitespace(unsigned char c) { 909aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks return (CharInfo[c] & CHAR_VERT_WS) ? true : false; 910aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks} 911aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks 9125f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// isWhitespace - Return true if this character is horizontal or vertical 9135f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// whitespace: ' ', '\t', '\f', '\v', '\n', '\r'. Note that this returns false 9145f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// for '\0'. 9155f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencerstatic inline bool isWhitespace(unsigned char c) { 91695c062b94e2af5380617cf5dc115497ae7a5f817Hartmut Kaiser return (CharInfo[c] & (CHAR_HORZ_WS|CHAR_VERT_WS)) ? true : false; 9175f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 9185f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 9195f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// isNumberBody - Return true if this is the body character of an 9205f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// preprocessing number, which is [a-zA-Z0-9_.]. 9215f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencerstatic inline bool isNumberBody(unsigned char c) { 9221eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump return (CharInfo[c] & (CHAR_LETTER|CHAR_NUMBER|CHAR_UNDER|CHAR_PERIOD)) ? 92395c062b94e2af5380617cf5dc115497ae7a5f817Hartmut Kaiser true : false; 9245f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 9255f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 9262fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper/// isRawStringDelimBody - Return true if this is the body character of a 9272fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper/// raw string delimiter. 9282fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topperstatic inline bool isRawStringDelimBody(unsigned char c) { 9292fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper return (CharInfo[c] & 9302fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper (CHAR_LETTER|CHAR_NUMBER|CHAR_UNDER|CHAR_PERIOD|CHAR_RAWDEL)) ? 9312fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper true : false; 9322fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper} 9332fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper 9345f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 9355f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer//===----------------------------------------------------------------------===// 9365f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// Diagnostics forwarding code. 9375f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer//===----------------------------------------------------------------------===// 9385f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 939409a03671224d4e5bdab1594c43baf070148f830Chris Lattner/// GetMappedTokenLoc - If lexing out of a 'mapped buffer', where we pretend the 940433db06b614f26dc6829e86d6ff469e2cca7d4f9Chandler Carruth/// lexer buffer was all expanded at a single point, perform the mapping. 941409a03671224d4e5bdab1594c43baf070148f830Chris Lattner/// This is currently only used for _Pragma implementation, so it is the slow 942409a03671224d4e5bdab1594c43baf070148f830Chris Lattner/// path of the hot getSourceLocation method. Do not allow it to be inlined. 94314bd96571ef6f0e97dc79ec4d01b547d60e8fa68Chandler Carruthstatic LLVM_ATTRIBUTE_NOINLINE SourceLocation GetMappedTokenLoc( 94414bd96571ef6f0e97dc79ec4d01b547d60e8fa68Chandler Carruth Preprocessor &PP, SourceLocation FileLoc, unsigned CharNo, unsigned TokLen); 945409a03671224d4e5bdab1594c43baf070148f830Chris Lattnerstatic SourceLocation GetMappedTokenLoc(Preprocessor &PP, 946409a03671224d4e5bdab1594c43baf070148f830Chris Lattner SourceLocation FileLoc, 947de7aeefc5573d669ed476d7bda7a8940d3bcadb7Chris Lattner unsigned CharNo, unsigned TokLen) { 948433db06b614f26dc6829e86d6ff469e2cca7d4f9Chandler Carruth assert(FileLoc.isMacroID() && "Must be a macro expansion"); 9491eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 950409a03671224d4e5bdab1594c43baf070148f830Chris Lattner // Otherwise, we're lexing "mapped tokens". This is used for things like 951433db06b614f26dc6829e86d6ff469e2cca7d4f9Chandler Carruth // _Pragma handling. Combine the expansion location of FileLoc with the 952df7c17a8d02fe09a3466786bae3e40fc3252687aChris Lattner // spelling location. 953e7fb48466afcbf2c4ccdfa658824282fdc3c512cChris Lattner SourceManager &SM = PP.getSourceManager(); 9541eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 955433db06b614f26dc6829e86d6ff469e2cca7d4f9Chandler Carruth // Create a new SLoc which is expanded from Expansion(FileLoc) but whose 956df7c17a8d02fe09a3466786bae3e40fc3252687aChris Lattner // characters come from spelling(FileLoc)+Offset. 957e7fb48466afcbf2c4ccdfa658824282fdc3c512cChris Lattner SourceLocation SpellingLoc = SM.getSpellingLoc(FileLoc); 958bcc2a67e5180612417727cbdd8afd0f79fdf726dChris Lattner SpellingLoc = SpellingLoc.getFileLocWithOffset(CharNo); 9591eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 960e7fb48466afcbf2c4ccdfa658824282fdc3c512cChris Lattner // Figure out the expansion loc range, which is the range covered by the 961e7fb48466afcbf2c4ccdfa658824282fdc3c512cChris Lattner // original _Pragma(...) sequence. 962e7fb48466afcbf2c4ccdfa658824282fdc3c512cChris Lattner std::pair<SourceLocation,SourceLocation> II = 963999f739404edf2078cf9f9c28b4dc45c19765842Chandler Carruth SM.getImmediateExpansionRange(FileLoc); 9641eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 965bf340e452339e374ea6eef78c1f0a2abdd16c5a3Chandler Carruth return SM.createExpansionLoc(SpellingLoc, II.first, II.second, TokLen); 966409a03671224d4e5bdab1594c43baf070148f830Chris Lattner} 967409a03671224d4e5bdab1594c43baf070148f830Chris Lattner 9685f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// getSourceLocation - Return a source location identifier for the specified 9695f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// offset in the current file. 970de7aeefc5573d669ed476d7bda7a8940d3bcadb7Chris LattnerSourceLocation Lexer::getSourceLocation(const char *Loc, 971de7aeefc5573d669ed476d7bda7a8940d3bcadb7Chris Lattner unsigned TokLen) const { 972448cec4c1c3705f6f49ffdefb58a7329942a2dd8Chris Lattner assert(Loc >= BufferStart && Loc <= BufferEnd && 9735f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer "Location out of range for this buffer!"); 9749dc1f530c086d2c16f8cba758b0f59a5bf41323aChris Lattner 9759dc1f530c086d2c16f8cba758b0f59a5bf41323aChris Lattner // In the normal case, we're just lexing from a simple file buffer, return 9769dc1f530c086d2c16f8cba758b0f59a5bf41323aChris Lattner // the file id from FileLoc with the offset specified. 977448cec4c1c3705f6f49ffdefb58a7329942a2dd8Chris Lattner unsigned CharNo = Loc-BufferStart; 9789dc1f530c086d2c16f8cba758b0f59a5bf41323aChris Lattner if (FileLoc.isFileID()) 979bcc2a67e5180612417727cbdd8afd0f79fdf726dChris Lattner return FileLoc.getFileLocWithOffset(CharNo); 9801eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 9812b2453a7d8fe732561795431f39ceb2b2a832d84Chris Lattner // Otherwise, this is the _Pragma lexer case, which pretends that all of the 9822b2453a7d8fe732561795431f39ceb2b2a832d84Chris Lattner // tokens are lexed from where the _Pragma was defined. 983168ae2d44a443da75ea85db5f3b5081eb0bce113Chris Lattner assert(PP && "This doesn't work on raw lexers"); 984de7aeefc5573d669ed476d7bda7a8940d3bcadb7Chris Lattner return GetMappedTokenLoc(*PP, FileLoc, CharNo, TokLen); 9855f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 9865f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 9875f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// Diag - Forwarding function for diagnostics. This translate a source 9885f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// position in the current buffer into a SourceLocation object for rendering. 9893cbfe2c4159e0a219ae660d50625c013aa4afbd0Chris LattnerDiagnosticBuilder Lexer::Diag(const char *Loc, unsigned DiagID) const { 9903692b09faa9fe346f39bc922db6dce48cdcc3f63Chris Lattner return PP->Diag(getSourceLocation(Loc), DiagID); 9915f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 9925f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 9935f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer//===----------------------------------------------------------------------===// 9945f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// Trigraph and Escaped Newline Handling Code. 9955f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer//===----------------------------------------------------------------------===// 9965f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 9975f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// GetTrigraphCharForLetter - Given a character that occurs after a ?? pair, 9985f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// return the decoded trigraph letter it corresponds to, or '\0' if nothing. 9995f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencerstatic char GetTrigraphCharForLetter(char Letter) { 10005f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer switch (Letter) { 10015f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer default: return 0; 10025f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '=': return '#'; 10035f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case ')': return ']'; 10045f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '(': return '['; 10055f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '!': return '|'; 10065f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '\'': return '^'; 10075f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '>': return '}'; 10085f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '/': return '\\'; 10095f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '<': return '{'; 10105f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '-': return '~'; 10115f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 10125f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 10135f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 10145f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// DecodeTrigraphChar - If the specified character is a legal trigraph when 10155f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// prefixed with ??, emit a trigraph warning. If trigraphs are enabled, 10165f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// return the result character. Finally, emit a warning about trigraph use 10175f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// whether trigraphs are enabled or not. 10185f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencerstatic char DecodeTrigraphChar(const char *CP, Lexer *L) { 10195f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer char Res = GetTrigraphCharForLetter(*CP); 10203692b09faa9fe346f39bc922db6dce48cdcc3f63Chris Lattner if (!Res || !L) return Res; 10211eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 10223692b09faa9fe346f39bc922db6dce48cdcc3f63Chris Lattner if (!L->getFeatures().Trigraphs) { 102374d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner if (!L->isLexingRawMode()) 102474d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner L->Diag(CP-2, diag::trigraph_ignored); 10253692b09faa9fe346f39bc922db6dce48cdcc3f63Chris Lattner return 0; 10265f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 10271eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 102874d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner if (!L->isLexingRawMode()) 10295f9e272e632e951b1efe824cd16acb4d96077930Chris Lattner L->Diag(CP-2, diag::trigraph_converted) << StringRef(&Res, 1); 10305f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer return Res; 10315f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 10325f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 103324f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner/// getEscapedNewLineSize - Return the size of the specified escaped newline, 103424f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner/// or 0 if it is not an escaped newline. P[-1] is known to be a "\" or a 10351eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump/// trigraph equivalent on entry to this function. 103624f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattnerunsigned Lexer::getEscapedNewLineSize(const char *Ptr) { 103724f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner unsigned Size = 0; 103824f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner while (isWhitespace(Ptr[Size])) { 103924f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner ++Size; 10401eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 104124f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner if (Ptr[Size-1] != '\n' && Ptr[Size-1] != '\r') 104224f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner continue; 104324f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner 104424f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner // If this is a \r\n or \n\r, skip the other half. 104524f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner if ((Ptr[Size] == '\r' || Ptr[Size] == '\n') && 104624f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner Ptr[Size-1] != Ptr[Size]) 104724f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner ++Size; 10481eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 104924f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner return Size; 10501eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump } 10511eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 105224f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner // Not an escaped newline, must be a \t or something else. 105324f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner return 0; 105424f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner} 105524f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner 1056033749571f8d4c804eeb357c70b06424aa24503bChris Lattner/// SkipEscapedNewLines - If P points to an escaped newline (or a series of 1057033749571f8d4c804eeb357c70b06424aa24503bChris Lattner/// them), skip over them and return the first non-escaped-newline found, 1058033749571f8d4c804eeb357c70b06424aa24503bChris Lattner/// otherwise return P. 1059033749571f8d4c804eeb357c70b06424aa24503bChris Lattnerconst char *Lexer::SkipEscapedNewLines(const char *P) { 1060033749571f8d4c804eeb357c70b06424aa24503bChris Lattner while (1) { 1061033749571f8d4c804eeb357c70b06424aa24503bChris Lattner const char *AfterEscape; 1062033749571f8d4c804eeb357c70b06424aa24503bChris Lattner if (*P == '\\') { 1063033749571f8d4c804eeb357c70b06424aa24503bChris Lattner AfterEscape = P+1; 1064033749571f8d4c804eeb357c70b06424aa24503bChris Lattner } else if (*P == '?') { 1065033749571f8d4c804eeb357c70b06424aa24503bChris Lattner // If not a trigraph for escape, bail out. 1066033749571f8d4c804eeb357c70b06424aa24503bChris Lattner if (P[1] != '?' || P[2] != '/') 1067033749571f8d4c804eeb357c70b06424aa24503bChris Lattner return P; 1068033749571f8d4c804eeb357c70b06424aa24503bChris Lattner AfterEscape = P+3; 1069033749571f8d4c804eeb357c70b06424aa24503bChris Lattner } else { 1070033749571f8d4c804eeb357c70b06424aa24503bChris Lattner return P; 1071033749571f8d4c804eeb357c70b06424aa24503bChris Lattner } 10721eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 1073033749571f8d4c804eeb357c70b06424aa24503bChris Lattner unsigned NewLineSize = Lexer::getEscapedNewLineSize(AfterEscape); 1074033749571f8d4c804eeb357c70b06424aa24503bChris Lattner if (NewLineSize == 0) return P; 1075033749571f8d4c804eeb357c70b06424aa24503bChris Lattner P = AfterEscape+NewLineSize; 1076033749571f8d4c804eeb357c70b06424aa24503bChris Lattner } 1077033749571f8d4c804eeb357c70b06424aa24503bChris Lattner} 1078033749571f8d4c804eeb357c70b06424aa24503bChris Lattner 1079aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks/// \brief Checks that the given token is the first token that occurs after the 1080aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks/// given location (this excludes comments and whitespace). Returns the location 1081aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks/// immediately after the specified token. If the token is not found or the 1082aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks/// location is inside a macro, the returned source location will be invalid. 1083aca25bccefe56121b686706afc84c8cb5d46e65bAnna ZaksSourceLocation Lexer::findLocationAfterToken(SourceLocation Loc, 1084aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks tok::TokenKind TKind, 1085aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks const SourceManager &SM, 1086aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks const LangOptions &LangOpts, 1087aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks bool SkipTrailingWhitespaceAndNewLine) { 1088aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks if (Loc.isMacroID()) { 1089aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks if (!Lexer::isAtEndOfMacroExpansion(Loc, SM, LangOpts)) 1090aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks return SourceLocation(); 1091aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks Loc = SM.getExpansionRange(Loc).second; 1092aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks } 1093aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks Loc = Lexer::getLocForEndOfToken(Loc, 0, SM, LangOpts); 1094aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks 1095aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks // Break down the source location. 1096aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(Loc); 1097aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks 1098aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks // Try to load the file buffer. 1099aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks bool InvalidTemp = false; 1100aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks llvm::StringRef File = SM.getBufferData(LocInfo.first, &InvalidTemp); 1101aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks if (InvalidTemp) 1102aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks return SourceLocation(); 1103aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks 1104aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks const char *TokenBegin = File.data() + LocInfo.second; 1105aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks 1106aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks // Lex from the start of the given location. 1107aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks Lexer lexer(SM.getLocForStartOfFile(LocInfo.first), LangOpts, File.begin(), 1108aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks TokenBegin, File.end()); 1109aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks // Find the token. 1110aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks Token Tok; 1111aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks lexer.LexFromRawLexer(Tok); 1112aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks if (Tok.isNot(TKind)) 1113aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks return SourceLocation(); 1114aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks SourceLocation TokenLoc = Tok.getLocation(); 1115aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks 1116aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks // Calculate how much whitespace needs to be skipped if any. 1117aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks unsigned NumWhitespaceChars = 0; 1118aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks if (SkipTrailingWhitespaceAndNewLine) { 1119aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks const char *TokenEnd = SM.getCharacterData(TokenLoc) + 1120aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks Tok.getLength(); 1121aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks unsigned char C = *TokenEnd; 1122aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks while (isHorizontalWhitespace(C)) { 1123aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks C = *(++TokenEnd); 1124aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks NumWhitespaceChars++; 1125aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks } 1126aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks if (isVerticalWhitespace(C)) 1127aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks NumWhitespaceChars++; 1128aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks } 1129aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks 1130aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks return TokenLoc.getFileLocWithOffset(Tok.getLength() + NumWhitespaceChars); 1131aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks} 113224f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner 11335f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// getCharAndSizeSlow - Peek a single 'character' from the specified buffer, 11345f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// get its size, and return it. This is tricky in several cases: 11355f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// 1. If currently at the start of a trigraph, we warn about the trigraph, 11365f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// then either return the trigraph (skipping 3 chars) or the '?', 11375f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// depending on whether trigraphs are enabled or not. 11385f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// 2. If this is an escaped newline (potentially with whitespace between 11395f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// the backslash and newline), implicitly skip the newline and return 11405f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// the char after it. 11415f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// 3. If this is a UCN, return it. FIXME: C++ UCN's? 11425f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// 11435f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// This handles the slow/uncommon case of the getCharAndSize method. Here we 11445f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// know that we can accumulate into Size, and that we have already incremented 11455f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// Ptr by Size bytes. 11465f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// 11475f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// NOTE: When this method is updated, getCharAndSizeSlowNoWarn (below) should 11485f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// be updated to match. 11495f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// 11505f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencerchar Lexer::getCharAndSizeSlow(const char *Ptr, unsigned &Size, 1151d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner Token *Tok) { 11525f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If we have a slash, look for an escaped newline. 11535f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (Ptr[0] == '\\') { 11545f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer ++Size; 11555f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer ++Ptr; 11565f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid SpencerSlash: 11575f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Common case, backslash-char where the char is not whitespace. 11585f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (!isWhitespace(Ptr[0])) return '\\'; 11591eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 11605636a3b6ece2c1f413464b72545e08eb0b7f06e4Chris Lattner // See if we have optional whitespace characters between the slash and 11615636a3b6ece2c1f413464b72545e08eb0b7f06e4Chris Lattner // newline. 116224f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner if (unsigned EscapedNewLineSize = getEscapedNewLineSize(Ptr)) { 116324f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner // Remember that this token needs to be cleaned. 116424f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner if (Tok) Tok->setFlag(Token::NeedsCleaning); 116524f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner 116624f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner // Warn if there was whitespace between the backslash and newline. 11675636a3b6ece2c1f413464b72545e08eb0b7f06e4Chris Lattner if (Ptr[0] != '\n' && Ptr[0] != '\r' && Tok && !isLexingRawMode()) 116824f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner Diag(Ptr, diag::backslash_newline_space); 11691eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 117024f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner // Found backslash<whitespace><newline>. Parse the char after it. 117124f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner Size += EscapedNewLineSize; 117224f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner Ptr += EscapedNewLineSize; 117324f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner // Use slow version to accumulate a correct size field. 117424f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner return getCharAndSizeSlow(Ptr, Size, Tok); 117524f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner } 11761eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 11775f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Otherwise, this is not an escaped newline, just return the slash. 11785f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer return '\\'; 11795f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 11801eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 11815f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If this is a trigraph, process it. 11825f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (Ptr[0] == '?' && Ptr[1] == '?') { 11835f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If this is actually a legal trigraph (not something like "??x"), emit 11845f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // a trigraph warning. If so, and if trigraphs are enabled, return it. 11855f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (char C = DecodeTrigraphChar(Ptr+2, Tok ? this : 0)) { 11865f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Remember that this token needs to be cleaned. 1187d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner if (Tok) Tok->setFlag(Token::NeedsCleaning); 11885f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 11895f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Ptr += 3; 11905f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Size += 3; 11915f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (C == '\\') goto Slash; 11925f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer return C; 11935f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 11945f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 11951eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 11965f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If this is neither, return a single character. 11975f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer ++Size; 11985f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer return *Ptr; 11995f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 12005f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 12015f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 12025f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// getCharAndSizeSlowNoWarn - Handle the slow/uncommon case of the 12035f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// getCharAndSizeNoWarn method. Here we know that we can accumulate into Size, 12045f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// and that we have already incremented Ptr by Size bytes. 12055f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// 12065f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// NOTE: When this method is updated, getCharAndSizeSlow (above) should 12075f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// be updated to match. 12085f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencerchar Lexer::getCharAndSizeSlowNoWarn(const char *Ptr, unsigned &Size, 12095f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer const LangOptions &Features) { 12105f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If we have a slash, look for an escaped newline. 12115f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (Ptr[0] == '\\') { 12125f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer ++Size; 12135f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer ++Ptr; 12145f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid SpencerSlash: 12155f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Common case, backslash-char where the char is not whitespace. 12165f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (!isWhitespace(Ptr[0])) return '\\'; 12171eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 12185f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // See if we have optional whitespace characters followed by a newline. 121924f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner if (unsigned EscapedNewLineSize = getEscapedNewLineSize(Ptr)) { 122024f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner // Found backslash<whitespace><newline>. Parse the char after it. 122124f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner Size += EscapedNewLineSize; 122224f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner Ptr += EscapedNewLineSize; 12231eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 122424f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner // Use slow version to accumulate a correct size field. 122524f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner return getCharAndSizeSlowNoWarn(Ptr, Size, Features); 122624f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner } 12271eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 12285f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Otherwise, this is not an escaped newline, just return the slash. 12295f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer return '\\'; 12305f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 12311eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 12325f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If this is a trigraph, process it. 12335f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (Features.Trigraphs && Ptr[0] == '?' && Ptr[1] == '?') { 12345f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If this is actually a legal trigraph (not something like "??x"), return 12355f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // it. 12365f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (char C = GetTrigraphCharForLetter(Ptr[2])) { 12375f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Ptr += 3; 12385f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Size += 3; 12395f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (C == '\\') goto Slash; 12405f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer return C; 12415f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 12425f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 12431eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 12445f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If this is neither, return a single character. 12455f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer ++Size; 12465f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer return *Ptr; 12475f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 12485f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 12495f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer//===----------------------------------------------------------------------===// 12505f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// Helper methods for lexing. 12515f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer//===----------------------------------------------------------------------===// 12525f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 1253f4f6c9db68465b886ec2e596feaa6ecc782395a4Douglas Gregor/// \brief Routine that indiscriminately skips bytes in the source file. 1254f4f6c9db68465b886ec2e596feaa6ecc782395a4Douglas Gregorvoid Lexer::SkipBytes(unsigned Bytes, bool StartOfLine) { 1255f4f6c9db68465b886ec2e596feaa6ecc782395a4Douglas Gregor BufferPtr += Bytes; 1256f4f6c9db68465b886ec2e596feaa6ecc782395a4Douglas Gregor if (BufferPtr > BufferEnd) 1257f4f6c9db68465b886ec2e596feaa6ecc782395a4Douglas Gregor BufferPtr = BufferEnd; 1258f4f6c9db68465b886ec2e596feaa6ecc782395a4Douglas Gregor IsAtStartOfLine = StartOfLine; 1259f4f6c9db68465b886ec2e596feaa6ecc782395a4Douglas Gregor} 1260f4f6c9db68465b886ec2e596feaa6ecc782395a4Douglas Gregor 1261d217773f106856a11879ec79dc468efefaf2ee75Chris Lattnervoid Lexer::LexIdentifier(Token &Result, const char *CurPtr) { 12625f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Match [_A-Za-z0-9]*, we have already matched [_A-Za-z$] 12635f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer unsigned Size; 12645f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer unsigned char C = *CurPtr++; 1265cd991dbb12f24368753cef086c7ad3ec203c9ea6Chris Lattner while (isIdentifierBody(C)) 12665f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer C = *CurPtr++; 1267cd991dbb12f24368753cef086c7ad3ec203c9ea6Chris Lattner 12685f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer --CurPtr; // Back up over the skipped character. 12695f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 12705f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Fast path, no $,\,? in identifier found. '\' might be an escaped newline 12715f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // or UCN, and ? might be a trigraph for '\', an escaped newline or UCN. 12725f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // FIXME: UCNs. 1273cd991dbb12f24368753cef086c7ad3ec203c9ea6Chris Lattner // 1274cd991dbb12f24368753cef086c7ad3ec203c9ea6Chris Lattner // TODO: Could merge these checks into a CharInfo flag to make the comparison 1275cd991dbb12f24368753cef086c7ad3ec203c9ea6Chris Lattner // cheaper 12765f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (C != '\\' && C != '?' && (C != '$' || !Features.DollarIdents)) { 12775f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid SpencerFinishIdentifier: 12785f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer const char *IdStart = BufferPtr; 1279c4bf2b9afb7d47445a9dc6bc848657098a4e3851Abramo Bagnara FormTokenWithChars(Result, CurPtr, tok::raw_identifier); 1280c4bf2b9afb7d47445a9dc6bc848657098a4e3851Abramo Bagnara Result.setRawIdentifierData(IdStart); 12811eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 12825f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If we are in raw mode, return this identifier raw. There is no need to 12835f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // look up identifier information or attempt to macro expand it. 1284c4bf2b9afb7d47445a9dc6bc848657098a4e3851Abramo Bagnara if (LexingRawMode) 1285c4bf2b9afb7d47445a9dc6bc848657098a4e3851Abramo Bagnara return; 12861eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 1287c4bf2b9afb7d47445a9dc6bc848657098a4e3851Abramo Bagnara // Fill in Result.IdentifierInfo and update the token kind, 1288c4bf2b9afb7d47445a9dc6bc848657098a4e3851Abramo Bagnara // looking up the identifier in the identifier table. 1289c4bf2b9afb7d47445a9dc6bc848657098a4e3851Abramo Bagnara IdentifierInfo *II = PP->LookUpIdentifierInfo(Result); 12901eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 12915f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Finally, now that we know we have an identifier, pass this off to the 12925f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // preprocessor, which may macro expand it or something. 1293d1186fa38166a581b51975f0382a45fc3a0733d0Chris Lattner if (II->isHandleIdentifierCase()) 12946a170eb3ea6d6319277becabef68eb1a26bf8766Chris Lattner PP->HandleIdentifier(Result); 12956aa52ec6b969faabf3764baf79d89810b8249a7eDouglas Gregor 12966a170eb3ea6d6319277becabef68eb1a26bf8766Chris Lattner return; 12975f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 12981eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 12995f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Otherwise, $,\,? in identifier found. Enter slower path. 13001eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 13015f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer C = getCharAndSize(CurPtr, Size); 13025f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer while (1) { 13035f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (C == '$') { 13045f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If we hit a $ and they are not supported in identifiers, we are done. 13055f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (!Features.DollarIdents) goto FinishIdentifier; 13061eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 13075f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Otherwise, emit a diagnostic and continue. 130874d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner if (!isLexingRawMode()) 130974d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner Diag(CurPtr, diag::ext_dollar_in_identifier); 13105f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(CurPtr, Size, Result); 13115f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer C = getCharAndSize(CurPtr, Size); 13125f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer continue; 13135f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } else if (!isIdentifierBody(C)) { // FIXME: UCNs. 13145f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Found end of identifier. 13155f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer goto FinishIdentifier; 13165f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 13175f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 13185f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Otherwise, this character is good, consume it. 13195f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(CurPtr, Size, Result); 13205f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 13215f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer C = getCharAndSize(CurPtr, Size); 13225f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer while (isIdentifierBody(C)) { // FIXME: UCNs. 13235f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(CurPtr, Size, Result); 13245f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer C = getCharAndSize(CurPtr, Size); 13255f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 13265f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 13275f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 13285f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 1329a75ec43d625753b4439b0d6f70bd988444c74617Douglas Gregor/// isHexaLiteral - Return true if Start points to a hex constant. 13304a551000bee716ac8b1bbe16134a53f0ad221a5aChris Lattner/// in microsoft mode (where this is supposed to be several different tokens). 13316ab55ebab20086f725c4017b48ef8d7691ef870fChris Lattnerstatic bool isHexaLiteral(const char *Start, const LangOptions &Features) { 13326ab55ebab20086f725c4017b48ef8d7691ef870fChris Lattner unsigned Size; 13336ab55ebab20086f725c4017b48ef8d7691ef870fChris Lattner char C1 = Lexer::getCharAndSizeNoWarn(Start, Size, Features); 13346ab55ebab20086f725c4017b48ef8d7691ef870fChris Lattner if (C1 != '0') 13356ab55ebab20086f725c4017b48ef8d7691ef870fChris Lattner return false; 13366ab55ebab20086f725c4017b48ef8d7691ef870fChris Lattner char C2 = Lexer::getCharAndSizeNoWarn(Start + Size, Size, Features); 13376ab55ebab20086f725c4017b48ef8d7691ef870fChris Lattner return (C2 == 'x' || C2 == 'X'); 1338a75ec43d625753b4439b0d6f70bd988444c74617Douglas Gregor} 13395f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 13405253c7ff266ae79308050c9f43d60dd1a67c5fb9Nate Begeman/// LexNumericConstant - Lex the remainder of a integer or floating point 13415f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// constant. From[-1] is the first character lexed. Return the end of the 13425f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// constant. 1343d217773f106856a11879ec79dc468efefaf2ee75Chris Lattnervoid Lexer::LexNumericConstant(Token &Result, const char *CurPtr) { 13445f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer unsigned Size; 13455f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer char C = getCharAndSize(CurPtr, Size); 13465f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer char PrevCh = 0; 13475f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer while (isNumberBody(C)) { // FIXME: UCNs? 13485f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(CurPtr, Size, Result); 13495f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer PrevCh = C; 13505f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer C = getCharAndSize(CurPtr, Size); 13515f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 13521eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 13535f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If we fell out, check for a sign, due to 1e+12. If we have one, continue. 1354b2f4a20ddc281b194caa00e850ad74759e0d50baChris Lattner if ((C == '-' || C == '+') && (PrevCh == 'E' || PrevCh == 'e')) { 1355b2f4a20ddc281b194caa00e850ad74759e0d50baChris Lattner // If we are in Microsoft mode, don't continue if the constant is hex. 1356b2f4a20ddc281b194caa00e850ad74759e0d50baChris Lattner // For example, MSVC will accept the following as 3 tokens: 0x1234567e+1 13576ab55ebab20086f725c4017b48ef8d7691ef870fChris Lattner if (!Features.Microsoft || !isHexaLiteral(BufferPtr, Features)) 1358b2f4a20ddc281b194caa00e850ad74759e0d50baChris Lattner return LexNumericConstant(Result, ConsumeChar(CurPtr, Size, Result)); 1359b2f4a20ddc281b194caa00e850ad74759e0d50baChris Lattner } 13605f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 13615f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If we have a hex FP constant, continue. 13628c723404c34e9347fae473489194e8c200c4ac83Sean Hunt if ((C == '-' || C == '+') && (PrevCh == 'P' || PrevCh == 'p') && 1363b2f4a20ddc281b194caa00e850ad74759e0d50baChris Lattner !Features.CPlusPlus0x) 13645f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer return LexNumericConstant(Result, ConsumeChar(CurPtr, Size, Result)); 13651eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 13665f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Update the location of token as well as BufferPtr. 136747246be8ac5b0ddde6c402b8fc6946b6135487b5Chris Lattner const char *TokStart = BufferPtr; 13689e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner FormTokenWithChars(Result, CurPtr, tok::numeric_constant); 136947246be8ac5b0ddde6c402b8fc6946b6135487b5Chris Lattner Result.setLiteralData(TokStart); 13705f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 13715f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 13725f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// LexStringLiteral - Lex the remainder of a string literal, after having lexed 13735cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor/// either " or L" or u8" or u" or U". 13745cee1195584fa8672253139c86e922daeda69b9eDouglas Gregorvoid Lexer::LexStringLiteral(Token &Result, const char *CurPtr, 13755cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor tok::TokenKind Kind) { 13765f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer const char *NulCharacter = 0; // Does this string contain the \0 character? 13771eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 13785f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer char C = getAndAdvanceChar(CurPtr, Result); 13795f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer while (C != '"') { 1380571339c215dc3fa44acccb89c7c6b1d5b962b7f2Chris Lattner // Skip escaped characters. Escaped newlines will already be processed by 1381571339c215dc3fa44acccb89c7c6b1d5b962b7f2Chris Lattner // getAndAdvanceChar. 1382571339c215dc3fa44acccb89c7c6b1d5b962b7f2Chris Lattner if (C == '\\') 13835f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer C = getAndAdvanceChar(CurPtr, Result); 138433611e0d5ab1372608a7649b1877cd4300621c71Douglas Gregor 1385571339c215dc3fa44acccb89c7c6b1d5b962b7f2Chris Lattner if (C == '\n' || C == '\r' || // Newline. 138633611e0d5ab1372608a7649b1877cd4300621c71Douglas Gregor (C == 0 && CurPtr-1 == BufferEnd)) { // End of file. 13877d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis if (!isLexingRawMode() && !Features.AsmPreprocessor) 1388ff1ed985ede24b9a45e52d726db95767e35fe21eArgyrios Kyrtzidis Diag(BufferPtr, diag::warn_unterminated_string); 13899e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner FormTokenWithChars(Result, CurPtr-1, tok::unknown); 13905f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer return; 13915f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 1392571339c215dc3fa44acccb89c7c6b1d5b962b7f2Chris Lattner 13937d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis if (C == 0) { 13947d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis if (isCodeCompletionPoint(CurPtr-1)) { 13957d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis PP->CodeCompleteNaturalLanguage(); 13967d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis FormTokenWithChars(Result, CurPtr-1, tok::unknown); 13977d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis return cutOffLexing(); 13987d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis } 13997d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis 1400571339c215dc3fa44acccb89c7c6b1d5b962b7f2Chris Lattner NulCharacter = CurPtr-1; 14017d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis } 14025f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer C = getAndAdvanceChar(CurPtr, Result); 14035f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 14041eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 14055f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If a nul character existed in the string, warn about it. 140674d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner if (NulCharacter && !isLexingRawMode()) 140774d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner Diag(NulCharacter, diag::null_in_string); 14085f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 14095f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Update the location of the token as well as the BufferPtr instance var. 141047246be8ac5b0ddde6c402b8fc6946b6135487b5Chris Lattner const char *TokStart = BufferPtr; 14115cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor FormTokenWithChars(Result, CurPtr, Kind); 141247246be8ac5b0ddde6c402b8fc6946b6135487b5Chris Lattner Result.setLiteralData(TokStart); 14135f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 14145f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 14152fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper/// LexRawStringLiteral - Lex the remainder of a raw string literal, after 14162fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper/// having lexed R", LR", u8R", uR", or UR". 14172fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Toppervoid Lexer::LexRawStringLiteral(Token &Result, const char *CurPtr, 14182fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper tok::TokenKind Kind) { 14192fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper // This function doesn't use getAndAdvanceChar because C++0x [lex.pptoken]p3: 14202fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper // Between the initial and final double quote characters of the raw string, 14212fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper // any transformations performed in phases 1 and 2 (trigraphs, 14222fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper // universal-character-names, and line splicing) are reverted. 14232fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper 14242fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper unsigned PrefixLen = 0; 14252fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper 14262fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper while (PrefixLen != 16 && isRawStringDelimBody(CurPtr[PrefixLen])) 14272fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper ++PrefixLen; 14282fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper 14292fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper // If the last character was not a '(', then we didn't lex a valid delimiter. 14302fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper if (CurPtr[PrefixLen] != '(') { 14312fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper if (!isLexingRawMode()) { 14322fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper const char *PrefixEnd = &CurPtr[PrefixLen]; 14332fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper if (PrefixLen == 16) { 14342fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper Diag(PrefixEnd, diag::err_raw_delim_too_long); 14352fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper } else { 14362fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper Diag(PrefixEnd, diag::err_invalid_char_raw_delim) 14372fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper << StringRef(PrefixEnd, 1); 14382fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper } 14392fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper } 14402fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper 14412fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper // Search for the next '"' in hopes of salvaging the lexer. Unfortunately, 14422fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper // it's possible the '"' was intended to be part of the raw string, but 14432fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper // there's not much we can do about that. 14442fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper while (1) { 14452fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper char C = *CurPtr++; 14462fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper 14472fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper if (C == '"') 14482fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper break; 14492fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper if (C == 0 && CurPtr-1 == BufferEnd) { 14502fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper --CurPtr; 14512fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper break; 14522fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper } 14532fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper } 14542fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper 14552fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper FormTokenWithChars(Result, CurPtr, tok::unknown); 14562fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper return; 14572fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper } 14582fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper 14592fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper // Save prefix and move CurPtr past it 14602fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper const char *Prefix = CurPtr; 14612fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper CurPtr += PrefixLen + 1; // skip over prefix and '(' 14622fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper 14632fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper while (1) { 14642fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper char C = *CurPtr++; 14652fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper 14662fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper if (C == ')') { 14672fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper // Check for prefix match and closing quote. 14682fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper if (strncmp(CurPtr, Prefix, PrefixLen) == 0 && CurPtr[PrefixLen] == '"') { 14692fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper CurPtr += PrefixLen + 1; // skip over prefix and '"' 14702fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper break; 14712fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper } 14722fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper } else if (C == 0 && CurPtr-1 == BufferEnd) { // End of file. 14732fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper if (!isLexingRawMode()) 14742fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper Diag(BufferPtr, diag::err_unterminated_raw_string) 14752fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper << StringRef(Prefix, PrefixLen); 14762fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper FormTokenWithChars(Result, CurPtr-1, tok::unknown); 14772fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper return; 14782fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper } 14792fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper } 14802fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper 14812fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper // Update the location of token as well as BufferPtr. 14822fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper const char *TokStart = BufferPtr; 14832fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper FormTokenWithChars(Result, CurPtr, Kind); 14842fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper Result.setLiteralData(TokStart); 14852fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper} 14862fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper 14875f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// LexAngledStringLiteral - Lex the remainder of an angled string literal, 14885f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// after having lexed the '<' character. This is used for #include filenames. 1489d217773f106856a11879ec79dc468efefaf2ee75Chris Lattnervoid Lexer::LexAngledStringLiteral(Token &Result, const char *CurPtr) { 14905f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer const char *NulCharacter = 0; // Does this string contain the \0 character? 14919cb51ce03caf0755759f7e67aea7d4be684ff738Chris Lattner const char *AfterLessPos = CurPtr; 14925f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer char C = getAndAdvanceChar(CurPtr, Result); 14935f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer while (C != '>') { 14945f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Skip escaped characters. 14955f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (C == '\\') { 14965f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Skip the escaped character. 14975f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer C = getAndAdvanceChar(CurPtr, Result); 14985f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } else if (C == '\n' || C == '\r' || // Newline. 14997d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis (C == 0 && (CurPtr-1 == BufferEnd || // End of file. 15007d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis isCodeCompletionPoint(CurPtr-1)))) { 15019cb51ce03caf0755759f7e67aea7d4be684ff738Chris Lattner // If the filename is unterminated, then it must just be a lone < 15029cb51ce03caf0755759f7e67aea7d4be684ff738Chris Lattner // character. Return this as such. 15039cb51ce03caf0755759f7e67aea7d4be684ff738Chris Lattner FormTokenWithChars(Result, AfterLessPos, tok::less); 15045f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer return; 15055f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } else if (C == 0) { 15065f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer NulCharacter = CurPtr-1; 15075f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 15085f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer C = getAndAdvanceChar(CurPtr, Result); 15095f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 15101eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 15115f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If a nul character existed in the string, warn about it. 151274d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner if (NulCharacter && !isLexingRawMode()) 151374d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner Diag(NulCharacter, diag::null_in_string); 15141eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 15155f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Update the location of token as well as BufferPtr. 151647246be8ac5b0ddde6c402b8fc6946b6135487b5Chris Lattner const char *TokStart = BufferPtr; 15179e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner FormTokenWithChars(Result, CurPtr, tok::angle_string_literal); 151847246be8ac5b0ddde6c402b8fc6946b6135487b5Chris Lattner Result.setLiteralData(TokStart); 15195f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 15205f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 15215f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 15225f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// LexCharConstant - Lex the remainder of a character constant, after having 15235cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor/// lexed either ' or L' or u' or U'. 15245cee1195584fa8672253139c86e922daeda69b9eDouglas Gregorvoid Lexer::LexCharConstant(Token &Result, const char *CurPtr, 15255cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor tok::TokenKind Kind) { 15265f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer const char *NulCharacter = 0; // Does this character contain the \0 character? 15275f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 15285f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer char C = getAndAdvanceChar(CurPtr, Result); 15295f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (C == '\'') { 153033ab3f6c7e3065550de202088f8f335ecfa16ae1Chris Lattner if (!isLexingRawMode() && !Features.AsmPreprocessor) 153174d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner Diag(BufferPtr, diag::err_empty_character); 15329e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner FormTokenWithChars(Result, CurPtr, tok::unknown); 15335f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer return; 15345f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 15351eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 1536d80f786689d608e5c22d6e1045884de7aff76c40Chris Lattner while (C != '\'') { 1537d80f786689d608e5c22d6e1045884de7aff76c40Chris Lattner // Skip escaped characters. 1538d80f786689d608e5c22d6e1045884de7aff76c40Chris Lattner if (C == '\\') { 1539d80f786689d608e5c22d6e1045884de7aff76c40Chris Lattner // Skip the escaped character. 1540d80f786689d608e5c22d6e1045884de7aff76c40Chris Lattner // FIXME: UCN's 15415f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer C = getAndAdvanceChar(CurPtr, Result); 1542d80f786689d608e5c22d6e1045884de7aff76c40Chris Lattner } else if (C == '\n' || C == '\r' || // Newline. 1543d80f786689d608e5c22d6e1045884de7aff76c40Chris Lattner (C == 0 && CurPtr-1 == BufferEnd)) { // End of file. 15447d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis if (!isLexingRawMode() && !Features.AsmPreprocessor) 1545ff1ed985ede24b9a45e52d726db95767e35fe21eArgyrios Kyrtzidis Diag(BufferPtr, diag::warn_unterminated_char); 1546d80f786689d608e5c22d6e1045884de7aff76c40Chris Lattner FormTokenWithChars(Result, CurPtr-1, tok::unknown); 1547d80f786689d608e5c22d6e1045884de7aff76c40Chris Lattner return; 1548d80f786689d608e5c22d6e1045884de7aff76c40Chris Lattner } else if (C == 0) { 15497d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis if (isCodeCompletionPoint(CurPtr-1)) { 15507d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis PP->CodeCompleteNaturalLanguage(); 15517d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis FormTokenWithChars(Result, CurPtr-1, tok::unknown); 15527d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis return cutOffLexing(); 15537d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis } 15547d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis 1555d80f786689d608e5c22d6e1045884de7aff76c40Chris Lattner NulCharacter = CurPtr-1; 1556d80f786689d608e5c22d6e1045884de7aff76c40Chris Lattner } 1557d80f786689d608e5c22d6e1045884de7aff76c40Chris Lattner C = getAndAdvanceChar(CurPtr, Result); 15585f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 15591eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 1560d80f786689d608e5c22d6e1045884de7aff76c40Chris Lattner // If a nul character existed in the character, warn about it. 156174d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner if (NulCharacter && !isLexingRawMode()) 156274d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner Diag(NulCharacter, diag::null_in_char); 15635f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 15645f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Update the location of token as well as BufferPtr. 156547246be8ac5b0ddde6c402b8fc6946b6135487b5Chris Lattner const char *TokStart = BufferPtr; 15665cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor FormTokenWithChars(Result, CurPtr, Kind); 156747246be8ac5b0ddde6c402b8fc6946b6135487b5Chris Lattner Result.setLiteralData(TokStart); 15685f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 15695f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 15705f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// SkipWhitespace - Efficiently skip over a series of whitespace characters. 15715f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// Update BufferPtr to point to the next non-whitespace character and return. 1572d88dc48e33d71732708960170e57a3d1bdc8f847Chris Lattner/// 1573d88dc48e33d71732708960170e57a3d1bdc8f847Chris Lattner/// This method forms a token and returns true if KeepWhitespaceMode is enabled. 1574d88dc48e33d71732708960170e57a3d1bdc8f847Chris Lattner/// 1575d88dc48e33d71732708960170e57a3d1bdc8f847Chris Lattnerbool Lexer::SkipWhitespace(Token &Result, const char *CurPtr) { 15765f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Whitespace - Skip it, then return the token after the whitespace. 15775f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer unsigned char Char = *CurPtr; // Skip consequtive spaces efficiently. 15785f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer while (1) { 15795f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Skip horizontal whitespace very aggressively. 15805f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer while (isHorizontalWhitespace(Char)) 15815f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Char = *++CurPtr; 15821eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 1583ddd3e8b90a92c39f620bda7a0945320c0b9e60dbDaniel Dunbar // Otherwise if we have something other than whitespace, we're done. 15845f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (Char != '\n' && Char != '\r') 15855f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 15861eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 15875f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (ParsingPreprocessorDirective) { 15885f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // End of preprocessor directive line, let LexTokenInternal handle this. 15895f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer BufferPtr = CurPtr; 1590d88dc48e33d71732708960170e57a3d1bdc8f847Chris Lattner return false; 15915f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 15921eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 15935f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // ok, but handle newline. 15945f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // The returned token is at the start of the line. 1595d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner Result.setFlag(Token::StartOfLine); 15965f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // No leading whitespace seen so far. 1597d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner Result.clearFlag(Token::LeadingSpace); 15985f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Char = *++CurPtr; 15995f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 16005f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 16015f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If this isn't immediately after a newline, there is leading space. 16025f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer char PrevChar = CurPtr[-1]; 16035f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (PrevChar != '\n' && PrevChar != '\r') 1604d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner Result.setFlag(Token::LeadingSpace); 16055f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 1606d88dc48e33d71732708960170e57a3d1bdc8f847Chris Lattner // If the client wants us to return whitespace, return it now. 1607d88dc48e33d71732708960170e57a3d1bdc8f847Chris Lattner if (isKeepWhitespaceMode()) { 16089e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner FormTokenWithChars(Result, CurPtr, tok::unknown); 1609d88dc48e33d71732708960170e57a3d1bdc8f847Chris Lattner return true; 1610d88dc48e33d71732708960170e57a3d1bdc8f847Chris Lattner } 16111eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 16125f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer BufferPtr = CurPtr; 1613d88dc48e33d71732708960170e57a3d1bdc8f847Chris Lattner return false; 16145f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 16155f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 16165f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// SkipBCPLComment - We have just read the // characters from input. Skip until 16175f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// we find the newline character thats terminate the comment. Then update 1618046c2277dcbcc8eb89dbb5b1b8c5226b7cb81635Chris Lattner/// BufferPtr and return. 1619046c2277dcbcc8eb89dbb5b1b8c5226b7cb81635Chris Lattner/// 1620046c2277dcbcc8eb89dbb5b1b8c5226b7cb81635Chris Lattner/// If we're in KeepCommentMode or any CommentHandler has inserted 1621046c2277dcbcc8eb89dbb5b1b8c5226b7cb81635Chris Lattner/// some tokens, this will store the first token and return true. 1622d217773f106856a11879ec79dc468efefaf2ee75Chris Lattnerbool Lexer::SkipBCPLComment(Token &Result, const char *CurPtr) { 16235f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If BCPL comments aren't explicitly enabled for this language, emit an 16245f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // extension warning. 162574d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner if (!Features.BCPLComment && !isLexingRawMode()) { 16265f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Diag(BufferPtr, diag::ext_bcpl_comment); 16271eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 16285f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Mark them enabled so we only emit one warning for this translation 16295f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // unit. 16305f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Features.BCPLComment = true; 16315f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 16321eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 16335f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Scan over the body of the comment. The common case, when scanning, is that 16345f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // the comment contains normal ascii characters with nothing interesting in 16355f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // them. As such, optimize for this case with the inner loop. 16365f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer char C; 16375f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer do { 16385f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer C = *CurPtr; 16395f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // FIXME: Speedup BCPL comment lexing. Just scan for a \n or \r character. 16405f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If we find a \n character, scan backwards, checking to see if it's an 16415f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // escaped newline, like we do for block comments. 16421eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 16435f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Skip over characters in the fast loop. 16445f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer while (C != 0 && // Potentially EOF. 16455f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer C != '\\' && // Potentially escaped newline. 16465f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer C != '?' && // Potentially trigraph. 16475f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer C != '\n' && C != '\r') // Newline or DOS-style newline. 16485f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer C = *++CurPtr; 16495f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 16505f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If this is a newline, we're done. 16515f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (C == '\n' || C == '\r') 16525f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; // Found the newline? Break out! 16531eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 16545f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Otherwise, this is a hard case. Fall back on getAndAdvanceChar to 1655bc3e9842212b78bcf49682641307e4f3419ecc5eChris Lattner // properly decode the character. Read it in raw mode to avoid emitting 1656bc3e9842212b78bcf49682641307e4f3419ecc5eChris Lattner // diagnostics about things like trigraphs. If we see an escaped newline, 1657bc3e9842212b78bcf49682641307e4f3419ecc5eChris Lattner // we'll handle it below. 16585f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer const char *OldPtr = CurPtr; 1659bc3e9842212b78bcf49682641307e4f3419ecc5eChris Lattner bool OldRawMode = isLexingRawMode(); 1660bc3e9842212b78bcf49682641307e4f3419ecc5eChris Lattner LexingRawMode = true; 16615f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer C = getAndAdvanceChar(CurPtr, Result); 1662bc3e9842212b78bcf49682641307e4f3419ecc5eChris Lattner LexingRawMode = OldRawMode; 1663ead616c5d8681a41b11273327813e61bda01907aChris Lattner 1664ead616c5d8681a41b11273327813e61bda01907aChris Lattner // If the char that we finally got was a \n, then we must have had something 1665ead616c5d8681a41b11273327813e61bda01907aChris Lattner // like \<newline><newline>. We don't want to have consumed the second 1666ead616c5d8681a41b11273327813e61bda01907aChris Lattner // newline, we want CurPtr, to end up pointing to it down below. 1667ead616c5d8681a41b11273327813e61bda01907aChris Lattner if (C == '\n' || C == '\r') { 1668ead616c5d8681a41b11273327813e61bda01907aChris Lattner --CurPtr; 1669ead616c5d8681a41b11273327813e61bda01907aChris Lattner C = 'x'; // doesn't matter what this is. 1670ead616c5d8681a41b11273327813e61bda01907aChris Lattner } 16711eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 16725f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If we read multiple characters, and one of those characters was a \r or 16735f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // \n, then we had an escaped newline within the comment. Emit diagnostic 16745f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // unless the next line is also a // comment. 16755f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (CurPtr != OldPtr+1 && C != '/' && CurPtr[0] != '/') { 16765f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer for (; OldPtr != CurPtr; ++OldPtr) 16775f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (OldPtr[0] == '\n' || OldPtr[0] == '\r') { 16785f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Okay, we found a // comment that ends in a newline, if the next 16795f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // line is also a // comment, but has spaces, don't emit a diagnostic. 16805f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (isspace(C)) { 16815f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer const char *ForwardPtr = CurPtr; 16825f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer while (isspace(*ForwardPtr)) // Skip whitespace. 16835f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer ++ForwardPtr; 16845f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (ForwardPtr[0] == '/' && ForwardPtr[1] == '/') 16855f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 16865f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 16871eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 168874d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner if (!isLexingRawMode()) 168974d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner Diag(OldPtr-1, diag::ext_multi_line_bcpl_comment); 16905f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 16915f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 16925f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 16931eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 169455817afdf9d453a443262a733f6caf6692dca118Douglas Gregor if (CurPtr == BufferEnd+1) { 169555817afdf9d453a443262a733f6caf6692dca118Douglas Gregor --CurPtr; 169655817afdf9d453a443262a733f6caf6692dca118Douglas Gregor break; 169755817afdf9d453a443262a733f6caf6692dca118Douglas Gregor } 16987d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis 16997d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis if (C == '\0' && isCodeCompletionPoint(CurPtr-1)) { 17007d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis PP->CodeCompleteNaturalLanguage(); 17017d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis cutOffLexing(); 17027d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis return false; 17037d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis } 17047d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis 17055f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } while (C != '\n' && C != '\r'); 17065f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 17073d0ad58b28e0d50fca7f21c6a078b05370510288Chris Lattner // Found but did not consume the newline. Notify comment handlers about the 17083d0ad58b28e0d50fca7f21c6a078b05370510288Chris Lattner // comment unless we're in a #if 0 block. 17093d0ad58b28e0d50fca7f21c6a078b05370510288Chris Lattner if (PP && !isLexingRawMode() && 17103d0ad58b28e0d50fca7f21c6a078b05370510288Chris Lattner PP->HandleComment(Result, SourceRange(getSourceLocation(BufferPtr), 17113d0ad58b28e0d50fca7f21c6a078b05370510288Chris Lattner getSourceLocation(CurPtr)))) { 1712046c2277dcbcc8eb89dbb5b1b8c5226b7cb81635Chris Lattner BufferPtr = CurPtr; 1713046c2277dcbcc8eb89dbb5b1b8c5226b7cb81635Chris Lattner return true; // A token has to be returned. 1714046c2277dcbcc8eb89dbb5b1b8c5226b7cb81635Chris Lattner } 17151eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 17165f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If we are returning comments as tokens, return this comment as a token. 1717fa95a019da00b926d64ff83358ba73bbc6ae1e37Chris Lattner if (inKeepCommentMode()) 17185f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer return SaveBCPLComment(Result, CurPtr); 17195f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 17205f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If we are inside a preprocessor directive and we see the end of line, 172184021556baceb76eedf7d44be8ba71d9b8cfaccePeter Collingbourne // return immediately, so that the lexer can return this as an EOD token. 17225f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (ParsingPreprocessorDirective || CurPtr == BufferEnd) { 17235f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer BufferPtr = CurPtr; 17242d3818959df7b71bfcf8c2f2b573856bfc4c3a21Chris Lattner return false; 17255f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 17261eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 17275f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Otherwise, eat the \n character. We don't care if this is a \n\r or 17287a4f004a77421af876152281b92c4f8ea24afb2dChris Lattner // \r\n sequence. This is an efficiency hack (because we know the \n can't 1729d88dc48e33d71732708960170e57a3d1bdc8f847Chris Lattner // contribute to another token), it isn't needed for correctness. Note that 1730d88dc48e33d71732708960170e57a3d1bdc8f847Chris Lattner // this is ok even in KeepWhitespaceMode, because we would have returned the 1731d88dc48e33d71732708960170e57a3d1bdc8f847Chris Lattner /// comment above in that mode. 17325f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer ++CurPtr; 17331eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 17345f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // The next returned token is at the start of the line. 1735d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner Result.setFlag(Token::StartOfLine); 17365f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // No leading whitespace seen so far. 1737d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner Result.clearFlag(Token::LeadingSpace); 17385f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer BufferPtr = CurPtr; 17392d3818959df7b71bfcf8c2f2b573856bfc4c3a21Chris Lattner return false; 17405f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 17415f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 17425f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// SaveBCPLComment - If in save-comment mode, package up this BCPL comment in 17435f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// an appropriate way and return it. 1744d217773f106856a11879ec79dc468efefaf2ee75Chris Lattnerbool Lexer::SaveBCPLComment(Token &Result, const char *CurPtr) { 17459e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner // If we're not in a preprocessor directive, just return the // comment 17469e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner // directly. 17479e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner FormTokenWithChars(Result, CurPtr, tok::comment); 17481eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 17499e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner if (!ParsingPreprocessorDirective) 17509e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner return true; 17511eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 17529e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner // If this BCPL-style comment is in a macro definition, transmogrify it into 17539e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner // a C-style block comment. 1754453091cc2082e207ea2c2dda645a9bc01b37fb0cDouglas Gregor bool Invalid = false; 1755453091cc2082e207ea2c2dda645a9bc01b37fb0cDouglas Gregor std::string Spelling = PP->getSpelling(Result, &Invalid); 1756453091cc2082e207ea2c2dda645a9bc01b37fb0cDouglas Gregor if (Invalid) 1757453091cc2082e207ea2c2dda645a9bc01b37fb0cDouglas Gregor return true; 1758453091cc2082e207ea2c2dda645a9bc01b37fb0cDouglas Gregor 17599e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner assert(Spelling[0] == '/' && Spelling[1] == '/' && "Not bcpl comment?"); 17609e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Spelling[1] = '*'; // Change prefix to "/*". 17619e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Spelling += "*/"; // add suffix. 17621eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 17639e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Result.setKind(tok::comment); 176447246be8ac5b0ddde6c402b8fc6946b6135487b5Chris Lattner PP->CreateString(&Spelling[0], Spelling.size(), Result, 176547246be8ac5b0ddde6c402b8fc6946b6135487b5Chris Lattner Result.getLocation()); 17662d3818959df7b71bfcf8c2f2b573856bfc4c3a21Chris Lattner return true; 17675f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 17685f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 17695f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// isBlockCommentEndOfEscapedNewLine - Return true if the specified newline 17705f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// character (either \n or \r) is part of an escaped newline sequence. Issue a 177147a2b40b7a3d9c89d6c3d12e5a81e2498c207750Chris Lattner/// diagnostic if so. We know that the newline is inside of a block comment. 17721eb4433ac451dc16f4133a88af2d002ac26c58efMike Stumpstatic bool isEndOfBlockCommentWithEscapedNewLine(const char *CurPtr, 17735f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Lexer *L) { 17745f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer assert(CurPtr[0] == '\n' || CurPtr[0] == '\r'); 17751eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 17765f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Back up off the newline. 17775f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer --CurPtr; 17781eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 17795f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If this is a two-character newline sequence, skip the other character. 17805f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (CurPtr[0] == '\n' || CurPtr[0] == '\r') { 17815f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // \n\n or \r\r -> not escaped newline. 17825f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (CurPtr[0] == CurPtr[1]) 17835f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer return false; 17845f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // \n\r or \r\n -> skip the newline. 17855f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer --CurPtr; 17865f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 17871eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 17885f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If we have horizontal whitespace, skip over it. We allow whitespace 17895f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // between the slash and newline. 17905f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer bool HasSpace = false; 17915f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer while (isHorizontalWhitespace(*CurPtr) || *CurPtr == 0) { 17925f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer --CurPtr; 17935f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer HasSpace = true; 17945f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 17951eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 17965f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If we have a slash, we know this is an escaped newline. 17975f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (*CurPtr == '\\') { 17985f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (CurPtr[-1] != '*') return false; 17995f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } else { 18005f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // It isn't a slash, is it the ?? / trigraph? 18015f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (CurPtr[0] != '/' || CurPtr[-1] != '?' || CurPtr[-2] != '?' || 18025f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr[-3] != '*') 18035f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer return false; 18041eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 18055f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // This is the trigraph ending the comment. Emit a stern warning! 18065f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr -= 2; 18075f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 18085f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If no trigraphs are enabled, warn that we ignored this trigraph and 18095f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // ignore this * character. 18105f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (!L->getFeatures().Trigraphs) { 181174d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner if (!L->isLexingRawMode()) 181274d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner L->Diag(CurPtr, diag::trigraph_ignored_block_comment); 18135f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer return false; 18145f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 181574d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner if (!L->isLexingRawMode()) 181674d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner L->Diag(CurPtr, diag::trigraph_ends_block_comment); 18175f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 18181eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 18195f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Warn about having an escaped newline between the */ characters. 182074d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner if (!L->isLexingRawMode()) 182174d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner L->Diag(CurPtr, diag::escaped_newline_block_comment_end); 18221eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 18235f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If there was space between the backslash and newline, warn about it. 182474d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner if (HasSpace && !L->isLexingRawMode()) 182574d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner L->Diag(CurPtr, diag::backslash_newline_space); 18261eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 18275f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer return true; 18285f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 18295f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 18305f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer#ifdef __SSE2__ 18315f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer#include <emmintrin.h> 18325f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer#elif __ALTIVEC__ 18335f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer#include <altivec.h> 18345f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer#undef bool 18355f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer#endif 18365f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 18375f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// SkipBlockComment - We have just read the /* characters from input. Read 18385f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// until we find the */ characters that terminate the comment. Note that we 18395f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// don't bother decoding trigraphs or escaped newlines in block comments, 18405f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// because they cannot cause the comment to end. The only thing that can 18415f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// happen is the comment could end with an escaped newline between the */ end 18425f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// of comment. 18432d3818959df7b71bfcf8c2f2b573856bfc4c3a21Chris Lattner/// 1844046c2277dcbcc8eb89dbb5b1b8c5226b7cb81635Chris Lattner/// If we're in KeepCommentMode or any CommentHandler has inserted 1845046c2277dcbcc8eb89dbb5b1b8c5226b7cb81635Chris Lattner/// some tokens, this will store the first token and return true. 1846d217773f106856a11879ec79dc468efefaf2ee75Chris Lattnerbool Lexer::SkipBlockComment(Token &Result, const char *CurPtr) { 18475f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Scan one character past where we should, looking for a '/' character. Once 1848fc8f0e14ad142ed811e90fbd9a30e419e301c717Chris Lattner // we find it, check to see if it was preceded by a *. This common 18495f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // optimization helps people who like to put a lot of * characters in their 18505f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // comments. 18518146b6851e873eab71341f1b1d3198894bc4c0acChris Lattner 18528146b6851e873eab71341f1b1d3198894bc4c0acChris Lattner // The first character we get with newlines and trigraphs skipped to handle 18538146b6851e873eab71341f1b1d3198894bc4c0acChris Lattner // the degenerate /*/ case below correctly if the * has an escaped newline 18548146b6851e873eab71341f1b1d3198894bc4c0acChris Lattner // after it. 18558146b6851e873eab71341f1b1d3198894bc4c0acChris Lattner unsigned CharSize; 18568146b6851e873eab71341f1b1d3198894bc4c0acChris Lattner unsigned char C = getCharAndSize(CurPtr, CharSize); 18578146b6851e873eab71341f1b1d3198894bc4c0acChris Lattner CurPtr += CharSize; 18585f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (C == 0 && CurPtr == BufferEnd+1) { 18597d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis if (!isLexingRawMode()) 18600af574270d3be2b0e73a3379dfaa633746f8fc6fChris Lattner Diag(BufferPtr, diag::err_unterminated_block_comment); 186131f0ecae4b2d4d268d04c562a43d9fbfee96f189Chris Lattner --CurPtr; 18621eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 186331f0ecae4b2d4d268d04c562a43d9fbfee96f189Chris Lattner // KeepWhitespaceMode should return this broken comment as a token. Since 186431f0ecae4b2d4d268d04c562a43d9fbfee96f189Chris Lattner // it isn't a well formed comment, just return it as an 'unknown' token. 186531f0ecae4b2d4d268d04c562a43d9fbfee96f189Chris Lattner if (isKeepWhitespaceMode()) { 18669e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner FormTokenWithChars(Result, CurPtr, tok::unknown); 186731f0ecae4b2d4d268d04c562a43d9fbfee96f189Chris Lattner return true; 186831f0ecae4b2d4d268d04c562a43d9fbfee96f189Chris Lattner } 18691eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 187031f0ecae4b2d4d268d04c562a43d9fbfee96f189Chris Lattner BufferPtr = CurPtr; 18712d3818959df7b71bfcf8c2f2b573856bfc4c3a21Chris Lattner return false; 18725f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 18731eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 18748146b6851e873eab71341f1b1d3198894bc4c0acChris Lattner // Check to see if the first character after the '/*' is another /. If so, 18758146b6851e873eab71341f1b1d3198894bc4c0acChris Lattner // then this slash does not end the block comment, it is part of it. 18768146b6851e873eab71341f1b1d3198894bc4c0acChris Lattner if (C == '/') 18778146b6851e873eab71341f1b1d3198894bc4c0acChris Lattner C = *CurPtr++; 18781eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 18795f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer while (1) { 18805f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Skip over all non-interesting characters until we find end of buffer or a 18815f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // (probably ending) '/' character. 18827d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis if (CurPtr + 24 < BufferEnd && 18837d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis // If there is a code-completion point avoid the fast scan because it 18847d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis // doesn't check for '\0'. 18857d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis !(PP && PP->getCodeCompletionFileLoc() == FileLoc)) { 18865f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // While not aligned to a 16-byte boundary. 18875f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer while (C != '/' && ((intptr_t)CurPtr & 0x0F) != 0) 18885f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer C = *CurPtr++; 18891eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 18905f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (C == '/') goto FoundSlash; 18915f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 18925f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer#ifdef __SSE2__ 18935f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer __m128i Slashes = _mm_set_epi8('/', '/', '/', '/', '/', '/', '/', '/', 18945f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer '/', '/', '/', '/', '/', '/', '/', '/'); 18955f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer while (CurPtr+16 <= BufferEnd && 18965f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer _mm_movemask_epi8(_mm_cmpeq_epi8(*(__m128i*)CurPtr, Slashes)) == 0) 18975f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr += 16; 18985f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer#elif __ALTIVEC__ 18995f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer __vector unsigned char Slashes = { 19001eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump '/', '/', '/', '/', '/', '/', '/', '/', 19015f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer '/', '/', '/', '/', '/', '/', '/', '/' 19025f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer }; 19035f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer while (CurPtr+16 <= BufferEnd && 19045f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer !vec_any_eq(*(vector unsigned char*)CurPtr, Slashes)) 19055f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr += 16; 19061eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump#else 19075f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Scan for '/' quickly. Many block comments are very large. 19085f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer while (CurPtr[0] != '/' && 19095f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr[1] != '/' && 19105f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr[2] != '/' && 19115f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr[3] != '/' && 19125f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr+4 < BufferEnd) { 19135f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr += 4; 19145f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 19155f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer#endif 19161eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 19175f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // It has to be one of the bytes scanned, increment to it and read one. 19185f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer C = *CurPtr++; 19195f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 19201eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 19215f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Loop to scan the remainder. 19225f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer while (C != '/' && C != '\0') 19235f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer C = *CurPtr++; 19241eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 19255f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer FoundSlash: 19265f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (C == '/') { 19275f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (CurPtr[-2] == '*') // We found the final */. We're done! 19285f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 19291eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 19305f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if ((CurPtr[-2] == '\n' || CurPtr[-2] == '\r')) { 19315f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (isEndOfBlockCommentWithEscapedNewLine(CurPtr-2, this)) { 19325f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // We found the final */, though it had an escaped newline between the 19335f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // * and /. We're done! 19345f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 19355f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 19365f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 19375f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (CurPtr[0] == '*' && CurPtr[1] != '/') { 19385f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If this is a /* inside of the comment, emit a warning. Don't do this 19395f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // if this is a /*/, which will end the comment. This misses cases with 19405f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // embedded escaped newlines, but oh well. 194174d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner if (!isLexingRawMode()) 194274d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner Diag(CurPtr-1, diag::warn_nested_block_comment); 19435f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 19445f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } else if (C == 0 && CurPtr == BufferEnd+1) { 19457d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis if (!isLexingRawMode()) 194674d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner Diag(BufferPtr, diag::err_unterminated_block_comment); 19475f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Note: the user probably forgot a */. We could continue immediately 19485f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // after the /*, but this would involve lexing a lot of what really is the 19495f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // comment, which surely would confuse the parser. 195031f0ecae4b2d4d268d04c562a43d9fbfee96f189Chris Lattner --CurPtr; 19511eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 195231f0ecae4b2d4d268d04c562a43d9fbfee96f189Chris Lattner // KeepWhitespaceMode should return this broken comment as a token. Since 195331f0ecae4b2d4d268d04c562a43d9fbfee96f189Chris Lattner // it isn't a well formed comment, just return it as an 'unknown' token. 195431f0ecae4b2d4d268d04c562a43d9fbfee96f189Chris Lattner if (isKeepWhitespaceMode()) { 19559e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner FormTokenWithChars(Result, CurPtr, tok::unknown); 195631f0ecae4b2d4d268d04c562a43d9fbfee96f189Chris Lattner return true; 195731f0ecae4b2d4d268d04c562a43d9fbfee96f189Chris Lattner } 19581eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 195931f0ecae4b2d4d268d04c562a43d9fbfee96f189Chris Lattner BufferPtr = CurPtr; 19602d3818959df7b71bfcf8c2f2b573856bfc4c3a21Chris Lattner return false; 19617d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis } else if (C == '\0' && isCodeCompletionPoint(CurPtr-1)) { 19627d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis PP->CodeCompleteNaturalLanguage(); 19637d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis cutOffLexing(); 19647d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis return false; 19655f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 19667d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis 19675f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer C = *CurPtr++; 19685f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 19691eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 19703d0ad58b28e0d50fca7f21c6a078b05370510288Chris Lattner // Notify comment handlers about the comment unless we're in a #if 0 block. 19713d0ad58b28e0d50fca7f21c6a078b05370510288Chris Lattner if (PP && !isLexingRawMode() && 19723d0ad58b28e0d50fca7f21c6a078b05370510288Chris Lattner PP->HandleComment(Result, SourceRange(getSourceLocation(BufferPtr), 19733d0ad58b28e0d50fca7f21c6a078b05370510288Chris Lattner getSourceLocation(CurPtr)))) { 1974046c2277dcbcc8eb89dbb5b1b8c5226b7cb81635Chris Lattner BufferPtr = CurPtr; 1975046c2277dcbcc8eb89dbb5b1b8c5226b7cb81635Chris Lattner return true; // A token has to be returned. 1976046c2277dcbcc8eb89dbb5b1b8c5226b7cb81635Chris Lattner } 19772e22253e03e175144aeb9d13350a12fd83f858beDouglas Gregor 19785f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If we are returning comments as tokens, return this comment as a token. 1979fa95a019da00b926d64ff83358ba73bbc6ae1e37Chris Lattner if (inKeepCommentMode()) { 19809e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner FormTokenWithChars(Result, CurPtr, tok::comment); 19812d3818959df7b71bfcf8c2f2b573856bfc4c3a21Chris Lattner return true; 19825f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 19835f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 19845f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // It is common for the tokens immediately after a /**/ comment to be 19855f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // whitespace. Instead of going through the big switch, handle it 1986d88dc48e33d71732708960170e57a3d1bdc8f847Chris Lattner // efficiently now. This is safe even in KeepWhitespaceMode because we would 1987d88dc48e33d71732708960170e57a3d1bdc8f847Chris Lattner // have already returned above with the comment as a token. 19885f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (isHorizontalWhitespace(*CurPtr)) { 1989d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner Result.setFlag(Token::LeadingSpace); 19905f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer SkipWhitespace(Result, CurPtr+1); 19912d3818959df7b71bfcf8c2f2b573856bfc4c3a21Chris Lattner return false; 19925f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 19935f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 19945f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Otherwise, just return so that the next character will be lexed as a token. 19955f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer BufferPtr = CurPtr; 1996d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner Result.setFlag(Token::LeadingSpace); 19972d3818959df7b71bfcf8c2f2b573856bfc4c3a21Chris Lattner return false; 19985f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 19995f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 20005f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer//===----------------------------------------------------------------------===// 20015f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// Primary Lexing Entry Points 20025f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer//===----------------------------------------------------------------------===// 20035f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 20045f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// ReadToEndOfLine - Read the rest of the current preprocessor line as an 20055f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// uninterpreted string. This switches the lexer out of directive mode. 20065f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencerstd::string Lexer::ReadToEndOfLine() { 20075f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer assert(ParsingPreprocessorDirective && ParsingFilename == false && 20085f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer "Must be in a preprocessing directive!"); 20095f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer std::string Result; 2010d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner Token Tmp; 20115f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 20125f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // CurPtr - Cache BufferPtr in an automatic variable. 20135f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer const char *CurPtr = BufferPtr; 20145f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer while (1) { 20155f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer char Char = getAndAdvanceChar(CurPtr, Tmp); 20165f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer switch (Char) { 20175f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer default: 20185f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Result += Char; 20195f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 20205f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case 0: // Null. 20215f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Found end of file? 20225f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (CurPtr-1 != BufferEnd) { 20237d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis if (isCodeCompletionPoint(CurPtr-1)) { 20247d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis PP->CodeCompleteNaturalLanguage(); 20257d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis cutOffLexing(); 20267d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis return Result; 20277d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis } 20287d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis 20295f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Nope, normal character, continue. 20305f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Result += Char; 20315f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 20325f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 20335f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // FALL THROUGH. 20345f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '\r': 20355f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '\n': 20365f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Okay, we found the end of the line. First, back up past the \0, \r, \n. 20375f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer assert(CurPtr[-1] == Char && "Trigraphs for newline?"); 20385f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer BufferPtr = CurPtr-1; 20391eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 204084021556baceb76eedf7d44be8ba71d9b8cfaccePeter Collingbourne // Next, lex the character, which should handle the EOD transition. 20415f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Lex(Tmp); 204255817afdf9d453a443262a733f6caf6692dca118Douglas Gregor if (Tmp.is(tok::code_completion)) { 20437d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis if (PP) 20447d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis PP->CodeCompleteNaturalLanguage(); 204555817afdf9d453a443262a733f6caf6692dca118Douglas Gregor Lex(Tmp); 204655817afdf9d453a443262a733f6caf6692dca118Douglas Gregor } 204784021556baceb76eedf7d44be8ba71d9b8cfaccePeter Collingbourne assert(Tmp.is(tok::eod) && "Unexpected token!"); 20481eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 20495f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Finally, we're done, return the string we found. 20505f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer return Result; 20515f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 20525f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 20535f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 20545f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 20555f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// LexEndOfFile - CurPtr points to the end of this file. Handle this 20565f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// condition, reporting diagnostics and handling other edge cases as required. 20575f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// This returns true if Result contains a token, false if PP.Lex should be 20585f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// called again. 2059d217773f106856a11879ec79dc468efefaf2ee75Chris Lattnerbool Lexer::LexEndOfFile(Token &Result, const char *CurPtr) { 20605f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If we hit the end of the file while parsing a preprocessor directive, 20615f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // end the preprocessor directive first. The next token returned will 20625f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // then be the end of file. 20635f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (ParsingPreprocessorDirective) { 20645f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Done parsing the "line". 20655f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer ParsingPreprocessorDirective = false; 20665f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Update the location of token as well as BufferPtr. 206784021556baceb76eedf7d44be8ba71d9b8cfaccePeter Collingbourne FormTokenWithChars(Result, CurPtr, tok::eod); 20681eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 20695f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Restore comment saving mode, in case it was disabled for directive. 2070f744d133e8d36eec65828d046ee695bda238125cChris Lattner SetCommentRetentionState(PP->getCommentRetentionState()); 20715f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer return true; // Have a token. 20721eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump } 207386d9a52c24d390631a888d4ff812e1b15445e0a0Douglas Gregor 207486d9a52c24d390631a888d4ff812e1b15445e0a0Douglas Gregor // If we are in raw mode, return this event as an EOF token. Let the caller 207586d9a52c24d390631a888d4ff812e1b15445e0a0Douglas Gregor // that put us in raw mode handle the event. 207686d9a52c24d390631a888d4ff812e1b15445e0a0Douglas Gregor if (isLexingRawMode()) { 207786d9a52c24d390631a888d4ff812e1b15445e0a0Douglas Gregor Result.startToken(); 207886d9a52c24d390631a888d4ff812e1b15445e0a0Douglas Gregor BufferPtr = BufferEnd; 207986d9a52c24d390631a888d4ff812e1b15445e0a0Douglas Gregor FormTokenWithChars(Result, BufferEnd, tok::eof); 208086d9a52c24d390631a888d4ff812e1b15445e0a0Douglas Gregor return true; 208186d9a52c24d390631a888d4ff812e1b15445e0a0Douglas Gregor } 208281b747b7fcc91c2fba9a3183d8fac80adbfc1d3eDouglas Gregor 2083f44e854ed1e3aa86d2ed6d615ccd109d50ddcff9Douglas Gregor // Issue diagnostics for unterminated #if and missing newline. 2084f44e854ed1e3aa86d2ed6d615ccd109d50ddcff9Douglas Gregor 20855f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If we are in a #if directive, emit an error. 20865f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer while (!ConditionalStack.empty()) { 20877d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis if (PP->getCodeCompletionFileLoc() != FileLoc) 20882d474ba9e8ae43a1a5a9f72718c0d79092b9453fDouglas Gregor PP->Diag(ConditionalStack.back().IfLoc, 20892d474ba9e8ae43a1a5a9f72718c0d79092b9453fDouglas Gregor diag::err_pp_unterminated_conditional); 20905f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer ConditionalStack.pop_back(); 20915f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 20921eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 2093b25e5d79d9d1967df058a242e96a62d0d0ace074Chris Lattner // C99 5.1.1.2p2: If the file is non-empty and didn't end in a newline, issue 2094b25e5d79d9d1967df058a242e96a62d0d0ace074Chris Lattner // a pedwarn. 2095b25e5d79d9d1967df058a242e96a62d0d0ace074Chris Lattner if (CurPtr != BufferStart && (CurPtr[-1] != '\n' && CurPtr[-1] != '\r')) 209620d0ee5d23c0f0e879ce1711d11df3118b3fb033Mike Stump Diag(BufferEnd, diag::ext_no_newline_eof) 2097849b243d4065f56742a4677d6dc8277609a151f8Douglas Gregor << FixItHint::CreateInsertion(getSourceLocation(BufferEnd), "\n"); 20981eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 20995f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer BufferPtr = CurPtr; 21005f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 21015f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Finally, let the preprocessor handle this. 2102168ae2d44a443da75ea85db5f3b5081eb0bce113Chris Lattner return PP->HandleEndOfFile(Result); 21035f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 21045f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 21055f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// isNextPPTokenLParen - Return 1 if the next unexpanded token lexed from 21065f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// the specified lexer will return a tok::l_paren token, 0 if it is something 21075f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// else and 2 if there are no more tokens in the buffer controlled by the 21085f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// lexer. 21095f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencerunsigned Lexer::isNextPPTokenLParen() { 21105f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer assert(!LexingRawMode && "How can we expand a macro from a skipping buffer?"); 21111eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 21125f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Switch to 'skipping' mode. This will ensure that we can lex a token 21135f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // without emitting diagnostics, disables macro expansion, and will cause EOF 21145f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // to return an EOF token instead of popping the include stack. 21155f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer LexingRawMode = true; 21161eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 21175f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Save state that can be changed while lexing so that we can restore it. 21185f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer const char *TmpBufferPtr = BufferPtr; 2119a864cf7c1d774a0f790bfc46befc87d9dbf1f65cChris Lattner bool inPPDirectiveMode = ParsingPreprocessorDirective; 21201eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 2121d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner Token Tok; 21225f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Tok.startToken(); 21235f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer LexTokenInternal(Tok); 21241eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 21255f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Restore state that may have changed. 21265f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer BufferPtr = TmpBufferPtr; 2127a864cf7c1d774a0f790bfc46befc87d9dbf1f65cChris Lattner ParsingPreprocessorDirective = inPPDirectiveMode; 21281eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 21295f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Restore the lexer back to non-skipping mode. 21305f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer LexingRawMode = false; 21311eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 213222f6bbcafa8871f4f20c4402d9cbc5c024fee99aChris Lattner if (Tok.is(tok::eof)) 21335f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer return 2; 213422f6bbcafa8871f4f20c4402d9cbc5c024fee99aChris Lattner return Tok.is(tok::l_paren); 21355f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 21365f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 213734f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner/// FindConflictEnd - Find the end of a version control conflict marker. 213834f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattnerstatic const char *FindConflictEnd(const char *CurPtr, const char *BufferEnd) { 21395f9e272e632e951b1efe824cd16acb4d96077930Chris Lattner StringRef RestOfBuffer(CurPtr+7, BufferEnd-CurPtr-7); 214034f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner size_t Pos = RestOfBuffer.find(">>>>>>>"); 21415f9e272e632e951b1efe824cd16acb4d96077930Chris Lattner while (Pos != StringRef::npos) { 214234f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner // Must occur at start of line. 214334f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner if (RestOfBuffer[Pos-1] != '\r' && 214434f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner RestOfBuffer[Pos-1] != '\n') { 214534f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner RestOfBuffer = RestOfBuffer.substr(Pos+7); 21463d488990e13835cb35538b2f0cd53dbda92e36beChris Lattner Pos = RestOfBuffer.find(">>>>>>>"); 214734f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner continue; 214834f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner } 214934f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner return RestOfBuffer.data()+Pos; 215034f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner } 215134f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner return 0; 215234f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner} 215334f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner 215434f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner/// IsStartOfConflictMarker - If the specified pointer is the start of a version 215534f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner/// control conflict marker like '<<<<<<<', recognize it as such, emit an error 215634f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner/// and recover nicely. This returns true if it is a conflict marker and false 215734f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner/// if not. 215834f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattnerbool Lexer::IsStartOfConflictMarker(const char *CurPtr) { 215934f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner // Only a conflict marker if it starts at the beginning of a line. 216034f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner if (CurPtr != BufferStart && 216134f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner CurPtr[-1] != '\n' && CurPtr[-1] != '\r') 216234f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner return false; 216334f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner 216434f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner // Check to see if we have <<<<<<<. 216534f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner if (BufferEnd-CurPtr < 8 || 21665f9e272e632e951b1efe824cd16acb4d96077930Chris Lattner StringRef(CurPtr, 7) != "<<<<<<<") 216734f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner return false; 216834f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner 216934f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner // If we have a situation where we don't care about conflict markers, ignore 217034f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner // it. 217134f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner if (IsInConflictMarker || isLexingRawMode()) 217234f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner return false; 217334f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner 217434f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner // Check to see if there is a >>>>>>> somewhere in the buffer at the start of 217534f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner // a line to terminate this conflict marker. 21763d488990e13835cb35538b2f0cd53dbda92e36beChris Lattner if (FindConflictEnd(CurPtr, BufferEnd)) { 217734f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner // We found a match. We are really in a conflict marker. 217834f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner // Diagnose this, and ignore to the end of line. 217934f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner Diag(CurPtr, diag::err_conflict_marker); 218034f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner IsInConflictMarker = true; 218134f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner 218234f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner // Skip ahead to the end of line. We know this exists because the 218334f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner // end-of-conflict marker starts with \r or \n. 218434f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner while (*CurPtr != '\r' && *CurPtr != '\n') { 218534f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner assert(CurPtr != BufferEnd && "Didn't find end of line"); 218634f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner ++CurPtr; 218734f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner } 218834f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner BufferPtr = CurPtr; 218934f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner return true; 219034f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner } 219134f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner 219234f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner // No end of conflict marker found. 219334f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner return false; 219434f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner} 219534f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner 219634f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner 219734f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner/// HandleEndOfConflictMarker - If this is a '=======' or '|||||||' or '>>>>>>>' 219834f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner/// marker, then it is the end of a conflict marker. Handle it by ignoring up 219934f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner/// until the end of the line. This returns true if it is a conflict marker and 220034f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner/// false if not. 220134f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattnerbool Lexer::HandleEndOfConflictMarker(const char *CurPtr) { 220234f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner // Only a conflict marker if it starts at the beginning of a line. 220334f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner if (CurPtr != BufferStart && 220434f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner CurPtr[-1] != '\n' && CurPtr[-1] != '\r') 220534f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner return false; 220634f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner 220734f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner // If we have a situation where we don't care about conflict markers, ignore 220834f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner // it. 220934f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner if (!IsInConflictMarker || isLexingRawMode()) 221034f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner return false; 221134f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner 221234f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner // Check to see if we have the marker (7 characters in a row). 221334f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner for (unsigned i = 1; i != 7; ++i) 221434f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner if (CurPtr[i] != CurPtr[0]) 221534f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner return false; 221634f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner 221734f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner // If we do have it, search for the end of the conflict marker. This could 221834f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner // fail if it got skipped with a '#if 0' or something. Note that CurPtr might 221934f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner // be the end of conflict marker. 222034f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner if (const char *End = FindConflictEnd(CurPtr, BufferEnd)) { 222134f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner CurPtr = End; 222234f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner 222334f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner // Skip ahead to the end of line. 222434f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner while (CurPtr != BufferEnd && *CurPtr != '\r' && *CurPtr != '\n') 222534f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner ++CurPtr; 222634f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner 222734f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner BufferPtr = CurPtr; 222834f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner 222934f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner // No longer in the conflict marker. 223034f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner IsInConflictMarker = false; 223134f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner return true; 223234f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner } 223334f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner 223434f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner return false; 223534f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner} 223634f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner 22377d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidisbool Lexer::isCodeCompletionPoint(const char *CurPtr) const { 22387d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis if (PP && PP->isCodeCompletionEnabled()) { 22397d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis SourceLocation Loc = FileLoc.getFileLocWithOffset(CurPtr-BufferStart); 22407d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis return Loc == PP->getCodeCompletionLoc(); 22417d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis } 22427d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis 22437d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis return false; 22447d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis} 22457d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis 22465f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 22475f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// LexTokenInternal - This implements a simple C family lexer. It is an 22485f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// extremely performance critical piece of code. This assumes that the buffer 2249efb173ddd95325c7fd3c15070392b27c07a49a85Chris Lattner/// has a null character at the end of the file. This returns a preprocessing 2250efb173ddd95325c7fd3c15070392b27c07a49a85Chris Lattner/// token, not a normal token, as such, it is an internal interface. It assumes 2251efb173ddd95325c7fd3c15070392b27c07a49a85Chris Lattner/// that the Flags of result have been cleared before calling this. 2252d217773f106856a11879ec79dc468efefaf2ee75Chris Lattnervoid Lexer::LexTokenInternal(Token &Result) { 22535f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid SpencerLexNextToken: 22545f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // New token, can't need cleaning yet. 2255d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner Result.clearFlag(Token::NeedsCleaning); 22565f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Result.setIdentifierInfo(0); 22571eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 22585f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // CurPtr - Cache BufferPtr in an automatic variable. 22595f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer const char *CurPtr = BufferPtr; 22605f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 22615f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Small amounts of horizontal whitespace is very common between tokens. 22625f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if ((*CurPtr == ' ') || (*CurPtr == '\t')) { 22635f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer ++CurPtr; 22645f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer while ((*CurPtr == ' ') || (*CurPtr == '\t')) 22655f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer ++CurPtr; 22661eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 2267d88dc48e33d71732708960170e57a3d1bdc8f847Chris Lattner // If we are keeping whitespace and other tokens, just return what we just 2268d88dc48e33d71732708960170e57a3d1bdc8f847Chris Lattner // skipped. The next lexer invocation will return the token after the 2269d88dc48e33d71732708960170e57a3d1bdc8f847Chris Lattner // whitespace. 2270d88dc48e33d71732708960170e57a3d1bdc8f847Chris Lattner if (isKeepWhitespaceMode()) { 22719e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner FormTokenWithChars(Result, CurPtr, tok::unknown); 2272d88dc48e33d71732708960170e57a3d1bdc8f847Chris Lattner return; 2273d88dc48e33d71732708960170e57a3d1bdc8f847Chris Lattner } 22741eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 22755f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer BufferPtr = CurPtr; 2276d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner Result.setFlag(Token::LeadingSpace); 22775f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 22781eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 22795f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer unsigned SizeTmp, SizeTmp2; // Temporaries for use in cases below. 22801eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 22815f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Read a character, advancing over it. 22825f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer char Char = getAndAdvanceChar(CurPtr, Result); 22839e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner tok::TokenKind Kind; 22841eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 22855f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer switch (Char) { 22865f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case 0: // Null. 22875f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Found end of file? 22885f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (CurPtr-1 == BufferEnd) { 22895f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Read the PP instance variable into an automatic variable, because 22905f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // LexEndOfFile will often delete 'this'. 2291168ae2d44a443da75ea85db5f3b5081eb0bce113Chris Lattner Preprocessor *PPCache = PP; 22925f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (LexEndOfFile(Result, CurPtr-1)) // Retreat back into the file. 22935f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer return; // Got a token to return. 2294168ae2d44a443da75ea85db5f3b5081eb0bce113Chris Lattner assert(PPCache && "Raw buffer::LexEndOfFile should return a token"); 2295168ae2d44a443da75ea85db5f3b5081eb0bce113Chris Lattner return PPCache->Lex(Result); 22965f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 22971eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 22987d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis // Check if we are performing code completion. 22997d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis if (isCodeCompletionPoint(CurPtr-1)) { 23007d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis // Return the code-completion token. 23017d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis Result.startToken(); 23027d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis FormTokenWithChars(Result, CurPtr, tok::code_completion); 23037d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis return; 23047d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis } 23057d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis 230674d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner if (!isLexingRawMode()) 230774d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner Diag(CurPtr-1, diag::null_in_file); 2308d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner Result.setFlag(Token::LeadingSpace); 2309d88dc48e33d71732708960170e57a3d1bdc8f847Chris Lattner if (SkipWhitespace(Result, CurPtr)) 2310d88dc48e33d71732708960170e57a3d1bdc8f847Chris Lattner return; // KeepWhitespaceMode 23111eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 23125f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer goto LexNextToken; // GCC isn't tail call eliminating. 2313a2bf105739cef7aea8e4d219629c627fcd11742eChris Lattner 2314a2bf105739cef7aea8e4d219629c627fcd11742eChris Lattner case 26: // DOS & CP/M EOF: "^Z". 2315a2bf105739cef7aea8e4d219629c627fcd11742eChris Lattner // If we're in Microsoft extensions mode, treat this as end of file. 2316a2bf105739cef7aea8e4d219629c627fcd11742eChris Lattner if (Features.Microsoft) { 2317a2bf105739cef7aea8e4d219629c627fcd11742eChris Lattner // Read the PP instance variable into an automatic variable, because 2318a2bf105739cef7aea8e4d219629c627fcd11742eChris Lattner // LexEndOfFile will often delete 'this'. 2319a2bf105739cef7aea8e4d219629c627fcd11742eChris Lattner Preprocessor *PPCache = PP; 2320a2bf105739cef7aea8e4d219629c627fcd11742eChris Lattner if (LexEndOfFile(Result, CurPtr-1)) // Retreat back into the file. 2321a2bf105739cef7aea8e4d219629c627fcd11742eChris Lattner return; // Got a token to return. 2322a2bf105739cef7aea8e4d219629c627fcd11742eChris Lattner assert(PPCache && "Raw buffer::LexEndOfFile should return a token"); 2323a2bf105739cef7aea8e4d219629c627fcd11742eChris Lattner return PPCache->Lex(Result); 2324a2bf105739cef7aea8e4d219629c627fcd11742eChris Lattner } 2325a2bf105739cef7aea8e4d219629c627fcd11742eChris Lattner // If Microsoft extensions are disabled, this is just random garbage. 2326a2bf105739cef7aea8e4d219629c627fcd11742eChris Lattner Kind = tok::unknown; 2327a2bf105739cef7aea8e4d219629c627fcd11742eChris Lattner break; 2328a2bf105739cef7aea8e4d219629c627fcd11742eChris Lattner 23295f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '\n': 23305f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '\r': 23315f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If we are inside a preprocessor directive and we see the end of line, 233284021556baceb76eedf7d44be8ba71d9b8cfaccePeter Collingbourne // we know we are done with the directive, so return an EOD token. 23335f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (ParsingPreprocessorDirective) { 23345f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Done parsing the "line". 23355f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer ParsingPreprocessorDirective = false; 23361eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 23375f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Restore comment saving mode, in case it was disabled for directive. 2338f744d133e8d36eec65828d046ee695bda238125cChris Lattner SetCommentRetentionState(PP->getCommentRetentionState()); 23391eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 23405f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Since we consumed a newline, we are back at the start of a line. 23415f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer IsAtStartOfLine = true; 23421eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 234384021556baceb76eedf7d44be8ba71d9b8cfaccePeter Collingbourne Kind = tok::eod; 23445f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 23455f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 23465f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // The returned token is at the start of the line. 2347d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner Result.setFlag(Token::StartOfLine); 23485f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // No leading whitespace seen so far. 2349d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner Result.clearFlag(Token::LeadingSpace); 23501eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 2351d88dc48e33d71732708960170e57a3d1bdc8f847Chris Lattner if (SkipWhitespace(Result, CurPtr)) 2352d88dc48e33d71732708960170e57a3d1bdc8f847Chris Lattner return; // KeepWhitespaceMode 23535f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer goto LexNextToken; // GCC isn't tail call eliminating. 23545f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case ' ': 23555f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '\t': 23565f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '\f': 23575f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '\v': 23588133cfcc0b672cc86d1ef73ce0c61eccecdb3690Chris Lattner SkipHorizontalWhitespace: 2359d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner Result.setFlag(Token::LeadingSpace); 2360d88dc48e33d71732708960170e57a3d1bdc8f847Chris Lattner if (SkipWhitespace(Result, CurPtr)) 2361d88dc48e33d71732708960170e57a3d1bdc8f847Chris Lattner return; // KeepWhitespaceMode 23628133cfcc0b672cc86d1ef73ce0c61eccecdb3690Chris Lattner 23638133cfcc0b672cc86d1ef73ce0c61eccecdb3690Chris Lattner SkipIgnoredUnits: 23648133cfcc0b672cc86d1ef73ce0c61eccecdb3690Chris Lattner CurPtr = BufferPtr; 23651eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 23668133cfcc0b672cc86d1ef73ce0c61eccecdb3690Chris Lattner // If the next token is obviously a // or /* */ comment, skip it efficiently 23678133cfcc0b672cc86d1ef73ce0c61eccecdb3690Chris Lattner // too (without going through the big switch stmt). 23688402c73dd880e8af46c826d873681820aebe32ebChris Lattner if (CurPtr[0] == '/' && CurPtr[1] == '/' && !inKeepCommentMode() && 23692ed42287b9b88a0bb08ab10e95e306da3f294ba9Daniel Dunbar Features.BCPLComment && !Features.TraditionalCPP) { 2370046c2277dcbcc8eb89dbb5b1b8c5226b7cb81635Chris Lattner if (SkipBCPLComment(Result, CurPtr+2)) 2371046c2277dcbcc8eb89dbb5b1b8c5226b7cb81635Chris Lattner return; // There is a token to return. 23728133cfcc0b672cc86d1ef73ce0c61eccecdb3690Chris Lattner goto SkipIgnoredUnits; 2373fa95a019da00b926d64ff83358ba73bbc6ae1e37Chris Lattner } else if (CurPtr[0] == '/' && CurPtr[1] == '*' && !inKeepCommentMode()) { 2374046c2277dcbcc8eb89dbb5b1b8c5226b7cb81635Chris Lattner if (SkipBlockComment(Result, CurPtr+2)) 2375046c2277dcbcc8eb89dbb5b1b8c5226b7cb81635Chris Lattner return; // There is a token to return. 23768133cfcc0b672cc86d1ef73ce0c61eccecdb3690Chris Lattner goto SkipIgnoredUnits; 23778133cfcc0b672cc86d1ef73ce0c61eccecdb3690Chris Lattner } else if (isHorizontalWhitespace(*CurPtr)) { 23788133cfcc0b672cc86d1ef73ce0c61eccecdb3690Chris Lattner goto SkipHorizontalWhitespace; 23798133cfcc0b672cc86d1ef73ce0c61eccecdb3690Chris Lattner } 23805f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer goto LexNextToken; // GCC isn't tail call eliminating. 2381a2bf105739cef7aea8e4d219629c627fcd11742eChris Lattner 23823a5707766850f9ee9daa35299794328b5caf96dcChris Lattner // C99 6.4.4.1: Integer Constants. 23833a5707766850f9ee9daa35299794328b5caf96dcChris Lattner // C99 6.4.4.2: Floating Constants. 23843a5707766850f9ee9daa35299794328b5caf96dcChris Lattner case '0': case '1': case '2': case '3': case '4': 23853a5707766850f9ee9daa35299794328b5caf96dcChris Lattner case '5': case '6': case '7': case '8': case '9': 23863a5707766850f9ee9daa35299794328b5caf96dcChris Lattner // Notify MIOpt that we read a non-whitespace/non-comment token. 23873a5707766850f9ee9daa35299794328b5caf96dcChris Lattner MIOpt.ReadToken(); 23883a5707766850f9ee9daa35299794328b5caf96dcChris Lattner return LexNumericConstant(Result, CurPtr); 23891eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 23905cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor case 'u': // Identifier (uber) or C++0x UTF-8 or UTF-16 string literal 23915cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor // Notify MIOpt that we read a non-whitespace/non-comment token. 23925cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor MIOpt.ReadToken(); 23935cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor 23945cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor if (Features.CPlusPlus0x) { 23955cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor Char = getCharAndSize(CurPtr, SizeTmp); 23965cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor 23975cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor // UTF-16 string literal 23985cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor if (Char == '"') 23995cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor return LexStringLiteral(Result, ConsumeChar(CurPtr, SizeTmp, Result), 24005cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor tok::utf16_string_literal); 24015cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor 24025cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor // UTF-16 character constant 24035cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor if (Char == '\'') 24045cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor return LexCharConstant(Result, ConsumeChar(CurPtr, SizeTmp, Result), 24055cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor tok::utf16_char_constant); 24065cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor 24072fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper // UTF-16 raw string literal 24082fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper if (Char == 'R' && getCharAndSize(CurPtr + SizeTmp, SizeTmp2) == '"') 24092fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper return LexRawStringLiteral(Result, 24102fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 24112fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper SizeTmp2, Result), 24122fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper tok::utf16_string_literal); 24132fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper 24142fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper if (Char == '8') { 24152fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper char Char2 = getCharAndSize(CurPtr + SizeTmp, SizeTmp2); 24162fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper 24172fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper // UTF-8 string literal 24182fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper if (Char2 == '"') 24192fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper return LexStringLiteral(Result, 24202fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 24212fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper SizeTmp2, Result), 24222fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper tok::utf8_string_literal); 24232fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper 24242fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper if (Char2 == 'R') { 24252fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper unsigned SizeTmp3; 24262fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper char Char3 = getCharAndSize(CurPtr + SizeTmp + SizeTmp2, SizeTmp3); 24272fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper // UTF-8 raw string literal 24282fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper if (Char3 == '"') { 24292fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper return LexRawStringLiteral(Result, 24302fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper ConsumeChar(ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 24312fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper SizeTmp2, Result), 24322fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper SizeTmp3, Result), 24332fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper tok::utf8_string_literal); 24342fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper } 24352fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper } 24362fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper } 24375cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor } 24385cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor 24395cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor // treat u like the start of an identifier. 24405cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor return LexIdentifier(Result, CurPtr); 24415cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor 24425cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor case 'U': // Identifier (Uber) or C++0x UTF-32 string literal 24435cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor // Notify MIOpt that we read a non-whitespace/non-comment token. 24445cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor MIOpt.ReadToken(); 24455cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor 24465cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor if (Features.CPlusPlus0x) { 24475cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor Char = getCharAndSize(CurPtr, SizeTmp); 24485cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor 24495cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor // UTF-32 string literal 24505cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor if (Char == '"') 24515cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor return LexStringLiteral(Result, ConsumeChar(CurPtr, SizeTmp, Result), 24525cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor tok::utf32_string_literal); 24535cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor 24545cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor // UTF-32 character constant 24555cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor if (Char == '\'') 24565cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor return LexCharConstant(Result, ConsumeChar(CurPtr, SizeTmp, Result), 24575cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor tok::utf32_char_constant); 24582fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper 24592fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper // UTF-32 raw string literal 24602fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper if (Char == 'R' && getCharAndSize(CurPtr + SizeTmp, SizeTmp2) == '"') 24612fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper return LexRawStringLiteral(Result, 24622fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 24632fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper SizeTmp2, Result), 24642fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper tok::utf32_string_literal); 24655cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor } 24665cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor 24675cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor // treat U like the start of an identifier. 24685cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor return LexIdentifier(Result, CurPtr); 24695cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor 24702fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper case 'R': // Identifier or C++0x raw string literal 24712fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper // Notify MIOpt that we read a non-whitespace/non-comment token. 24722fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper MIOpt.ReadToken(); 24732fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper 24742fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper if (Features.CPlusPlus0x) { 24752fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper Char = getCharAndSize(CurPtr, SizeTmp); 24762fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper 24772fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper if (Char == '"') 24782fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper return LexRawStringLiteral(Result, 24792fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper ConsumeChar(CurPtr, SizeTmp, Result), 24802fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper tok::string_literal); 24812fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper } 24822fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper 24832fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper // treat R like the start of an identifier. 24842fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper return LexIdentifier(Result, CurPtr); 24852fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper 24863a5707766850f9ee9daa35299794328b5caf96dcChris Lattner case 'L': // Identifier (Loony) or wide literal (L'x' or L"xyz"). 24875f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Notify MIOpt that we read a non-whitespace/non-comment token. 24885f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer MIOpt.ReadToken(); 24895f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Char = getCharAndSize(CurPtr, SizeTmp); 24905f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 24915f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Wide string literal. 24925f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (Char == '"') 24935f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer return LexStringLiteral(Result, ConsumeChar(CurPtr, SizeTmp, Result), 24945cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor tok::wide_string_literal); 24955f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 24962fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper // Wide raw string literal. 24972fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper if (Features.CPlusPlus0x && Char == 'R' && 24982fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper getCharAndSize(CurPtr + SizeTmp, SizeTmp2) == '"') 24992fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper return LexRawStringLiteral(Result, 25002fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 25012fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper SizeTmp2, Result), 25022fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper tok::wide_string_literal); 25032fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper 25045f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Wide character constant. 25055f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (Char == '\'') 25065cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor return LexCharConstant(Result, ConsumeChar(CurPtr, SizeTmp, Result), 25075cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor tok::wide_char_constant); 25085f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // FALL THROUGH, treating L like the start of an identifier. 25091eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 25105f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // C99 6.4.2: Identifiers. 25115f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': case 'G': 25125f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case 'H': case 'I': case 'J': case 'K': /*'L'*/case 'M': case 'N': 25132fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper case 'O': case 'P': case 'Q': /*'R'*/case 'S': case 'T': /*'U'*/ 25145f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case 'V': case 'W': case 'X': case 'Y': case 'Z': 25155f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': case 'g': 25165f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case 'h': case 'i': case 'j': case 'k': case 'l': case 'm': case 'n': 25175cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor case 'o': case 'p': case 'q': case 'r': case 's': case 't': /*'u'*/ 25185f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case 'v': case 'w': case 'x': case 'y': case 'z': 25195f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '_': 25205f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Notify MIOpt that we read a non-whitespace/non-comment token. 25215f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer MIOpt.ReadToken(); 25225f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer return LexIdentifier(Result, CurPtr); 25233a5707766850f9ee9daa35299794328b5caf96dcChris Lattner 25243a5707766850f9ee9daa35299794328b5caf96dcChris Lattner case '$': // $ in identifiers. 25253a5707766850f9ee9daa35299794328b5caf96dcChris Lattner if (Features.DollarIdents) { 252674d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner if (!isLexingRawMode()) 252774d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner Diag(CurPtr-1, diag::ext_dollar_in_identifier); 25283a5707766850f9ee9daa35299794328b5caf96dcChris Lattner // Notify MIOpt that we read a non-whitespace/non-comment token. 25293a5707766850f9ee9daa35299794328b5caf96dcChris Lattner MIOpt.ReadToken(); 25303a5707766850f9ee9daa35299794328b5caf96dcChris Lattner return LexIdentifier(Result, CurPtr); 25313a5707766850f9ee9daa35299794328b5caf96dcChris Lattner } 25321eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 25339e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::unknown; 25343a5707766850f9ee9daa35299794328b5caf96dcChris Lattner break; 25351eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 25365f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // C99 6.4.4: Character Constants. 25375f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '\'': 25385f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Notify MIOpt that we read a non-whitespace/non-comment token. 25395f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer MIOpt.ReadToken(); 25405cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor return LexCharConstant(Result, CurPtr, tok::char_constant); 25415f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 25425f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // C99 6.4.5: String Literals. 25435f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '"': 25445f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Notify MIOpt that we read a non-whitespace/non-comment token. 25455f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer MIOpt.ReadToken(); 25465cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor return LexStringLiteral(Result, CurPtr, tok::string_literal); 25475f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 25485f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // C99 6.4.6: Punctuators. 25495f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '?': 25509e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::question; 25515f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 25525f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '[': 25539e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::l_square; 25545f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 25555f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case ']': 25569e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::r_square; 25575f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 25585f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '(': 25599e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::l_paren; 25605f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 25615f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case ')': 25629e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::r_paren; 25635f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 25645f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '{': 25659e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::l_brace; 25665f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 25675f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '}': 25689e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::r_brace; 25695f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 25705f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '.': 25715f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Char = getCharAndSize(CurPtr, SizeTmp); 25725f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (Char >= '0' && Char <= '9') { 25735f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Notify MIOpt that we read a non-whitespace/non-comment token. 25745f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer MIOpt.ReadToken(); 25755f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 25765f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer return LexNumericConstant(Result, ConsumeChar(CurPtr, SizeTmp, Result)); 25775f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } else if (Features.CPlusPlus && Char == '*') { 25789e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::periodstar; 25795f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr += SizeTmp; 25805f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } else if (Char == '.' && 25815f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer getCharAndSize(CurPtr+SizeTmp, SizeTmp2) == '.') { 25829e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::ellipsis; 25835f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 25845f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer SizeTmp2, Result); 25855f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } else { 25869e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::period; 25875f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 25885f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 25895f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '&': 25905f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Char = getCharAndSize(CurPtr, SizeTmp); 25915f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (Char == '&') { 25929e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::ampamp; 25935f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 25945f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } else if (Char == '=') { 25959e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::ampequal; 25965f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 25975f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } else { 25989e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::amp; 25995f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 26005f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 26011eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump case '*': 26025f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (getCharAndSize(CurPtr, SizeTmp) == '=') { 26039e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::starequal; 26045f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 26055f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } else { 26069e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::star; 26075f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 26085f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 26095f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '+': 26105f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Char = getCharAndSize(CurPtr, SizeTmp); 26115f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (Char == '+') { 26125f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 26139e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::plusplus; 26145f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } else if (Char == '=') { 26155f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 26169e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::plusequal; 26175f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } else { 26189e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::plus; 26195f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 26205f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 26215f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '-': 26225f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Char = getCharAndSize(CurPtr, SizeTmp); 26239e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner if (Char == '-') { // -- 26245f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 26259e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::minusminus; 26261eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump } else if (Char == '>' && Features.CPlusPlus && 26279e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner getCharAndSize(CurPtr+SizeTmp, SizeTmp2) == '*') { // C++ ->* 26285f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 26295f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer SizeTmp2, Result); 26309e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::arrowstar; 26319e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner } else if (Char == '>') { // -> 26325f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 26339e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::arrow; 26349e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner } else if (Char == '=') { // -= 26355f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 26369e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::minusequal; 26375f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } else { 26389e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::minus; 26395f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 26405f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 26415f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '~': 26429e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::tilde; 26435f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 26445f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '!': 26455f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (getCharAndSize(CurPtr, SizeTmp) == '=') { 26469e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::exclaimequal; 26475f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 26485f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } else { 26499e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::exclaim; 26505f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 26515f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 26525f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '/': 26535f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // 6.4.9: Comments 26545f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Char = getCharAndSize(CurPtr, SizeTmp); 26555f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (Char == '/') { // BCPL comment. 26568402c73dd880e8af46c826d873681820aebe32ebChris Lattner // Even if BCPL comments are disabled (e.g. in C89 mode), we generally 26578402c73dd880e8af46c826d873681820aebe32ebChris Lattner // want to lex this as a comment. There is one problem with this though, 26588402c73dd880e8af46c826d873681820aebe32ebChris Lattner // that in one particular corner case, this can change the behavior of the 26598402c73dd880e8af46c826d873681820aebe32ebChris Lattner // resultant program. For example, In "foo //**/ bar", C89 would lex 26608402c73dd880e8af46c826d873681820aebe32ebChris Lattner // this as "foo / bar" and langauges with BCPL comments would lex it as 26618402c73dd880e8af46c826d873681820aebe32ebChris Lattner // "foo". Check to see if the character after the second slash is a '*'. 26628402c73dd880e8af46c826d873681820aebe32ebChris Lattner // If so, we will lex that as a "/" instead of the start of a comment. 26632ed42287b9b88a0bb08ab10e95e306da3f294ba9Daniel Dunbar // However, we never do this in -traditional-cpp mode. 26642ed42287b9b88a0bb08ab10e95e306da3f294ba9Daniel Dunbar if ((Features.BCPLComment || 26652ed42287b9b88a0bb08ab10e95e306da3f294ba9Daniel Dunbar getCharAndSize(CurPtr+SizeTmp, SizeTmp2) != '*') && 26662ed42287b9b88a0bb08ab10e95e306da3f294ba9Daniel Dunbar !Features.TraditionalCPP) { 26678402c73dd880e8af46c826d873681820aebe32ebChris Lattner if (SkipBCPLComment(Result, ConsumeChar(CurPtr, SizeTmp, Result))) 2668046c2277dcbcc8eb89dbb5b1b8c5226b7cb81635Chris Lattner return; // There is a token to return. 26691eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 26708402c73dd880e8af46c826d873681820aebe32ebChris Lattner // It is common for the tokens immediately after a // comment to be 26718402c73dd880e8af46c826d873681820aebe32ebChris Lattner // whitespace (indentation for the next line). Instead of going through 26728402c73dd880e8af46c826d873681820aebe32ebChris Lattner // the big switch, handle it efficiently now. 26738402c73dd880e8af46c826d873681820aebe32ebChris Lattner goto SkipIgnoredUnits; 26748402c73dd880e8af46c826d873681820aebe32ebChris Lattner } 26758402c73dd880e8af46c826d873681820aebe32ebChris Lattner } 26761eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 26778402c73dd880e8af46c826d873681820aebe32ebChris Lattner if (Char == '*') { // /**/ comment. 26785f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (SkipBlockComment(Result, ConsumeChar(CurPtr, SizeTmp, Result))) 2679046c2277dcbcc8eb89dbb5b1b8c5226b7cb81635Chris Lattner return; // There is a token to return. 26802d3818959df7b71bfcf8c2f2b573856bfc4c3a21Chris Lattner goto LexNextToken; // GCC isn't tail call eliminating. 26818402c73dd880e8af46c826d873681820aebe32ebChris Lattner } 26821eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 26838402c73dd880e8af46c826d873681820aebe32ebChris Lattner if (Char == '=') { 26845f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 26859e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::slashequal; 26865f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } else { 26879e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::slash; 26885f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 26895f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 26905f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '%': 26915f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Char = getCharAndSize(CurPtr, SizeTmp); 26925f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (Char == '=') { 26939e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::percentequal; 26945f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 26955f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } else if (Features.Digraphs && Char == '>') { 26969e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::r_brace; // '%>' -> '}' 26975f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 26985f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } else if (Features.Digraphs && Char == ':') { 26995f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 27005f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Char = getCharAndSize(CurPtr, SizeTmp); 27015f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (Char == '%' && getCharAndSize(CurPtr+SizeTmp, SizeTmp2) == ':') { 27029e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::hashhash; // '%:%:' -> '##' 27035f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 27045f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer SizeTmp2, Result); 27055f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } else if (Char == '@' && Features.Microsoft) { // %:@ -> #@ -> Charize 27065f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 270774d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner if (!isLexingRawMode()) 270874d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner Diag(BufferPtr, diag::charize_microsoft_ext); 27099e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::hashat; 2710e91e93225db2e66906878513c6ef4dd6a7ee2b6aChris Lattner } else { // '%:' -> '#' 27115f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // We parsed a # character. If this occurs at the start of the line, 27125f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // it's actually the start of a preprocessing directive. Callback to 27135f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // the preprocessor to handle it. 27145f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // FIXME: -fpreprocessed mode?? 2715766703b76dc1eb0c49d11cafc2a17c7407876ebcChris Lattner if (Result.isAtStartOfLine() && !LexingRawMode && !Is_PragmaLexer) { 2716e91e93225db2e66906878513c6ef4dd6a7ee2b6aChris Lattner FormTokenWithChars(Result, CurPtr, tok::hash); 2717168ae2d44a443da75ea85db5f3b5081eb0bce113Chris Lattner PP->HandleDirective(Result); 27181eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 27195f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // As an optimization, if the preprocessor didn't switch lexers, tail 27205f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // recurse. 2721168ae2d44a443da75ea85db5f3b5081eb0bce113Chris Lattner if (PP->isCurrentLexer(this)) { 27225f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Start a new token. If this is a #include or something, the PP may 27235f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // want us starting at the beginning of the line again. If so, set 2724515f43f9f23de50d155b481b8774ec40bdfd7ff2Chris Lattner // the StartOfLine flag and clear LeadingSpace. 27255f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (IsAtStartOfLine) { 2726d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner Result.setFlag(Token::StartOfLine); 2727515f43f9f23de50d155b481b8774ec40bdfd7ff2Chris Lattner Result.clearFlag(Token::LeadingSpace); 27285f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer IsAtStartOfLine = false; 27295f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 27305f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer goto LexNextToken; // GCC isn't tail call eliminating. 27315f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 27321eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 2733168ae2d44a443da75ea85db5f3b5081eb0bce113Chris Lattner return PP->Lex(Result); 27345f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 27351eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 2736e91e93225db2e66906878513c6ef4dd6a7ee2b6aChris Lattner Kind = tok::hash; 27375f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 27385f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } else { 27399e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::percent; 27405f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 27415f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 27425f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '<': 27435f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Char = getCharAndSize(CurPtr, SizeTmp); 27445f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (ParsingFilename) { 27459cb51ce03caf0755759f7e67aea7d4be684ff738Chris Lattner return LexAngledStringLiteral(Result, CurPtr); 27465f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } else if (Char == '<') { 274734f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner char After = getCharAndSize(CurPtr+SizeTmp, SizeTmp2); 274834f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner if (After == '=') { 274934f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner Kind = tok::lesslessequal; 275034f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 275134f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner SizeTmp2, Result); 275234f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner } else if (After == '<' && IsStartOfConflictMarker(CurPtr-1)) { 275334f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner // If this is actually a '<<<<<<<' version control conflict marker, 275434f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner // recognize it as such and recover nicely. 275534f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner goto LexNextToken; 27561b791d6465d42a9763927be1dd8af229efcbbf5ePeter Collingbourne } else if (Features.CUDA && After == '<') { 27571b791d6465d42a9763927be1dd8af229efcbbf5ePeter Collingbourne Kind = tok::lesslessless; 27581b791d6465d42a9763927be1dd8af229efcbbf5ePeter Collingbourne CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 27591b791d6465d42a9763927be1dd8af229efcbbf5ePeter Collingbourne SizeTmp2, Result); 276034f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner } else { 276134f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 276234f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner Kind = tok::lessless; 276334f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner } 27645f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } else if (Char == '=') { 27655f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 27669e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::lessequal; 27679e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner } else if (Features.Digraphs && Char == ':') { // '<:' -> '[' 276887a1e19735a27d84edc2ac1331c040e2fb4c3b1aRichard Smith if (Features.CPlusPlus0x && 276987a1e19735a27d84edc2ac1331c040e2fb4c3b1aRichard Smith getCharAndSize(CurPtr + SizeTmp, SizeTmp2) == ':') { 277087a1e19735a27d84edc2ac1331c040e2fb4c3b1aRichard Smith // C++0x [lex.pptoken]p3: 277187a1e19735a27d84edc2ac1331c040e2fb4c3b1aRichard Smith // Otherwise, if the next three characters are <:: and the subsequent 277287a1e19735a27d84edc2ac1331c040e2fb4c3b1aRichard Smith // character is neither : nor >, the < is treated as a preprocessor 277387a1e19735a27d84edc2ac1331c040e2fb4c3b1aRichard Smith // token by itself and not as the first character of the alternative 277487a1e19735a27d84edc2ac1331c040e2fb4c3b1aRichard Smith // token <:. 277587a1e19735a27d84edc2ac1331c040e2fb4c3b1aRichard Smith unsigned SizeTmp3; 277687a1e19735a27d84edc2ac1331c040e2fb4c3b1aRichard Smith char After = getCharAndSize(CurPtr + SizeTmp + SizeTmp2, SizeTmp3); 277787a1e19735a27d84edc2ac1331c040e2fb4c3b1aRichard Smith if (After != ':' && After != '>') { 277887a1e19735a27d84edc2ac1331c040e2fb4c3b1aRichard Smith Kind = tok::less; 277987a1e19735a27d84edc2ac1331c040e2fb4c3b1aRichard Smith break; 278087a1e19735a27d84edc2ac1331c040e2fb4c3b1aRichard Smith } 278187a1e19735a27d84edc2ac1331c040e2fb4c3b1aRichard Smith } 278287a1e19735a27d84edc2ac1331c040e2fb4c3b1aRichard Smith 27835f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 27849e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::l_square; 27859e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner } else if (Features.Digraphs && Char == '%') { // '<%' -> '{' 27865f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 27879e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::l_brace; 27885f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } else { 27899e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::less; 27905f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 27915f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 27925f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '>': 27935f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Char = getCharAndSize(CurPtr, SizeTmp); 27945f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (Char == '=') { 27955f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 27969e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::greaterequal; 27975f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } else if (Char == '>') { 279834f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner char After = getCharAndSize(CurPtr+SizeTmp, SizeTmp2); 279934f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner if (After == '=') { 280034f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 280134f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner SizeTmp2, Result); 280234f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner Kind = tok::greatergreaterequal; 280334f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner } else if (After == '>' && HandleEndOfConflictMarker(CurPtr-1)) { 280434f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner // If this is '>>>>>>>' and we're in a conflict marker, ignore it. 280534f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner goto LexNextToken; 28061b791d6465d42a9763927be1dd8af229efcbbf5ePeter Collingbourne } else if (Features.CUDA && After == '>') { 28071b791d6465d42a9763927be1dd8af229efcbbf5ePeter Collingbourne Kind = tok::greatergreatergreater; 28081b791d6465d42a9763927be1dd8af229efcbbf5ePeter Collingbourne CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 28091b791d6465d42a9763927be1dd8af229efcbbf5ePeter Collingbourne SizeTmp2, Result); 281034f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner } else { 281134f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 281234f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner Kind = tok::greatergreater; 281334f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner } 281434f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner 28155f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } else { 28169e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::greater; 28175f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 28185f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 28195f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '^': 28205f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Char = getCharAndSize(CurPtr, SizeTmp); 28215f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (Char == '=') { 28225f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 28239e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::caretequal; 28245f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } else { 28259e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::caret; 28265f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 28275f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 28285f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '|': 28295f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Char = getCharAndSize(CurPtr, SizeTmp); 28305f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (Char == '=') { 28319e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::pipeequal; 28325f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 28335f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } else if (Char == '|') { 283434f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner // If this is '|||||||' and we're in a conflict marker, ignore it. 283534f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner if (CurPtr[1] == '|' && HandleEndOfConflictMarker(CurPtr-1)) 283634f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner goto LexNextToken; 28379e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::pipepipe; 28385f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 28395f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } else { 28409e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::pipe; 28415f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 28425f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 28435f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case ':': 28445f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Char = getCharAndSize(CurPtr, SizeTmp); 28455f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (Features.Digraphs && Char == '>') { 28469e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::r_square; // ':>' -> ']' 28475f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 28485f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } else if (Features.CPlusPlus && Char == ':') { 28499e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::coloncolon; 28505f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 28511eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump } else { 28529e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::colon; 28535f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 28545f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 28555f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case ';': 28569e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::semi; 28575f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 28585f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '=': 28595f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Char = getCharAndSize(CurPtr, SizeTmp); 28605f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (Char == '=') { 286134f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner // If this is '=======' and we're in a conflict marker, ignore it. 286234f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner if (CurPtr[1] == '=' && HandleEndOfConflictMarker(CurPtr-1)) 286334f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner goto LexNextToken; 286434f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner 28659e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::equalequal; 28665f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 28671eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump } else { 28689e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::equal; 28695f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 28705f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 28715f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case ',': 28729e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::comma; 28735f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 28745f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '#': 28755f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Char = getCharAndSize(CurPtr, SizeTmp); 28765f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (Char == '#') { 28779e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::hashhash; 28785f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 28795f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } else if (Char == '@' && Features.Microsoft) { // #@ -> Charize 28809e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::hashat; 288174d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner if (!isLexingRawMode()) 288274d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner Diag(BufferPtr, diag::charize_microsoft_ext); 28835f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 28845f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } else { 28855f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // We parsed a # character. If this occurs at the start of the line, 28865f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // it's actually the start of a preprocessing directive. Callback to 28875f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // the preprocessor to handle it. 28885f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // FIXME: -fpreprocessed mode?? 2889766703b76dc1eb0c49d11cafc2a17c7407876ebcChris Lattner if (Result.isAtStartOfLine() && !LexingRawMode && !Is_PragmaLexer) { 2890e91e93225db2e66906878513c6ef4dd6a7ee2b6aChris Lattner FormTokenWithChars(Result, CurPtr, tok::hash); 2891168ae2d44a443da75ea85db5f3b5081eb0bce113Chris Lattner PP->HandleDirective(Result); 28921eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 28935f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // As an optimization, if the preprocessor didn't switch lexers, tail 28945f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // recurse. 2895168ae2d44a443da75ea85db5f3b5081eb0bce113Chris Lattner if (PP->isCurrentLexer(this)) { 28965f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Start a new token. If this is a #include or something, the PP may 28975f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // want us starting at the beginning of the line again. If so, set 2898515f43f9f23de50d155b481b8774ec40bdfd7ff2Chris Lattner // the StartOfLine flag and clear LeadingSpace. 28995f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (IsAtStartOfLine) { 2900d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner Result.setFlag(Token::StartOfLine); 2901515f43f9f23de50d155b481b8774ec40bdfd7ff2Chris Lattner Result.clearFlag(Token::LeadingSpace); 29025f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer IsAtStartOfLine = false; 29035f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 29045f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer goto LexNextToken; // GCC isn't tail call eliminating. 29055f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 2906168ae2d44a443da75ea85db5f3b5081eb0bce113Chris Lattner return PP->Lex(Result); 29075f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 29081eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 2909e91e93225db2e66906878513c6ef4dd6a7ee2b6aChris Lattner Kind = tok::hash; 29105f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 29115f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 29125f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 29133a5707766850f9ee9daa35299794328b5caf96dcChris Lattner case '@': 29143a5707766850f9ee9daa35299794328b5caf96dcChris Lattner // Objective C support. 29153a5707766850f9ee9daa35299794328b5caf96dcChris Lattner if (CurPtr[-1] == '@' && Features.ObjC1) 29169e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::at; 29173a5707766850f9ee9daa35299794328b5caf96dcChris Lattner else 29189e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::unknown; 29193a5707766850f9ee9daa35299794328b5caf96dcChris Lattner break; 29201eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 29215f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '\\': 29225f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // FIXME: UCN's. 29235f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // FALL THROUGH. 29245f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer default: 29259e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::unknown; 29265f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 29275f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 29281eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 29295f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Notify MIOpt that we read a non-whitespace/non-comment token. 29305f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer MIOpt.ReadToken(); 29315f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 29325f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Update the location of token as well as BufferPtr. 29339e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner FormTokenWithChars(Result, CurPtr, Kind); 29345f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 2935