1f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot//===--- CommentLexer.h - Lexer for structured comments ---------*- C++ -*-===// 2f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot// 3f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot// The LLVM Compiler Infrastructure 4f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot// 5f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot// This file is distributed under the University of Illinois Open Source 6f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot// License. See LICENSE.TXT for details. 7f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot// 8f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot//===----------------------------------------------------------------------===// 9f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot// 10f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot// This file defines lexer for structured comments and supporting token class. 11f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot// 12f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot//===----------------------------------------------------------------------===// 13f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot 14f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot#ifndef LLVM_CLANG_AST_COMMENTLEXER_H 15f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot#define LLVM_CLANG_AST_COMMENTLEXER_H 16f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot 17f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot#include "clang/Basic/Diagnostic.h" 18f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot#include "clang/Basic/SourceManager.h" 19f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot#include "llvm/ADT/SmallString.h" 20f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot#include "llvm/ADT/StringRef.h" 21f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot#include "llvm/Support/Allocator.h" 22f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot#include "llvm/Support/raw_ostream.h" 23f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot 24f3014761c955345d6e05491608e73228d014afbandroid-build-team Robotnamespace clang { 25f3014761c955345d6e05491608e73228d014afbandroid-build-team Robotnamespace comments { 26f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot 27f3014761c955345d6e05491608e73228d014afbandroid-build-team Robotclass Lexer; 28f3014761c955345d6e05491608e73228d014afbandroid-build-team Robotclass TextTokenRetokenizer; 29f3014761c955345d6e05491608e73228d014afbandroid-build-team Robotstruct CommandInfo; 30f3014761c955345d6e05491608e73228d014afbandroid-build-team Robotclass CommandTraits; 31f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot 32f3014761c955345d6e05491608e73228d014afbandroid-build-team Robotnamespace tok { 33f3014761c955345d6e05491608e73228d014afbandroid-build-team Robotenum TokenKind { 34f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot eof, 35f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot newline, 36f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot text, 37f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot unknown_command, // Command that does not have an ID. 38f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot backslash_command, // Command with an ID, that used backslash marker. 39f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot at_command, // Command with an ID, that used 'at' marker. 40f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot verbatim_block_begin, 41f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot verbatim_block_line, 42f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot verbatim_block_end, 43f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot verbatim_line_name, 44f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot verbatim_line_text, 45f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot html_start_tag, // <tag 46f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot html_ident, // attr 47f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot html_equals, // = 48f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot html_quoted_string, // "blah\"blah" or 'blah\'blah' 49f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot html_greater, // > 50f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot html_slash_greater, // /> 51f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot html_end_tag // </tag 52f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot}; 53f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot} // end namespace tok 54f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot 55f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot/// \brief Comment token. 56f3014761c955345d6e05491608e73228d014afbandroid-build-team Robotclass Token { 57f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot friend class Lexer; 58f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot friend class TextTokenRetokenizer; 59f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot 60f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot /// The location of the token. 61f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot SourceLocation Loc; 62f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot 63f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot /// The actual kind of the token. 64f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot tok::TokenKind Kind; 65f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot 66f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot /// Length of the token spelling in comment. Can be 0 for synthenized 67f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot /// tokens. 68f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot unsigned Length; 69f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot 70f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot /// Contains text value associated with a token. 71f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot const char *TextPtr; 72f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot 73f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot /// Integer value associated with a token. 74f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot /// 75f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot /// If the token is a konwn command, contains command ID and TextPtr is 76f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot /// unused (command spelling can be found with CommandTraits). Otherwise, 77f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot /// contains the length of the string that starts at TextPtr. 78f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot unsigned IntVal; 79f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot 80f3014761c955345d6e05491608e73228d014afbandroid-build-team Robotpublic: 81f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot SourceLocation getLocation() const LLVM_READONLY { return Loc; } 82f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot void setLocation(SourceLocation SL) { Loc = SL; } 83f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot 84f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot SourceLocation getEndLocation() const LLVM_READONLY { 85f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot if (Length == 0 || Length == 1) 86f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot return Loc; 87f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot return Loc.getLocWithOffset(Length - 1); 88f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot } 89f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot 90f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot tok::TokenKind getKind() const LLVM_READONLY { return Kind; } 91f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot void setKind(tok::TokenKind K) { Kind = K; } 92f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot 93f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot bool is(tok::TokenKind K) const LLVM_READONLY { return Kind == K; } 94f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot bool isNot(tok::TokenKind K) const LLVM_READONLY { return Kind != K; } 95f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot 96f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot unsigned getLength() const LLVM_READONLY { return Length; } 97f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot void setLength(unsigned L) { Length = L; } 98f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot 99f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot StringRef getText() const LLVM_READONLY { 100f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot assert(is(tok::text)); 101f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot return StringRef(TextPtr, IntVal); 102f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot } 103f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot 104f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot void setText(StringRef Text) { 105f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot assert(is(tok::text)); 106f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot TextPtr = Text.data(); 107f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot IntVal = Text.size(); 108f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot } 109f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot 110f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot StringRef getUnknownCommandName() const LLVM_READONLY { 111f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot assert(is(tok::unknown_command)); 112f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot return StringRef(TextPtr, IntVal); 113f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot } 114f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot 115f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot void setUnknownCommandName(StringRef Name) { 116f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot assert(is(tok::unknown_command)); 117f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot TextPtr = Name.data(); 118f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot IntVal = Name.size(); 119f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot } 120f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot 121f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot unsigned getCommandID() const LLVM_READONLY { 122f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot assert(is(tok::backslash_command) || is(tok::at_command)); 123f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot return IntVal; 124f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot } 125f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot 126f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot void setCommandID(unsigned ID) { 127f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot assert(is(tok::backslash_command) || is(tok::at_command)); 128f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot IntVal = ID; 129f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot } 130f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot 131f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot unsigned getVerbatimBlockID() const LLVM_READONLY { 132f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot assert(is(tok::verbatim_block_begin) || is(tok::verbatim_block_end)); 133f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot return IntVal; 134f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot } 135f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot 136f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot void setVerbatimBlockID(unsigned ID) { 137f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot assert(is(tok::verbatim_block_begin) || is(tok::verbatim_block_end)); 138f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot IntVal = ID; 139f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot } 140f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot 141f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot StringRef getVerbatimBlockText() const LLVM_READONLY { 142f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot assert(is(tok::verbatim_block_line)); 143f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot return StringRef(TextPtr, IntVal); 144f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot } 145f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot 146f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot void setVerbatimBlockText(StringRef Text) { 147f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot assert(is(tok::verbatim_block_line)); 148f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot TextPtr = Text.data(); 149f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot IntVal = Text.size(); 150f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot } 151f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot 152f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot unsigned getVerbatimLineID() const LLVM_READONLY { 153f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot assert(is(tok::verbatim_line_name)); 154f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot return IntVal; 155f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot } 156f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot 157f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot void setVerbatimLineID(unsigned ID) { 158f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot assert(is(tok::verbatim_line_name)); 159f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot IntVal = ID; 160f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot } 161f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot 162f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot StringRef getVerbatimLineText() const LLVM_READONLY { 163f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot assert(is(tok::verbatim_line_text)); 164f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot return StringRef(TextPtr, IntVal); 165f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot } 166f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot 167f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot void setVerbatimLineText(StringRef Text) { 168f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot assert(is(tok::verbatim_line_text)); 169f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot TextPtr = Text.data(); 170f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot IntVal = Text.size(); 171f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot } 172f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot 173f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot StringRef getHTMLTagStartName() const LLVM_READONLY { 174f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot assert(is(tok::html_start_tag)); 175f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot return StringRef(TextPtr, IntVal); 176f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot } 177f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot 178f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot void setHTMLTagStartName(StringRef Name) { 179f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot assert(is(tok::html_start_tag)); 180f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot TextPtr = Name.data(); 181f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot IntVal = Name.size(); 182f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot } 183f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot 184f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot StringRef getHTMLIdent() const LLVM_READONLY { 185f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot assert(is(tok::html_ident)); 186f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot return StringRef(TextPtr, IntVal); 187f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot } 188f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot 189f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot void setHTMLIdent(StringRef Name) { 190f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot assert(is(tok::html_ident)); 191f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot TextPtr = Name.data(); 192f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot IntVal = Name.size(); 193f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot } 194f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot 195f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot StringRef getHTMLQuotedString() const LLVM_READONLY { 196f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot assert(is(tok::html_quoted_string)); 197f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot return StringRef(TextPtr, IntVal); 198f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot } 199f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot 200f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot void setHTMLQuotedString(StringRef Str) { 201f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot assert(is(tok::html_quoted_string)); 202f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot TextPtr = Str.data(); 203f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot IntVal = Str.size(); 204f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot } 205f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot 206f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot StringRef getHTMLTagEndName() const LLVM_READONLY { 207f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot assert(is(tok::html_end_tag)); 208f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot return StringRef(TextPtr, IntVal); 209f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot } 210f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot 211f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot void setHTMLTagEndName(StringRef Name) { 212f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot assert(is(tok::html_end_tag)); 213f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot TextPtr = Name.data(); 214f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot IntVal = Name.size(); 215f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot } 216f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot 217f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot void dump(const Lexer &L, const SourceManager &SM) const; 218f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot}; 219f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot 220f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot/// \brief Comment lexer. 221f3014761c955345d6e05491608e73228d014afbandroid-build-team Robotclass Lexer { 222f3014761c955345d6e05491608e73228d014afbandroid-build-team Robotprivate: 223f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot Lexer(const Lexer &) = delete; 224f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot void operator=(const Lexer &) = delete; 225f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot 226f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot /// Allocator for strings that are semantic values of tokens and have to be 227f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot /// computed (for example, resolved decimal character references). 228f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot llvm::BumpPtrAllocator &Allocator; 229f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot 230f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot DiagnosticsEngine &Diags; 231f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot 232f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot const CommandTraits &Traits; 233f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot 234f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot const char *const BufferStart; 235f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot const char *const BufferEnd; 236f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot SourceLocation FileLoc; 237f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot 238f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot const char *BufferPtr; 239f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot 240f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot /// One past end pointer for the current comment. For BCPL comments points 241f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot /// to newline or BufferEnd, for C comments points to star in '*/'. 242f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot const char *CommentEnd; 243f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot 244f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot enum LexerCommentState { 245f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot LCS_BeforeComment, 246f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot LCS_InsideBCPLComment, 247f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot LCS_InsideCComment, 248f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot LCS_BetweenComments 249f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot }; 250f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot 251f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot /// Low-level lexer state, track if we are inside or outside of comment. 252f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot LexerCommentState CommentState; 253f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot 254f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot enum LexerState { 255f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot /// Lexing normal comment text 256f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot LS_Normal, 257f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot 258f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot /// Finished lexing verbatim block beginning command, will lex first body 259f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot /// line. 260f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot LS_VerbatimBlockFirstLine, 261f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot 262f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot /// Lexing verbatim block body line-by-line, skipping line-starting 263f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot /// decorations. 264f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot LS_VerbatimBlockBody, 265f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot 266f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot /// Finished lexing verbatim line beginning command, will lex text (one 267f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot /// line). 268f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot LS_VerbatimLineText, 269f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot 270f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot /// Finished lexing \verbatim <TAG \endverbatim part, lexing tag attributes. 271f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot LS_HTMLStartTag, 272f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot 273f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot /// Finished lexing \verbatim </TAG \endverbatim part, lexing '>'. 274f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot LS_HTMLEndTag 275f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot }; 276f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot 277f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot /// Current lexing mode. 278f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot LexerState State; 279f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot 280f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot /// If State is LS_VerbatimBlock, contains the name of verbatim end 281f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot /// command, including command marker. 282f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot SmallString<16> VerbatimBlockEndCommandName; 283f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot 284f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot /// Given a character reference name (e.g., "lt"), return the character that 285f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot /// it stands for (e.g., "<"). 286f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot StringRef resolveHTMLNamedCharacterReference(StringRef Name) const; 287f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot 288f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot /// Given a Unicode codepoint as base-10 integer, return the character. 289f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot StringRef resolveHTMLDecimalCharacterReference(StringRef Name) const; 290f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot 291f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot /// Given a Unicode codepoint as base-16 integer, return the character. 292f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot StringRef resolveHTMLHexCharacterReference(StringRef Name) const; 293f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot 294f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot void formTokenWithChars(Token &Result, const char *TokEnd, 295f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot tok::TokenKind Kind); 296f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot 297f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot void formTextToken(Token &Result, const char *TokEnd) { 298f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot StringRef Text(BufferPtr, TokEnd - BufferPtr); 299f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot formTokenWithChars(Result, TokEnd, tok::text); 300f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot Result.setText(Text); 301f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot } 302f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot 303f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot SourceLocation getSourceLocation(const char *Loc) const { 304f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot assert(Loc >= BufferStart && Loc <= BufferEnd && 305f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot "Location out of range for this buffer!"); 306f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot 307f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot const unsigned CharNo = Loc - BufferStart; 308f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot return FileLoc.getLocWithOffset(CharNo); 309f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot } 310f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot 311f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) { 312f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot return Diags.Report(Loc, DiagID); 313f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot } 314f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot 315f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot /// Eat string matching regexp \code \s*\* \endcode. 316f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot void skipLineStartingDecorations(); 317f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot 318f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot /// Lex stuff inside comments. CommentEnd should be set correctly. 319f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot void lexCommentText(Token &T); 320f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot 321f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot void setupAndLexVerbatimBlock(Token &T, 322f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot const char *TextBegin, 323f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot char Marker, const CommandInfo *Info); 324f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot 325f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot void lexVerbatimBlockFirstLine(Token &T); 326f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot 327f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot void lexVerbatimBlockBody(Token &T); 328f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot 329f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot void setupAndLexVerbatimLine(Token &T, const char *TextBegin, 330f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot const CommandInfo *Info); 331f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot 332f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot void lexVerbatimLineText(Token &T); 333f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot 334f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot void lexHTMLCharacterReference(Token &T); 335f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot 336f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot void setupAndLexHTMLStartTag(Token &T); 337f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot 338f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot void lexHTMLStartTag(Token &T); 339f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot 340f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot void setupAndLexHTMLEndTag(Token &T); 341f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot 342f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot void lexHTMLEndTag(Token &T); 343f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot 344f3014761c955345d6e05491608e73228d014afbandroid-build-team Robotpublic: 345f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot Lexer(llvm::BumpPtrAllocator &Allocator, DiagnosticsEngine &Diags, 346f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot const CommandTraits &Traits, 347f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot SourceLocation FileLoc, 348f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot const char *BufferStart, const char *BufferEnd); 349f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot 350f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot void lex(Token &T); 351f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot 352f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot StringRef getSpelling(const Token &Tok, 353f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot const SourceManager &SourceMgr, 354f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot bool *Invalid = nullptr) const; 355f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot}; 356f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot 357f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot} // end namespace comments 358f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot} // end namespace clang 359f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot 360f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot#endif 361f3014761c955345d6e05491608e73228d014afbandroid-build-team Robot 362