Token.h revision 1eb4433ac451dc16f4133a88af2d002ac26c58ef
1//===--- Token.h - Token interface ------------------------------*- C++ -*-===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file defines the Token interface. 11// 12//===----------------------------------------------------------------------===// 13 14#ifndef LLVM_CLANG_TOKEN_H 15#define LLVM_CLANG_TOKEN_H 16 17#include "clang/Basic/TemplateKinds.h" 18#include "clang/Basic/TokenKinds.h" 19#include "clang/Basic/SourceLocation.h" 20#include <cstdlib> 21 22namespace clang { 23 24class IdentifierInfo; 25 26/// Token - This structure provides full information about a lexed token. 27/// It is not intended to be space efficient, it is intended to return as much 28/// information as possible about each returned token. This is expected to be 29/// compressed into a smaller form if memory footprint is important. 30/// 31/// The parser can create a special "annotation token" representing a stream of 32/// tokens that were parsed and semantically resolved, e.g.: "foo::MyClass<int>" 33/// can be represented by a single typename annotation token that carries 34/// information about the SourceRange of the tokens and the type object. 35class Token { 36 /// The location of the token. 37 SourceLocation Loc; 38 39 // Conceptually these next two fields could be in a union. However, this 40 // causes gcc 4.2 to pessimize LexTokenInternal, a very performance critical 41 // routine. Keeping as separate members with casts until a more beautiful fix 42 // presents itself. 43 44 /// UintData - This holds either the length of the token text, when 45 /// a normal token, or the end of the SourceRange when an annotation 46 /// token. 47 unsigned UintData; 48 49 /// PtrData - This is a union of four different pointer types, which depends 50 /// on what type of token this is: 51 /// Identifiers, keywords, etc: 52 /// This is an IdentifierInfo*, which contains the uniqued identifier 53 /// spelling. 54 /// Literals: isLiteral() returns true. 55 /// This is a pointer to the start of the token in a text buffer, which 56 /// may be dirty (have trigraphs / escaped newlines). 57 /// Annotations (resolved type names, C++ scopes, etc): isAnnotation(). 58 /// This is a pointer to sema-specific data for the annotation token. 59 /// Other: 60 /// This is null. 61 void *PtrData; 62 63 /// Kind - The actual flavor of token this is. 64 /// 65 unsigned Kind : 8; // DON'T make Kind a 'tok::TokenKind'; 66 // MSVC will treat it as a signed char and 67 // TokenKinds > 127 won't be handled correctly. 68 69 /// Flags - Bits we track about this token, members of the TokenFlags enum. 70 unsigned Flags : 8; 71public: 72 73 // Various flags set per token: 74 enum TokenFlags { 75 StartOfLine = 0x01, // At start of line or only after whitespace. 76 LeadingSpace = 0x02, // Whitespace exists before this token. 77 DisableExpand = 0x04, // This identifier may never be macro expanded. 78 NeedsCleaning = 0x08 // Contained an escaped newline or trigraph. 79 }; 80 81 tok::TokenKind getKind() const { return (tok::TokenKind)Kind; } 82 void setKind(tok::TokenKind K) { Kind = K; } 83 84 /// is/isNot - Predicates to check if this token is a specific kind, as in 85 /// "if (Tok.is(tok::l_brace)) {...}". 86 bool is(tok::TokenKind K) const { return Kind == (unsigned) K; } 87 bool isNot(tok::TokenKind K) const { return Kind != (unsigned) K; } 88 89 /// isLiteral - Return true if this is a "literal", like a numeric 90 /// constant, string, etc. 91 bool isLiteral() const { 92 return is(tok::numeric_constant) || is(tok::char_constant) || 93 is(tok::string_literal) || is(tok::wide_string_literal) || 94 is(tok::angle_string_literal); 95 } 96 97 bool isAnnotation() const { 98 return is(tok::annot_typename) || 99 is(tok::annot_cxxscope) || 100 is(tok::annot_template_id); 101 } 102 103 /// getLocation - Return a source location identifier for the specified 104 /// offset in the current file. 105 SourceLocation getLocation() const { return Loc; } 106 unsigned getLength() const { 107 assert(!isAnnotation() && "Annotation tokens have no length field"); 108 return UintData; 109 } 110 111 void setLocation(SourceLocation L) { Loc = L; } 112 void setLength(unsigned Len) { 113 assert(!isAnnotation() && "Annotation tokens have no length field"); 114 UintData = Len; 115 } 116 117 SourceLocation getAnnotationEndLoc() const { 118 assert(isAnnotation() && "Used AnnotEndLocID on non-annotation token"); 119 return SourceLocation::getFromRawEncoding(UintData); 120 } 121 void setAnnotationEndLoc(SourceLocation L) { 122 assert(isAnnotation() && "Used AnnotEndLocID on non-annotation token"); 123 UintData = L.getRawEncoding(); 124 } 125 126 /// getAnnotationRange - SourceRange of the group of tokens that this 127 /// annotation token represents. 128 SourceRange getAnnotationRange() const { 129 return SourceRange(getLocation(), getAnnotationEndLoc()); 130 } 131 void setAnnotationRange(SourceRange R) { 132 setLocation(R.getBegin()); 133 setAnnotationEndLoc(R.getEnd()); 134 } 135 136 const char *getName() const { 137 return tok::getTokenName( (tok::TokenKind) Kind); 138 } 139 140 /// startToken - Reset all flags to cleared. 141 /// 142 void startToken() { 143 Kind = tok::unknown; 144 Flags = 0; 145 PtrData = 0; 146 Loc = SourceLocation(); 147 } 148 149 IdentifierInfo *getIdentifierInfo() const { 150 assert(!isAnnotation() && "Used IdentInfo on annotation token!"); 151 if (isLiteral()) return 0; 152 return (IdentifierInfo*) PtrData; 153 } 154 void setIdentifierInfo(IdentifierInfo *II) { 155 PtrData = (void*) II; 156 } 157 158 /// getLiteralData - For a literal token (numeric constant, string, etc), this 159 /// returns a pointer to the start of it in the text buffer if known, null 160 /// otherwise. 161 const char *getLiteralData() const { 162 assert(isLiteral() && "Cannot get literal data of non-literal"); 163 return reinterpret_cast<const char*>(PtrData); 164 } 165 void setLiteralData(const char *Ptr) { 166 assert(isLiteral() && "Cannot set literal data of non-literal"); 167 PtrData = (void*)Ptr; 168 } 169 170 void *getAnnotationValue() const { 171 assert(isAnnotation() && "Used AnnotVal on non-annotation token"); 172 return PtrData; 173 } 174 void setAnnotationValue(void *val) { 175 assert(isAnnotation() && "Used AnnotVal on non-annotation token"); 176 PtrData = val; 177 } 178 179 /// setFlag - Set the specified flag. 180 void setFlag(TokenFlags Flag) { 181 Flags |= Flag; 182 } 183 184 /// clearFlag - Unset the specified flag. 185 void clearFlag(TokenFlags Flag) { 186 Flags &= ~Flag; 187 } 188 189 /// getFlags - Return the internal represtation of the flags. 190 /// Only intended for low-level operations such as writing tokens to 191 // disk. 192 unsigned getFlags() const { 193 return Flags; 194 } 195 196 /// setFlagValue - Set a flag to either true or false. 197 void setFlagValue(TokenFlags Flag, bool Val) { 198 if (Val) 199 setFlag(Flag); 200 else 201 clearFlag(Flag); 202 } 203 204 /// isAtStartOfLine - Return true if this token is at the start of a line. 205 /// 206 bool isAtStartOfLine() const { return (Flags & StartOfLine) ? true : false; } 207 208 /// hasLeadingSpace - Return true if this token has whitespace before it. 209 /// 210 bool hasLeadingSpace() const { return (Flags & LeadingSpace) ? true : false; } 211 212 /// isExpandDisabled - Return true if this identifier token should never 213 /// be expanded in the future, due to C99 6.10.3.4p2. 214 bool isExpandDisabled() const { 215 return (Flags & DisableExpand) ? true : false; 216 } 217 218 /// isObjCAtKeyword - Return true if we have an ObjC keyword identifier. 219 bool isObjCAtKeyword(tok::ObjCKeywordKind objcKey) const; 220 221 /// getObjCKeywordID - Return the ObjC keyword kind. 222 tok::ObjCKeywordKind getObjCKeywordID() const; 223 224 /// needsCleaning - Return true if this token has trigraphs or escaped 225 /// newlines in it. 226 /// 227 bool needsCleaning() const { return (Flags & NeedsCleaning) ? true : false; } 228}; 229 230/// PPConditionalInfo - Information about the conditional stack (#if directives) 231/// currently active. 232struct PPConditionalInfo { 233 /// IfLoc - Location where the conditional started. 234 /// 235 SourceLocation IfLoc; 236 237 /// WasSkipping - True if this was contained in a skipping directive, e.g. 238 /// in a "#if 0" block. 239 bool WasSkipping; 240 241 /// FoundNonSkip - True if we have emitted tokens already, and now we're in 242 /// an #else block or something. Only useful in Skipping blocks. 243 bool FoundNonSkip; 244 245 /// FoundElse - True if we've seen a #else in this block. If so, 246 /// #elif/#else directives are not allowed. 247 bool FoundElse; 248}; 249 250/// TemplateIdAnnotation - Information about a template-id annotation 251/// token, which contains the template declaration, template 252/// arguments, whether those template arguments were types or 253/// expressions, and the source locations for important tokens. All of 254/// the information about template arguments is allocated directly 255/// after this structure. 256struct TemplateIdAnnotation { 257 /// TemplateNameLoc - The location of the template name within the 258 /// source. 259 SourceLocation TemplateNameLoc; 260 261 /// FIXME: Temporarily stores the name of a specialization 262 IdentifierInfo *Name; 263 264 /// The declaration of the template corresponding to the 265 /// template-name. This is an Action::DeclTy*. 266 void *Template; 267 268 /// The kind of template that Template refers to. 269 TemplateNameKind Kind; 270 271 /// The location of the '<' before the template argument 272 /// list. 273 SourceLocation LAngleLoc; 274 275 /// The location of the '>' after the template argument 276 /// list. 277 SourceLocation RAngleLoc; 278 279 /// NumArgs - The number of template arguments. 280 unsigned NumArgs; 281 282 /// \brief Retrieves a pointer to the template arguments 283 void **getTemplateArgs() { return (void **)(this + 1); } 284 285 /// \brief Retrieves a pointer to the array of template argument 286 /// locations. 287 SourceLocation *getTemplateArgLocations() { 288 return (SourceLocation *)(getTemplateArgs() + NumArgs); 289 } 290 291 /// \brief Retrieves a pointer to the array of flags that states 292 /// whether the template arguments are types. 293 bool *getTemplateArgIsType() { 294 return (bool *)(getTemplateArgLocations() + NumArgs); 295 } 296 297 static TemplateIdAnnotation* Allocate(unsigned NumArgs) { 298 TemplateIdAnnotation *TemplateId 299 = (TemplateIdAnnotation *)std::malloc(sizeof(TemplateIdAnnotation) + 300 sizeof(void*) * NumArgs + 301 sizeof(SourceLocation) * NumArgs + 302 sizeof(bool) * NumArgs); 303 TemplateId->NumArgs = NumArgs; 304 return TemplateId; 305 } 306 307 void Destroy() { free(this); } 308}; 309 310} // end namespace clang 311 312#endif 313