Token.h revision 79ed16e2e605d67a12cccdcf9ad1b231175da1a6
1//===--- Token.h - Token interface ------------------------------*- C++ -*-===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10//  This file defines the Token interface.
11//
12//===----------------------------------------------------------------------===//
13
14#ifndef LLVM_CLANG_TOKEN_H
15#define LLVM_CLANG_TOKEN_H
16
17#include "clang/Basic/TokenKinds.h"
18#include "clang/Basic/SourceLocation.h"
19
20namespace clang {
21
22class IdentifierInfo;
23
24/// Token - This structure provides full information about a lexed token.
25/// It is not intended to be space efficient, it is intended to return as much
26/// information as possible about each returned token.  This is expected to be
27/// compressed into a smaller form if memory footprint is important.
28///
29/// The parser can create a special "annotation token" representing a stream of
30/// tokens that were parsed and semantically resolved, e.g.: "foo::MyClass<int>"
31/// can be represented by a single typename annotation token that carries
32/// information about the SourceRange of the tokens and the type object.
33class Token {
34  /// The location of the token.
35  SourceLocation Loc;
36
37  // Conceptually these next two fields could be in a union with
38  // access depending on isAnnotationToken(). However, this causes gcc
39  // 4.2 to pessimize LexTokenInternal, a very performance critical
40  // routine. Keeping as separate members with casts until a more
41  // beautiful fix presents itself.
42
43  /// UintData - This holds either the length of the token text, when
44  /// a normal token, or the end of the SourceRange when an annotation
45  /// token.
46  unsigned UintData;
47
48  /// PtrData - For normal tokens, this points to the uniqued
49  /// information for the identifier (if an identifier token) or
50  /// null. For annotation tokens, this points to information specific
51  /// to the annotation token.
52  void *PtrData;
53
54  /// Kind - The actual flavor of token this is.
55  ///
56  unsigned Kind : 8;  // DON'T make Kind a 'tok::TokenKind';
57                      // MSVC will treat it as a signed char and
58                      // TokenKinds > 127 won't be handled correctly.
59
60  /// Flags - Bits we track about this token, members of the TokenFlags enum.
61  unsigned Flags : 8;
62public:
63
64  // Various flags set per token:
65  enum TokenFlags {
66    StartOfLine   = 0x01,  // At start of line or only after whitespace.
67    LeadingSpace  = 0x02,  // Whitespace exists before this token.
68    DisableExpand = 0x04,  // This identifier may never be macro expanded.
69    NeedsCleaning = 0x08   // Contained an escaped newline or trigraph.
70  };
71
72  tok::TokenKind getKind() const { return (tok::TokenKind)Kind; }
73  void setKind(tok::TokenKind K) { Kind = K; }
74
75  /// is/isNot - Predicates to check if this token is a specific kind, as in
76  /// "if (Tok.is(tok::l_brace)) {...}".
77  bool is(tok::TokenKind K) const { return Kind == (unsigned) K; }
78  bool isNot(tok::TokenKind K) const { return Kind != (unsigned) K; }
79
80  bool isAnnotationToken() const {
81    return is(tok::annot_typename) ||
82           is(tok::annot_cxxscope) ||
83           is(tok::annot_template_id);
84  }
85
86  /// getLocation - Return a source location identifier for the specified
87  /// offset in the current file.
88  SourceLocation getLocation() const { return Loc; }
89  unsigned getLength() const {
90    assert(!isAnnotationToken() && "Annotation tokens have no length field");
91    return UintData;
92  }
93
94  void setLocation(SourceLocation L) { Loc = L; }
95  void setLength(unsigned Len) {
96    assert(!isAnnotationToken() && "Annotation tokens have no length field");
97    UintData = Len;
98  }
99
100  SourceLocation getAnnotationEndLoc() const {
101    assert(isAnnotationToken() && "Used AnnotEndLocID on non-annotation token");
102    return SourceLocation::getFromRawEncoding(UintData);
103  }
104  void setAnnotationEndLoc(SourceLocation L) {
105    assert(isAnnotationToken() && "Used AnnotEndLocID on non-annotation token");
106    UintData = L.getRawEncoding();
107  }
108
109  /// getAnnotationRange - SourceRange of the group of tokens that this
110  /// annotation token represents.
111  SourceRange getAnnotationRange() const {
112    return SourceRange(getLocation(), getAnnotationEndLoc());
113  }
114  void setAnnotationRange(SourceRange R) {
115    setLocation(R.getBegin());
116    setAnnotationEndLoc(R.getEnd());
117  }
118
119  const char *getName() const {
120    return tok::getTokenName( (tok::TokenKind) Kind);
121  }
122
123  /// startToken - Reset all flags to cleared.
124  ///
125  void startToken() {
126    Kind = tok::unknown;
127    Flags = 0;
128    PtrData = 0;
129    Loc = SourceLocation();
130  }
131
132  IdentifierInfo *getIdentifierInfo() const {
133    assert(!isAnnotationToken() && "Used IdentInfo on annotation token");
134    return (IdentifierInfo*) PtrData;
135  }
136  void setIdentifierInfo(IdentifierInfo *II) {
137    PtrData = (void*) II;
138  }
139
140  void *getAnnotationValue() const {
141    assert(isAnnotationToken() && "Used AnnotVal on non-annotation token");
142    return PtrData;
143  }
144  void setAnnotationValue(void *val) {
145    assert(isAnnotationToken() && "Used AnnotVal on non-annotation token");
146    PtrData = val;
147  }
148
149  /// setFlag - Set the specified flag.
150  void setFlag(TokenFlags Flag) {
151    Flags |= Flag;
152  }
153
154  /// clearFlag - Unset the specified flag.
155  void clearFlag(TokenFlags Flag) {
156    Flags &= ~Flag;
157  }
158
159  /// getFlags - Return the internal represtation of the flags.
160  ///  Only intended for low-level operations such as writing tokens to
161  //   disk.
162  unsigned getFlags() const {
163    return Flags;
164  }
165
166  /// setFlagValue - Set a flag to either true or false.
167  void setFlagValue(TokenFlags Flag, bool Val) {
168    if (Val)
169      setFlag(Flag);
170    else
171      clearFlag(Flag);
172  }
173
174  /// isAtStartOfLine - Return true if this token is at the start of a line.
175  ///
176  bool isAtStartOfLine() const { return (Flags & StartOfLine) ? true : false; }
177
178  /// hasLeadingSpace - Return true if this token has whitespace before it.
179  ///
180  bool hasLeadingSpace() const { return (Flags & LeadingSpace) ? true : false; }
181
182  /// isExpandDisabled - Return true if this identifier token should never
183  /// be expanded in the future, due to C99 6.10.3.4p2.
184  bool isExpandDisabled() const {
185    return (Flags & DisableExpand) ? true : false;
186  }
187
188  /// isObjCAtKeyword - Return true if we have an ObjC keyword identifier.
189  bool isObjCAtKeyword(tok::ObjCKeywordKind objcKey) const;
190
191  /// getObjCKeywordID - Return the ObjC keyword kind.
192  tok::ObjCKeywordKind getObjCKeywordID() const;
193
194  /// needsCleaning - Return true if this token has trigraphs or escaped
195  /// newlines in it.
196  ///
197  bool needsCleaning() const { return (Flags & NeedsCleaning) ? true : false; }
198};
199
200/// PPConditionalInfo - Information about the conditional stack (#if directives)
201/// currently active.
202struct PPConditionalInfo {
203  /// IfLoc - Location where the conditional started.
204  ///
205  SourceLocation IfLoc;
206
207  /// WasSkipping - True if this was contained in a skipping directive, e.g.
208  /// in a "#if 0" block.
209  bool WasSkipping;
210
211  /// FoundNonSkip - True if we have emitted tokens already, and now we're in
212  /// an #else block or something.  Only useful in Skipping blocks.
213  bool FoundNonSkip;
214
215  /// FoundElse - True if we've seen a #else in this block.  If so,
216  /// #elif/#else directives are not allowed.
217  bool FoundElse;
218};
219
220/// TemplateIdAnnotation - Information about a template-id annotation
221/// token, which contains the template declaration, template
222/// arguments, and the source locations for important tokens.
223struct TemplateIdAnnotation {
224  /// TemplateNameLoc - The location of the template name within the
225  /// source.
226  SourceLocation TemplateNameLoc;
227
228  /// Template - The declaration of the template corresponding to the
229  /// template-name. This is an Action::DeclTy*.
230  void *Template;
231
232  /// LAngleLoc - The location of the '<' before the template argument
233  /// list.
234  SourceLocation LAngleLoc;
235
236  /// NumArgs - The number of template arguments. The arguments
237  /// themselves are Action::TemplateArgTy pointers allocated directly
238  /// following the TemplateIdAnnotation structure.
239  unsigned NumArgs;
240};
241
242}  // end namespace clang
243
244#endif
245