1324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver/** \file 2324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * Defines the interface for an ANTLR3 common token stream. Custom token streams should create 3324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * one of these and then override any functions by installing their own pointers 4324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * to implement the various functions. 5324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 6324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver#ifndef _ANTLR3_TOKENSTREAM_H 7324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver#define _ANTLR3_TOKENSTREAM_H 8324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 9324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver// [The "BSD licence"] 10324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver// Copyright (c) 2005-2009 Jim Idle, Temporal Wave LLC 11324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver// http://www.temporal-wave.com 12324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver// http://www.linkedin.com/in/jimidle 13324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver// 14324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver// All rights reserved. 15324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver// 16324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver// Redistribution and use in source and binary forms, with or without 17324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver// modification, are permitted provided that the following conditions 18324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver// are met: 19324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver// 1. Redistributions of source code must retain the above copyright 20324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver// notice, this list of conditions and the following disclaimer. 21324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver// 2. Redistributions in binary form must reproduce the above copyright 22324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver// notice, this list of conditions and the following disclaimer in the 23324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver// documentation and/or other materials provided with the distribution. 24324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver// 3. The name of the author may not be used to endorse or promote products 25324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver// derived from this software without specific prior written permission. 26324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver// 27324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 28324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 29324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver// OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 30324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver// IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 31324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 32324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver// NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 33324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 34324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 35324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 36324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver// THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 37324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 38324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver#include <antlr3defs.h> 39324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver#include <antlr3string.h> 40324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver#include <antlr3collections.h> 41324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver#include <antlr3input.h> 42324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver#include <antlr3commontoken.h> 43324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver#include <antlr3bitset.h> 44324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver#include <antlr3debugeventlistener.h> 45324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 46324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver#ifdef __cplusplus 47324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverextern "C" { 48324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver#endif 49324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 50324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver/** Definition of a token source, which has a pointer to a function that 51324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * returns the next token (using a token factory if it is going to be 52324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * efficient) and a pointer to an ANTLR3_INPUT_STREAM. This is slightly 53324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * different to the Java interface because we have no way to implement 54324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * multiple interfaces without defining them in the interface structure 55324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * or casting (void *), which is too convoluted. 56324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 57324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruvertypedef struct ANTLR3_TOKEN_SOURCE_struct 58324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver{ 59324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /** Pointer to a function that returns the next token in the stream. 60324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 61324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver pANTLR3_COMMON_TOKEN (*nextToken)(struct ANTLR3_TOKEN_SOURCE_struct * tokenSource); 62324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 63324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /** Whoever is providing tokens, needs to provide a string factory too 64324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 65324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver pANTLR3_STRING_FACTORY strFactory; 66324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 67324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /** A special pre-allocated token, which signifies End Of Tokens. Because this must 68324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * be set up with the current input index and so on, we embed the structure and 69324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * return the address of it. It is marked as factoryMade, so that it is never 70324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * attempted to be freed. 71324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 72324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver ANTLR3_COMMON_TOKEN eofToken; 73324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 74324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /// A special pre-allocated token, which is returned by mTokens() if the 75324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /// lexer rule said to just skip the generated token altogether. 76324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /// Having this single token stops us wasting memory by have the token factory 77324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /// actually create something that we are going to SKIP(); anyway. 78324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /// 79324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver ANTLR3_COMMON_TOKEN skipToken; 80324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 81324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /** Whatever is supplying the token source interface, needs a pointer to 82324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * itself so that this pointer can be passed to it when the nextToken 83324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * function is called. 84324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 85324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver void * super; 86324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 87324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /** When the token source is constructed, it is populated with the file 88324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * name from whence the tokens were produced by the lexer. This pointer is a 89324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * copy of the one supplied by the CharStream (and may be NULL) so should 90324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * not be manipulated other than to copy or print it. 91324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 92324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver pANTLR3_STRING fileName; 93324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver} 94324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver ANTLR3_TOKEN_SOURCE; 95324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 96324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver/** Definition of the ANTLR3 common token stream interface. 97324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * \remark 98324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * Much of the documentation for this interface is stolen from Ter's Java implementation. 99324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 100324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruvertypedef struct ANTLR3_TOKEN_STREAM_struct 101324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver{ 102324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /** Pointer to the token source for this stream 103324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 104324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver pANTLR3_TOKEN_SOURCE tokenSource; 105324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 106324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /** Whatever is providing this interface needs a pointer to itself 107324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * so that this can be passed back to it whenever the api functions 108324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * are called. 109324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 110324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver void * super; 111324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 112324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /** All input streams implement the ANTLR3_INT_STREAM interface... 113324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 114324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver pANTLR3_INT_STREAM istream; 115324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 116324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /// Debugger interface, is this is a debugging token stream 117324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /// 118324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver pANTLR3_DEBUG_EVENT_LISTENER debugger; 119324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 120324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /// Indicates the initial stream state for dbgConsume() 121324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /// 122324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver ANTLR3_BOOLEAN initialStreamState; 123324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 124324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /** Get Token at current input pointer + i ahead where i=1 is next Token. 125324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * i<0 indicates tokens in the past. So -1 is previous token and -2 is 126324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * two tokens ago. LT(0) is undefined. For i>=n, return Token.EOFToken. 127324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * Return null for LT(0) and any index that results in an absolute address 128324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * that is negative. 129324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 130324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver pANTLR3_COMMON_TOKEN (*_LT) (struct ANTLR3_TOKEN_STREAM_struct * tokenStream, ANTLR3_INT32 k); 131324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 132324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /** Get a token at an absolute index i; 0..n-1. This is really only 133324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * needed for profiling and debugging and token stream rewriting. 134324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * If you don't want to buffer up tokens, then this method makes no 135324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * sense for you. Naturally you can't use the rewrite stream feature. 136324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * I believe DebugTokenStream can easily be altered to not use 137324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * this method, removing the dependency. 138324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 139324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver pANTLR3_COMMON_TOKEN (*get) (struct ANTLR3_TOKEN_STREAM_struct * tokenStream, ANTLR3_UINT32 i); 140324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 141324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /** Where is this stream pulling tokens from? This is not the name, but 142324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * a pointer into an interface that contains a ANTLR3_TOKEN_SOURCE interface. 143324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * The Token Source interface contains a pointer to the input stream and a pointer 144324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * to a function that returns the next token. 145324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 146324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver pANTLR3_TOKEN_SOURCE (*getTokenSource) (struct ANTLR3_TOKEN_STREAM_struct * tokenStream); 147324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 148324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /** Function that installs a token source for teh stream 149324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 150324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver void (*setTokenSource) (struct ANTLR3_TOKEN_STREAM_struct * tokenStream, 151324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver pANTLR3_TOKEN_SOURCE tokenSource); 152324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 153324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /** Return the text of all the tokens in the stream, as the old tramp in 154324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * Leeds market used to say; "Get the lot!" 155324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 156324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver pANTLR3_STRING (*toString) (struct ANTLR3_TOKEN_STREAM_struct * tokenStream); 157324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 158324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /** Return the text of all tokens from start to stop, inclusive. 159324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * If the stream does not buffer all the tokens then it can just 160324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * return an empty ANTLR3_STRING or NULL; Grammars should not access $ruleLabel.text in 161324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * an action in that case. 162324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 163324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver pANTLR3_STRING (*toStringSS) (struct ANTLR3_TOKEN_STREAM_struct * tokenStream, ANTLR3_UINT32 start, ANTLR3_UINT32 stop); 164324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 165324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /** Because the user is not required to use a token with an index stored 166324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * in it, we must provide a means for two token objects themselves to 167324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * indicate the start/end location. Most often this will just delegate 168324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * to the other toString(int,int). This is also parallel with 169324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * the pTREENODE_STREAM->toString(Object,Object). 170324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 171324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver pANTLR3_STRING (*toStringTT) (struct ANTLR3_TOKEN_STREAM_struct * tokenStream, pANTLR3_COMMON_TOKEN start, pANTLR3_COMMON_TOKEN stop); 172324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 173324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 174324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /** Function that sets the token stream into debugging mode 175324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 176324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver void (*setDebugListener) (struct ANTLR3_TOKEN_STREAM_struct * tokenStream, pANTLR3_DEBUG_EVENT_LISTENER debugger); 177324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 178324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 179324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 180324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /** Function that knows how to free the memory for an ANTLR3_TOKEN_STREAM 181324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 182324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver void (*free) (struct ANTLR3_TOKEN_STREAM_struct * tokenStream); 183324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver} 184324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver ANTLR3_TOKEN_STREAM; 185324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 186324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver/** Common token stream is an implementation of ANTLR_TOKEN_STREAM for the default 187324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * parsers and recognizers. You may of course build your own implementation if 188324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * you are so inclined. 189324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 190324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruvertypedef struct ANTLR3_COMMON_TOKEN_STREAM_struct 191324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver{ 192324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /** The ANTLR3_TOKEN_STREAM interface implementation, which also includes 193324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * the intstream implementation. We could duplicate the pANTLR_INT_STREAM 194324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * in this interface and initialize it to a copy, but this could be confusing 195324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * it just results in one more level of indirection and I think that with 196324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * judicial use of 'const' later, the optimizer will do decent job. 197324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 198324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver pANTLR3_TOKEN_STREAM tstream; 199324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 200324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /** Whatever is supplying the COMMON_TOKEN_STREAM needs a pointer to itself 201324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * so that this can be accessed by any of the API functions which it implements. 202324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 203324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver void * super; 204324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 205324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /** Records every single token pulled from the source indexed by the token index. 206324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * There might be more efficient ways to do this, such as referencing directly in to 207324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * the token factory pools, but for now this is convenient and the ANTLR3_LIST is not 208324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * a huge overhead as it only stores pointers anyway, but allows for iterations and 209324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * so on. 210324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 211324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver pANTLR3_VECTOR tokens; 212324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 213324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /** Override map of tokens. If a token type has an entry in here, then 214324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * the pointer in the table points to an int, being the override channel number 215324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * that should always be used for this token type. 216324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 217324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver pANTLR3_LIST channelOverrides; 218324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 219324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /** Discared set. If a token has an entry in this table, then it is thrown 220324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * away (data pointer is always NULL). 221324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 222324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver pANTLR3_LIST discardSet; 223324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 224324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /* The channel number that this token stream is tuned to. For instance, whitespace 225324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * is usually tuned to channel 99, which no token stream would normally tune to and 226324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * so it is thrown away. 227324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 228324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver ANTLR3_UINT32 channel; 229324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 230324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /** If this flag is set to ANTLR3_TRUE, then tokens that the stream sees that are not 231324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * in the channel that this stream is tuned to, are not tracked in the 232324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * tokens table. When set to false, ALL tokens are added to the tracking. 233324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 234324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver ANTLR3_BOOLEAN discardOffChannel; 235324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 236324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /** The index into the tokens list of the current token (the next one that will be 237324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * consumed. p = -1 indicates that the token list is empty. 238324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 239324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver ANTLR3_INT32 p; 240324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 241324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /** A simple filter mechanism whereby you can tell this token stream 242324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * to force all tokens of type ttype to be on channel. For example, 243324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * when interpreting, we cannot exec actions so we need to tell 244324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * the stream to force all WS and NEWLINE to be a different, ignored 245324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * channel. 246324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 247324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver void (*setTokenTypeChannel) (struct ANTLR3_COMMON_TOKEN_STREAM_struct * tokenStream, 248324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver ANTLR3_UINT32 ttype, ANTLR3_UINT32 channel); 249324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 250324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /** Add a particular token type to the discard set. If a token is found to belong 251324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * to this set, then it is skipped/thrown away 252324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 253324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver void (*discardTokenType) (struct ANTLR3_COMMON_TOKEN_STREAM_struct * tokenStream, ANTLR3_INT32 ttype); 254324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 255324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /** Signal to discard off channel tokens from here on in. 256324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 257324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver void (*discardOffChannelToks)(struct ANTLR3_COMMON_TOKEN_STREAM_struct * tokenStream, ANTLR3_BOOLEAN discard); 258324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 259324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /** Function that returns a pointer to the ANTLR3_LIST of all tokens 260324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * in the stream (this causes the buffer to fill if we have not get any yet) 261324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 262324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver pANTLR3_VECTOR (*getTokens) (struct ANTLR3_COMMON_TOKEN_STREAM_struct * tokenStream); 263324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 264324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /** Function that returns all the tokens between a start and a stop index. 265324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * TODO: This is a new list (Ack! Maybe this is a reason to have factories for LISTS and HASHTABLES etc :-( come back to this) 266324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 267324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver pANTLR3_LIST (*getTokenRange) (struct ANTLR3_COMMON_TOKEN_STREAM_struct * tokenStream, ANTLR3_UINT32 start, ANTLR3_UINT32 stop); 268324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 269324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /** Function that returns all the tokens indicated by the specified bitset, within a range of tokens 270324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 271324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver pANTLR3_LIST (*getTokensSet) (struct ANTLR3_COMMON_TOKEN_STREAM_struct * tokenStream, 272324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver ANTLR3_UINT32 start, ANTLR3_UINT32 stop, pANTLR3_BITSET types); 273324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 274324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /** Function that returns all the tokens indicated by being a member of the supplied List 275324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 276324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver pANTLR3_LIST (*getTokensList) (struct ANTLR3_COMMON_TOKEN_STREAM_struct * tokenStream, 277324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver ANTLR3_UINT32 start, ANTLR3_UINT32 stop, pANTLR3_LIST list); 278324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 279324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /** Function that returns all tokens of a certain type within a range. 280324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 281324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver pANTLR3_LIST (*getTokensType) (struct ANTLR3_COMMON_TOKEN_STREAM_struct * tokenStream, 282324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver ANTLR3_UINT32 start, ANTLR3_UINT32 stop, ANTLR3_UINT32 type); 283324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 284324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /** Function that resets the token stream so that it can be reused, but 285324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * but that does not free up any resources, such as the token factory 286324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * the factory pool and so on. This prevents the need to keep freeing 287324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * and reallocating the token pools if the thing you are building is 288324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * a multi-shot dameon or somethign like that. It is much faster to 289324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * just reuse all the vectors. 290324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 291324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver void (*reset) (struct ANTLR3_COMMON_TOKEN_STREAM_struct * tokenStream); 292324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 293324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /** Function that knows how to free an ANTLR3_COMMON_TOKEN_STREAM 294324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 295324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver void (*free) (struct ANTLR3_COMMON_TOKEN_STREAM_struct * tokenStream); 296324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver} 297324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver ANTLR3_COMMON_TOKEN_STREAM; 298324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 299324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver#ifdef __cplusplus 300324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver} 301324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver#endif 302324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 303324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver#endif 304