1324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver/// \file 2324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver/// Default implementation of CommonTokenStream 3324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver/// 4324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 5324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver// [The "BSD licence"] 6324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver// Copyright (c) 2005-2009 Jim Idle, Temporal Wave LLC 7324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver// http://www.temporal-wave.com 8324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver// http://www.linkedin.com/in/jimidle 9324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver// 10324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver// All rights reserved. 11324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver// 12324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver// Redistribution and use in source and binary forms, with or without 13324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver// modification, are permitted provided that the following conditions 14324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver// are met: 15324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver// 1. Redistributions of source code must retain the above copyright 16324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver// notice, this list of conditions and the following disclaimer. 17324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver// 2. Redistributions in binary form must reproduce the above copyright 18324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver// notice, this list of conditions and the following disclaimer in the 19324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver// documentation and/or other materials provided with the distribution. 20324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver// 3. The name of the author may not be used to endorse or promote products 21324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver// derived from this software without specific prior written permission. 22324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver// 23324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 24324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 25324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver// OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 26324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver// IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 27324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 28324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver// NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 29324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 30324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 31324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 32324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver// THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 34324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver#include <antlr3tokenstream.h> 35324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 36324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver#ifdef ANTLR3_WINDOWS 37324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver#pragma warning( disable : 4100 ) 38324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver#endif 39324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 40324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver// COMMON_TOKEN_STREAM API 41324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver// 42324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverstatic void setTokenTypeChannel (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 ttype, ANTLR3_UINT32 channel); 43324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverstatic void discardTokenType (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_INT32 ttype); 44324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverstatic void discardOffChannel (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_BOOLEAN discard); 45324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverstatic pANTLR3_VECTOR getTokens (pANTLR3_COMMON_TOKEN_STREAM cts); 46324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverstatic pANTLR3_LIST getTokenRange (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop); 47324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverstatic pANTLR3_LIST getTokensSet (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, pANTLR3_BITSET types); 48324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverstatic pANTLR3_LIST getTokensList (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, pANTLR3_LIST list); 49324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverstatic pANTLR3_LIST getTokensType (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, ANTLR3_UINT32 type); 50324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverstatic void reset (pANTLR3_COMMON_TOKEN_STREAM cts); 51324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 52324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver// TOKEN_STREAM API 53324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver// 54324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverstatic pANTLR3_COMMON_TOKEN tokLT (pANTLR3_TOKEN_STREAM ts, ANTLR3_INT32 k); 55324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverstatic pANTLR3_COMMON_TOKEN dbgTokLT (pANTLR3_TOKEN_STREAM ts, ANTLR3_INT32 k); 56324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverstatic pANTLR3_COMMON_TOKEN get (pANTLR3_TOKEN_STREAM ts, ANTLR3_UINT32 i); 57324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverstatic pANTLR3_TOKEN_SOURCE getTokenSource (pANTLR3_TOKEN_STREAM ts); 58324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverstatic void setTokenSource (pANTLR3_TOKEN_STREAM ts, pANTLR3_TOKEN_SOURCE tokenSource); 59324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverstatic pANTLR3_STRING toString (pANTLR3_TOKEN_STREAM ts); 60324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverstatic pANTLR3_STRING toStringSS (pANTLR3_TOKEN_STREAM ts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop); 61324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverstatic pANTLR3_STRING toStringTT (pANTLR3_TOKEN_STREAM ts, pANTLR3_COMMON_TOKEN start, pANTLR3_COMMON_TOKEN stop); 62324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverstatic void setDebugListener (pANTLR3_TOKEN_STREAM ts, pANTLR3_DEBUG_EVENT_LISTENER debugger); 63324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 64324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver// INT STREAM API 65324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver// 66324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverstatic void consume (pANTLR3_INT_STREAM is); 67324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverstatic void dbgConsume (pANTLR3_INT_STREAM is); 68324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverstatic ANTLR3_UINT32 _LA (pANTLR3_INT_STREAM is, ANTLR3_INT32 i); 69324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverstatic ANTLR3_UINT32 dbgLA (pANTLR3_INT_STREAM is, ANTLR3_INT32 i); 70324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverstatic ANTLR3_MARKER mark (pANTLR3_INT_STREAM is); 71324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverstatic ANTLR3_MARKER dbgMark (pANTLR3_INT_STREAM is); 72324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverstatic void release (pANTLR3_INT_STREAM is, ANTLR3_MARKER mark); 73324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverstatic ANTLR3_UINT32 size (pANTLR3_INT_STREAM is); 74324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverstatic ANTLR3_MARKER tindex (pANTLR3_INT_STREAM is); 75324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverstatic void rewindStream (pANTLR3_INT_STREAM is, ANTLR3_MARKER marker); 76324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverstatic void dbgRewindStream (pANTLR3_INT_STREAM is, ANTLR3_MARKER marker); 77324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverstatic void rewindLast (pANTLR3_INT_STREAM is); 78324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverstatic void dbgRewindLast (pANTLR3_INT_STREAM is); 79324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverstatic void seek (pANTLR3_INT_STREAM is, ANTLR3_MARKER index); 80324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverstatic void dbgSeek (pANTLR3_INT_STREAM is, ANTLR3_MARKER index); 81324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverstatic pANTLR3_STRING getSourceName (pANTLR3_INT_STREAM is); 82324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverstatic void antlr3TokenStreamFree (pANTLR3_TOKEN_STREAM stream); 83324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverstatic void antlr3CTSFree (pANTLR3_COMMON_TOKEN_STREAM stream); 84324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 85324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver// Helpers 86324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver// 87324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverstatic void fillBuffer (pANTLR3_COMMON_TOKEN_STREAM tokenStream); 88324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverstatic ANTLR3_UINT32 skipOffTokenChannels (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 i); 89324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverstatic ANTLR3_UINT32 skipOffTokenChannelsReverse (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 i); 90324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverstatic pANTLR3_COMMON_TOKEN LB (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 i); 91324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 92324c4644fee44b9898524c09511bd33c3f12e2dfBen GruverANTLR3_API pANTLR3_TOKEN_STREAM 93324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverantlr3TokenStreamNew() 94324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver{ 95324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver pANTLR3_TOKEN_STREAM stream; 96324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 97324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver // Memory for the interface structure 98324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver // 99324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver stream = (pANTLR3_TOKEN_STREAM) ANTLR3_MALLOC(sizeof(ANTLR3_TOKEN_STREAM)); 100324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 101324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (stream == NULL) 102324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 103324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return NULL; 104324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 105324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 106324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver // Install basic API 107324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver // 108324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver stream->free = antlr3TokenStreamFree; 109324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 110324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 111324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return stream; 112324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver} 113324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 114324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverstatic void 115324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverantlr3TokenStreamFree(pANTLR3_TOKEN_STREAM stream) 116324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver{ 117324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver ANTLR3_FREE(stream); 118324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver} 119324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 120324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverstatic void 121324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverantlr3CTSFree (pANTLR3_COMMON_TOKEN_STREAM stream) 122324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver{ 123324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver // We only free up our subordinate interfaces if they belong 124324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver // to us, otherwise we let whoever owns them deal with them. 125324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver // 126324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (stream->tstream->super == stream) 127324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 128324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (stream->tstream->istream->super == stream->tstream) 129324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 130324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver stream->tstream->istream->free(stream->tstream->istream); 131324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver stream->tstream->istream = NULL; 132324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 133324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver stream->tstream->free(stream->tstream); 134324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 135324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 136324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver // Now we free our own resources 137324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver // 138324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (stream->tokens != NULL) 139324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 140324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver stream->tokens->free(stream->tokens); 141324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver stream->tokens = NULL; 142324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 143324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (stream->discardSet != NULL) 144324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 145324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver stream->discardSet->free(stream->discardSet); 146324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver stream->discardSet = NULL; 147324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 148324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (stream->channelOverrides != NULL) 149324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 150324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver stream->channelOverrides->free(stream->channelOverrides); 151324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver stream->channelOverrides = NULL; 152324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 153324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 154324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver // Free our memory now 155324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver // 156324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver ANTLR3_FREE(stream); 157324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver} 158324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 159324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver// Reset a token stream so it can be used again and can reuse it's 160324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver// resources. 161324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver// 162324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverstatic void 163324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverreset (pANTLR3_COMMON_TOKEN_STREAM cts) 164324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver{ 165324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 166324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver // Free any resources that ar most like specifc to the 167324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver // run we just did. 168324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver // 169324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (cts->discardSet != NULL) 170324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 171324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver cts->discardSet->free(cts->discardSet); 172324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver cts->discardSet = NULL; 173324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 174324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (cts->channelOverrides != NULL) 175324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 176324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver cts->channelOverrides->free(cts->channelOverrides); 177324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver cts->channelOverrides = NULL; 178324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 179324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 180324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver // Now, if there were any existing tokens in the stream, 181324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver // then we just reset the vector count so that it starts 182324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver // again. We must traverse the entries unfortunately as 183324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver // there may be free pointers for custom token types and 184324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver // so on. However that is just a quick NULL check on the 185324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver // vector entries. 186324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver // 187324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (cts->tokens != NULL) 188324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 189324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver cts->tokens->clear(cts->tokens); 190324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 191324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver else 192324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 193324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /* Install the token tracking tables 194324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 195324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver cts->tokens = antlr3VectorNew(0); 196324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 197324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 198324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver // Reset to defaults 199324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver // 200324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver cts->discardOffChannel = ANTLR3_FALSE; 201324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver cts->channel = ANTLR3_TOKEN_DEFAULT_CHANNEL; 202324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver cts->p = -1; 203324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver} 204324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 205324c4644fee44b9898524c09511bd33c3f12e2dfBen GruverANTLR3_API pANTLR3_COMMON_TOKEN_STREAM 206324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverantlr3CommonTokenDebugStreamSourceNew(ANTLR3_UINT32 hint, pANTLR3_TOKEN_SOURCE source, pANTLR3_DEBUG_EVENT_LISTENER debugger) 207324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver{ 208324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver pANTLR3_COMMON_TOKEN_STREAM stream; 209324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 210324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver // Create a standard token stream 211324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver // 212324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver stream = antlr3CommonTokenStreamSourceNew(hint, source); 213324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 214324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver // Install the debugger object 215324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver // 216324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver stream->tstream->debugger = debugger; 217324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 218324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver // Override standard token stream methods with debugging versions 219324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver // 220324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver stream->tstream->initialStreamState = ANTLR3_FALSE; 221324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 222324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver stream->tstream->_LT = dbgTokLT; 223324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 224324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver stream->tstream->istream->consume = dbgConsume; 225324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver stream->tstream->istream->_LA = dbgLA; 226324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver stream->tstream->istream->mark = dbgMark; 227324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver stream->tstream->istream->rewind = dbgRewindStream; 228324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver stream->tstream->istream->rewindLast = dbgRewindLast; 229324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver stream->tstream->istream->seek = dbgSeek; 230324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 231324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return stream; 232324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver} 233324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 234324c4644fee44b9898524c09511bd33c3f12e2dfBen GruverANTLR3_API pANTLR3_COMMON_TOKEN_STREAM 235324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverantlr3CommonTokenStreamSourceNew(ANTLR3_UINT32 hint, pANTLR3_TOKEN_SOURCE source) 236324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver{ 237324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver pANTLR3_COMMON_TOKEN_STREAM stream; 238324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 239324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver stream = antlr3CommonTokenStreamNew(hint); 240324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 241324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver stream->channel = ANTLR3_TOKEN_DEFAULT_CHANNEL; 242324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 243324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver stream->channelOverrides = NULL; 244324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver stream->discardSet = NULL; 245324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver stream->discardOffChannel = ANTLR3_FALSE; 246324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 247324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver stream->tstream->setTokenSource(stream->tstream, source); 248324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 249324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver stream->free = antlr3CTSFree; 250324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return stream; 251324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver} 252324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 253324c4644fee44b9898524c09511bd33c3f12e2dfBen GruverANTLR3_API pANTLR3_COMMON_TOKEN_STREAM 254324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverantlr3CommonTokenStreamNew(ANTLR3_UINT32 hint) 255324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver{ 256324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver pANTLR3_COMMON_TOKEN_STREAM stream; 257324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 258324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /* Memory for the interface structure 259324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 260324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver stream = (pANTLR3_COMMON_TOKEN_STREAM) ANTLR3_MALLOC(sizeof(ANTLR3_COMMON_TOKEN_STREAM)); 261324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 262324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (stream == NULL) 263324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 264324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return NULL; 265324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 266324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 267324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /* Create space for the token stream interface 268324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 269324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver stream->tstream = antlr3TokenStreamNew(); 270324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver stream->tstream->super = stream; 271324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 272324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /* Create space for the INT_STREAM interfacce 273324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 274324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver stream->tstream->istream = antlr3IntStreamNew(); 275324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver stream->tstream->istream->super = (stream->tstream); 276324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver stream->tstream->istream->type = ANTLR3_TOKENSTREAM; 277324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 278324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /* Install the token tracking tables 279324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 280324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver stream->tokens = antlr3VectorNew(0); 281324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 282324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /* Defaults 283324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 284324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver stream->p = -1; 285324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 286324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /* Install the common token stream API 287324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 288324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver stream->setTokenTypeChannel = setTokenTypeChannel; 289324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver stream->discardTokenType = discardTokenType; 290324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver stream->discardOffChannelToks = discardOffChannel; 291324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver stream->getTokens = getTokens; 292324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver stream->getTokenRange = getTokenRange; 293324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver stream->getTokensSet = getTokensSet; 294324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver stream->getTokensList = getTokensList; 295324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver stream->getTokensType = getTokensType; 296324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver stream->reset = reset; 297324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 298324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /* Install the token stream API 299324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 300324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver stream->tstream->_LT = tokLT; 301324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver stream->tstream->get = get; 302324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver stream->tstream->getTokenSource = getTokenSource; 303324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver stream->tstream->setTokenSource = setTokenSource; 304324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver stream->tstream->toString = toString; 305324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver stream->tstream->toStringSS = toStringSS; 306324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver stream->tstream->toStringTT = toStringTT; 307324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver stream->tstream->setDebugListener = setDebugListener; 308324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 309324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /* Install INT_STREAM interface 310324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 311324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver stream->tstream->istream->_LA = _LA; 312324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver stream->tstream->istream->mark = mark; 313324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver stream->tstream->istream->release = release; 314324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver stream->tstream->istream->size = size; 315324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver stream->tstream->istream->index = tindex; 316324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver stream->tstream->istream->rewind = rewindStream; 317324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver stream->tstream->istream->rewindLast= rewindLast; 318324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver stream->tstream->istream->seek = seek; 319324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver stream->tstream->istream->consume = consume; 320324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver stream->tstream->istream->getSourceName = getSourceName; 321324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 322324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return stream; 323324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver} 324324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 325324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver// Install a debug listener adn switch to debug mode methods 326324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver// 327324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverstatic void 328324c4644fee44b9898524c09511bd33c3f12e2dfBen GruversetDebugListener (pANTLR3_TOKEN_STREAM ts, pANTLR3_DEBUG_EVENT_LISTENER debugger) 329324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver{ 330324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver // Install the debugger object 331324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver // 332324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver ts->debugger = debugger; 333324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 334324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver // Override standard token stream methods with debugging versions 335324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver // 336324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver ts->initialStreamState = ANTLR3_FALSE; 337324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 338324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver ts->_LT = dbgTokLT; 339324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 340324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver ts->istream->consume = dbgConsume; 341324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver ts->istream->_LA = dbgLA; 342324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver ts->istream->mark = dbgMark; 343324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver ts->istream->rewind = dbgRewindStream; 344324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver ts->istream->rewindLast = dbgRewindLast; 345324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver ts->istream->seek = dbgSeek; 346324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver} 347324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 348324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver/** Get the ith token from the current position 1..n where k=1 is the 349324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver* first symbol of lookahead. 350324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver*/ 351324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverstatic pANTLR3_COMMON_TOKEN 352324c4644fee44b9898524c09511bd33c3f12e2dfBen GruvertokLT (pANTLR3_TOKEN_STREAM ts, ANTLR3_INT32 k) 353324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver{ 354324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver ANTLR3_INT32 i; 355324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver ANTLR3_INT32 n; 356324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver pANTLR3_COMMON_TOKEN_STREAM cts; 357324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 358324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver cts = (pANTLR3_COMMON_TOKEN_STREAM)ts->super; 359324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 360324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (k < 0) 361324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 362324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return LB(cts, -k); 363324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 364324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 365324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (cts->p == -1) 366324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 367324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver fillBuffer(cts); 368324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 369324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 370324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver // Here we used to check for k == 0 and return 0, but this seems 371324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver // a superfluous check to me. LT(k=0) is therefore just undefined 372324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver // and we won't waste the clock cycles on the check 373324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver // 374324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 375324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if ((cts->p + k - 1) >= (ANTLR3_INT32)ts->istream->cachedSize) 376324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 377324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver pANTLR3_COMMON_TOKEN teof = &(ts->tokenSource->eofToken); 378324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 379324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver teof->setStartIndex (teof, ts->istream->index (ts->istream)); 380324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver teof->setStopIndex (teof, ts->istream->index (ts->istream)); 381324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return teof; 382324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 383324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 384324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver i = cts->p; 385324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver n = 1; 386324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 387324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /* Need to find k good tokens, skipping ones that are off channel 388324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 389324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver while ( n < k) 390324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 391324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /* Skip off-channel tokens */ 392324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver i = skipOffTokenChannels(cts, i+1); /* leave p on valid token */ 393324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver n++; 394324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 395324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if ( (ANTLR3_UINT32) i >= ts->istream->cachedSize) 396324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 397324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver pANTLR3_COMMON_TOKEN teof = &(ts->tokenSource->eofToken); 398324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 399324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver teof->setStartIndex (teof, ts->istream->index(ts->istream)); 400324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver teof->setStopIndex (teof, ts->istream->index(ts->istream)); 401324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return teof; 402324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 403324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 404324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver // Here the token must be in the input vector. Rather then incur 405324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver // function call penalty, we just return the pointer directly 406324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver // from the vector 407324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver // 408324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return (pANTLR3_COMMON_TOKEN)cts->tokens->elements[i].element; 409324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver //return (pANTLR3_COMMON_TOKEN)cts->tokens->get(cts->tokens, i); 410324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver} 411324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 412324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver/// Debug only method to flag consumption of initial off-channel 413324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver/// tokens in the input stream 414324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver/// 415324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverstatic void 416324c4644fee44b9898524c09511bd33c3f12e2dfBen GruverconsumeInitialHiddenTokens(pANTLR3_INT_STREAM is) 417324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver{ 418324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver ANTLR3_MARKER first; 419324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver ANTLR3_INT32 i; 420324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver pANTLR3_TOKEN_STREAM ts; 421324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 422324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver ts = (pANTLR3_TOKEN_STREAM) is->super; 423324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver first = is->index(is); 424324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 425324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver for (i=0; i<first; i++) 426324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 427324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver ts->debugger->consumeHiddenToken(ts->debugger, ts->get(ts, i)); 428324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 429324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 430324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver ts->initialStreamState = ANTLR3_FALSE; 431324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 432324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver} 433324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 434324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver/// As per the normal tokLT but sends information to the debugger 435324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver/// 436324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverstatic pANTLR3_COMMON_TOKEN 437324c4644fee44b9898524c09511bd33c3f12e2dfBen GruverdbgTokLT (pANTLR3_TOKEN_STREAM ts, ANTLR3_INT32 k) 438324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver{ 439324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (ts->initialStreamState == ANTLR3_TRUE) 440324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 441324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver consumeInitialHiddenTokens(ts->istream); 442324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 443324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return tokLT(ts, k); 444324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver} 445324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 446324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver#ifdef ANTLR3_WINDOWS 447324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /* When fully optimized VC7 complains about non reachable code. 448324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * Not yet sure if this is an optimizer bug, or a bug in the flow analysis 449324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 450324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver#pragma warning( disable : 4702 ) 451324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver#endif 452324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 453324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverstatic pANTLR3_COMMON_TOKEN 454324c4644fee44b9898524c09511bd33c3f12e2dfBen GruverLB(pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_INT32 k) 455324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver{ 456324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver ANTLR3_INT32 i; 457324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver ANTLR3_INT32 n; 458324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 459324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (cts->p == -1) 460324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 461324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver fillBuffer(cts); 462324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 463324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (k == 0) 464324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 465324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return NULL; 466324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 467324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if ((cts->p - k) < 0) 468324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 469324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return NULL; 470324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 471324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 472324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver i = cts->p; 473324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver n = 1; 474324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 475324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /* Need to find k good tokens, going backwards, skipping ones that are off channel 476324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 477324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver while (n <= (ANTLR3_INT32) k) 478324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 479324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /* Skip off-channel tokens 480324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 481324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 482324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver i = skipOffTokenChannelsReverse(cts, i - 1); /* leave p on valid token */ 483324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver n++; 484324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 485324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (i < 0) 486324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 487324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return NULL; 488324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 489324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver // Here the token must be in the input vector. Rather then incut 490324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver // function call penalty, we jsut return the pointer directly 491324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver // from the vector 492324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver // 493324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return (pANTLR3_COMMON_TOKEN)cts->tokens->elements[i].element; 494324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver} 495324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 496324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverstatic pANTLR3_COMMON_TOKEN 497324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverget (pANTLR3_TOKEN_STREAM ts, ANTLR3_UINT32 i) 498324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver{ 499324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver pANTLR3_COMMON_TOKEN_STREAM cts; 500324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 501324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver cts = (pANTLR3_COMMON_TOKEN_STREAM)ts->super; 502324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 503324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return (pANTLR3_COMMON_TOKEN)(cts->tokens->get(cts->tokens, i)); /* Token index is zero based but vectors are 1 based */ 504324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver} 505324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 506324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverstatic pANTLR3_TOKEN_SOURCE 507324c4644fee44b9898524c09511bd33c3f12e2dfBen GruvergetTokenSource (pANTLR3_TOKEN_STREAM ts) 508324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver{ 509324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return ts->tokenSource; 510324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver} 511324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 512324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverstatic void 513324c4644fee44b9898524c09511bd33c3f12e2dfBen GruversetTokenSource ( pANTLR3_TOKEN_STREAM ts, 514324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver pANTLR3_TOKEN_SOURCE tokenSource) 515324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver{ 516324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver ts->tokenSource = tokenSource; 517324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver} 518324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 519324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverstatic pANTLR3_STRING 520324c4644fee44b9898524c09511bd33c3f12e2dfBen GruvertoString (pANTLR3_TOKEN_STREAM ts) 521324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver{ 522324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver pANTLR3_COMMON_TOKEN_STREAM cts; 523324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 524324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver cts = (pANTLR3_COMMON_TOKEN_STREAM)ts->super; 525324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 526324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (cts->p == -1) 527324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 528324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver fillBuffer(cts); 529324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 530324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 531324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return ts->toStringSS(ts, 0, ts->istream->size(ts->istream)); 532324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver} 533324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 534324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverstatic pANTLR3_STRING 535324c4644fee44b9898524c09511bd33c3f12e2dfBen GruvertoStringSS(pANTLR3_TOKEN_STREAM ts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop) 536324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver{ 537324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver pANTLR3_STRING string; 538324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver pANTLR3_TOKEN_SOURCE tsource; 539324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver pANTLR3_COMMON_TOKEN tok; 540324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver ANTLR3_UINT32 i; 541324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver pANTLR3_COMMON_TOKEN_STREAM cts; 542324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 543324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver cts = (pANTLR3_COMMON_TOKEN_STREAM) ts->super; 544324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 545324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (cts->p == -1) 546324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 547324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver fillBuffer(cts); 548324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 549324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (stop >= ts->istream->size(ts->istream)) 550324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 551324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver stop = ts->istream->size(ts->istream) - 1; 552324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 553324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 554324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /* Who is giving us these tokens? 555324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 556324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver tsource = ts->getTokenSource(ts); 557324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 558324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (tsource != NULL && cts->tokens != NULL) 559324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 560324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /* Finally, let's get a string 561324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 562324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver string = tsource->strFactory->newRaw(tsource->strFactory); 563324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 564324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver for (i = start; i <= stop; i++) 565324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 566324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver tok = ts->get(ts, i); 567324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (tok != NULL) 568324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 569324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver string->appendS(string, tok->getText(tok)); 570324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 571324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 572324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 573324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return string; 574324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 575324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return NULL; 576324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 577324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver} 578324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 579324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverstatic pANTLR3_STRING 580324c4644fee44b9898524c09511bd33c3f12e2dfBen GruvertoStringTT (pANTLR3_TOKEN_STREAM ts, pANTLR3_COMMON_TOKEN start, pANTLR3_COMMON_TOKEN stop) 581324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver{ 582324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (start != NULL && stop != NULL) 583324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 584324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return ts->toStringSS(ts, (ANTLR3_UINT32)start->getTokenIndex(start), (ANTLR3_UINT32)stop->getTokenIndex(stop)); 585324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 586324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver else 587324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 588324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return NULL; 589324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 590324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver} 591324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 592324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver/** Move the input pointer to the next incoming token. The stream 593324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * must become active with LT(1) available. consume() simply 594324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * moves the input pointer so that LT(1) points at the next 595324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * input symbol. Consume at least one token. 596324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * 597324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * Walk past any token not on the channel the parser is listening to. 598324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 599324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverstatic void 600324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverconsume (pANTLR3_INT_STREAM is) 601324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver{ 602324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver pANTLR3_COMMON_TOKEN_STREAM cts; 603324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver pANTLR3_TOKEN_STREAM ts; 604324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 605324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver ts = (pANTLR3_TOKEN_STREAM) is->super; 606324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver cts = (pANTLR3_COMMON_TOKEN_STREAM) ts->super; 607324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 608324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if ((ANTLR3_UINT32)cts->p < cts->tokens->count) 609324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 610324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver cts->p++; 611324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver cts->p = skipOffTokenChannels(cts, cts->p); 612324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 613324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver} 614324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 615324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 616324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver/// As per ordinary consume but notifies the debugger about hidden 617324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver/// tokens and so on. 618324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver/// 619324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverstatic void 620324c4644fee44b9898524c09511bd33c3f12e2dfBen GruverdbgConsume (pANTLR3_INT_STREAM is) 621324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver{ 622324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver pANTLR3_TOKEN_STREAM ts; 623324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver ANTLR3_MARKER a; 624324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver ANTLR3_MARKER b; 625324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver pANTLR3_COMMON_TOKEN t; 626324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 627324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver ts = (pANTLR3_TOKEN_STREAM) is->super; 628324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 629324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (ts->initialStreamState == ANTLR3_TRUE) 630324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 631324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver consumeInitialHiddenTokens(is); 632324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 633324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 634324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver a = is->index(is); // Where are we right now? 635324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver t = ts->_LT(ts, 1); // Current token from stream 636324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 637324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver consume(is); // Standard consumer 638324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 639324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver b = is->index(is); // Where are we after consuming 1 on channel token? 640324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 641324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver ts->debugger->consumeToken(ts->debugger, t); // Tell the debugger that we consumed the first token 642324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 643324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (b>a+1) 644324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 645324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver // The standard consume caused the index to advance by more than 1, 646324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver // which can only happen if it skipped some off-channel tokens. 647324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver // we need to tell the debugger about those tokens. 648324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver // 649324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver ANTLR3_MARKER i; 650324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 651324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver for (i = a+1; i<b; i++) 652324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 653324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver ts->debugger->consumeHiddenToken(ts->debugger, ts->get(ts, (ANTLR3_UINT32)i)); 654324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 655324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 656324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 657324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver} 658324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 659324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver/** A simple filter mechanism whereby you can tell this token stream 660324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * to force all tokens of type ttype to be on channel. For example, 661324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * when interpreting, we cannot execute actions so we need to tell 662324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * the stream to force all WS and NEWLINE to be a different, ignored, 663324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * channel. 664324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 665324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverstatic void 666324c4644fee44b9898524c09511bd33c3f12e2dfBen GruversetTokenTypeChannel (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 ttype, ANTLR3_UINT32 channel) 667324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver{ 668324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (tokenStream->channelOverrides == NULL) 669324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 670324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver tokenStream->channelOverrides = antlr3ListNew(10); 671324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 672324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 673324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /* We add one to the channel so we can distinguish NULL as being no entry in the 674324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * table for a particular token type. 675324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 676324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver tokenStream->channelOverrides->put(tokenStream->channelOverrides, ttype, ANTLR3_FUNC_PTR((ANTLR3_UINT32)channel + 1), NULL); 677324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver} 678324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 679324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverstatic void 680324c4644fee44b9898524c09511bd33c3f12e2dfBen GruverdiscardTokenType (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 ttype) 681324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver{ 682324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (tokenStream->discardSet == NULL) 683324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 684324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver tokenStream->discardSet = antlr3ListNew(31); 685324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 686324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 687324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /* We add one to the channel so we can distinguish NULL as being no entry in the 688324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * table for a particular token type. We could use bitsets for this I suppose too. 689324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 690324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver tokenStream->discardSet->put(tokenStream->discardSet, ttype, ANTLR3_FUNC_PTR((ANTLR3_UINT32)ttype + 1), NULL); 691324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver} 692324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 693324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverstatic void 694324c4644fee44b9898524c09511bd33c3f12e2dfBen GruverdiscardOffChannel (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_BOOLEAN discard) 695324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver{ 696324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver tokenStream->discardOffChannel = discard; 697324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver} 698324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 699324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverstatic pANTLR3_VECTOR 700324c4644fee44b9898524c09511bd33c3f12e2dfBen GruvergetTokens (pANTLR3_COMMON_TOKEN_STREAM tokenStream) 701324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver{ 702324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (tokenStream->p == -1) 703324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 704324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver fillBuffer(tokenStream); 705324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 706324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 707324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return tokenStream->tokens; 708324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver} 709324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 710324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverstatic pANTLR3_LIST 711324c4644fee44b9898524c09511bd33c3f12e2dfBen GruvergetTokenRange (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 start, ANTLR3_UINT32 stop) 712324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver{ 713324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return tokenStream->getTokensSet(tokenStream, start, stop, NULL); 714324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver} 715324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver/** Given a start and stop index, return a List of all tokens in 716324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * the token type BitSet. Return null if no tokens were found. This 717324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * method looks at both on and off channel tokens. 718324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 719324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverstatic pANTLR3_LIST 720324c4644fee44b9898524c09511bd33c3f12e2dfBen GruvergetTokensSet (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, pANTLR3_BITSET types) 721324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver{ 722324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver pANTLR3_LIST filteredList; 723324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver ANTLR3_UINT32 i; 724324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver ANTLR3_UINT32 n; 725324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver pANTLR3_COMMON_TOKEN tok; 726324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 727324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (tokenStream->p == -1) 728324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 729324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver fillBuffer(tokenStream); 730324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 731324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (stop > tokenStream->tstream->istream->size(tokenStream->tstream->istream)) 732324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 733324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver stop = tokenStream->tstream->istream->size(tokenStream->tstream->istream); 734324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 735324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (start > stop) 736324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 737324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return NULL; 738324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 739324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 740324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /* We have the range set, now we need to iterate through the 741324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * installed tokens and create a new list with just the ones we want 742324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * in it. We are just moving pointers about really. 743324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 744324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver filteredList = antlr3ListNew((ANTLR3_UINT32)tokenStream->tstream->istream->size(tokenStream->tstream->istream)); 745324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 746324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver for (i = start, n = 0; i<= stop; i++) 747324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 748324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver tok = tokenStream->tstream->get(tokenStream->tstream, i); 749324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 750324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if ( types == NULL 751324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver || types->isMember(types, tok->getType(tok) == ANTLR3_TRUE) 752324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver ) 753324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 754324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver filteredList->put(filteredList, n++, (void *)tok, NULL); 755324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 756324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 757324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 758324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /* Did we get any then? 759324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 760324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (filteredList->size(filteredList) == 0) 761324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 762324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver filteredList->free(filteredList); 763324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver filteredList = NULL; 764324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 765324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 766324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return filteredList; 767324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver} 768324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 769324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverstatic pANTLR3_LIST 770324c4644fee44b9898524c09511bd33c3f12e2dfBen GruvergetTokensList (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, pANTLR3_LIST list) 771324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver{ 772324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver pANTLR3_BITSET bitSet; 773324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver pANTLR3_LIST newlist; 774324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 775324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver bitSet = antlr3BitsetList(list->table); 776324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 777324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver newlist = tokenStream->getTokensSet(tokenStream, start, stop, bitSet); 778324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 779324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver bitSet->free(bitSet); 780324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 781324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return newlist; 782324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 783324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver} 784324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 785324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverstatic pANTLR3_LIST 786324c4644fee44b9898524c09511bd33c3f12e2dfBen GruvergetTokensType (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, ANTLR3_UINT32 type) 787324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver{ 788324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver pANTLR3_BITSET bitSet; 789324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver pANTLR3_LIST newlist; 790324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 791324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver bitSet = antlr3BitsetOf(type, -1); 792324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver newlist = tokenStream->getTokensSet(tokenStream, start, stop, bitSet); 793324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 794324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver bitSet->free(bitSet); 795324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 796324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return newlist; 797324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver} 798324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 799324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverstatic ANTLR3_UINT32 800324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver_LA (pANTLR3_INT_STREAM is, ANTLR3_INT32 i) 801324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver{ 802324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver pANTLR3_TOKEN_STREAM ts; 803324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver pANTLR3_COMMON_TOKEN tok; 804324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 805324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver ts = (pANTLR3_TOKEN_STREAM) is->super; 806324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 807324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver tok = ts->_LT(ts, i); 808324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 809324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (tok != NULL) 810324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 811324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return tok->getType(tok); 812324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 813324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver else 814324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 815324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return ANTLR3_TOKEN_INVALID; 816324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 817324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver} 818324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 819324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver/// As per _LA() but for debug mode. 820324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver/// 821324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverstatic ANTLR3_UINT32 822324c4644fee44b9898524c09511bd33c3f12e2dfBen GruverdbgLA (pANTLR3_INT_STREAM is, ANTLR3_INT32 i) 823324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver{ 824324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver pANTLR3_TOKEN_STREAM ts; 825324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 826324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver ts = (pANTLR3_TOKEN_STREAM) is->super; 827324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 828324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (ts->initialStreamState == ANTLR3_TRUE) 829324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 830324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver consumeInitialHiddenTokens(is); 831324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 832324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver ts->debugger->LT(ts->debugger, i, tokLT(ts, i)); 833324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return _LA(is, i); 834324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver} 835324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 836324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverstatic ANTLR3_MARKER 837324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruvermark (pANTLR3_INT_STREAM is) 838324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver{ 839324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver is->lastMarker = is->index(is); 840324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return is->lastMarker; 841324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver} 842324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 843324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver/// As per mark() but with a call to tell the debugger we are doing this 844324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver/// 845324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverstatic ANTLR3_MARKER 846324c4644fee44b9898524c09511bd33c3f12e2dfBen GruverdbgMark (pANTLR3_INT_STREAM is) 847324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver{ 848324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver pANTLR3_TOKEN_STREAM ts; 849324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 850324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver ts = (pANTLR3_TOKEN_STREAM) is->super; 851324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 852324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver is->lastMarker = is->index(is); 853324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver ts->debugger->mark(ts->debugger, is->lastMarker); 854324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 855324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return is->lastMarker; 856324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver} 857324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 858324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverstatic void 859324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverrelease (pANTLR3_INT_STREAM is, ANTLR3_MARKER mark) 860324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver{ 861324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return; 862324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver} 863324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 864324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverstatic ANTLR3_UINT32 865324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruversize (pANTLR3_INT_STREAM is) 866324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver{ 867324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver pANTLR3_COMMON_TOKEN_STREAM cts; 868324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver pANTLR3_TOKEN_STREAM ts; 869324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 870324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (is->cachedSize > 0) 871324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 872324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return is->cachedSize; 873324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 874324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver ts = (pANTLR3_TOKEN_STREAM) is->super; 875324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver cts = (pANTLR3_COMMON_TOKEN_STREAM) ts->super; 876324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 877324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver is->cachedSize = cts->tokens->count; 878324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return is->cachedSize; 879324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver} 880324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 881324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverstatic ANTLR3_MARKER 882324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruvertindex (pANTLR3_INT_STREAM is) 883324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver{ 884324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver pANTLR3_COMMON_TOKEN_STREAM cts; 885324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver pANTLR3_TOKEN_STREAM ts; 886324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 887324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver ts = (pANTLR3_TOKEN_STREAM) is->super; 888324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver cts = (pANTLR3_COMMON_TOKEN_STREAM) ts->super; 889324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 890324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return cts->p; 891324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver} 892324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 893324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverstatic void 894324c4644fee44b9898524c09511bd33c3f12e2dfBen GruverdbgRewindLast (pANTLR3_INT_STREAM is) 895324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver{ 896324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver pANTLR3_TOKEN_STREAM ts; 897324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 898324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver ts = (pANTLR3_TOKEN_STREAM) is->super; 899324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 900324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver ts->debugger->rewindLast(ts->debugger); 901324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 902324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver is->rewind(is, is->lastMarker); 903324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver} 904324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverstatic void 905324c4644fee44b9898524c09511bd33c3f12e2dfBen GruverrewindLast (pANTLR3_INT_STREAM is) 906324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver{ 907324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver is->rewind(is, is->lastMarker); 908324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver} 909324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverstatic void 910324c4644fee44b9898524c09511bd33c3f12e2dfBen GruverrewindStream (pANTLR3_INT_STREAM is, ANTLR3_MARKER marker) 911324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver{ 912324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver is->seek(is, (ANTLR3_UINT32)(marker)); 913324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver} 914324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverstatic void 915324c4644fee44b9898524c09511bd33c3f12e2dfBen GruverdbgRewindStream (pANTLR3_INT_STREAM is, ANTLR3_MARKER marker) 916324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver{ 917324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver pANTLR3_TOKEN_STREAM ts; 918324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 919324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver ts = (pANTLR3_TOKEN_STREAM) is->super; 920324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 921324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver ts->debugger->rewind(ts->debugger, marker); 922324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 923324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver is->seek(is, (ANTLR3_UINT32)(marker)); 924324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver} 925324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 926324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverstatic void 927324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverseek (pANTLR3_INT_STREAM is, ANTLR3_MARKER index) 928324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver{ 929324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver pANTLR3_COMMON_TOKEN_STREAM cts; 930324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver pANTLR3_TOKEN_STREAM ts; 931324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 932324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver ts = (pANTLR3_TOKEN_STREAM) is->super; 933324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver cts = (pANTLR3_COMMON_TOKEN_STREAM) ts->super; 934324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 935324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver cts->p = (ANTLR3_UINT32)index; 936324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver} 937324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverstatic void 938324c4644fee44b9898524c09511bd33c3f12e2dfBen GruverdbgSeek (pANTLR3_INT_STREAM is, ANTLR3_MARKER index) 939324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver{ 940324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver // TODO: Implement seek in debugger when Ter adds it to Java 941324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver // 942324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver seek(is, index); 943324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver} 944324c4644fee44b9898524c09511bd33c3f12e2dfBen GruverANTLR3_API void 945324c4644fee44b9898524c09511bd33c3f12e2dfBen GruverfillBufferExt(pANTLR3_COMMON_TOKEN_STREAM tokenStream) 946324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver{ 947324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver fillBuffer(tokenStream); 948324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver} 949324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverstatic void 950324c4644fee44b9898524c09511bd33c3f12e2dfBen GruverfillBuffer(pANTLR3_COMMON_TOKEN_STREAM tokenStream) { 951324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver ANTLR3_UINT32 index; 952324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver pANTLR3_COMMON_TOKEN tok; 953324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver ANTLR3_BOOLEAN discard; 954324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver void * channelI; 955324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 956324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /* Start at index 0 of course 957324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 958324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver index = 0; 959324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 960324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /* Pick out the next token from the token source 961324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * Remember we just get a pointer (reference if you like) here 962324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * and so if we store it anywhere, we don't set any pointers to auto free it. 963324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 964324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver tok = tokenStream->tstream->tokenSource->nextToken(tokenStream->tstream->tokenSource); 965324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 966324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver while (tok != NULL && tok->type != ANTLR3_TOKEN_EOF) 967324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 968324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver discard = ANTLR3_FALSE; /* Assume we are not discarding */ 969324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 970324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /* I employ a bit of a trick, or perhaps hack here. Rather than 971324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * store a pointer to a structure in the override map and discard set 972324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * we store the value + 1 cast to a void *. Hence on systems where NULL = (void *)0 973324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * we can distinguish "not being there" from "being channel or type 0" 974324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 975324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 976324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (tokenStream->discardSet != NULL 977324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver && tokenStream->discardSet->get(tokenStream->discardSet, tok->getType(tok)) != NULL) 978324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 979324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver discard = ANTLR3_TRUE; 980324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 981324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver else if ( tokenStream->discardOffChannel == ANTLR3_TRUE 982324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver && tok->getChannel(tok) != tokenStream->channel 983324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver ) 984324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 985324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver discard = ANTLR3_TRUE; 986324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 987324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver else if (tokenStream->channelOverrides != NULL) 988324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 989324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /* See if this type is in the override map 990324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 991324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver channelI = tokenStream->channelOverrides->get(tokenStream->channelOverrides, tok->getType(tok) + 1); 992324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 993324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (channelI != NULL) 994324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 995324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /* Override found 996324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 997324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver tok->setChannel(tok, ANTLR3_UINT32_CAST(channelI) - 1); 998324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 999324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 1000324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1001324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /* If not discarding it, add it to the list at the current index 1002324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 1003324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (discard == ANTLR3_FALSE) 1004324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 1005324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /* Add it, indicating that we will delete it and the table should not 1006324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 1007324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver tok->setTokenIndex(tok, index); 1008324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver tokenStream->p++; 1009324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver tokenStream->tokens->add(tokenStream->tokens, (void *) tok, NULL); 1010324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver index++; 1011324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 1012324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1013324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver tok = tokenStream->tstream->tokenSource->nextToken(tokenStream->tstream->tokenSource); 1014324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 1015324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1016324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /* Cache the size so we don't keep doing indirect method calls. We do this as 1017324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * early as possible so that anything after this may utilize the cached value. 1018324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 1019324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver tokenStream->tstream->istream->cachedSize = tokenStream->tokens->count; 1020324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1021324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /* Set the consume pointer to the first token that is on our channel 1022324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 1023324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver tokenStream->p = 0; 1024324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver tokenStream->p = skipOffTokenChannels(tokenStream, tokenStream->p); 1025324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1026324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver} 1027324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1028324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver/// Given a starting index, return the index of the first on-channel 1029324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver/// token. 1030324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver/// 1031324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverstatic ANTLR3_UINT32 1032324c4644fee44b9898524c09511bd33c3f12e2dfBen GruverskipOffTokenChannels(pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 i) { 1033324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver ANTLR3_INT32 n; 1034324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver pANTLR3_COMMON_TOKEN tok; 1035324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1036324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver n = tokenStream->tstream->istream->cachedSize; 1037324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1038324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver while (i < n) 1039324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 1040324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver tok = (pANTLR3_COMMON_TOKEN)tokenStream->tokens->elements[i].element; 1041324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1042324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (tok->channel!= tokenStream->channel) 1043324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 1044324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver i++; 1045324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 1046324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver else 1047324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 1048324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return i; 1049324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 1050324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 1051324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return i; 1052324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver} 1053324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1054324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverstatic ANTLR3_UINT32 1055324c4644fee44b9898524c09511bd33c3f12e2dfBen GruverskipOffTokenChannelsReverse(pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 x) 1056324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver{ 1057324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver pANTLR3_COMMON_TOKEN tok; 1058324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1059324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver while (x >= 0) 1060324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 1061324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver tok = (pANTLR3_COMMON_TOKEN)tokenStream->tokens->elements[x].element; 1062324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1063324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if ((tok->channel != tokenStream->channel)) 1064324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 1065324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver x--; 1066324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 1067324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver else 1068324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 1069324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return x; 1070324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 1071324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 1072324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return x; 1073324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver} 1074324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1075324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver/// Return a string that represents the name assoicated with the input source 1076324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver/// 1077324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver/// /param[in] is The ANTLR3_INT_STREAM interface that is representing this token stream. 1078324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver/// 1079324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver/// /returns 1080324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver/// /implements ANTLR3_INT_STREAM_struct::getSourceName() 1081324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver/// 1082324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverstatic pANTLR3_STRING 1083324c4644fee44b9898524c09511bd33c3f12e2dfBen GruvergetSourceName (pANTLR3_INT_STREAM is) 1084324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver{ 1085324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver // Slightly convoluted as we must trace back to the lexer's input source 1086324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver // via the token source. The streamName that is here is not initialized 1087324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver // because this is a token stream, not a file or string stream, which are the 1088324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver // only things that have a context for a source name. 1089324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver // 1090324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return ((pANTLR3_TOKEN_STREAM)(is->super))->tokenSource->fileName; 1091324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver} 1092