1/// \file 2/// Default implementation of CommonTokenStream 3/// 4 5// [The "BSD licence"] 6// Copyright (c) 2005-2009 Jim Idle, Temporal Wave LLC 7// http://www.temporal-wave.com 8// http://www.linkedin.com/in/jimidle 9// 10// All rights reserved. 11// 12// Redistribution and use in source and binary forms, with or without 13// modification, are permitted provided that the following conditions 14// are met: 15// 1. Redistributions of source code must retain the above copyright 16// notice, this list of conditions and the following disclaimer. 17// 2. Redistributions in binary form must reproduce the above copyright 18// notice, this list of conditions and the following disclaimer in the 19// documentation and/or other materials provided with the distribution. 20// 3. The name of the author may not be used to endorse or promote products 21// derived from this software without specific prior written permission. 22// 23// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 24// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 25// OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 26// IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 27// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 28// NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 29// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 30// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 31// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 32// THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 34#include <antlr3tokenstream.h> 35 36#ifdef ANTLR3_WINDOWS 37#pragma warning( disable : 4100 ) 38#endif 39 40// COMMON_TOKEN_STREAM API 41// 42static void setTokenTypeChannel (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 ttype, ANTLR3_UINT32 channel); 43static void discardTokenType (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_INT32 ttype); 44static void discardOffChannel (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_BOOLEAN discard); 45static pANTLR3_VECTOR getTokens (pANTLR3_COMMON_TOKEN_STREAM cts); 46static pANTLR3_LIST getTokenRange (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop); 47static pANTLR3_LIST getTokensSet (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, pANTLR3_BITSET types); 48static pANTLR3_LIST getTokensList (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, pANTLR3_LIST list); 49static pANTLR3_LIST getTokensType (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, ANTLR3_UINT32 type); 50static void reset (pANTLR3_COMMON_TOKEN_STREAM cts); 51 52// TOKEN_STREAM API 53// 54static pANTLR3_COMMON_TOKEN tokLT (pANTLR3_TOKEN_STREAM ts, ANTLR3_INT32 k); 55static pANTLR3_COMMON_TOKEN dbgTokLT (pANTLR3_TOKEN_STREAM ts, ANTLR3_INT32 k); 56static pANTLR3_COMMON_TOKEN get (pANTLR3_TOKEN_STREAM ts, ANTLR3_UINT32 i); 57static pANTLR3_TOKEN_SOURCE getTokenSource (pANTLR3_TOKEN_STREAM ts); 58static void setTokenSource (pANTLR3_TOKEN_STREAM ts, pANTLR3_TOKEN_SOURCE tokenSource); 59static pANTLR3_STRING toString (pANTLR3_TOKEN_STREAM ts); 60static pANTLR3_STRING toStringSS (pANTLR3_TOKEN_STREAM ts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop); 61static pANTLR3_STRING toStringTT (pANTLR3_TOKEN_STREAM ts, pANTLR3_COMMON_TOKEN start, pANTLR3_COMMON_TOKEN stop); 62static void setDebugListener (pANTLR3_TOKEN_STREAM ts, pANTLR3_DEBUG_EVENT_LISTENER debugger); 63 64// INT STREAM API 65// 66static void consume (pANTLR3_INT_STREAM is); 67static void dbgConsume (pANTLR3_INT_STREAM is); 68static ANTLR3_UINT32 _LA (pANTLR3_INT_STREAM is, ANTLR3_INT32 i); 69static ANTLR3_UINT32 dbgLA (pANTLR3_INT_STREAM is, ANTLR3_INT32 i); 70static ANTLR3_MARKER mark (pANTLR3_INT_STREAM is); 71static ANTLR3_MARKER dbgMark (pANTLR3_INT_STREAM is); 72static void release (pANTLR3_INT_STREAM is, ANTLR3_MARKER mark); 73static ANTLR3_UINT32 size (pANTLR3_INT_STREAM is); 74static ANTLR3_MARKER tindex (pANTLR3_INT_STREAM is); 75static void rewindStream (pANTLR3_INT_STREAM is, ANTLR3_MARKER marker); 76static void dbgRewindStream (pANTLR3_INT_STREAM is, ANTLR3_MARKER marker); 77static void rewindLast (pANTLR3_INT_STREAM is); 78static void dbgRewindLast (pANTLR3_INT_STREAM is); 79static void seek (pANTLR3_INT_STREAM is, ANTLR3_MARKER index); 80static void dbgSeek (pANTLR3_INT_STREAM is, ANTLR3_MARKER index); 81static pANTLR3_STRING getSourceName (pANTLR3_INT_STREAM is); 82static void antlr3TokenStreamFree (pANTLR3_TOKEN_STREAM stream); 83static void antlr3CTSFree (pANTLR3_COMMON_TOKEN_STREAM stream); 84 85// Helpers 86// 87static void fillBuffer (pANTLR3_COMMON_TOKEN_STREAM tokenStream); 88static ANTLR3_UINT32 skipOffTokenChannels (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 i); 89static ANTLR3_UINT32 skipOffTokenChannelsReverse (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 i); 90static pANTLR3_COMMON_TOKEN LB (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 i); 91 92ANTLR3_API pANTLR3_TOKEN_STREAM 93antlr3TokenStreamNew() 94{ 95 pANTLR3_TOKEN_STREAM stream; 96 97 // Memory for the interface structure 98 // 99 stream = (pANTLR3_TOKEN_STREAM) ANTLR3_MALLOC(sizeof(ANTLR3_TOKEN_STREAM)); 100 101 if (stream == NULL) 102 { 103 return NULL; 104 } 105 106 // Install basic API 107 // 108 stream->free = antlr3TokenStreamFree; 109 110 111 return stream; 112} 113 114static void 115antlr3TokenStreamFree(pANTLR3_TOKEN_STREAM stream) 116{ 117 ANTLR3_FREE(stream); 118} 119 120static void 121antlr3CTSFree (pANTLR3_COMMON_TOKEN_STREAM stream) 122{ 123 // We only free up our subordinate interfaces if they belong 124 // to us, otherwise we let whoever owns them deal with them. 125 // 126 if (stream->tstream->super == stream) 127 { 128 if (stream->tstream->istream->super == stream->tstream) 129 { 130 stream->tstream->istream->free(stream->tstream->istream); 131 stream->tstream->istream = NULL; 132 } 133 stream->tstream->free(stream->tstream); 134 } 135 136 // Now we free our own resources 137 // 138 if (stream->tokens != NULL) 139 { 140 stream->tokens->free(stream->tokens); 141 stream->tokens = NULL; 142 } 143 if (stream->discardSet != NULL) 144 { 145 stream->discardSet->free(stream->discardSet); 146 stream->discardSet = NULL; 147 } 148 if (stream->channelOverrides != NULL) 149 { 150 stream->channelOverrides->free(stream->channelOverrides); 151 stream->channelOverrides = NULL; 152 } 153 154 // Free our memory now 155 // 156 ANTLR3_FREE(stream); 157} 158 159// Reset a token stream so it can be used again and can reuse it's 160// resources. 161// 162static void 163reset (pANTLR3_COMMON_TOKEN_STREAM cts) 164{ 165 166 // Free any resources that ar most like specifc to the 167 // run we just did. 168 // 169 if (cts->discardSet != NULL) 170 { 171 cts->discardSet->free(cts->discardSet); 172 cts->discardSet = NULL; 173 } 174 if (cts->channelOverrides != NULL) 175 { 176 cts->channelOverrides->free(cts->channelOverrides); 177 cts->channelOverrides = NULL; 178 } 179 180 // Now, if there were any existing tokens in the stream, 181 // then we just reset the vector count so that it starts 182 // again. We must traverse the entries unfortunately as 183 // there may be free pointers for custom token types and 184 // so on. However that is just a quick NULL check on the 185 // vector entries. 186 // 187 if (cts->tokens != NULL) 188 { 189 cts->tokens->clear(cts->tokens); 190 } 191 else 192 { 193 /* Install the token tracking tables 194 */ 195 cts->tokens = antlr3VectorNew(0); 196 } 197 198 // Reset to defaults 199 // 200 cts->discardOffChannel = ANTLR3_FALSE; 201 cts->channel = ANTLR3_TOKEN_DEFAULT_CHANNEL; 202 cts->p = -1; 203} 204 205ANTLR3_API pANTLR3_COMMON_TOKEN_STREAM 206antlr3CommonTokenDebugStreamSourceNew(ANTLR3_UINT32 hint, pANTLR3_TOKEN_SOURCE source, pANTLR3_DEBUG_EVENT_LISTENER debugger) 207{ 208 pANTLR3_COMMON_TOKEN_STREAM stream; 209 210 // Create a standard token stream 211 // 212 stream = antlr3CommonTokenStreamSourceNew(hint, source); 213 214 // Install the debugger object 215 // 216 stream->tstream->debugger = debugger; 217 218 // Override standard token stream methods with debugging versions 219 // 220 stream->tstream->initialStreamState = ANTLR3_FALSE; 221 222 stream->tstream->_LT = dbgTokLT; 223 224 stream->tstream->istream->consume = dbgConsume; 225 stream->tstream->istream->_LA = dbgLA; 226 stream->tstream->istream->mark = dbgMark; 227 stream->tstream->istream->rewind = dbgRewindStream; 228 stream->tstream->istream->rewindLast = dbgRewindLast; 229 stream->tstream->istream->seek = dbgSeek; 230 231 return stream; 232} 233 234ANTLR3_API pANTLR3_COMMON_TOKEN_STREAM 235antlr3CommonTokenStreamSourceNew(ANTLR3_UINT32 hint, pANTLR3_TOKEN_SOURCE source) 236{ 237 pANTLR3_COMMON_TOKEN_STREAM stream; 238 239 stream = antlr3CommonTokenStreamNew(hint); 240 241 stream->channel = ANTLR3_TOKEN_DEFAULT_CHANNEL; 242 243 stream->channelOverrides = NULL; 244 stream->discardSet = NULL; 245 stream->discardOffChannel = ANTLR3_FALSE; 246 247 stream->tstream->setTokenSource(stream->tstream, source); 248 249 stream->free = antlr3CTSFree; 250 return stream; 251} 252 253ANTLR3_API pANTLR3_COMMON_TOKEN_STREAM 254antlr3CommonTokenStreamNew(ANTLR3_UINT32 hint) 255{ 256 pANTLR3_COMMON_TOKEN_STREAM stream; 257 258 /* Memory for the interface structure 259 */ 260 stream = (pANTLR3_COMMON_TOKEN_STREAM) ANTLR3_MALLOC(sizeof(ANTLR3_COMMON_TOKEN_STREAM)); 261 262 if (stream == NULL) 263 { 264 return NULL; 265 } 266 267 /* Create space for the token stream interface 268 */ 269 stream->tstream = antlr3TokenStreamNew(); 270 stream->tstream->super = stream; 271 272 /* Create space for the INT_STREAM interfacce 273 */ 274 stream->tstream->istream = antlr3IntStreamNew(); 275 stream->tstream->istream->super = (stream->tstream); 276 stream->tstream->istream->type = ANTLR3_TOKENSTREAM; 277 278 /* Install the token tracking tables 279 */ 280 stream->tokens = antlr3VectorNew(0); 281 282 /* Defaults 283 */ 284 stream->p = -1; 285 286 /* Install the common token stream API 287 */ 288 stream->setTokenTypeChannel = setTokenTypeChannel; 289 stream->discardTokenType = discardTokenType; 290 stream->discardOffChannelToks = discardOffChannel; 291 stream->getTokens = getTokens; 292 stream->getTokenRange = getTokenRange; 293 stream->getTokensSet = getTokensSet; 294 stream->getTokensList = getTokensList; 295 stream->getTokensType = getTokensType; 296 stream->reset = reset; 297 298 /* Install the token stream API 299 */ 300 stream->tstream->_LT = tokLT; 301 stream->tstream->get = get; 302 stream->tstream->getTokenSource = getTokenSource; 303 stream->tstream->setTokenSource = setTokenSource; 304 stream->tstream->toString = toString; 305 stream->tstream->toStringSS = toStringSS; 306 stream->tstream->toStringTT = toStringTT; 307 stream->tstream->setDebugListener = setDebugListener; 308 309 /* Install INT_STREAM interface 310 */ 311 stream->tstream->istream->_LA = _LA; 312 stream->tstream->istream->mark = mark; 313 stream->tstream->istream->release = release; 314 stream->tstream->istream->size = size; 315 stream->tstream->istream->index = tindex; 316 stream->tstream->istream->rewind = rewindStream; 317 stream->tstream->istream->rewindLast= rewindLast; 318 stream->tstream->istream->seek = seek; 319 stream->tstream->istream->consume = consume; 320 stream->tstream->istream->getSourceName = getSourceName; 321 322 return stream; 323} 324 325// Install a debug listener adn switch to debug mode methods 326// 327static void 328setDebugListener (pANTLR3_TOKEN_STREAM ts, pANTLR3_DEBUG_EVENT_LISTENER debugger) 329{ 330 // Install the debugger object 331 // 332 ts->debugger = debugger; 333 334 // Override standard token stream methods with debugging versions 335 // 336 ts->initialStreamState = ANTLR3_FALSE; 337 338 ts->_LT = dbgTokLT; 339 340 ts->istream->consume = dbgConsume; 341 ts->istream->_LA = dbgLA; 342 ts->istream->mark = dbgMark; 343 ts->istream->rewind = dbgRewindStream; 344 ts->istream->rewindLast = dbgRewindLast; 345 ts->istream->seek = dbgSeek; 346} 347 348/** Get the ith token from the current position 1..n where k=1 is the 349* first symbol of lookahead. 350*/ 351static pANTLR3_COMMON_TOKEN 352tokLT (pANTLR3_TOKEN_STREAM ts, ANTLR3_INT32 k) 353{ 354 ANTLR3_INT32 i; 355 ANTLR3_INT32 n; 356 pANTLR3_COMMON_TOKEN_STREAM cts; 357 358 cts = (pANTLR3_COMMON_TOKEN_STREAM)ts->super; 359 360 if (k < 0) 361 { 362 return LB(cts, -k); 363 } 364 365 if (cts->p == -1) 366 { 367 fillBuffer(cts); 368 } 369 370 // Here we used to check for k == 0 and return 0, but this seems 371 // a superfluous check to me. LT(k=0) is therefore just undefined 372 // and we won't waste the clock cycles on the check 373 // 374 375 if ((cts->p + k - 1) >= (ANTLR3_INT32)ts->istream->cachedSize) 376 { 377 pANTLR3_COMMON_TOKEN teof = &(ts->tokenSource->eofToken); 378 379 teof->setStartIndex (teof, ts->istream->index (ts->istream)); 380 teof->setStopIndex (teof, ts->istream->index (ts->istream)); 381 return teof; 382 } 383 384 i = cts->p; 385 n = 1; 386 387 /* Need to find k good tokens, skipping ones that are off channel 388 */ 389 while ( n < k) 390 { 391 /* Skip off-channel tokens */ 392 i = skipOffTokenChannels(cts, i+1); /* leave p on valid token */ 393 n++; 394 } 395 if ( (ANTLR3_UINT32) i >= ts->istream->cachedSize) 396 { 397 pANTLR3_COMMON_TOKEN teof = &(ts->tokenSource->eofToken); 398 399 teof->setStartIndex (teof, ts->istream->index(ts->istream)); 400 teof->setStopIndex (teof, ts->istream->index(ts->istream)); 401 return teof; 402 } 403 404 // Here the token must be in the input vector. Rather then incur 405 // function call penalty, we just return the pointer directly 406 // from the vector 407 // 408 return (pANTLR3_COMMON_TOKEN)cts->tokens->elements[i].element; 409 //return (pANTLR3_COMMON_TOKEN)cts->tokens->get(cts->tokens, i); 410} 411 412/// Debug only method to flag consumption of initial off-channel 413/// tokens in the input stream 414/// 415static void 416consumeInitialHiddenTokens(pANTLR3_INT_STREAM is) 417{ 418 ANTLR3_MARKER first; 419 ANTLR3_INT32 i; 420 pANTLR3_TOKEN_STREAM ts; 421 422 ts = (pANTLR3_TOKEN_STREAM) is->super; 423 first = is->index(is); 424 425 for (i=0; i<first; i++) 426 { 427 ts->debugger->consumeHiddenToken(ts->debugger, ts->get(ts, i)); 428 } 429 430 ts->initialStreamState = ANTLR3_FALSE; 431 432} 433 434/// As per the normal tokLT but sends information to the debugger 435/// 436static pANTLR3_COMMON_TOKEN 437dbgTokLT (pANTLR3_TOKEN_STREAM ts, ANTLR3_INT32 k) 438{ 439 if (ts->initialStreamState == ANTLR3_TRUE) 440 { 441 consumeInitialHiddenTokens(ts->istream); 442 } 443 return tokLT(ts, k); 444} 445 446#ifdef ANTLR3_WINDOWS 447 /* When fully optimized VC7 complains about non reachable code. 448 * Not yet sure if this is an optimizer bug, or a bug in the flow analysis 449 */ 450#pragma warning( disable : 4702 ) 451#endif 452 453static pANTLR3_COMMON_TOKEN 454LB(pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_INT32 k) 455{ 456 ANTLR3_INT32 i; 457 ANTLR3_INT32 n; 458 459 if (cts->p == -1) 460 { 461 fillBuffer(cts); 462 } 463 if (k == 0) 464 { 465 return NULL; 466 } 467 if ((cts->p - k) < 0) 468 { 469 return NULL; 470 } 471 472 i = cts->p; 473 n = 1; 474 475 /* Need to find k good tokens, going backwards, skipping ones that are off channel 476 */ 477 while (n <= (ANTLR3_INT32) k) 478 { 479 /* Skip off-channel tokens 480 */ 481 482 i = skipOffTokenChannelsReverse(cts, i - 1); /* leave p on valid token */ 483 n++; 484 } 485 if (i < 0) 486 { 487 return NULL; 488 } 489 // Here the token must be in the input vector. Rather then incut 490 // function call penalty, we jsut return the pointer directly 491 // from the vector 492 // 493 return (pANTLR3_COMMON_TOKEN)cts->tokens->elements[i].element; 494} 495 496static pANTLR3_COMMON_TOKEN 497get (pANTLR3_TOKEN_STREAM ts, ANTLR3_UINT32 i) 498{ 499 pANTLR3_COMMON_TOKEN_STREAM cts; 500 501 cts = (pANTLR3_COMMON_TOKEN_STREAM)ts->super; 502 503 return (pANTLR3_COMMON_TOKEN)(cts->tokens->get(cts->tokens, i)); /* Token index is zero based but vectors are 1 based */ 504} 505 506static pANTLR3_TOKEN_SOURCE 507getTokenSource (pANTLR3_TOKEN_STREAM ts) 508{ 509 return ts->tokenSource; 510} 511 512static void 513setTokenSource ( pANTLR3_TOKEN_STREAM ts, 514 pANTLR3_TOKEN_SOURCE tokenSource) 515{ 516 ts->tokenSource = tokenSource; 517} 518 519static pANTLR3_STRING 520toString (pANTLR3_TOKEN_STREAM ts) 521{ 522 pANTLR3_COMMON_TOKEN_STREAM cts; 523 524 cts = (pANTLR3_COMMON_TOKEN_STREAM)ts->super; 525 526 if (cts->p == -1) 527 { 528 fillBuffer(cts); 529 } 530 531 return ts->toStringSS(ts, 0, ts->istream->size(ts->istream)); 532} 533 534static pANTLR3_STRING 535toStringSS(pANTLR3_TOKEN_STREAM ts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop) 536{ 537 pANTLR3_STRING string; 538 pANTLR3_TOKEN_SOURCE tsource; 539 pANTLR3_COMMON_TOKEN tok; 540 ANTLR3_UINT32 i; 541 pANTLR3_COMMON_TOKEN_STREAM cts; 542 543 cts = (pANTLR3_COMMON_TOKEN_STREAM) ts->super; 544 545 if (cts->p == -1) 546 { 547 fillBuffer(cts); 548 } 549 if (stop >= ts->istream->size(ts->istream)) 550 { 551 stop = ts->istream->size(ts->istream) - 1; 552 } 553 554 /* Who is giving us these tokens? 555 */ 556 tsource = ts->getTokenSource(ts); 557 558 if (tsource != NULL && cts->tokens != NULL) 559 { 560 /* Finally, let's get a string 561 */ 562 string = tsource->strFactory->newRaw(tsource->strFactory); 563 564 for (i = start; i <= stop; i++) 565 { 566 tok = ts->get(ts, i); 567 if (tok != NULL) 568 { 569 string->appendS(string, tok->getText(tok)); 570 } 571 } 572 573 return string; 574 } 575 return NULL; 576 577} 578 579static pANTLR3_STRING 580toStringTT (pANTLR3_TOKEN_STREAM ts, pANTLR3_COMMON_TOKEN start, pANTLR3_COMMON_TOKEN stop) 581{ 582 if (start != NULL && stop != NULL) 583 { 584 return ts->toStringSS(ts, (ANTLR3_UINT32)start->getTokenIndex(start), (ANTLR3_UINT32)stop->getTokenIndex(stop)); 585 } 586 else 587 { 588 return NULL; 589 } 590} 591 592/** Move the input pointer to the next incoming token. The stream 593 * must become active with LT(1) available. consume() simply 594 * moves the input pointer so that LT(1) points at the next 595 * input symbol. Consume at least one token. 596 * 597 * Walk past any token not on the channel the parser is listening to. 598 */ 599static void 600consume (pANTLR3_INT_STREAM is) 601{ 602 pANTLR3_COMMON_TOKEN_STREAM cts; 603 pANTLR3_TOKEN_STREAM ts; 604 605 ts = (pANTLR3_TOKEN_STREAM) is->super; 606 cts = (pANTLR3_COMMON_TOKEN_STREAM) ts->super; 607 608 if ((ANTLR3_UINT32)cts->p < cts->tokens->count) 609 { 610 cts->p++; 611 cts->p = skipOffTokenChannels(cts, cts->p); 612 } 613} 614 615 616/// As per ordinary consume but notifies the debugger about hidden 617/// tokens and so on. 618/// 619static void 620dbgConsume (pANTLR3_INT_STREAM is) 621{ 622 pANTLR3_TOKEN_STREAM ts; 623 ANTLR3_MARKER a; 624 ANTLR3_MARKER b; 625 pANTLR3_COMMON_TOKEN t; 626 627 ts = (pANTLR3_TOKEN_STREAM) is->super; 628 629 if (ts->initialStreamState == ANTLR3_TRUE) 630 { 631 consumeInitialHiddenTokens(is); 632 } 633 634 a = is->index(is); // Where are we right now? 635 t = ts->_LT(ts, 1); // Current token from stream 636 637 consume(is); // Standard consumer 638 639 b = is->index(is); // Where are we after consuming 1 on channel token? 640 641 ts->debugger->consumeToken(ts->debugger, t); // Tell the debugger that we consumed the first token 642 643 if (b>a+1) 644 { 645 // The standard consume caused the index to advance by more than 1, 646 // which can only happen if it skipped some off-channel tokens. 647 // we need to tell the debugger about those tokens. 648 // 649 ANTLR3_MARKER i; 650 651 for (i = a+1; i<b; i++) 652 { 653 ts->debugger->consumeHiddenToken(ts->debugger, ts->get(ts, (ANTLR3_UINT32)i)); 654 } 655 656 } 657} 658 659/** A simple filter mechanism whereby you can tell this token stream 660 * to force all tokens of type ttype to be on channel. For example, 661 * when interpreting, we cannot execute actions so we need to tell 662 * the stream to force all WS and NEWLINE to be a different, ignored, 663 * channel. 664 */ 665static void 666setTokenTypeChannel (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 ttype, ANTLR3_UINT32 channel) 667{ 668 if (tokenStream->channelOverrides == NULL) 669 { 670 tokenStream->channelOverrides = antlr3ListNew(10); 671 } 672 673 /* We add one to the channel so we can distinguish NULL as being no entry in the 674 * table for a particular token type. 675 */ 676 tokenStream->channelOverrides->put(tokenStream->channelOverrides, ttype, ANTLR3_FUNC_PTR((ANTLR3_UINT32)channel + 1), NULL); 677} 678 679static void 680discardTokenType (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 ttype) 681{ 682 if (tokenStream->discardSet == NULL) 683 { 684 tokenStream->discardSet = antlr3ListNew(31); 685 } 686 687 /* We add one to the channel so we can distinguish NULL as being no entry in the 688 * table for a particular token type. We could use bitsets for this I suppose too. 689 */ 690 tokenStream->discardSet->put(tokenStream->discardSet, ttype, ANTLR3_FUNC_PTR((ANTLR3_UINT32)ttype + 1), NULL); 691} 692 693static void 694discardOffChannel (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_BOOLEAN discard) 695{ 696 tokenStream->discardOffChannel = discard; 697} 698 699static pANTLR3_VECTOR 700getTokens (pANTLR3_COMMON_TOKEN_STREAM tokenStream) 701{ 702 if (tokenStream->p == -1) 703 { 704 fillBuffer(tokenStream); 705 } 706 707 return tokenStream->tokens; 708} 709 710static pANTLR3_LIST 711getTokenRange (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 start, ANTLR3_UINT32 stop) 712{ 713 return tokenStream->getTokensSet(tokenStream, start, stop, NULL); 714} 715/** Given a start and stop index, return a List of all tokens in 716 * the token type BitSet. Return null if no tokens were found. This 717 * method looks at both on and off channel tokens. 718 */ 719static pANTLR3_LIST 720getTokensSet (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, pANTLR3_BITSET types) 721{ 722 pANTLR3_LIST filteredList; 723 ANTLR3_UINT32 i; 724 ANTLR3_UINT32 n; 725 pANTLR3_COMMON_TOKEN tok; 726 727 if (tokenStream->p == -1) 728 { 729 fillBuffer(tokenStream); 730 } 731 if (stop > tokenStream->tstream->istream->size(tokenStream->tstream->istream)) 732 { 733 stop = tokenStream->tstream->istream->size(tokenStream->tstream->istream); 734 } 735 if (start > stop) 736 { 737 return NULL; 738 } 739 740 /* We have the range set, now we need to iterate through the 741 * installed tokens and create a new list with just the ones we want 742 * in it. We are just moving pointers about really. 743 */ 744 filteredList = antlr3ListNew((ANTLR3_UINT32)tokenStream->tstream->istream->size(tokenStream->tstream->istream)); 745 746 for (i = start, n = 0; i<= stop; i++) 747 { 748 tok = tokenStream->tstream->get(tokenStream->tstream, i); 749 750 if ( types == NULL 751 || types->isMember(types, tok->getType(tok) == ANTLR3_TRUE) 752 ) 753 { 754 filteredList->put(filteredList, n++, (void *)tok, NULL); 755 } 756 } 757 758 /* Did we get any then? 759 */ 760 if (filteredList->size(filteredList) == 0) 761 { 762 filteredList->free(filteredList); 763 filteredList = NULL; 764 } 765 766 return filteredList; 767} 768 769static pANTLR3_LIST 770getTokensList (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, pANTLR3_LIST list) 771{ 772 pANTLR3_BITSET bitSet; 773 pANTLR3_LIST newlist; 774 775 bitSet = antlr3BitsetList(list->table); 776 777 newlist = tokenStream->getTokensSet(tokenStream, start, stop, bitSet); 778 779 bitSet->free(bitSet); 780 781 return newlist; 782 783} 784 785static pANTLR3_LIST 786getTokensType (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, ANTLR3_UINT32 type) 787{ 788 pANTLR3_BITSET bitSet; 789 pANTLR3_LIST newlist; 790 791 bitSet = antlr3BitsetOf(type, -1); 792 newlist = tokenStream->getTokensSet(tokenStream, start, stop, bitSet); 793 794 bitSet->free(bitSet); 795 796 return newlist; 797} 798 799static ANTLR3_UINT32 800_LA (pANTLR3_INT_STREAM is, ANTLR3_INT32 i) 801{ 802 pANTLR3_TOKEN_STREAM ts; 803 pANTLR3_COMMON_TOKEN tok; 804 805 ts = (pANTLR3_TOKEN_STREAM) is->super; 806 807 tok = ts->_LT(ts, i); 808 809 if (tok != NULL) 810 { 811 return tok->getType(tok); 812 } 813 else 814 { 815 return ANTLR3_TOKEN_INVALID; 816 } 817} 818 819/// As per _LA() but for debug mode. 820/// 821static ANTLR3_UINT32 822dbgLA (pANTLR3_INT_STREAM is, ANTLR3_INT32 i) 823{ 824 pANTLR3_TOKEN_STREAM ts; 825 826 ts = (pANTLR3_TOKEN_STREAM) is->super; 827 828 if (ts->initialStreamState == ANTLR3_TRUE) 829 { 830 consumeInitialHiddenTokens(is); 831 } 832 ts->debugger->LT(ts->debugger, i, tokLT(ts, i)); 833 return _LA(is, i); 834} 835 836static ANTLR3_MARKER 837mark (pANTLR3_INT_STREAM is) 838{ 839 is->lastMarker = is->index(is); 840 return is->lastMarker; 841} 842 843/// As per mark() but with a call to tell the debugger we are doing this 844/// 845static ANTLR3_MARKER 846dbgMark (pANTLR3_INT_STREAM is) 847{ 848 pANTLR3_TOKEN_STREAM ts; 849 850 ts = (pANTLR3_TOKEN_STREAM) is->super; 851 852 is->lastMarker = is->index(is); 853 ts->debugger->mark(ts->debugger, is->lastMarker); 854 855 return is->lastMarker; 856} 857 858static void 859release (pANTLR3_INT_STREAM is, ANTLR3_MARKER mark) 860{ 861 return; 862} 863 864static ANTLR3_UINT32 865size (pANTLR3_INT_STREAM is) 866{ 867 pANTLR3_COMMON_TOKEN_STREAM cts; 868 pANTLR3_TOKEN_STREAM ts; 869 870 if (is->cachedSize > 0) 871 { 872 return is->cachedSize; 873 } 874 ts = (pANTLR3_TOKEN_STREAM) is->super; 875 cts = (pANTLR3_COMMON_TOKEN_STREAM) ts->super; 876 877 is->cachedSize = cts->tokens->count; 878 return is->cachedSize; 879} 880 881static ANTLR3_MARKER 882tindex (pANTLR3_INT_STREAM is) 883{ 884 pANTLR3_COMMON_TOKEN_STREAM cts; 885 pANTLR3_TOKEN_STREAM ts; 886 887 ts = (pANTLR3_TOKEN_STREAM) is->super; 888 cts = (pANTLR3_COMMON_TOKEN_STREAM) ts->super; 889 890 return cts->p; 891} 892 893static void 894dbgRewindLast (pANTLR3_INT_STREAM is) 895{ 896 pANTLR3_TOKEN_STREAM ts; 897 898 ts = (pANTLR3_TOKEN_STREAM) is->super; 899 900 ts->debugger->rewindLast(ts->debugger); 901 902 is->rewind(is, is->lastMarker); 903} 904static void 905rewindLast (pANTLR3_INT_STREAM is) 906{ 907 is->rewind(is, is->lastMarker); 908} 909static void 910rewindStream (pANTLR3_INT_STREAM is, ANTLR3_MARKER marker) 911{ 912 is->seek(is, (ANTLR3_UINT32)(marker)); 913} 914static void 915dbgRewindStream (pANTLR3_INT_STREAM is, ANTLR3_MARKER marker) 916{ 917 pANTLR3_TOKEN_STREAM ts; 918 919 ts = (pANTLR3_TOKEN_STREAM) is->super; 920 921 ts->debugger->rewind(ts->debugger, marker); 922 923 is->seek(is, (ANTLR3_UINT32)(marker)); 924} 925 926static void 927seek (pANTLR3_INT_STREAM is, ANTLR3_MARKER index) 928{ 929 pANTLR3_COMMON_TOKEN_STREAM cts; 930 pANTLR3_TOKEN_STREAM ts; 931 932 ts = (pANTLR3_TOKEN_STREAM) is->super; 933 cts = (pANTLR3_COMMON_TOKEN_STREAM) ts->super; 934 935 cts->p = (ANTLR3_UINT32)index; 936} 937static void 938dbgSeek (pANTLR3_INT_STREAM is, ANTLR3_MARKER index) 939{ 940 // TODO: Implement seek in debugger when Ter adds it to Java 941 // 942 seek(is, index); 943} 944ANTLR3_API void 945fillBufferExt(pANTLR3_COMMON_TOKEN_STREAM tokenStream) 946{ 947 fillBuffer(tokenStream); 948} 949static void 950fillBuffer(pANTLR3_COMMON_TOKEN_STREAM tokenStream) { 951 ANTLR3_UINT32 index; 952 pANTLR3_COMMON_TOKEN tok; 953 ANTLR3_BOOLEAN discard; 954 void * channelI; 955 956 /* Start at index 0 of course 957 */ 958 index = 0; 959 960 /* Pick out the next token from the token source 961 * Remember we just get a pointer (reference if you like) here 962 * and so if we store it anywhere, we don't set any pointers to auto free it. 963 */ 964 tok = tokenStream->tstream->tokenSource->nextToken(tokenStream->tstream->tokenSource); 965 966 while (tok != NULL && tok->type != ANTLR3_TOKEN_EOF) 967 { 968 discard = ANTLR3_FALSE; /* Assume we are not discarding */ 969 970 /* I employ a bit of a trick, or perhaps hack here. Rather than 971 * store a pointer to a structure in the override map and discard set 972 * we store the value + 1 cast to a void *. Hence on systems where NULL = (void *)0 973 * we can distinguish "not being there" from "being channel or type 0" 974 */ 975 976 if (tokenStream->discardSet != NULL 977 && tokenStream->discardSet->get(tokenStream->discardSet, tok->getType(tok)) != NULL) 978 { 979 discard = ANTLR3_TRUE; 980 } 981 else if ( tokenStream->discardOffChannel == ANTLR3_TRUE 982 && tok->getChannel(tok) != tokenStream->channel 983 ) 984 { 985 discard = ANTLR3_TRUE; 986 } 987 else if (tokenStream->channelOverrides != NULL) 988 { 989 /* See if this type is in the override map 990 */ 991 channelI = tokenStream->channelOverrides->get(tokenStream->channelOverrides, tok->getType(tok) + 1); 992 993 if (channelI != NULL) 994 { 995 /* Override found 996 */ 997 tok->setChannel(tok, ANTLR3_UINT32_CAST(channelI) - 1); 998 } 999 } 1000 1001 /* If not discarding it, add it to the list at the current index 1002 */ 1003 if (discard == ANTLR3_FALSE) 1004 { 1005 /* Add it, indicating that we will delete it and the table should not 1006 */ 1007 tok->setTokenIndex(tok, index); 1008 tokenStream->p++; 1009 tokenStream->tokens->add(tokenStream->tokens, (void *) tok, NULL); 1010 index++; 1011 } 1012 1013 tok = tokenStream->tstream->tokenSource->nextToken(tokenStream->tstream->tokenSource); 1014 } 1015 1016 /* Cache the size so we don't keep doing indirect method calls. We do this as 1017 * early as possible so that anything after this may utilize the cached value. 1018 */ 1019 tokenStream->tstream->istream->cachedSize = tokenStream->tokens->count; 1020 1021 /* Set the consume pointer to the first token that is on our channel 1022 */ 1023 tokenStream->p = 0; 1024 tokenStream->p = skipOffTokenChannels(tokenStream, tokenStream->p); 1025 1026} 1027 1028/// Given a starting index, return the index of the first on-channel 1029/// token. 1030/// 1031static ANTLR3_UINT32 1032skipOffTokenChannels(pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 i) { 1033 ANTLR3_INT32 n; 1034 pANTLR3_COMMON_TOKEN tok; 1035 1036 n = tokenStream->tstream->istream->cachedSize; 1037 1038 while (i < n) 1039 { 1040 tok = (pANTLR3_COMMON_TOKEN)tokenStream->tokens->elements[i].element; 1041 1042 if (tok->channel!= tokenStream->channel) 1043 { 1044 i++; 1045 } 1046 else 1047 { 1048 return i; 1049 } 1050 } 1051 return i; 1052} 1053 1054static ANTLR3_UINT32 1055skipOffTokenChannelsReverse(pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 x) 1056{ 1057 pANTLR3_COMMON_TOKEN tok; 1058 1059 while (x >= 0) 1060 { 1061 tok = (pANTLR3_COMMON_TOKEN)tokenStream->tokens->elements[x].element; 1062 1063 if ((tok->channel != tokenStream->channel)) 1064 { 1065 x--; 1066 } 1067 else 1068 { 1069 return x; 1070 } 1071 } 1072 return x; 1073} 1074 1075/// Return a string that represents the name assoicated with the input source 1076/// 1077/// /param[in] is The ANTLR3_INT_STREAM interface that is representing this token stream. 1078/// 1079/// /returns 1080/// /implements ANTLR3_INT_STREAM_struct::getSourceName() 1081/// 1082static pANTLR3_STRING 1083getSourceName (pANTLR3_INT_STREAM is) 1084{ 1085 // Slightly convoluted as we must trace back to the lexer's input source 1086 // via the token source. The streamName that is here is not initialized 1087 // because this is a token stream, not a file or string stream, which are the 1088 // only things that have a context for a source name. 1089 // 1090 return ((pANTLR3_TOKEN_STREAM)(is->super))->tokenSource->fileName; 1091} 1092