13eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang/* ANTLRTokenBuffer.C 23eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang * 33eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang * SOFTWARE RIGHTS 43eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang * 53eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang * We reserve no LEGAL rights to the Purdue Compiler Construction Tool 63eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang * Set (PCCTS) -- PCCTS is in the public domain. An individual or 73eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang * company may do whatever they wish with source code distributed with 83eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang * PCCTS or the code generated by PCCTS, including the incorporation of 93eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang * PCCTS, or its output, into commerical software. 103eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang * 113eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang * We encourage users to develop software with PCCTS. However, we do ask 123eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang * that credit is given to us for developing PCCTS. By "credit", 133eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang * we mean that if you incorporate our source code into one of your 143eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang * programs (commercial product, research project, or otherwise) that you 153eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang * acknowledge this fact somewhere in the documentation, research report, 163eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang * etc... If you like PCCTS and have developed a nice tool with the 173eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang * output, please mention that you developed it using PCCTS. In 183eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang * addition, we ask that this header remain intact in our source code. 193eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang * As long as these guidelines are kept, we expect to continue enhancing 203eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang * this system and expect to make other tools available as they are 213eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang * completed. 223eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang * 233eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang * ANTLR 1.33 243eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang * Terence Parr 253eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang * Parr Research Corporation 263eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang * with Purdue University and AHPCRC, University of Minnesota 273eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang * 1989-1998 283eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang */ 293eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang 303eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwangtypedef int ANTLRTokenType; // fool AToken.h into compiling 313eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang 323eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwangclass ANTLRParser; /* MR1 */ 333eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang 343eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang#define ANTLR_SUPPORT_CODE 353eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang 363eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang#include "pcctscfg.h" 373eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang 383eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang#include ATOKENBUFFER_H 393eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwangtypedef ANTLRAbstractToken *_ANTLRTokenPtr; 403eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang 413eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang#if defined(DBG_TBUF)||defined(DBG_TBUF_MARK_REW) 423eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwangstatic unsigned char test[1000]; 433eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang#endif 443eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang 453eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang#ifdef DBG_REFCOUNTTOKEN 463eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwangint ANTLRCommonToken::ctor = 0; 473eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwangint ANTLRCommonToken::dtor = 0; 483eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang#endif 493eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang 503eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwangANTLRTokenBuffer:: 513eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwangANTLRTokenBuffer(ANTLRTokenStream *_input, int _k, int _chunk_size_formal) /* MR14 */ 523eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang{ 533eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang this->input = _input; 543eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang this->k = _k; 553eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang buffer_size = chunk_size = _chunk_size_formal; 563eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang buffer = (_ANTLRTokenPtr *) 573eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang calloc(chunk_size+1,sizeof(_ANTLRTokenPtr )); 583eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang if ( buffer == NULL ) { 593eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang panic("cannot alloc token buffer"); 603eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang } 613eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang buffer++; // leave the first elem empty so tp-1 is valid ptr 623eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang 633eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang tp = &buffer[0]; 643eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang last = tp-1; 653eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang next = &buffer[0]; 663eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang num_markers = 0; 673eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang end_of_buffer = &buffer[buffer_size-1]; 683eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang // BUGBUG -- threshold = &buffer[(int)(buffer_size*(1.0/2.0))]; 693eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang threshold = &buffer[(int)(buffer_size / 2)]; 703eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang _deleteTokens = 1; // assume we delete tokens 713eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang parser=NULL; // MR5 - uninitialized reference 723eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang} 733eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang 743eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwangstatic void f() {;} 753eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwangANTLRTokenBuffer:: 763eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang~ANTLRTokenBuffer() 773eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang{ 783eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang f(); 793eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang // Delete all remaining tokens (from 0..last inclusive) 803eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang if ( _deleteTokens ) 813eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang { 823eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang _ANTLRTokenPtr *z; 833eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang for (z=buffer; z<=last; z++) 843eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang { 853eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang (*z)->deref(); 863eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang// z->deref(); 873eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang#ifdef DBG_REFCOUNTTOKEN 883eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang fprintf(stderr, "##########dtor: deleting token '%s' (ref %d)\n", 893eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang ((ANTLRCommonToken *)*z)->getText(), (*z)->nref()); 903eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang#endif 913eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang if ( (*z)->nref()==0 ) 923eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang { 933eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang delete (*z); 943eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang } 953eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang } 963eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang } 973eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang 983eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang if ( buffer!=NULL ) free((char *)(buffer-1)); 993eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang} 1003eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang 1013eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang#if defined(DBG_TBUF)||defined(DBG_TBUF_MARK_REW) 1023eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang#include "pccts_stdio.h" 1033eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwangPCCTS_NAMESPACE_STD 1043eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang#endif 1053eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang 1063eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang_ANTLRTokenPtr ANTLRTokenBuffer:: 1073eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwanggetToken() 1083eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang{ 1093eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang if ( tp <= last ) // is there any buffered lookahead still to be read? 1103eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang { 1113eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang return *tp++; // read buffered lookahead 1123eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang } 1133eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang // out of buffered lookahead, get some more "real" 1143eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang // input from getANTLRToken() 1153eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang if ( num_markers==0 ) 1163eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang { 1173eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang if( next > threshold ) 1183eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang { 1193eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang#ifdef DBG_TBUF 1203eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwangfprintf(stderr,"getToken: next > threshold (high water is %d)\n", threshold-buffer); 1213eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang#endif 1223eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang makeRoom(); 1233eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang } 1243eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang } 1253eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang else { 1263eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang if ( next > end_of_buffer ) 1273eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang { 1283eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang#ifdef DBG_TBUF 1293eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwangfprintf(stderr,"getToken: next > end_of_buffer (size is %d)\n", buffer_size); 1303eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang#endif 1313eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang extendBuffer(); 1323eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang } 1333eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang } 1343eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang *next = getANTLRToken(); 1353eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang (*next)->ref(); // say we have a copy of this pointer in buffer 1363eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang last = next; 1373eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang next++; 1383eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang tp = last; 1393eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang return *tp++; 1403eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang} 1413eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang 1423eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwangvoid ANTLRTokenBuffer:: 1433eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwangrewind(int pos) 1443eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang{ 1453eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang#if defined(DBG_TBUF)||defined(DBG_TBUF_MARK_REW) 1463eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang fprintf(stderr, "rewind(%d)[nm=%d,from=%d,%d.n=%d]\n", pos, num_markers, tp-buffer,pos,test[pos]); 1473eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang test[pos]--; 1483eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang#endif 1493eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang tp = &buffer[pos]; 1503eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang num_markers--; 1513eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang} 1523eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang 1533eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang/* 1543eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang * This function is used to specify that the token pointers read 1553eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang * by the ANTLRTokenBuffer should be buffered up (to be reused later). 1563eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang */ 1573eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwangint ANTLRTokenBuffer:: 1583eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwangmark() 1593eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang{ 1603eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang#if defined(DBG_TBUF)||defined(DBG_TBUF_MARK_REW) 1613eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang test[tp-buffer]++; 1623eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang fprintf(stderr,"mark(%d)[nm=%d,%d.n=%d]\n",tp-buffer,num_markers+1,tp-buffer,test[tp-buffer]); 1633eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang#endif 1643eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang num_markers++; 1653eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang return tp - buffer; 1663eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang} 1673eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang 1683eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang/* 1693eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang * returns the token pointer n positions ahead. 1703eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang * This implies that bufferedToken(1) gets the NEXT symbol of lookahead. 1713eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang * This is used in conjunction with the ANTLRParser lookahead buffer. 1723eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang * 1733eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang * No markers are set or anything. A bunch of input is buffered--that's all. 1743eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang * The tp pointer is left alone as the lookahead has not been advanced 1753eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang * with getToken(). The next call to getToken() will find a token 1763eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang * in the buffer and won't have to call getANTLRToken(). 1773eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang * 1783eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang * If this is called before a consume() is done, how_many_more_i_need is 1793eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang * set to 'n'. 1803eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang */ 1813eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang_ANTLRTokenPtr ANTLRTokenBuffer:: 1823eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwangbufferedToken(int n) 1833eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang{ 1843eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang// int how_many_more_i_need = (last-tp < 0) ? n : n-(last-tp)-1; 1853eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang int how_many_more_i_need = (tp > last) ? n : n-(last-tp)-1; 1863eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang // Make sure that at least n tokens are available in the buffer 1873eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang#ifdef DBG_TBUF 1883eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang fprintf(stderr, "bufferedToken(%d)\n", n); 1893eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang#endif 1903eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang for (int i=1; i<=how_many_more_i_need; i++) 1913eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang { 1923eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang if ( next > end_of_buffer ) // buffer overflow? 1933eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang { 1943eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang extendBuffer(); 1953eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang } 1963eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang *next = getANTLRToken(); 1973eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang (*next)->ref(); // say we have a copy of this pointer in buffer 1983eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang last = next; 1993eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang next++; 2003eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang } 2013eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang return tp[n - 1]; 2023eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang} 2033eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang 2043eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang/* If no markers are set, the none of the input needs to be saved (except 2053eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang * for the lookahead Token pointers). We save only k-1 token pointers as 2063eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang * we are guaranteed to do a getANTLRToken() right after this because otherwise 2073eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang * we wouldn't have needed to extend the buffer. 2083eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang * 2093eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang * If there are markers in the buffer, we need to save things and so 2103eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang * extendBuffer() is called. 2113eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang */ 2123eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwangvoid ANTLRTokenBuffer:: 2133eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwangmakeRoom() 2143eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang{ 2153eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang#ifdef DBG_TBUF 2163eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang fprintf(stderr, "in makeRoom.................\n"); 2173eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang fprintf(stderr, "num_markers==%d\n", num_markers); 2183eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang#endif 2193eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang/* 2203eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang if ( num_markers == 0 ) 2213eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang { 2223eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang*/ 2233eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang#ifdef DBG_TBUF 2243eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang fprintf(stderr, "moving lookahead and resetting next\n"); 2253eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang 2263eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang _ANTLRTokenPtr *r; 2273eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang fprintf(stderr, "tbuf = ["); 2283eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang for (r=buffer; r<=last; r++) 2293eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang { 2303eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang if ( *r==NULL ) fprintf(stderr, " xxx"); 2313eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang else fprintf(stderr, " '%s'", ((ANTLRCommonToken *)*r)->getText()); 2323eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang } 2333eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang fprintf(stderr, " ]\n"); 2343eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang 2353eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang fprintf(stderr, 2363eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang "before: tp=%d, last=%d, next=%d, threshold=%d\n",tp-buffer,last-buffer,next-buffer,threshold-buffer); 2373eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang#endif 2383eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang 2393eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang // Delete all tokens from 0..last-(k-1) inclusive 2403eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang if ( _deleteTokens ) 2413eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang { 2423eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang _ANTLRTokenPtr *z; 2433eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang for (z=buffer; z<=last-(k-1); z++) 2443eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang { 2453eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang (*z)->deref(); 2463eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang// z->deref(); 2473eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang#ifdef DBG_REFCOUNTTOKEN 2483eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang fprintf(stderr, "##########makeRoom: deleting token '%s' (ref %d)\n", 2493eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang ((ANTLRCommonToken *)*z)->getText(), (*z)->nref()); 2503eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang#endif 2513eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang if ( (*z)->nref()==0 ) 2523eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang { 2533eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang delete (*z); 2543eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang } 2553eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang } 2563eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang } 2573eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang 2583eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang // reset the buffer to initial conditions, but move k-1 symbols 2593eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang // to the beginning of buffer and put new input symbol at k 2603eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang _ANTLRTokenPtr *p = buffer, *q = last-(k-1)+1; 2613eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang// ANTLRAbstractToken **p = buffer, **q = end_of_buffer-(k-1)+1; 2623eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang#ifdef DBG_TBUF 2633eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang fprintf(stderr, "lookahead buffer = ["); 2643eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang#endif 2653eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang for (int i=1; i<=(k-1); i++) 2663eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang { 2673eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang *p++ = *q++; 2683eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang#ifdef DBG_TBUF 2693eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang fprintf(stderr, 2703eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang " '%s'", ((ANTLRCommonToken *)buffer[i-1])->getText()); 2713eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang#endif 2723eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang } 2733eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang#ifdef DBG_TBUF 2743eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang fprintf(stderr, " ]\n"); 2753eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang#endif 2763eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang next = &buffer[k-1]; 2773eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang tp = &buffer[k-1]; // tp points to what will be filled in next 2783eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang last = tp-1; 2793eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang#ifdef DBG_TBUF 2803eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang fprintf(stderr, 2813eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang "after: tp=%d, last=%d, next=%d\n", 2823eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang tp-buffer, last-buffer, next-buffer); 2833eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang#endif 2843eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang/* 2853eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang } 2863eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang else { 2873eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang extendBuffer(); 2883eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang } 2893eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang*/ 2903eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang} 2913eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang 2923eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang/* This function extends 'buffer' by chunk_size and returns with all 2933eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang * pointers at the same relative positions in the buffer (the buffer base 2943eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang * address could have changed in realloc()) except that 'next' comes 2953eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang * back set to where the next token should be stored. All other pointers 2963eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang * are untouched. 2973eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang */ 2983eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwangvoid 2993eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwangANTLRTokenBuffer:: 3003eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwangextendBuffer() 3013eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang{ 3023eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang int save_last = last-buffer, save_tp = tp-buffer, save_next = next-buffer; 3033eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang#ifdef DBG_TBUF 3043eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang fprintf(stderr, "extending physical buffer\n"); 3053eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang#endif 3063eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang buffer_size += chunk_size; 3073eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang buffer = (_ANTLRTokenPtr *) 3083eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang realloc((char *)(buffer-1), 3093eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang (buffer_size+1)*sizeof(_ANTLRTokenPtr )); 3103eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang if ( buffer == NULL ) { 3113eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang panic("cannot alloc token buffer"); 3123eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang } 3133eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang buffer++; // leave the first elem empty so tp-1 is valid ptr 3143eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang 3153eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang tp = buffer + save_tp; // put the pointers back to same relative position 3163eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang last = buffer + save_last; 3173eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang next = buffer + save_next; 3183eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang end_of_buffer = &buffer[buffer_size-1]; 3193eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang // BUGBUG -- threshold = &buffer[(int)(buffer_size*(1.0/2.0))]; 3203eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang threshold = &buffer[(int)(buffer_size / 2)]; 3213eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang 3223eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang/* 3233eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang // zero out new token ptrs so we'll know if something to delete in buffer 3243eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang ANTLRAbstractToken **p = end_of_buffer-chunk_size+1; 3253eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang for (; p<=end_of_buffer; p++) *p = NULL; 3263eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang*/ 3273eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang} 3283eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang 3293eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwangANTLRParser * ANTLRTokenBuffer:: // MR1 3303eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwangsetParser(ANTLRParser *p) { // MR1 3313eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang ANTLRParser *old=parser; // MR1 3323eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang parser=p; // MR1 3333eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang input->setParser(p); // MR1 3343eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang return old; // MR1 3353eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang} // MR1 3363eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang // MR1 3373eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwangANTLRParser * ANTLRTokenBuffer:: // MR1 3383eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwanggetParser() { // MR1 3393eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang return parser; // MR1 3403eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang} // MR1 3413eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang 3423eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang/* to avoid having to link in another file just for the smart token ptr 3433eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang * stuff, we include it here. Ugh. 3443eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang */ 3453eb9473ea9a949badfe06ae61d2d3fcfa53651c7qwang#include ATOKPTR_C 346