13447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein/* 23447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein [The "BSD license"] 33447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein Copyright (c) 2005-2009 Terence Parr 43447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein All rights reserved. 53447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 63447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein Redistribution and use in source and binary forms, with or without 73447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein modification, are permitted provided that the following conditions 83447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein are met: 93447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 1. Redistributions of source code must retain the above copyright 103447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein notice, this list of conditions and the following disclaimer. 113447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 2. Redistributions in binary form must reproduce the above copyright 123447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein notice, this list of conditions and the following disclaimer in the 133447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein documentation and/or other materials provided with the distribution. 143447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 3. The name of the author may not be used to endorse or promote products 153447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein derived from this software without specific prior written permission. 163447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 173447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 183447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 193447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 203447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 213447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 223447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 233447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 243447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 253447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 263447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 273447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein */ 283447a5916aa62f44de24cc441fc9987116ddff52Andrew Sappersteinpackage org.antlr.runtime; 293447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 303447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein/** A lexer is recognizer that draws input symbols from a character stream. 313447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein * lexer grammars result in a subclass of this object. A Lexer object 323447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein * uses simplified match() and error recovery mechanisms in the interest 333447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein * of speed. 343447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein */ 353447a5916aa62f44de24cc441fc9987116ddff52Andrew Sappersteinpublic abstract class Lexer extends BaseRecognizer implements TokenSource { 363447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein /** Where is the lexer drawing characters from? */ 373447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein protected CharStream input; 383447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 393447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein public Lexer() { 403447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 413447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 423447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein public Lexer(CharStream input) { 433447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein this.input = input; 443447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 453447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 463447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein public Lexer(CharStream input, RecognizerSharedState state) { 473447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein super(state); 483447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein this.input = input; 493447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 503447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 513447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein public void reset() { 523447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein super.reset(); // reset all recognizer state variables 533447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein // wack Lexer state variables 543447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein if ( input!=null ) { 553447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein input.seek(0); // rewind the input 563447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 573447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein if ( state==null ) { 583447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein return; // no shared state work to do 593447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 603447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein state.token = null; 613447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein state.type = Token.INVALID_TOKEN_TYPE; 623447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein state.channel = Token.DEFAULT_CHANNEL; 633447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein state.tokenStartCharIndex = -1; 643447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein state.tokenStartCharPositionInLine = -1; 653447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein state.tokenStartLine = -1; 663447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein state.text = null; 673447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 683447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 693447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein /** Return a token from this source; i.e., match a token on the char 703447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein * stream. 713447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein */ 723447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein public Token nextToken() { 733447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein while (true) { 743447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein state.token = null; 753447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein state.channel = Token.DEFAULT_CHANNEL; 763447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein state.tokenStartCharIndex = input.index(); 773447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein state.tokenStartCharPositionInLine = input.getCharPositionInLine(); 783447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein state.tokenStartLine = input.getLine(); 793447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein state.text = null; 803447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein if ( input.LA(1)==CharStream.EOF ) { 813447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein Token eof = new CommonToken((CharStream)input,Token.EOF, 823447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein Token.DEFAULT_CHANNEL, 833447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein input.index(),input.index()); 843447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein eof.setLine(getLine()); 853447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein eof.setCharPositionInLine(getCharPositionInLine()); 863447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein return eof; 873447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 883447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein try { 893447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein mTokens(); 903447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein if ( state.token==null ) { 913447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein emit(); 923447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 933447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein else if ( state.token==Token.SKIP_TOKEN ) { 943447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein continue; 953447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 963447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein return state.token; 973447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 98324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver catch (MismatchedRangeException re) { 99324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver reportError(re); 100324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver // matchRange() routine has already called recover() 1013447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 102324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver catch (MismatchedTokenException re) { 1033447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein reportError(re); 1043447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein // match() routine has already called recover() 1053447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 106324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver catch (RecognitionException re) { 107324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver reportError(re); 108324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver recover(re); // throw out current char and try again 109324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 1103447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 1113447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 1123447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 1133447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein /** Instruct the lexer to skip creating a token for current lexer rule 1143447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein * and look for another token. nextToken() knows to keep looking when 1153447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein * a lexer rule finishes with token set to SKIP_TOKEN. Recall that 1163447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein * if token==null at end of any token rule, it creates one for you 1173447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein * and emits it. 1183447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein */ 1193447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein public void skip() { 1203447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein state.token = Token.SKIP_TOKEN; 1213447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 1223447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 1233447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein /** This is the lexer entry point that sets instance var 'token' */ 1243447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein public abstract void mTokens() throws RecognitionException; 1253447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 1263447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein /** Set the char stream and reset the lexer */ 1273447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein public void setCharStream(CharStream input) { 1283447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein this.input = null; 1293447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein reset(); 1303447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein this.input = input; 1313447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 1323447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 1333447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein public CharStream getCharStream() { 1343447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein return this.input; 1353447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 1363447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 1373447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein public String getSourceName() { 1383447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein return input.getSourceName(); 1393447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 1403447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 1413447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein /** Currently does not support multiple emits per nextToken invocation 1423447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein * for efficiency reasons. Subclass and override this method and 1433447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein * nextToken (to push tokens into a list and pull from that list rather 1443447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein * than a single variable as this implementation does). 1453447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein */ 1463447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein public void emit(Token token) { 1473447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein state.token = token; 1483447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 1493447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 1503447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein /** The standard method called to automatically emit a token at the 1513447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein * outermost lexical rule. The token object should point into the 1523447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein * char buffer start..stop. If there is a text override in 'text', 1533447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein * use that to set the token's text. Override this method to emit 1543447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein * custom Token objects. 1553447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein * 1563447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein * If you are building trees, then you should also override 1573447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein * Parser or TreeParser.getMissingSymbol(). 1583447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein */ 1593447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein public Token emit() { 1603447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein Token t = new CommonToken(input, state.type, state.channel, state.tokenStartCharIndex, getCharIndex()-1); 1613447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein t.setLine(state.tokenStartLine); 1623447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein t.setText(state.text); 1633447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein t.setCharPositionInLine(state.tokenStartCharPositionInLine); 1643447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein emit(t); 1653447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein return t; 1663447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 1673447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 1683447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein public void match(String s) throws MismatchedTokenException { 1693447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein int i = 0; 1703447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein while ( i<s.length() ) { 1713447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein if ( input.LA(1)!=s.charAt(i) ) { 1723447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein if ( state.backtracking>0 ) { 1733447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein state.failed = true; 1743447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein return; 1753447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 1763447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein MismatchedTokenException mte = 1773447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein new MismatchedTokenException(s.charAt(i), input); 1783447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein recover(mte); 1793447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein throw mte; 1803447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 1813447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein i++; 1823447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein input.consume(); 1833447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein state.failed = false; 1843447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 1853447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 1863447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 1873447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein public void matchAny() { 1883447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein input.consume(); 1893447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 1903447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 1913447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein public void match(int c) throws MismatchedTokenException { 1923447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein if ( input.LA(1)!=c ) { 1933447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein if ( state.backtracking>0 ) { 1943447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein state.failed = true; 1953447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein return; 1963447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 1973447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein MismatchedTokenException mte = 1983447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein new MismatchedTokenException(c, input); 1993447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein recover(mte); // don't really recover; just consume in lexer 2003447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein throw mte; 2013447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 2023447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein input.consume(); 2033447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein state.failed = false; 2043447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 2053447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 2063447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein public void matchRange(int a, int b) 2073447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein throws MismatchedRangeException 2083447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein { 2093447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein if ( input.LA(1)<a || input.LA(1)>b ) { 2103447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein if ( state.backtracking>0 ) { 2113447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein state.failed = true; 2123447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein return; 2133447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 2143447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein MismatchedRangeException mre = 2153447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein new MismatchedRangeException(a,b,input); 2163447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein recover(mre); 2173447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein throw mre; 2183447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 2193447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein input.consume(); 2203447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein state.failed = false; 2213447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 2223447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 2233447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein public int getLine() { 2243447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein return input.getLine(); 2253447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 2263447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 2273447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein public int getCharPositionInLine() { 2283447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein return input.getCharPositionInLine(); 2293447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 2303447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 2313447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein /** What is the index of the current character of lookahead? */ 2323447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein public int getCharIndex() { 2333447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein return input.index(); 2343447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 2353447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 2363447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein /** Return the text matched so far for the current token or any 2373447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein * text override. 2383447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein */ 2393447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein public String getText() { 2403447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein if ( state.text!=null ) { 2413447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein return state.text; 2423447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 2433447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein return input.substring(state.tokenStartCharIndex,getCharIndex()-1); 2443447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 2453447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 2463447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein /** Set the complete text of this token; it wipes any previous 2473447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein * changes to the text. 2483447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein */ 2493447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein public void setText(String text) { 2503447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein state.text = text; 2513447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 2523447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 2533447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein public void reportError(RecognitionException e) { 2543447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein /** TODO: not thought about recovery in lexer yet. 2553447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein * 2563447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein // if we've already reported an error and have not matched a token 2573447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein // yet successfully, don't report any errors. 2583447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein if ( errorRecovery ) { 2593447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein //System.err.print("[SPURIOUS] "); 2603447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein return; 2613447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 2623447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein errorRecovery = true; 2633447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein */ 2643447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 2653447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein displayRecognitionError(this.getTokenNames(), e); 2663447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 2673447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 2683447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein public String getErrorMessage(RecognitionException e, String[] tokenNames) { 2693447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein String msg = null; 2703447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein if ( e instanceof MismatchedTokenException ) { 2713447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein MismatchedTokenException mte = (MismatchedTokenException)e; 2723447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein msg = "mismatched character "+getCharErrorDisplay(e.c)+" expecting "+getCharErrorDisplay(mte.expecting); 2733447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 2743447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein else if ( e instanceof NoViableAltException ) { 2753447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein NoViableAltException nvae = (NoViableAltException)e; 2763447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein // for development, can add "decision=<<"+nvae.grammarDecisionDescription+">>" 2773447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein // and "(decision="+nvae.decisionNumber+") and 2783447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein // "state "+nvae.stateNumber 2793447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein msg = "no viable alternative at character "+getCharErrorDisplay(e.c); 2803447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 2813447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein else if ( e instanceof EarlyExitException ) { 2823447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein EarlyExitException eee = (EarlyExitException)e; 2833447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein // for development, can add "(decision="+eee.decisionNumber+")" 2843447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein msg = "required (...)+ loop did not match anything at character "+getCharErrorDisplay(e.c); 2853447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 2863447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein else if ( e instanceof MismatchedNotSetException ) { 2873447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein MismatchedNotSetException mse = (MismatchedNotSetException)e; 2883447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein msg = "mismatched character "+getCharErrorDisplay(e.c)+" expecting set "+mse.expecting; 2893447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 2903447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein else if ( e instanceof MismatchedSetException ) { 2913447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein MismatchedSetException mse = (MismatchedSetException)e; 2923447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein msg = "mismatched character "+getCharErrorDisplay(e.c)+" expecting set "+mse.expecting; 2933447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 2943447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein else if ( e instanceof MismatchedRangeException ) { 2953447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein MismatchedRangeException mre = (MismatchedRangeException)e; 2963447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein msg = "mismatched character "+getCharErrorDisplay(e.c)+" expecting set "+ 2973447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein getCharErrorDisplay(mre.a)+".."+getCharErrorDisplay(mre.b); 2983447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 2993447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein else { 3003447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein msg = super.getErrorMessage(e, tokenNames); 3013447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 3023447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein return msg; 3033447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 3043447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 3053447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein public String getCharErrorDisplay(int c) { 3063447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein String s = String.valueOf((char)c); 3073447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein switch ( c ) { 3083447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein case Token.EOF : 3093447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein s = "<EOF>"; 3103447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein break; 3113447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein case '\n' : 3123447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein s = "\\n"; 3133447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein break; 3143447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein case '\t' : 3153447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein s = "\\t"; 3163447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein break; 3173447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein case '\r' : 3183447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein s = "\\r"; 3193447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein break; 3203447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 3213447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein return "'"+s+"'"; 3223447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 3233447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 3243447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein /** Lexers can normally match any char in it's vocabulary after matching 3253447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein * a token, so do the easy thing and just kill a character and hope 3263447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein * it all works out. You can instead use the rule invocation stack 3273447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein * to do sophisticated error recovery if you are in a fragment rule. 3283447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein */ 3293447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein public void recover(RecognitionException re) { 3303447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein //System.out.println("consuming char "+(char)input.LA(1)+" during recovery"); 3313447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein //re.printStackTrace(); 3323447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein input.consume(); 3333447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 3343447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 3353447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein public void traceIn(String ruleName, int ruleIndex) { 3363447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein String inputSymbol = ((char)input.LT(1))+" line="+getLine()+":"+getCharPositionInLine(); 3373447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein super.traceIn(ruleName, ruleIndex, inputSymbol); 3383447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 3393447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 3403447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein public void traceOut(String ruleName, int ruleIndex) { 3413447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein String inputSymbol = ((char)input.LT(1))+" line="+getLine()+":"+getCharPositionInLine(); 3423447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein super.traceOut(ruleName, ruleIndex, inputSymbol); 3433447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 3443447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein} 345