1/*
2 * [The "BSD licence"]
3 * Copyright (c) 2010 Ben Gruver
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 *    notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 *    notice, this list of conditions and the following disclaimer in the
13 *    documentation and/or other materials provided with the distribution.
14 * 3. The name of the author may not be used to endorse or promote products
15 *    derived from this software without specific prior written permission.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 * INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 */
28
29import org.antlr.runtime.ANTLRInputStream;
30import org.antlr.runtime.CommonToken;
31import org.antlr.runtime.CommonTokenStream;
32import org.antlr.runtime.RecognitionException;
33import org.jf.smali.expectedTokensTestGrammarLexer;
34import org.jf.smali.expectedTokensTestGrammarParser;
35import org.jf.smali.smaliFlexLexer;
36import org.jf.smali.smaliParser;
37import org.junit.Assert;
38import org.junit.Test;
39
40import java.io.File;
41import java.io.IOException;
42import java.io.InputStream;
43import java.util.HashMap;
44import java.util.List;
45
46import static org.jf.smali.expectedTokensTestGrammarParser.ExpectedToken;
47
48public class LexerTest {
49    private static final HashMap<String, Integer> tokenTypesByName;
50
51    static {
52        tokenTypesByName = new HashMap<String, Integer>();
53
54        for (int i=0; i<smaliParser.tokenNames.length; i++) {
55            tokenTypesByName.put(smaliParser.tokenNames[i], i);
56        }
57    }
58
59    @Test
60    public void DirectiveTest() {
61        runTest("DirectiveTest");
62    }
63
64    @Test
65    public void ByteLiteralTest() {
66        runTest("ByteLiteralTest");
67    }
68
69    @Test
70    public void ShortLiteralTest() {
71        runTest("ShortLiteralTest");
72    }
73
74    @Test
75    public void IntegerLiteralTest() {
76        runTest("IntegerLiteralTest");
77    }
78
79    @Test
80    public void LongLiteralTest() {
81        runTest("LongLiteralTest");
82    }
83
84    @Test
85    public void FloatLiteralTest() {
86        runTest("FloatLiteralTest");
87    }
88
89    @Test
90    public void CharLiteralTest() {
91        runTest("CharLiteralTest");
92    }
93
94    @Test
95    public void StringLiteralTest() {
96        runTest("StringLiteralTest");
97    }
98
99    @Test
100    public void MiscTest() {
101        runTest("MiscTest");
102    }
103
104    @Test
105    public void CommentTest() {
106        runTest("CommentTest", false);
107    }
108
109    @Test
110    public void InstructionTest() {
111        runTest("InstructionTest", true);
112    }
113
114    @Test
115    public void TypeAndIdentifierTest() {
116        runTest("TypeAndIdentifierTest");
117    }
118
119    @Test
120    public void SymbolTest() {
121        runTest("SymbolTest", false);
122    }
123
124    @Test
125    public void RealSmaliFileTest() {
126        runTest("RealSmaliFileTest", true);
127    }
128
129    public void runTest(String test) {
130        runTest(test, true);
131    }
132
133    public void runTest(String test, boolean discardHiddenTokens) {
134        String smaliFile = String.format("LexerTest%s%s.smali", File.separatorChar, test);
135        String tokensFile = String.format("LexerTest%s%s.tokens", File.separatorChar, test);
136
137        expectedTokensTestGrammarLexer expectedTokensLexer = null;
138        try {
139            expectedTokensLexer = new expectedTokensTestGrammarLexer(new ANTLRInputStream(
140                    LexerTest.class.getClassLoader().getResourceAsStream(tokensFile)));
141        } catch (IOException ex) {
142            throw new RuntimeException(ex);
143        }
144
145        CommonTokenStream expectedTokensStream = new CommonTokenStream(expectedTokensLexer);
146
147        expectedTokensTestGrammarParser expectedTokensParser =
148                new expectedTokensTestGrammarParser(expectedTokensStream);
149        try {
150            expectedTokensParser.top();
151        } catch (RecognitionException ex) {
152            throw new RuntimeException(ex);
153        }
154
155        List<ExpectedToken> expectedTokens = expectedTokensParser.getExpectedTokens();
156
157        InputStream smaliStream = LexerTest.class.getClassLoader().getResourceAsStream(smaliFile);
158        if (smaliStream == null) {
159            Assert.fail("Could not load " + smaliFile);
160        }
161        smaliFlexLexer lexer = new smaliFlexLexer(smaliStream);
162        lexer.setSourceFile(new File(test + ".smali"));
163        lexer.setSuppressErrors(true);
164
165        CommonTokenStream tokenStream = new CommonTokenStream(lexer);
166        tokenStream.fill();
167        List tokens = tokenStream.getTokens();
168
169        int expectedTokenIndex = 0;
170        CommonToken token;
171        for (int i=0; i<tokens.size()-1; i++) {
172            token = (CommonToken)tokens.get(i);
173
174            if (discardHiddenTokens && token.getChannel() == smaliParser.HIDDEN) {
175                continue;
176            }
177
178            if (expectedTokenIndex >= expectedTokens.size()) {
179                Assert.fail("Too many tokens");
180            }
181
182            if (token.getType() == smaliParser.INVALID_TOKEN) {
183                Assert.assertTrue("Encountered an INVALID_TOKEN not on the error channel",
184                        token.getChannel() == smaliParser.ERROR_CHANNEL);
185            }
186
187            ExpectedToken expectedToken = expectedTokens.get(expectedTokenIndex++);
188            if (!tokenTypesByName.containsKey(expectedToken.tokenName)) {
189                Assert.fail("Unknown token: " + expectedToken.tokenName);
190            }
191            int expectedTokenType = tokenTypesByName.get(expectedToken.tokenName);
192
193            if (token.getType() != expectedTokenType) {
194                Assert.fail(String.format("Invalid token at index %d. Expecting %s, got %s(%s)",
195                        expectedTokenIndex-1, expectedToken.tokenName, getTokenName(token.getType()), token.getText()));
196            }
197
198            if (expectedToken.tokenText != null) {
199                if (!expectedToken.tokenText.equals(token.getText())) {
200                    Assert.fail(
201                            String.format("Invalid token text at index %d. Expecting text \"%s\", got \"%s\"",
202                                    expectedTokenIndex - 1, expectedToken.tokenText, token.getText()));
203                }
204            }
205        }
206
207        if (expectedTokenIndex < expectedTokens.size()) {
208            Assert.fail(String.format("Not enough tokens. Expecting %d tokens, but got %d", expectedTokens.size(),
209                    expectedTokenIndex));
210        }
211    }
212
213
214
215    private static String getTokenName(int tokenType) {
216        return smaliParser.tokenNames[tokenType];
217    }
218}
219