LexerTest.java revision 22532f9dd9daf18b86c6e46b6984012e8ebd21af 
1/*
2 * [The "BSD licence"]
3 * Copyright (c) 2010 Ben Gruver
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 *    notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 *    notice, this list of conditions and the following disclaimer in the
13 *    documentation and/or other materials provided with the distribution.
14 * 3. The name of the author may not be used to endorse or promote products
15 *    derived from this software without specific prior written permission.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 * INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 */
28
29import org.antlr.runtime.ANTLRInputStream;
30import org.antlr.runtime.CommonToken;
31import org.antlr.runtime.CommonTokenStream;
32import org.antlr.runtime.RecognitionException;
33import org.jf.smali.*;
34import static org.jf.smali.expectedTokensTestGrammarParser.ExpectedToken;
35import org.junit.Assert;
36import org.junit.Test;
37
38import java.io.*;
39import java.util.HashMap;
40import java.util.List;
41
42public class LexerTest {
43    private static final HashMap<String, Integer> tokenTypesByName;
44
45    static {
46        tokenTypesByName = new HashMap<String, Integer>();
47
48        for (int i=0; i<smaliParser.tokenNames.length; i++) {
49            tokenTypesByName.put(smaliParser.tokenNames[i], i);
50        }
51    }
52
53    @Test
54    public void DirectiveTest() {
55        runTest("DirectiveTest");
56    }
57
58    @Test
59    public void ByteLiteralTest() {
60        runTest("ByteLiteralTest");
61    }
62
63    @Test
64    public void ShortLiteralTest() {
65        runTest("ShortLiteralTest");
66    }
67
68    @Test
69    public void IntegerLiteralTest() {
70        runTest("IntegerLiteralTest");
71    }
72
73    @Test
74    public void LongLiteralTest() {
75        runTest("LongLiteralTest");
76    }
77
78    @Test
79    public void FloatLiteralTest() {
80        runTest("FloatLiteralTest");
81    }
82
83    @Test
84    public void CharLiteralTest() {
85        runTest("CharLiteralTest");
86    }
87
88    @Test
89    public void StringLiteralTest() {
90        runTest("StringLiteralTest");
91    }
92
93    @Test
94    public void MiscTest() {
95        runTest("MiscTest");
96    }
97
98    @Test
99    public void CommentTest() {
100        runTest("CommentTest", false);
101    }
102
103    @Test
104    public void InstructionTest() {
105        runTest("InstructionTest", true);
106    }
107
108    @Test
109    public void TypeAndIdentifierTest() {
110        runTest("TypeAndIdentifierTest");
111    }
112
113    @Test
114    public void SymbolTest() {
115        runTest("SymbolTest", false);
116    }
117
118    public void runTest(String test) {
119        runTest(test, true);
120    }
121
122    public void runTest(String test, boolean discardHiddenTokens) {
123        String smaliFile = String.format("LexerTest%s%s.smali", File.separatorChar, test);
124        String tokensFile = String.format("LexerTest%s%s.tokens", File.separatorChar, test);
125
126        expectedTokensTestGrammarLexer expectedTokensLexer = null;
127        try {
128            expectedTokensLexer = new expectedTokensTestGrammarLexer(new ANTLRInputStream(
129                    LexerTest.class.getClassLoader().getResourceAsStream(tokensFile)));
130        } catch (IOException ex) {
131            throw new RuntimeException(ex);
132        }
133
134        CommonTokenStream expectedTokensStream = new CommonTokenStream(expectedTokensLexer);
135
136        expectedTokensTestGrammarParser expectedTokensParser =
137                new expectedTokensTestGrammarParser(expectedTokensStream);
138        try {
139            expectedTokensParser.top();
140        } catch (RecognitionException ex) {
141            throw new RuntimeException(ex);
142        }
143
144        List<ExpectedToken> expectedTokens = expectedTokensParser.getExpectedTokens();
145
146        InputStream smaliStream = LexerTest.class.getClassLoader().getResourceAsStream(smaliFile);
147        if (smaliStream == null) {
148            Assert.fail("Could not load " + smaliFile);
149        }
150        smaliFlexLexer lexer = new smaliFlexLexer(smaliStream);
151
152        CommonTokenStream tokenStream = new CommonTokenStream(lexer);
153        List tokens = tokenStream.getTokens();
154
155        int expectedTokenIndex = 0;
156        CommonToken token;
157        for (int i=0; i<tokens.size(); i++) {
158            token = (CommonToken)tokens.get(i);
159
160            if (discardHiddenTokens && token.getChannel() == smaliLexer.HIDDEN) {
161                continue;
162            }
163
164            if (expectedTokenIndex >= expectedTokens.size()) {
165                Assert.fail("Too many tokens");
166            }
167
168            if (token.getType() == smaliParser.INVALID_TOKEN) {
169                Assert.assertTrue("Encountered an INVALID_TOKEN not on the error channel",
170                        token.getChannel() == smaliLexer.ERROR_CHANNEL);
171            }
172
173            ExpectedToken expectedToken = expectedTokens.get(expectedTokenIndex++);
174            if (!tokenTypesByName.containsKey(expectedToken.tokenName)) {
175                Assert.fail("Unknown token: " + expectedToken.tokenName);
176            }
177            int expectedTokenType = tokenTypesByName.get(expectedToken.tokenName);
178
179            if (token.getType() != expectedTokenType) {
180                Assert.fail(String.format("Invalid token at index %d. Expecting %s, got %s",
181                        expectedTokenIndex-1, expectedToken.tokenName, getTokenName(token.getType())));
182            }
183
184            if (expectedToken.tokenText != null) {
185                if (!expectedToken.tokenText.equals(token.getText())) {
186                    Assert.fail(
187                            String.format("Invalid token text at index %d. Expecting text \"%s\", got \"%s\"",
188                                    expectedTokenIndex - 1, expectedToken.tokenText, token.getText()));
189                }
190            }
191        }
192
193        if (expectedTokenIndex < expectedTokens.size()) {
194            Assert.fail(String.format("Not enough tokens. Expecting %d tokens, but got %d", expectedTokens.size(),
195                    expectedTokenIndex));
196        }
197    }
198
199
200
201    private static String getTokenName(int tokenType) {
202        return smaliParser.tokenNames[tokenType];
203    }
204}
205