LexerTest.java revision 48d5b730272ada20e5b8c0245d297b03dbbf6d02 
1/*
2 * [The "BSD licence"]
3 * Copyright (c) 2010 Ben Gruver
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 *    notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 *    notice, this list of conditions and the following disclaimer in the
13 *    documentation and/or other materials provided with the distribution.
14 * 3. The name of the author may not be used to endorse or promote products
15 *    derived from this software without specific prior written permission.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 * INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 */
28
29import org.antlr.runtime.ANTLRInputStream;
30import org.antlr.runtime.CommonToken;
31import org.antlr.runtime.CommonTokenStream;
32import org.antlr.runtime.RecognitionException;
33import org.jf.dexlib.Util.Utf8Utils;
34import org.jf.smali.*;
35import static org.jf.smali.expectedTokensTestGrammarParser.ExpectedToken;
36import org.junit.Assert;
37import org.junit.Test;
38
39import java.io.*;
40import java.util.HashMap;
41import java.util.List;
42
43public class LexerTest {
44    private static final HashMap<String, Integer> tokenTypesByName;
45
46    static {
47        tokenTypesByName = new HashMap<String, Integer>();
48
49        for (int i=0; i<smaliParser.tokenNames.length; i++) {
50            tokenTypesByName.put(smaliParser.tokenNames[i], i);
51        }
52    }
53
54    @Test
55    public void DirectiveTest() {
56        runTest("DirectiveTest");
57    }
58
59    @Test
60    public void ByteLiteralTest() {
61        runTest("ByteLiteralTest");
62    }
63
64    @Test
65    public void ShortLiteralTest() {
66        runTest("ShortLiteralTest");
67    }
68
69    @Test
70    public void IntegerLiteralTest() {
71        runTest("IntegerLiteralTest");
72    }
73
74    @Test
75    public void LongLiteralTest() {
76        runTest("LongLiteralTest");
77    }
78
79    @Test
80    public void FloatLiteralTest() {
81        runTest("FloatLiteralTest");
82    }
83
84    @Test
85    public void CharLiteralTest() {
86        runTest("CharLiteralTest");
87    }
88
89    @Test
90    public void StringLiteralTest() {
91        runTest("StringLiteralTest");
92    }
93
94    @Test
95    public void MiscTest() {
96        runTest("MiscTest");
97    }
98
99    @Test
100    public void CommentTest() {
101        runTest("CommentTest", false);
102    }
103
104    @Test
105    public void InstructionTest() {
106        runTest("InstructionTest", true);
107    }
108
109    @Test
110    public void TypeAndIdentifierTest() {
111        runTest("TypeAndIdentifierTest");
112    }
113
114    @Test
115    public void SymbolTest() {
116        runTest("SymbolTest", false);
117    }
118
119    @Test
120    public void RealSmaliFileTest() {
121        runTest("RealSmaliFileTest", true);
122    }
123
124    public void runTest(String test) {
125        runTest(test, true);
126    }
127
128    public void runTest(String test, boolean discardHiddenTokens) {
129        String smaliFile = String.format("LexerTest%s%s.smali", File.separatorChar, test);
130        String tokensFile = String.format("LexerTest%s%s.tokens", File.separatorChar, test);
131
132        expectedTokensTestGrammarLexer expectedTokensLexer = null;
133        try {
134            expectedTokensLexer = new expectedTokensTestGrammarLexer(new ANTLRInputStream(
135                    LexerTest.class.getClassLoader().getResourceAsStream(tokensFile)));
136        } catch (IOException ex) {
137            throw new RuntimeException(ex);
138        }
139
140        CommonTokenStream expectedTokensStream = new CommonTokenStream(expectedTokensLexer);
141
142        expectedTokensTestGrammarParser expectedTokensParser =
143                new expectedTokensTestGrammarParser(expectedTokensStream);
144        try {
145            expectedTokensParser.top();
146        } catch (RecognitionException ex) {
147            throw new RuntimeException(ex);
148        }
149
150        List<ExpectedToken> expectedTokens = expectedTokensParser.getExpectedTokens();
151
152        InputStream smaliStream = LexerTest.class.getClassLoader().getResourceAsStream(smaliFile);
153        if (smaliStream == null) {
154            Assert.fail("Could not load " + smaliFile);
155        }
156        smaliFlexLexer lexer = new smaliFlexLexer(smaliStream);
157        lexer.setSourceFile(new File(test + ".smali"));
158        lexer.setSuppressErrors(true);
159
160        CommonTokenStream tokenStream = new CommonTokenStream(lexer);
161        List tokens = tokenStream.getTokens();
162
163        int expectedTokenIndex = 0;
164        CommonToken token;
165        for (int i=0; i<tokens.size(); i++) {
166            token = (CommonToken)tokens.get(i);
167
168            if (discardHiddenTokens && token.getChannel() == smaliParser.HIDDEN) {
169                continue;
170            }
171
172            if (expectedTokenIndex >= expectedTokens.size()) {
173                Assert.fail("Too many tokens");
174            }
175
176            if (token.getType() == smaliParser.INVALID_TOKEN) {
177                Assert.assertTrue("Encountered an INVALID_TOKEN not on the error channel",
178                        token.getChannel() == smaliParser.ERROR_CHANNEL);
179            }
180
181            ExpectedToken expectedToken = expectedTokens.get(expectedTokenIndex++);
182            if (!tokenTypesByName.containsKey(expectedToken.tokenName)) {
183                Assert.fail("Unknown token: " + expectedToken.tokenName);
184            }
185            int expectedTokenType = tokenTypesByName.get(expectedToken.tokenName);
186
187            if (token.getType() != expectedTokenType) {
188                Assert.fail(String.format("Invalid token at index %d. Expecting %s, got %s(%s)",
189                        expectedTokenIndex-1, expectedToken.tokenName, getTokenName(token.getType()), token.getText()));
190            }
191
192            if (expectedToken.tokenText != null) {
193                if (!expectedToken.tokenText.equals(token.getText())) {
194                    Assert.fail(
195                            String.format("Invalid token text at index %d. Expecting text \"%s\", got \"%s\"",
196                                    expectedTokenIndex - 1, expectedToken.tokenText, token.getText()));
197                }
198            }
199        }
200
201        if (expectedTokenIndex < expectedTokens.size()) {
202            Assert.fail(String.format("Not enough tokens. Expecting %d tokens, but got %d", expectedTokens.size(),
203                    expectedTokenIndex));
204        }
205    }
206
207
208
209    private static String getTokenName(int tokenType) {
210        return smaliParser.tokenNames[tokenType];
211    }
212}
213