1 /*
2  * [The "BSD licence"]
3  * Copyright (c) 2010 Ben Gruver
4  * All rights reserved.
5  *
6  * Redistribution and use in source and binary forms, with or without
7  * modification, are permitted provided that the following conditions
8  * are met:
9  * 1. Redistributions of source code must retain the above copyright
10  *    notice, this list of conditions and the following disclaimer.
11  * 2. Redistributions in binary form must reproduce the above copyright
12  *    notice, this list of conditions and the following disclaimer in the
13  *    documentation and/or other materials provided with the distribution.
14  * 3. The name of the author may not be used to endorse or promote products
15  *    derived from this software without specific prior written permission.
16  *
17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25  * INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27  */
28 
29 import org.antlr.runtime.ANTLRInputStream;
30 import org.antlr.runtime.CommonToken;
31 import org.antlr.runtime.CommonTokenStream;
32 import org.antlr.runtime.RecognitionException;
33 import org.jf.smali.expectedTokensTestGrammarLexer;
34 import org.jf.smali.expectedTokensTestGrammarParser;
35 import org.jf.smali.smaliFlexLexer;
36 import org.jf.smali.smaliParser;
37 import org.junit.Assert;
38 import org.junit.Test;
39 
40 import java.io.File;
41 import java.io.IOException;
42 import java.io.InputStream;
43 import java.util.HashMap;
44 import java.util.List;
45 
46 import static org.jf.smali.expectedTokensTestGrammarParser.ExpectedToken;
47 
48 public class LexerTest {
49     private static final HashMap<String, Integer> tokenTypesByName;
50 
51     static {
52         tokenTypesByName = new HashMap<String, Integer>();
53 
54         for (int i=0; i<smaliParser.tokenNames.length; i++) {
tokenTypesByName.put(smaliParser.tokenNames[i], i)55             tokenTypesByName.put(smaliParser.tokenNames[i], i);
56         }
57     }
58 
59     @Test
DirectiveTest()60     public void DirectiveTest() {
61         runTest("DirectiveTest");
62     }
63 
64     @Test
ByteLiteralTest()65     public void ByteLiteralTest() {
66         runTest("ByteLiteralTest");
67     }
68 
69     @Test
ShortLiteralTest()70     public void ShortLiteralTest() {
71         runTest("ShortLiteralTest");
72     }
73 
74     @Test
IntegerLiteralTest()75     public void IntegerLiteralTest() {
76         runTest("IntegerLiteralTest");
77     }
78 
79     @Test
LongLiteralTest()80     public void LongLiteralTest() {
81         runTest("LongLiteralTest");
82     }
83 
84     @Test
FloatLiteralTest()85     public void FloatLiteralTest() {
86         runTest("FloatLiteralTest");
87     }
88 
89     @Test
CharLiteralTest()90     public void CharLiteralTest() {
91         runTest("CharLiteralTest");
92     }
93 
94     @Test
StringLiteralTest()95     public void StringLiteralTest() {
96         runTest("StringLiteralTest");
97     }
98 
99     @Test
MiscTest()100     public void MiscTest() {
101         runTest("MiscTest");
102     }
103 
104     @Test
CommentTest()105     public void CommentTest() {
106         runTest("CommentTest", false);
107     }
108 
109     @Test
InstructionTest()110     public void InstructionTest() {
111         runTest("InstructionTest", true);
112     }
113 
114     @Test
TypeAndIdentifierTest()115     public void TypeAndIdentifierTest() {
116         runTest("TypeAndIdentifierTest");
117     }
118 
119     @Test
SymbolTest()120     public void SymbolTest() {
121         runTest("SymbolTest", false);
122     }
123 
124     @Test
RealSmaliFileTest()125     public void RealSmaliFileTest() {
126         runTest("RealSmaliFileTest", true);
127     }
128 
runTest(String test)129     public void runTest(String test) {
130         runTest(test, true);
131     }
132 
runTest(String test, boolean discardHiddenTokens)133     public void runTest(String test, boolean discardHiddenTokens) {
134         String smaliFile = String.format("LexerTest%s%s.smali", File.separatorChar, test);
135         String tokensFile = String.format("LexerTest%s%s.tokens", File.separatorChar, test);
136 
137         expectedTokensTestGrammarLexer expectedTokensLexer = null;
138         try {
139             expectedTokensLexer = new expectedTokensTestGrammarLexer(new ANTLRInputStream(
140                     LexerTest.class.getClassLoader().getResourceAsStream(tokensFile)));
141         } catch (IOException ex) {
142             throw new RuntimeException(ex);
143         }
144 
145         CommonTokenStream expectedTokensStream = new CommonTokenStream(expectedTokensLexer);
146 
147         expectedTokensTestGrammarParser expectedTokensParser =
148                 new expectedTokensTestGrammarParser(expectedTokensStream);
149         try {
150             expectedTokensParser.top();
151         } catch (RecognitionException ex) {
152             throw new RuntimeException(ex);
153         }
154 
155         List<ExpectedToken> expectedTokens = expectedTokensParser.getExpectedTokens();
156 
157         InputStream smaliStream = LexerTest.class.getClassLoader().getResourceAsStream(smaliFile);
158         if (smaliStream == null) {
159             Assert.fail("Could not load " + smaliFile);
160         }
161         smaliFlexLexer lexer = new smaliFlexLexer(smaliStream);
162         lexer.setSourceFile(new File(test + ".smali"));
163         lexer.setSuppressErrors(true);
164 
165         CommonTokenStream tokenStream = new CommonTokenStream(lexer);
166         tokenStream.fill();
167         List tokens = tokenStream.getTokens();
168 
169         int expectedTokenIndex = 0;
170         CommonToken token;
171         for (int i=0; i<tokens.size()-1; i++) {
172             token = (CommonToken)tokens.get(i);
173 
174             if (discardHiddenTokens && token.getChannel() == smaliParser.HIDDEN) {
175                 continue;
176             }
177 
178             if (expectedTokenIndex >= expectedTokens.size()) {
179                 Assert.fail("Too many tokens");
180             }
181 
182             if (token.getType() == smaliParser.INVALID_TOKEN) {
183                 Assert.assertTrue("Encountered an INVALID_TOKEN not on the error channel",
184                         token.getChannel() == smaliParser.ERROR_CHANNEL);
185             }
186 
187             ExpectedToken expectedToken = expectedTokens.get(expectedTokenIndex++);
188             if (!tokenTypesByName.containsKey(expectedToken.tokenName)) {
189                 Assert.fail("Unknown token: " + expectedToken.tokenName);
190             }
191             int expectedTokenType = tokenTypesByName.get(expectedToken.tokenName);
192 
193             if (token.getType() != expectedTokenType) {
194                 Assert.fail(String.format("Invalid token at index %d. Expecting %s, got %s(%s)",
195                         expectedTokenIndex-1, expectedToken.tokenName, getTokenName(token.getType()), token.getText()));
196             }
197 
198             if (expectedToken.tokenText != null) {
199                 if (!expectedToken.tokenText.equals(token.getText())) {
200                     Assert.fail(
201                             String.format("Invalid token text at index %d. Expecting text \"%s\", got \"%s\"",
202                                     expectedTokenIndex - 1, expectedToken.tokenText, token.getText()));
203                 }
204             }
205         }
206 
207         if (expectedTokenIndex < expectedTokens.size()) {
208             Assert.fail(String.format("Not enough tokens. Expecting %d tokens, but got %d", expectedTokens.size(),
209                     expectedTokenIndex));
210         }
211     }
212 
213 
214 
getTokenName(int tokenType)215     private static String getTokenName(int tokenType) {
216         return smaliParser.tokenNames[tokenType];
217     }
218 }
219