1 /*
2  * Copyright (C) 2016 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 package com.android.tradefed.testtype;
17 
18 import com.android.tradefed.result.ITestLifeCycleReceiver;
19 import com.android.tradefed.result.TestDescription;
20 import com.android.tradefed.util.StreamUtil;
21 
22 import org.easymock.EasyMock;
23 import org.json.JSONException;
24 import org.json.JSONObject;
25 import org.junit.Assert;
26 import org.junit.Test;
27 import org.junit.runner.RunWith;
28 import org.junit.runners.JUnit4;
29 
30 import java.io.IOException;
31 import java.io.InputStream;
32 import java.text.ParseException;
33 import java.text.SimpleDateFormat;
34 import java.util.Collections;
35 import java.util.Date;
36 
37 /**
38  * Unit tests for {@link VtsMultiDeviceTestResultParser}.
39  */
40 @RunWith(JUnit4.class)
41 public class VtsMultiDeviceTestResultParserTest {
42     // Test file paths in the resources
43     private static final String OUTPUT_FILE_1 = "/testtype/vts_multi_device_test_parser_output.txt";
44     private static final String OUTPUT_FILE_2 =
45             "/testtype/vts_multi_device_test_parser_output_timeout.txt";
46     private static final String OUTPUT_FILE_3 =
47             "/testtype/vts_multi_device_test_parser_output_error.txt";
48     private static final String SUMMARY_FILE_NORMAL = "/testtype/test_run_summary_normal.json";
49     private static final String SUMMARY_FILE_CLASS_ERRORS =
50             "/testtype/test_run_summary_class_errors.json";
51 
52     // test name
53     private static final String RUN_NAME = "SampleLightFuzzTest";
54     private static final String TEST_NAME_1 = "testTurnOnLightBlackBoxFuzzing";
55     private static final String TEST_NAME_2 = "testTurnOnLightWhiteBoxFuzzing";
56 
57     // error messages in the summary files.
58     private static final String FAILURE_MESSAGE = "unit test";
59     private static final String CLASS_ERROR_MESSAGE = "class error";
60 
61     /**
62      * Test the run method with a normal input.
63      */
64     @Test
testRunTimeoutInput()65     public void testRunTimeoutInput() throws IOException {
66         String[] contents = getOutput(OUTPUT_FILE_2);
67         long totalTime = getTotalTime(contents);
68 
69         // prepare the mock object
70         ITestLifeCycleReceiver mockRunListener = EasyMock.createMock(ITestLifeCycleReceiver.class);
71         mockRunListener.testRunStarted(TEST_NAME_2, 2);
72         TestDescription test1 = new TestDescription(RUN_NAME, TEST_NAME_2);
73         mockRunListener.testStarted(test1);
74         mockRunListener.testFailed(test1, "TIMEOUT");
75         mockRunListener.testEnded(test1, Collections.emptyMap());
76 
77         TestDescription test2 = new TestDescription(RUN_NAME, TEST_NAME_1);
78         mockRunListener.testStarted(test2);
79         mockRunListener.testEnded(test2, Collections.emptyMap());
80         mockRunListener.testRunEnded(totalTime, Collections.emptyMap());
81 
82         EasyMock.replay(mockRunListener);
83         VtsMultiDeviceTestResultParser resultParser =
84                 new VtsMultiDeviceTestResultParser(mockRunListener, RUN_NAME);
85         resultParser.processNewLines(contents);
86         resultParser.done();
87         EasyMock.verify(mockRunListener);
88     }
89 
90     /**
91      * Test the run method with a normal input.
92      */
93     @Test
testRunNormalInput()94     public void testRunNormalInput() throws IOException {
95         String[] contents = getOutput(OUTPUT_FILE_1);
96         long totalTime = getTotalTime(contents);
97 
98         // prepare the mock object
99         ITestLifeCycleReceiver mockRunListener = EasyMock.createMock(ITestLifeCycleReceiver.class);
100         mockRunListener.testRunStarted(TEST_NAME_2, 2);
101         TestDescription test1 = new TestDescription(RUN_NAME, TEST_NAME_2);
102         mockRunListener.testStarted(test1);
103         mockRunListener.testEnded(test1, Collections.emptyMap());
104 
105         TestDescription test2 = new TestDescription(RUN_NAME, TEST_NAME_1);
106         mockRunListener.testStarted(test2);
107         mockRunListener.testEnded(test2, Collections.emptyMap());
108         mockRunListener.testRunEnded(totalTime, Collections.emptyMap());
109 
110         EasyMock.replay(mockRunListener);
111         VtsMultiDeviceTestResultParser resultParser =
112                 new VtsMultiDeviceTestResultParser(mockRunListener, RUN_NAME);
113         resultParser.processNewLines(contents);
114         resultParser.done();
115         EasyMock.verify(mockRunListener);
116     }
117 
118     /**
119      * Test the run method with a erroneous input.
120      */
121     @Test
testRunErrorInput()122     public void testRunErrorInput() throws IOException {
123         String[] contents = getOutput(OUTPUT_FILE_3);
124         long totalTime = getTotalTime(contents);
125 
126         // prepare the mock object
127         ITestLifeCycleReceiver mockRunListener = EasyMock.createMock(ITestLifeCycleReceiver.class);
128         mockRunListener.testRunStarted(null, 0);
129         mockRunListener.testRunEnded(totalTime, Collections.<String, String>emptyMap());
130 
131         EasyMock.replay(mockRunListener);
132         VtsMultiDeviceTestResultParser resultParser =
133                 new VtsMultiDeviceTestResultParser(mockRunListener, RUN_NAME);
134         resultParser.processNewLines(contents);
135         resultParser.done();
136         EasyMock.verify(mockRunListener);
137     }
138     /**
139      * @param contents The logs that are used for a test case.
140      * @return {long} total running time of the test.
141      */
getTotalTime(String[] contents)142     private long getTotalTime(String[] contents) {
143         Date startDate = getDate(contents, true);
144         Date endDate = getDate(contents, false);
145 
146         if (startDate == null || endDate == null) {
147             return 0;
148         }
149         return endDate.getTime() - startDate.getTime();
150     }
151 
152     /**
153      * Read a resource file as a string.
154      *
155      * @param resource the name of the resource.
156      * @return {String} the contents of the resource.
157      * @throws IOException if fails to read.
158      */
getResourceAsString(String resource)159     private String getResourceAsString(String resource) throws IOException {
160         InputStream input = getClass().getResourceAsStream(resource);
161         Assert.assertNotNull("Cannot load " + resource, input);
162         try {
163             return StreamUtil.getStringFromStream(input);
164         } finally {
165             input.close();
166         }
167     }
168 
169     /**
170      * Returns the sample shell output for a test command.
171      *
172      * @return {String} shell output
173      * @throws IOException
174      */
getOutput(String filePath)175     private String[] getOutput(String filePath) throws IOException {
176         return getResourceAsString(filePath).split("\n");
177     }
178 
179     /**
180      * Return the time in milliseconds to calculate the time elapsed in a particular test.
181      *
182      * @param lines The logs that need to be parsed.
183      * @param calculateStartDate flag which is true if we need to calculate start date.
184      * @return {Date} the start and end time corresponding to a test.
185      */
getDate(String[] lines, boolean calculateStartDate)186     private Date getDate(String[] lines, boolean calculateStartDate) {
187         Date date = null;
188         int begin = calculateStartDate ? 0 : lines.length - 1;
189         int diff = calculateStartDate ? 1 : -1;
190 
191         for (int index = begin; index >= 0 && index < lines.length; index += diff) {
192             lines[index].trim();
193             String[] toks = lines[index].split(" ");
194 
195             // set the start time from the first line
196             // the loop should continue if exception occurs, else it can break
197             if (toks.length < 3) {
198                 continue;
199             }
200             String time = toks[2];
201             SimpleDateFormat sdf = new SimpleDateFormat("HH:mm:ss.SSS");
202             try {
203                 date = sdf.parse(time);
204             } catch (ParseException e) {
205                 continue;
206             }
207             break;
208         }
209         return date;
210     }
211 
212     /*
213      * Test parsing a summary containing passing and failing records.
214      */
215     @Test
testNormalSummary()216     public void testNormalSummary() throws IOException, JSONException {
217         JSONObject object = new JSONObject(getResourceAsString(SUMMARY_FILE_NORMAL));
218 
219         ITestLifeCycleReceiver mockRunListener = EasyMock.createMock(ITestLifeCycleReceiver.class);
220         mockRunListener.testRunStarted(RUN_NAME, 2);
221         TestDescription test1 = new TestDescription(RUN_NAME, TEST_NAME_1);
222         mockRunListener.testStarted(test1, 1525425222367l);
223         mockRunListener.testEnded(test1, 1525425223793l, Collections.emptyMap());
224 
225         TestDescription test2 = new TestDescription(RUN_NAME, TEST_NAME_2);
226         mockRunListener.testStarted(test2, 1525425749536l);
227         mockRunListener.testFailed(test2, FAILURE_MESSAGE);
228         mockRunListener.testEnded(test2, 1525425749537l, Collections.emptyMap());
229         mockRunListener.testRunEnded(EasyMock.anyLong(), EasyMock.eq(Collections.emptyMap()));
230 
231         EasyMock.replay(mockRunListener);
232         VtsMultiDeviceTestResultParser resultParser =
233                 new VtsMultiDeviceTestResultParser(mockRunListener, RUN_NAME);
234         resultParser.processJsonFile(object);
235     }
236 
237     /*
238      * Test parsing a summary containing class error message.
239      */
240     @Test
testClassErrorSummary()241     public void testClassErrorSummary() throws IOException, JSONException {
242         JSONObject object = new JSONObject(getResourceAsString(SUMMARY_FILE_CLASS_ERRORS));
243 
244         ITestLifeCycleReceiver mockRunListener = EasyMock.createMock(ITestLifeCycleReceiver.class);
245         mockRunListener.testRunStarted(RUN_NAME, 1);
246         TestDescription test1 = new TestDescription(RUN_NAME, TEST_NAME_1);
247         mockRunListener.testStarted(test1, 1525424790227l);
248         mockRunListener.testFailed(test1, FAILURE_MESSAGE);
249         mockRunListener.testEnded(test1, 1525424790227l, Collections.emptyMap());
250         mockRunListener.testRunFailed(CLASS_ERROR_MESSAGE);
251         mockRunListener.testRunEnded(EasyMock.anyLong(), EasyMock.eq(Collections.emptyMap()));
252 
253         EasyMock.replay(mockRunListener);
254         VtsMultiDeviceTestResultParser resultParser =
255                 new VtsMultiDeviceTestResultParser(mockRunListener, RUN_NAME);
256         resultParser.processJsonFile(object);
257     }
258 }
259