1 /*
2  * Copyright (C) 2015 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package com.android.compatibility.common.tradefed.result;
18 
19 import com.android.compatibility.common.tradefed.build.CompatibilityBuildHelper;
20 import com.android.compatibility.common.tradefed.build.CompatibilityBuildProvider;
21 import com.android.compatibility.common.util.DeviceInfo;
22 import com.android.compatibility.common.util.ICaseResult;
23 import com.android.compatibility.common.util.IInvocationResult;
24 import com.android.compatibility.common.util.IModuleResult;
25 import com.android.compatibility.common.util.ITestResult;
26 import com.android.compatibility.common.util.TestStatus;
27 import com.android.tradefed.build.IBuildInfo;
28 import com.android.tradefed.config.OptionSetter;
29 import com.android.tradefed.invoker.IInvocationContext;
30 import com.android.tradefed.invoker.InvocationContext;
31 import com.android.tradefed.metrics.proto.MetricMeasurement.Metric;
32 import com.android.tradefed.result.ByteArrayInputStreamSource;
33 import com.android.tradefed.result.InputStreamSource;
34 import com.android.tradefed.result.LogDataType;
35 import com.android.tradefed.result.TestDescription;
36 import com.android.tradefed.util.AbiUtils;
37 import com.android.tradefed.util.FileUtil;
38 
39 import junit.framework.TestCase;
40 
41 import java.io.File;
42 import java.io.FileFilter;
43 import java.util.HashMap;
44 import java.util.List;
45 
46 /**
47  * Unit tests for {@link ResultReporter}
48  */
49 public class ResultReporterTest extends TestCase {
50 
51     private static final String ROOT_PROPERTY = "TESTS_ROOT";
52     private static final String SUITE_NAME = "TESTS";
53     private static final String BUILD_NUMBER = "2";
54     private static final String SUITE_PLAN = "cts";
55     private static final String DYNAMIC_CONFIG_URL = "";
56     private static final String ROOT_DIR_NAME = "root";
57     private static final String BASE_DIR_NAME = "android-tests";
58     private static final String TESTCASES = "testcases";
59     private static final String NAME = "ModuleName";
60     private static final String ABI = "mips64";
61     private static final String ID = AbiUtils.createId(ABI, NAME);
62     private static final String CLASS = "android.test.FoorBar";
63     private static final String METHOD_1 = "testBlah1";
64     private static final String METHOD_2 = "testBlah2";
65     private static final String METHOD_3 = "testBlah3";
66     private static final String TEST_1 = String.format("%s#%s", CLASS, METHOD_1);
67     private static final String TEST_2 = String.format("%s#%s", CLASS, METHOD_2);
68     private static final String TEST_3 = String.format("%s#%s", CLASS, METHOD_3);
69     private static final String STACK_TRACE = "Something small is not alright\n " +
70             "at four.big.insects.Marley.sing(Marley.java:10)";
71     private static final String RESULT_DIR = "result123";
72     private static final String[] FORMATTING_FILES = {
73         "compatibility_result.css",
74         "compatibility_result.xsl",
75         "logo.png"};
76 
77     private ResultReporter mReporter;
78     private IBuildInfo mBuildInfo;
79     private IInvocationContext mContext;
80     private CompatibilityBuildHelper mBuildHelper;
81 
82     private File mRoot = null;
83     private File mBase = null;
84     private File mTests = null;
85 
86     @Override
setUp()87     public void setUp() throws Exception {
88         mReporter = new ResultReporter();
89         mRoot = FileUtil.createTempDir(ROOT_DIR_NAME);
90         mBase = new File(mRoot, BASE_DIR_NAME);
91         mBase.mkdirs();
92         mTests = new File(mBase, TESTCASES);
93         mTests.mkdirs();
94         System.setProperty(ROOT_PROPERTY, mRoot.getAbsolutePath());
95         CompatibilityBuildProvider provider = new CompatibilityBuildProvider() {
96             @Override
97             protected String getSuiteInfoName() {
98                 return SUITE_NAME;
99             }
100             @Override
101             protected String getSuiteInfoBuildNumber() {
102                 return BUILD_NUMBER;
103             }
104             @Override
105             protected String getSuiteInfoVersion() {
106                 return BUILD_NUMBER;
107             }
108         };
109         OptionSetter setter = new OptionSetter(provider);
110         setter.setOptionValue("plan", SUITE_PLAN);
111         setter.setOptionValue("dynamic-config-url", DYNAMIC_CONFIG_URL);
112         mBuildInfo = provider.getBuild();
113         mBuildHelper = new CompatibilityBuildHelper(mBuildInfo);
114         mContext = new InvocationContext();
115         mContext.addDeviceBuildInfo("fakeDevice", mBuildInfo);
116     }
117 
118     @Override
tearDown()119     public void tearDown() throws Exception {
120         mReporter = null;
121         FileUtil.recursiveDelete(mRoot);
122     }
123 
testSetup()124     public void testSetup() throws Exception {
125         mReporter.invocationStarted(mContext);
126         // Should have created a directory for the logs
127         File[] children = mBuildHelper.getLogsDir().listFiles();
128         assertTrue("Didn't create logs dir", children.length == 1 && children[0].isDirectory());
129         // Should have created a directory for the results
130         children = mBuildHelper.getResultsDir().listFiles();
131         assertTrue("Didn't create results dir", children.length == 1 && children[0].isDirectory());
132         mReporter.invocationEnded(10);
133         // Should have created a zip file
134         children = mBuildHelper.getResultsDir().listFiles(new FileFilter() {
135             @Override
136             public boolean accept(File pathname) {
137                 return pathname.getName().endsWith(".zip");
138             }
139         });
140         assertTrue("Didn't create results zip",
141                 children.length == 1 && children[0].isFile() && children[0].length() > 0);
142     }
143 
testResultReporting()144     public void testResultReporting() throws Exception {
145         mReporter.invocationStarted(mContext);
146         mReporter.testRunStarted(ID, 2);
147         TestDescription test1 = new TestDescription(CLASS, METHOD_1);
148         mReporter.testStarted(test1);
149         mReporter.testEnded(test1, new HashMap<String, Metric>());
150         TestDescription test2 = new TestDescription(CLASS, METHOD_2);
151         mReporter.testStarted(test2);
152         mReporter.testFailed(test2, STACK_TRACE);
153         TestDescription test3 = new TestDescription(CLASS, METHOD_3);
154         mReporter.testStarted(test3);
155         mReporter.testFailed(test3, STACK_TRACE);
156         mReporter.testEnded(test3, new HashMap<String, Metric>());
157         mReporter.testRunEnded(10, new HashMap<String, Metric>());
158         mReporter.invocationEnded(10);
159         IInvocationResult result = mReporter.getResult();
160         assertEquals("Expected 1 pass", 1, result.countResults(TestStatus.PASS));
161         assertEquals("Expected 2 failures", 2, result.countResults(TestStatus.FAIL));
162         List<IModuleResult> modules = result.getModules();
163         assertEquals("Expected 1 module", 1, modules.size());
164         IModuleResult module = modules.get(0);
165         assertTrue(module.isDone());
166         assertEquals("Incorrect ID", ID, module.getId());
167         List<ICaseResult> caseResults = module.getResults();
168         assertEquals("Expected 1 test case", 1, caseResults.size());
169         ICaseResult caseResult = caseResults.get(0);
170         List<ITestResult> testResults = caseResult.getResults();
171         assertEquals("Expected 3 tests", 3, testResults.size());
172         ITestResult result1 = caseResult.getResult(METHOD_1);
173         assertNotNull(String.format("Expected result for %s", TEST_1), result1);
174         assertEquals(String.format("Expected pass for %s", TEST_1), TestStatus.PASS,
175                 result1.getResultStatus());
176         ITestResult result2 = caseResult.getResult(METHOD_2);
177         assertNotNull(String.format("Expected result for %s", TEST_2), result2);
178         assertEquals(String.format("Expected fail for %s", TEST_2), TestStatus.FAIL,
179                 result2.getResultStatus());
180         ITestResult result3 = caseResult.getResult(METHOD_3);
181         assertNotNull(String.format("Expected result for %s", TEST_3), result3);
182         assertEquals(String.format("Expected fail for %s", TEST_3), TestStatus.FAIL,
183                 result3.getResultStatus());
184     }
185 
makeTestRun(String[] methods, boolean[] passes)186     private void makeTestRun(String[] methods, boolean[] passes) {
187         mReporter.testRunStarted(ID, methods.length);
188 
189         for (int i = 0; i < methods.length; i++) {
190             TestDescription test = new TestDescription(CLASS, methods[i]);
191             mReporter.testStarted(test);
192             if (!passes[i]) {
193                 mReporter.testFailed(test, STACK_TRACE);
194             }
195             mReporter.testEnded(test, new HashMap<String, Metric>());
196         }
197 
198         mReporter.testRunEnded(10, new HashMap<String, Metric>());
199     }
200 
testRepeatedExecutions()201     public void testRepeatedExecutions() throws Exception {
202         String[] methods = new String[] {METHOD_1, METHOD_2, METHOD_3};
203 
204         mReporter.invocationStarted(mContext);
205 
206         makeTestRun(methods, new boolean[] {true, false, true});
207         makeTestRun(methods, new boolean[] {true, false, false});
208         makeTestRun(methods, new boolean[] {true, true, true});
209 
210         mReporter.invocationEnded(10);
211 
212         // Verification
213 
214         IInvocationResult result = mReporter.getResult();
215         assertEquals("Expected 1 pass", 1, result.countResults(TestStatus.PASS));
216         assertEquals("Expected 2 failures", 2, result.countResults(TestStatus.FAIL));
217         List<IModuleResult> modules = result.getModules();
218         assertEquals("Expected 1 module", 1, modules.size());
219         IModuleResult module = modules.get(0);
220         assertEquals("Incorrect ID", ID, module.getId());
221         List<ICaseResult> caseResults = module.getResults();
222         assertEquals("Expected 1 test case", 1, caseResults.size());
223         ICaseResult caseResult = caseResults.get(0);
224         List<ITestResult> testResults = caseResult.getResults();
225         assertEquals("Expected 3 tests", 3, testResults.size());
226 
227         // Test 1 details
228         ITestResult result1 = caseResult.getResult(METHOD_1);
229         assertNotNull(String.format("Expected result for %s", TEST_1), result1);
230         assertEquals(String.format("Expected pass for %s", TEST_1), TestStatus.PASS,
231                 result1.getResultStatus());
232 
233         // Test 2 details
234         ITestResult result2 = caseResult.getResult(METHOD_2);
235         assertNotNull(String.format("Expected result for %s", TEST_2), result2);
236         assertEquals(String.format("Expected fail for %s", TEST_2), TestStatus.FAIL,
237                 result2.getResultStatus());
238         // TODO: Define requirement. Should this result have multiple stack traces?
239         assertEquals(result2.getStackTrace(), STACK_TRACE);
240 
241         // Test 3 details
242         ITestResult result3 = caseResult.getResult(METHOD_3);
243         assertNotNull(String.format("Expected result for %s", TEST_3), result3);
244         assertEquals(String.format("Expected fail for %s", TEST_3), TestStatus.FAIL,
245                 result3.getResultStatus());
246         assertEquals(result3.getStackTrace(), STACK_TRACE);
247     }
248 
testRetry()249     public void testRetry() throws Exception {
250         mReporter.invocationStarted(mContext);
251 
252         // Set up IInvocationResult with existing results from previous session
253         mReporter.testRunStarted(ID, 2);
254         IInvocationResult invocationResult = mReporter.getResult();
255         IModuleResult moduleResult = invocationResult.getOrCreateModule(ID);
256         ICaseResult caseResult = moduleResult.getOrCreateResult(CLASS);
257         ITestResult testResult1 = caseResult.getOrCreateResult(METHOD_1);
258         testResult1.setResultStatus(TestStatus.PASS);
259         testResult1.setRetry(true);
260         ITestResult testResult2 = caseResult.getOrCreateResult(METHOD_2);
261         testResult2.setResultStatus(TestStatus.FAIL);
262         testResult2.setStackTrace(STACK_TRACE);
263         testResult2.setRetry(true);
264 
265         // Flip results for the current session
266         TestDescription test1 = new TestDescription(CLASS, METHOD_1);
267         mReporter.testStarted(test1);
268         mReporter.testFailed(test1, STACK_TRACE);
269         mReporter.testEnded(test1, new HashMap<String, Metric>());
270         TestDescription test2 = new TestDescription(CLASS, METHOD_2);
271         mReporter.testStarted(test2);
272         mReporter.testEnded(test2, new HashMap<String, Metric>());
273 
274         mReporter.testRunEnded(10, new HashMap<String, Metric>());
275         mReporter.invocationEnded(10);
276 
277         // Verification that results have been overwritten.
278         IInvocationResult result = mReporter.getResult();
279         assertEquals("Expected 1 pass", 1, result.countResults(TestStatus.PASS));
280         assertEquals("Expected 1 failure", 1, result.countResults(TestStatus.FAIL));
281         List<IModuleResult> modules = result.getModules();
282         assertEquals("Expected 1 module", 1, modules.size());
283         IModuleResult module = modules.get(0);
284         List<ICaseResult> cases = module.getResults();
285         assertEquals("Expected 1 test case", 1, cases.size());
286         ICaseResult case1 = cases.get(0);
287         List<ITestResult> testResults = case1.getResults();
288         assertEquals("Expected 2 tests", 2, testResults.size());
289 
290         // Test 1 details
291         ITestResult finalTestResult1 = case1.getResult(METHOD_1);
292         assertNotNull(String.format("Expected result for %s", TEST_1), finalTestResult1);
293         assertEquals(String.format("Expected fail for %s", TEST_1), TestStatus.FAIL,
294                 finalTestResult1.getResultStatus());
295         assertEquals(finalTestResult1.getStackTrace(), STACK_TRACE);
296 
297         // Test 2 details
298         ITestResult finalTestResult2 = case1.getResult(METHOD_2);
299         assertNotNull(String.format("Expected result for %s", TEST_2), finalTestResult2);
300         assertEquals(String.format("Expected pass for %s", TEST_2), TestStatus.PASS,
301                 finalTestResult2.getResultStatus());
302     }
303 
testRetryCanSetDone()304     public void testRetryCanSetDone() throws Exception {
305         mReporter.invocationStarted(mContext);
306         // Set mCanMarkDone directly (otherwise we must build result directory, write XML, and
307         // perform actual retry)
308         mReporter.mCanMarkDone = true;
309         // Set up IInvocationResult with existing results from previous session
310         IInvocationResult invocationResult = mReporter.getResult();
311         IModuleResult moduleResult = invocationResult.getOrCreateModule(ID);
312         moduleResult.initializeDone(false);
313         ICaseResult caseResult = moduleResult.getOrCreateResult(CLASS);
314         ITestResult testResult1 = caseResult.getOrCreateResult(METHOD_1);
315         testResult1.setResultStatus(TestStatus.PASS);
316         testResult1.setRetry(true);
317         ITestResult testResult2 = caseResult.getOrCreateResult(METHOD_2);
318         testResult2.setResultStatus(TestStatus.FAIL);
319         testResult2.setStackTrace(STACK_TRACE);
320         testResult2.setRetry(true);
321 
322         // Assume no additional filtering is applied to retry, and all tests for the module have
323         // been collected. Thus, module "done" value should switch.
324         mReporter.testRunStarted(ID, 1);
325 
326         TestDescription test2 = new TestDescription(CLASS, METHOD_2);
327         mReporter.testStarted(test2);
328         mReporter.testEnded(test2, new HashMap<String, Metric>());
329 
330         mReporter.testRunEnded(10, new HashMap<String, Metric>());
331         mReporter.invocationEnded(10);
332 
333         // Verification that results have been overwritten.
334         IInvocationResult result = mReporter.getResult();
335         assertEquals("Expected 2 pass", 2, result.countResults(TestStatus.PASS));
336         assertEquals("Expected 0 failures", 0, result.countResults(TestStatus.FAIL));
337         List<IModuleResult> modules = result.getModules();
338         assertEquals("Expected 1 module", 1, modules.size());
339         IModuleResult module = modules.get(0);
340         assertTrue("Module should be marked done", module.isDone());
341     }
342 
testRetryCannotSetDone()343     public void testRetryCannotSetDone() throws Exception {
344         mReporter.invocationStarted(mContext);
345         // Set mCanMarkDone directly (otherwise we must build result directory, write XML, and
346         // perform actual retry)
347         mReporter.mCanMarkDone = false;
348         // Set up IInvocationResult with existing results from previous session
349         IInvocationResult invocationResult = mReporter.getResult();
350         IModuleResult moduleResult = invocationResult.getOrCreateModule(ID);
351         moduleResult.setDone(false);
352         ICaseResult caseResult = moduleResult.getOrCreateResult(CLASS);
353         ITestResult testResult1 = caseResult.getOrCreateResult(METHOD_1);
354         testResult1.setResultStatus(TestStatus.PASS);
355         testResult1.setRetry(true);
356         ITestResult testResult2 = caseResult.getOrCreateResult(METHOD_2);
357         testResult2.setResultStatus(TestStatus.FAIL);
358         testResult2.setStackTrace(STACK_TRACE);
359         testResult2.setRetry(true);
360 
361         // Since using retry-type failed option, we only run previously failed test
362         // and don't run any non-executed tests, so module "done" value should not switch.
363         mReporter.testRunStarted(ID, 1);
364 
365         TestDescription test2 = new TestDescription(CLASS, METHOD_2);
366         mReporter.testStarted(test2);
367         mReporter.testEnded(test2, new HashMap<String, Metric>());
368 
369         mReporter.testRunEnded(10, new HashMap<String, Metric>());
370         mReporter.invocationEnded(10);
371 
372         // Verification that results have been overwritten.
373         IInvocationResult result = mReporter.getResult();
374         assertEquals("Expected 2 pass", 2, result.countResults(TestStatus.PASS));
375         assertEquals("Expected 0 failures", 0, result.countResults(TestStatus.FAIL));
376         List<IModuleResult> modules = result.getModules();
377         assertEquals("Expected 1 module", 1, modules.size());
378         IModuleResult module = modules.get(0);
379         assertFalse("Module should not be marked done", module.isDone());
380     }
381 
testResultReporting_moduleNotDone()382     public void testResultReporting_moduleNotDone() throws Exception {
383         mReporter.invocationStarted(mContext);
384         mReporter.testRunStarted(ID, 2);
385         TestDescription test1 = new TestDescription(CLASS, METHOD_1);
386         mReporter.testStarted(test1);
387         mReporter.testEnded(test1, new HashMap<String, Metric>());
388         mReporter.testRunFailed("error");
389         mReporter.testRunEnded(10, new HashMap<String, Metric>());
390         mReporter.invocationEnded(10);
391         IInvocationResult result = mReporter.getResult();
392         assertEquals("Expected 1 pass", 1, result.countResults(TestStatus.PASS));
393         assertEquals("Expected 0 failures", 0, result.countResults(TestStatus.FAIL));
394         List<IModuleResult> modules = result.getModules();
395         assertEquals("Expected 1 module", 1, modules.size());
396         IModuleResult module = modules.get(0);
397 
398         // Ensure module is reported as not done
399         assertFalse(module.isDone());
400         assertEquals("Incorrect ID", ID, module.getId());
401         List<ICaseResult> caseResults = module.getResults();
402         assertEquals("Expected 1 test case", 1, caseResults.size());
403         ICaseResult caseResult = caseResults.get(0);
404         List<ITestResult> testResults = caseResult.getResults();
405         assertEquals("Expected 1 tests", 1, testResults.size());
406         ITestResult result1 = caseResult.getResult(METHOD_1);
407         assertNotNull(String.format("Expected result for %s", TEST_1), result1);
408         assertEquals(String.format("Expected pass for %s", TEST_1), TestStatus.PASS,
409                 result1.getResultStatus());
410     }
411 
testResultReporting_moduleNotDone_noTests()412     public void testResultReporting_moduleNotDone_noTests() throws Exception {
413         mReporter.invocationStarted(mContext);
414         mReporter.testRunStarted(ID, 0);
415         mReporter.testRunFailed("error"); // test run failure should prevent marking module "done"
416         mReporter.testRunEnded(10, new HashMap<String, String>());
417         mReporter.invocationEnded(10);
418         IInvocationResult result = mReporter.getResult();
419         assertEquals("Expected 0 pass", 0, result.countResults(TestStatus.PASS));
420         assertEquals("Expected 0 failures", 0, result.countResults(TestStatus.FAIL));
421         List<IModuleResult> modules = result.getModules();
422         assertEquals("Expected 1 module", 1, modules.size());
423         IModuleResult module = modules.get(0);
424         assertEquals("Incorrect ID", ID, module.getId());
425         // Ensure module is reported as not done
426         assertFalse(module.isDone());
427     }
428 
testResultReporting_moduleDone_noTests()429     public void testResultReporting_moduleDone_noTests() throws Exception {
430         mReporter.invocationStarted(mContext);
431         mReporter.testRunStarted(ID, 0);
432         // Lack of test run failure should allow module to be marked "done"
433         mReporter.testRunEnded(10, new HashMap<String, String>());
434         mReporter.invocationEnded(10);
435         IInvocationResult result = mReporter.getResult();
436         assertEquals("Expected 0 pass", 0, result.countResults(TestStatus.PASS));
437         assertEquals("Expected 0 failures", 0, result.countResults(TestStatus.FAIL));
438         List<IModuleResult> modules = result.getModules();
439         assertEquals("Expected 1 module", 1, modules.size());
440         IModuleResult module = modules.get(0);
441         assertEquals("Incorrect ID", ID, module.getId());
442         // Ensure module is reported as done
443         assertTrue(module.isDone());
444     }
445 
testCopyFormattingFiles()446     public void testCopyFormattingFiles() throws Exception {
447         File resultDir = new File(mBuildHelper.getResultsDir(), RESULT_DIR);
448         resultDir.mkdirs();
449         ResultReporter.copyFormattingFiles(resultDir, SUITE_NAME);
450         for (String filename : FORMATTING_FILES) {
451             File file = new File(resultDir, filename);
452             assertTrue(String.format("%s (%s) was not created", filename, file.getAbsolutePath()),
453                     file.exists() && file.isFile() && file.length() > 0);
454         }
455     }
456 
457     /**
458      * Ensure that when {@link ResultReporter#testLog(String, LogDataType, InputStreamSource)} is
459      * called, a single invocation result folder is created and populated.
460      */
testTestLog()461     public void testTestLog() throws Exception {
462         InputStreamSource fakeData = new ByteArrayInputStreamSource("test".getBytes());
463         mReporter.invocationStarted(mContext);
464         mReporter.testLog("test1", LogDataType.LOGCAT, fakeData);
465         // date folder
466         assertEquals(1, mBuildHelper.getLogsDir().list().length);
467         // inv_ folder
468         assertEquals(1, mBuildHelper.getLogsDir().listFiles()[0].list().length);
469         // actual logs
470         assertEquals(1, mBuildHelper.getLogsDir().listFiles()[0].listFiles()[0].list().length);
471         mReporter.testLog("test2", LogDataType.LOGCAT, fakeData);
472         // date folder
473         assertEquals(1, mBuildHelper.getLogsDir().list().length);
474         // inv_ folder
475         assertEquals(1, mBuildHelper.getLogsDir().listFiles()[0].list().length);
476         // actual logs
477         assertEquals(2, mBuildHelper.getLogsDir().listFiles()[0].listFiles()[0].list().length);
478     }
479 
480     /**
481      * Ensure that when {@link ResultReporter#testLog(String, LogDataType, InputStreamSource)} is
482      * called for host-side device info, a device info file is created in the result
483      */
testTestLogWithDeviceInfo()484     public void testTestLogWithDeviceInfo() throws Exception {
485         InputStreamSource fakeData = new ByteArrayInputStreamSource("test".getBytes());
486         String deviceInfoName = String.format("Test%s", DeviceInfo.FILE_SUFFIX);
487         mReporter.invocationStarted(mContext);
488         mReporter.testLog(deviceInfoName, LogDataType.TEXT, fakeData);
489         File deviceInfoFolder = new File(mBuildHelper.getResultDir(), DeviceInfo.RESULT_DIR_NAME);
490         // assert device info folder was created
491         assertTrue(deviceInfoFolder.exists());
492         File[] deviceInfoFiles = deviceInfoFolder.listFiles();
493         // assert that one file was written to the folder
494         assertEquals(1, deviceInfoFiles.length);
495         File deviceInfoFile = deviceInfoFiles[0];
496         // assert device info file has been named correctly
497         assertEquals(deviceInfoName, deviceInfoFile.getName());
498         // assert contents of the file
499         assertEquals("test", FileUtil.readStringFromFile(deviceInfoFile));
500     }
501 
502     /** Ensure that the module is not marked done if any of the shard fails. */
testResultReporter_sharded()503     public void testResultReporter_sharded() throws Exception {
504         ResultReporter shard1 = new ResultReporter(mReporter);
505         ResultReporter shard2 = new ResultReporter(mReporter);
506 
507         mReporter.invocationStarted(mContext);
508         shard1.invocationStarted(mContext);
509         shard2.invocationStarted(mContext);
510 
511         // First shard is good
512         shard1.testRunStarted(ID, 1);
513         TestDescription test1 = new TestDescription(CLASS, METHOD_1);
514         shard1.testStarted(test1);
515         shard1.testEnded(test1, new HashMap<String, Metric>());
516         shard1.testRunEnded(10, new HashMap<String, Metric>());
517         shard1.invocationEnded(10);
518         // Second shard failed
519         shard2.testRunStarted(ID, 2);
520         TestDescription test2 = new TestDescription(CLASS, METHOD_2);
521         shard2.testStarted(test2);
522         shard2.testEnded(test2, new HashMap<String, Metric>());
523         shard2.testRunFailed("error");
524         shard2.testRunEnded(10, new HashMap<String, Metric>());
525         shard2.invocationEnded(10);
526 
527         IInvocationResult result = mReporter.getResult();
528         assertEquals("Expected 2 pass", 2, result.countResults(TestStatus.PASS));
529         assertEquals("Expected 0 failures", 0, result.countResults(TestStatus.FAIL));
530         List<IModuleResult> modules = result.getModules();
531         assertEquals("Expected 1 module", 1, modules.size());
532         IModuleResult module = modules.get(0);
533 
534         // Ensure module is seen as not done and failed
535         assertFalse(module.isDone());
536         assertTrue(module.isFailed());
537 
538         assertEquals("Incorrect ID", ID, module.getId());
539         List<ICaseResult> caseResults = module.getResults();
540         assertEquals("Expected 1 test run", 1, caseResults.size());
541         ICaseResult caseResult = caseResults.get(0);
542         List<ITestResult> testResults = caseResult.getResults();
543         assertEquals("Expected 2 test cases", 2, testResults.size());
544         ITestResult result1 = caseResult.getResult(METHOD_1);
545         assertNotNull(String.format("Expected result for %s", TEST_1), result1);
546         assertEquals(
547                 String.format("Expected pass for %s", TEST_1),
548                 TestStatus.PASS,
549                 result1.getResultStatus());
550     }
551 
552     /** Ensure that the run history of the current run is added to previous run history. */
testRetryWithRunHistory()553     public void testRetryWithRunHistory() throws Exception {
554         mReporter.invocationStarted(mContext);
555 
556         // Set up IInvocationResult with existing results from previous session
557         mReporter.testRunStarted(ID, 2);
558         IInvocationResult invocationResult = mReporter.getResult();
559         IModuleResult moduleResult = invocationResult.getOrCreateModule(ID);
560         ICaseResult caseResult = moduleResult.getOrCreateResult(CLASS);
561         ITestResult testResult1 = caseResult.getOrCreateResult(METHOD_1);
562         testResult1.setResultStatus(TestStatus.PASS);
563         testResult1.setRetry(true);
564         ITestResult testResult2 = caseResult.getOrCreateResult(METHOD_2);
565         testResult2.setResultStatus(TestStatus.FAIL);
566         testResult2.setStackTrace(STACK_TRACE);
567         testResult2.setRetry(true);
568         // Set up IInvocationResult with the run history of previous runs.
569         invocationResult.addInvocationInfo(
570                 "run_history", "[{\"startTime\":1,\"endTime\":2},{\"startTime\":3,\"endTime\":4}]");
571 
572         // Flip results for the current session
573         TestDescription test1 = new TestDescription(CLASS, METHOD_1);
574         mReporter.testStarted(test1);
575         mReporter.testFailed(test1, STACK_TRACE);
576         mReporter.testEnded(test1, new HashMap<String, Metric>());
577         TestDescription test2 = new TestDescription(CLASS, METHOD_2);
578         mReporter.testStarted(test2);
579         mReporter.testEnded(test2, new HashMap<String, Metric>());
580 
581         mReporter.testRunEnded(10, new HashMap<String, Metric>());
582         mReporter.invocationEnded(10);
583 
584         // Verification that results have been overwritten.
585         IInvocationResult result = mReporter.getResult();
586         assertEquals("Expected 1 pass", 1, result.countResults(TestStatus.PASS));
587         assertEquals("Expected 1 failure", 1, result.countResults(TestStatus.FAIL));
588         List<IModuleResult> modules = result.getModules();
589         assertEquals("Expected 1 module", 1, modules.size());
590         IModuleResult module = modules.get(0);
591         List<ICaseResult> cases = module.getResults();
592         assertEquals("Expected 1 test case", 1, cases.size());
593         ICaseResult case1 = cases.get(0);
594         List<ITestResult> testResults = case1.getResults();
595         assertEquals("Expected 2 tests", 2, testResults.size());
596 
597         long startTime = mReporter.getResult().getStartTime();
598         String expectedRunHistory =
599                 String.format(
600                         "[{\"startTime\":1,\"endTime\":2},"
601                                 + "{\"startTime\":3,\"endTime\":4},{\"startTime\":%d,\"endTime\":%d}]",
602                         startTime, startTime + 10);
603         assertEquals(expectedRunHistory, invocationResult.getInvocationInfo().get("run_history"));
604 
605         // Test 1 details
606         ITestResult finalTestResult1 = case1.getResult(METHOD_1);
607         assertNotNull(String.format("Expected result for %s", TEST_1), finalTestResult1);
608         assertEquals(
609                 String.format("Expected fail for %s", TEST_1),
610                 TestStatus.FAIL,
611                 finalTestResult1.getResultStatus());
612         assertEquals(finalTestResult1.getStackTrace(), STACK_TRACE);
613 
614         // Test 2 details
615         ITestResult finalTestResult2 = case1.getResult(METHOD_2);
616         assertNotNull(String.format("Expected result for %s", TEST_2), finalTestResult2);
617         assertEquals(
618                 String.format("Expected pass for %s", TEST_2),
619                 TestStatus.PASS,
620                 finalTestResult2.getResultStatus());
621     }
622 }
623