1 /*
2  * Copyright (C) 2018 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 package com.android.tradefed.testtype.suite;
17 
18 import static org.junit.Assert.assertEquals;
19 import static org.junit.Assert.assertTrue;
20 
21 import com.android.ddmlib.testrunner.TestResult.TestStatus;
22 import com.android.tradefed.config.IConfiguration;
23 import com.android.tradefed.device.DeviceNotAvailableException;
24 import com.android.tradefed.device.DeviceUnresponsiveException;
25 import com.android.tradefed.device.metric.IMetricCollector;
26 import com.android.tradefed.invoker.InvocationContext;
27 import com.android.tradefed.metrics.proto.MetricMeasurement.Metric;
28 import com.android.tradefed.testtype.IRemoteTest;
29 import com.android.tradefed.testtype.ITestFilterReceiver;
30 import com.android.tradefed.result.CollectingTestListener;
31 import com.android.tradefed.result.FileSystemLogSaver;
32 import com.android.tradefed.result.ITestInvocationListener;
33 import com.android.tradefed.result.TestDescription;
34 import com.android.tradefed.result.TestResult;
35 import com.android.tradefed.result.TestRunResult;
36 
37 import org.junit.Test;
38 import org.junit.runner.RunWith;
39 import org.junit.runners.JUnit4;
40 import org.mockito.Mockito;
41 
42 import java.util.ArrayList;
43 import java.util.Collections;
44 import java.util.HashMap;
45 import java.util.List;
46 import java.util.Map;
47 import java.util.Set;
48 
49 /** Unit tests for {@link com.android.tradefed.testtype.suite.GranularRetriableTestWrapper}. */
50 @RunWith(JUnit4.class)
51 public class GranularRetriableTestWrapperTest {
52 
53     private class BasicFakeTest implements IRemoteTest {
54 
55         protected ArrayList<TestDescription> mTestCases;
56         private Map<TestDescription, Boolean> mShouldFail;
57 
BasicFakeTest()58         public BasicFakeTest() {
59             mTestCases = new ArrayList<>();
60             TestDescription defaultTestCase = new TestDescription("ClassFoo", "TestFoo");
61             mTestCases.add(defaultTestCase);
62             mShouldFail = new HashMap<TestDescription, Boolean>();
63             mShouldFail.put(defaultTestCase, false);
64         }
65 
BasicFakeTest(ArrayList<TestDescription> testCases)66         public BasicFakeTest(ArrayList<TestDescription> testCases) {
67             mTestCases = testCases;
68             mShouldFail = new HashMap<TestDescription, Boolean>();
69             for (TestDescription testCase : testCases) {
70                 mShouldFail.put(testCase, false);
71             }
72         }
73 
setFailedTestCase(TestDescription testCase)74         public void setFailedTestCase(TestDescription testCase) {
75             mShouldFail.put(testCase, true);
76         }
77 
78         @Override
run(ITestInvocationListener listener)79         public void run(ITestInvocationListener listener) throws DeviceUnresponsiveException {
80             listener.testRunStarted("test run", mTestCases.size());
81             for (TestDescription td : mTestCases) {
82                 listener.testStarted(td);
83                 if (mShouldFail.get(td)) {
84                     listener.testFailed(td, String.format("Fake failure %s", td.toString()));
85                 }
86                 listener.testEnded(td, new HashMap<String, Metric>());
87             }
88             listener.testRunEnded(0, new HashMap<String, Metric>());
89         }
90     }
91 
92     private class FakeTest extends BasicFakeTest implements ITestFilterReceiver {
93 
FakeTest(ArrayList<TestDescription> testCases)94         public FakeTest(ArrayList<TestDescription> testCases) {
95             super(testCases);
96         }
97 
FakeTest()98         public FakeTest() {
99             super();
100         }
101 
102         @Override
addIncludeFilter(String filter)103         public void addIncludeFilter(String filter) {
104             String[] descriptionStr = filter.split("#");
105             mTestCases = new ArrayList<>();
106             mTestCases.add(new TestDescription(descriptionStr[0], descriptionStr[1]));
107         }
108 
109         @Override
addAllIncludeFilters(Set<String> filters)110         public void addAllIncludeFilters(Set<String> filters) {}
111 
112         @Override
addExcludeFilter(String filters)113         public void addExcludeFilter(String filters) {}
114 
115         @Override
addAllExcludeFilters(Set<String> filters)116         public void addAllExcludeFilters(Set<String> filters) {}
117     }
118 
createGranularTestWrapper( IRemoteTest test, int maxRunCount)119     public GranularRetriableTestWrapper createGranularTestWrapper(
120             IRemoteTest test, int maxRunCount) {
121         GranularRetriableTestWrapper granularTestWrapper =
122                 new GranularRetriableTestWrapper(test, null, null, maxRunCount);
123         granularTestWrapper.setModuleId("test module");
124         granularTestWrapper.setMarkTestsSkipped(false);
125         granularTestWrapper.setMetricCollectors(new ArrayList<IMetricCollector>());
126         // Setup InvocationContext.
127         granularTestWrapper.setInvocationContext(new InvocationContext());
128         // Setup logsaver.
129         granularTestWrapper.setLogSaver(new FileSystemLogSaver());
130         IConfiguration mockModuleConfiguration = Mockito.mock(IConfiguration.class);
131         granularTestWrapper.setModuleConfig(mockModuleConfiguration);
132         return granularTestWrapper;
133     }
134 
135     /**
136      * Test the intra module "run" triggers IRemoteTest run method with a dedicated ModuleListener.
137      */
138     @Test
testIntraModuleRun_pass()139     public void testIntraModuleRun_pass() throws Exception {
140         GranularRetriableTestWrapper granularTestWrapper =
141                 createGranularTestWrapper(new FakeTest(), 1);
142         assertEquals(0, granularTestWrapper.getTestRunResultCollector().size());
143         granularTestWrapper.intraModuleRun();
144         assertEquals(1, granularTestWrapper.getTestRunResultCollector().size());
145         Set<TestDescription> completedTests =
146                 granularTestWrapper.getFinalTestRunResult().getCompletedTests();
147         assertEquals(1, completedTests.size());
148         assertEquals("ClassFoo#TestFoo", completedTests.toArray()[0].toString());
149     }
150 
151     /**
152      * Test that the intra module "run" method catches DeviceNotAvailableException and raises it
153      * after record the tests.
154      */
155     @Test(expected = DeviceNotAvailableException.class)
testIntraModuleRun_catchDeviceNotAvailableException()156     public void testIntraModuleRun_catchDeviceNotAvailableException() throws Exception {
157         IRemoteTest mockTest = Mockito.mock(IRemoteTest.class);
158         Mockito.doThrow(new DeviceNotAvailableException("fake message", "serial"))
159                 .when(mockTest)
160                 .run(Mockito.any(ITestInvocationListener.class));
161         GranularRetriableTestWrapper granularTestWrapper = createGranularTestWrapper(mockTest, 1);
162         // Verify.
163         granularTestWrapper.intraModuleRun();
164     }
165 
166     /**
167      * Test that the intra module "run" method catches DeviceUnresponsiveException and doesn't raise
168      * it again.
169      */
170     @Test
testIntraModuleRun_catchDeviceUnresponsiveException()171     public void testIntraModuleRun_catchDeviceUnresponsiveException() throws Exception {
172         FakeTest test =
173                 new FakeTest() {
174                     @Override
175                     public void run(ITestInvocationListener listener)
176                             throws DeviceUnresponsiveException {
177                         listener.testRunStarted("test run", 1);
178                         throw new DeviceUnresponsiveException("fake message", "serial");
179                     }
180                 };
181         GranularRetriableTestWrapper granularTestWrapper = createGranularTestWrapper(test, 1);
182         granularTestWrapper.intraModuleRun();
183         TestRunResult finalResult = granularTestWrapper.getTestRunResultCollector().get(0);
184         assertTrue(finalResult.isRunFailure());
185     }
186 
187     /**
188      * Test that the "run" method has built-in retry logic and each run has an individual
189      * ModuleListener and TestRunResult.
190      */
191     @Test
testRun_withMultipleRun()192     public void testRun_withMultipleRun() throws Exception {
193         ArrayList<TestDescription> testCases = new ArrayList<>();
194         TestDescription fakeTestCase = new TestDescription("Class", "Test");
195         testCases.add(fakeTestCase);
196         FakeTest test = new FakeTest(testCases);
197         test.setFailedTestCase(fakeTestCase);
198 
199         int maxRunCount = 5;
200         GranularRetriableTestWrapper granularTestWrapper =
201                 createGranularTestWrapper(test, maxRunCount);
202         granularTestWrapper.run(new CollectingTestListener());
203         // Verify the test has run 5 times.
204         assertEquals(maxRunCount, granularTestWrapper.getTestRunResultCollector().size());
205         Map<TestDescription, TestResult> testResults =
206                 granularTestWrapper.getFinalTestRunResult().getTestResults();
207         testResults.containsKey(fakeTestCase);
208         // Verify the final TestRunResult is a merged value of every retried TestRunResults.
209         assertEquals(TestStatus.FAILURE, testResults.get(fakeTestCase).getStatus());
210         String stacktrace =
211                 String.join("\n", Collections.nCopies(maxRunCount, "Fake failure Class#Test"));
212         assertEquals(stacktrace, testResults.get(fakeTestCase).getStackTrace());
213     }
214 
215     /**
216      * Test that the "run" method only retry failed test cases, and merge the multiple test results
217      * to a single TestRunResult.
218      */
219     @Test
testRun_retryOnFailedTestCaseOnly()220     public void testRun_retryOnFailedTestCaseOnly() throws Exception {
221         ArrayList<TestDescription> testCases = new ArrayList<>();
222         TestDescription fakeTestCase1 = new TestDescription("Class1", "Test1");
223         TestDescription fakeTestCase2 = new TestDescription("Class2", "Test2");
224         testCases.add(fakeTestCase1);
225         testCases.add(fakeTestCase2);
226         FakeTest test = new FakeTest(testCases);
227         // Only the first testcase is failed and retried.
228         test.setFailedTestCase(fakeTestCase1);
229         // Run each testcases (if failed) max to 3 times.
230         int maxRunCount = 3;
231         GranularRetriableTestWrapper granularTestWrapper =
232                 createGranularTestWrapper(test, maxRunCount);
233         granularTestWrapper.run(new CollectingTestListener());
234 
235         TestRunResult finalResult = granularTestWrapper.getFinalTestRunResult();
236         assertEquals(
237                 TestStatus.FAILURE, finalResult.getTestResults().get(fakeTestCase1).getStatus());
238         assertEquals(
239                 TestStatus.PASSED, finalResult.getTestResults().get(fakeTestCase2).getStatus());
240         // Verify the test has 3 TestRunResults, indicating it runs 3 times. And only failed test
241         // case is in the retried TestRunResult.
242         assertEquals(maxRunCount, granularTestWrapper.getTestRunResultCollector().size());
243         List<TestRunResult> resultCollector = granularTestWrapper.getTestRunResultCollector();
244         TestRunResult latestRunResult = resultCollector.get(resultCollector.size() - 1);
245         assertEquals(1, latestRunResult.getNumTests());
246         latestRunResult.getTestResults().containsKey(fakeTestCase1);
247     }
248 
249     /**
250      * Test that if IRemoteTest doesn't implement ITestFilterReceiver, the "run" method will retry
251      * all test cases.
252      */
253     @Test
testRun_retryAllTestCasesIfNotSupportTestFilterReceiver()254     public void testRun_retryAllTestCasesIfNotSupportTestFilterReceiver() throws Exception {
255         ArrayList<TestDescription> testCases = new ArrayList<>();
256         TestDescription fakeTestCase1 = new TestDescription("Class1", "Test1");
257         TestDescription fakeTestCase2 = new TestDescription("Class2", "Test2");
258         testCases.add(fakeTestCase1);
259         testCases.add(fakeTestCase2);
260         BasicFakeTest test = new BasicFakeTest(testCases);
261         // Only the first testcase is failed.
262         test.setFailedTestCase(fakeTestCase1);
263         // Run each testcases (if has failure) max to 3 times.
264         int maxRunCount = 3;
265         GranularRetriableTestWrapper granularTestWrapper =
266                 createGranularTestWrapper(test, maxRunCount);
267         granularTestWrapper.run(new CollectingTestListener());
268         // Verify the test has 3 TestRunResults, indicating it runs 3 times. And all test cases
269         // are retried.
270         assertEquals(maxRunCount, granularTestWrapper.getTestRunResultCollector().size());
271         List<TestRunResult> resultCollector = granularTestWrapper.getTestRunResultCollector();
272         for (TestRunResult runResult : resultCollector) {
273             assertEquals(2, runResult.getNumTests());
274             assertEquals(
275                     TestStatus.FAILURE, runResult.getTestResults().get(fakeTestCase1).getStatus());
276             assertEquals(
277                     TestStatus.PASSED, runResult.getTestResults().get(fakeTestCase2).getStatus());
278         }
279     }
280 }
281