1 /*
2  * Copyright (C) 2018 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package com.android.tradefed.testtype.suite;
18 
19 import com.android.tradefed.config.IConfiguration;
20 import com.android.tradefed.device.DeviceNotAvailableException;
21 import com.android.tradefed.device.DeviceUnresponsiveException;
22 
23 import com.android.tradefed.device.metric.IMetricCollector;
24 import com.android.tradefed.invoker.IInvocationContext;
25 import com.android.tradefed.log.LogUtil.CLog;
26 import com.android.tradefed.result.ILogSaver;
27 import com.android.tradefed.result.ITestInvocationListener;
28 import com.android.tradefed.result.LogSaverResultForwarder;
29 import com.android.tradefed.result.TestDescription;
30 import com.android.tradefed.result.TestRunResult;
31 import com.android.tradefed.testtype.IRemoteTest;
32 import com.android.tradefed.testtype.ITestFilterReceiver;
33 
34 import com.google.common.annotations.VisibleForTesting;
35 
36 import java.util.ArrayList;
37 import java.util.List;
38 import java.util.Set;
39 
40 /**
41  * A wrapper class works on the {@link IRemoteTest} to granulate the IRemoteTest in testcase level.
42  * An IRemoteTest can contain multiple testcases. Previously, these testcases are treated as a
43  * whole: When IRemoteTest runs, all testcases will run. Some IRemoteTest (The ones that implements
44  * ITestFilterReceiver) can accept a whitelist of testcases and only run those testcases. This class
45  * takes advantage of the existing feature and provides a more flexible way to run test suite.
46  *
47  * <ul>
48  *   <li> Single testcase can be retried multiple times (within the same IRemoteTest run) to reduce
49  *       the non-test-error failure rates.
50  *   <li> The retried testcases are dynamically collected from previous run failures.
51  * </ul>
52  *
53  * <p>Note:
54  *
55  * <ul>
56  *   <li> The prerequisite to run a subset of test cases is that the test type should implement the
57  *       interface {@link ITestFilterReceiver}.
58  *   <li> X is customized max retry number.
59  * </ul>
60  */
61 public class GranularRetriableTestWrapper implements IRemoteTest {
62 
63     private boolean mIsGranulatedTestCaseRetriable;
64     private IRemoteTest mTest;
65     private boolean mSkipTestCases;
66     private List<IMetricCollector> mRunMetricCollectors;
67     private ITestInvocationListener mMainListener;
68     private TestFailureListener mFailureListener;
69     private IInvocationContext mModuleInvocationContext;
70     private IConfiguration mModuleConfiguration;
71     private List<TestRunResult> mTestRunResultCollector;
72     private List<ModuleListener> mModuleListenerCollector;
73     private List<ITestInvocationListener> mModuleLevelListeners;
74     private ILogSaver mLogSaver;
75     private String mModuleId;
76     private boolean mIsMetricCollectorInitialized;
77     private int mMaxRunLimit;
78 
GranularRetriableTestWrapper( IRemoteTest test, TestFailureListener failureListener, List<ITestInvocationListener> moduleLevelListeners, int maxRunLimit)79     public GranularRetriableTestWrapper(
80             IRemoteTest test,
81             TestFailureListener failureListener,
82             List<ITestInvocationListener> moduleLevelListeners,
83             int maxRunLimit) {
84         mTest = test;
85         mTestRunResultCollector = new ArrayList<TestRunResult>();
86         mFailureListener = failureListener;
87         mModuleListenerCollector = new ArrayList<ModuleListener>();
88         mIsMetricCollectorInitialized = false;
89         mModuleLevelListeners = moduleLevelListeners;
90         mMaxRunLimit = maxRunLimit;
91         // TODO(b/77548917): Right now we only support ITestFilterReciever. We should expect to
92         // support ITestFile*Filter*Receiver in the future.
93         if (test instanceof ITestFilterReceiver) {
94             mIsGranulatedTestCaseRetriable = true;
95         } else {
96             mIsGranulatedTestCaseRetriable = false;
97         }
98     }
99 
100     /**
101      * Set the {@link ModuleDefinition} name as a {@link GranularRetriableTestWrapper} attribute.
102      *
103      * @param moduleId the name of the moduleDefinition.
104      */
setModuleId(String moduleId)105     public void setModuleId(String moduleId) {
106         mModuleId = moduleId;
107     }
108 
109     /**
110      * Set the {@link ModuleDefinition} RunStrategy as a {@link GranularRetriableTestWrapper}
111      * attribute.
112      *
113      * @param skipTestCases whether the testcases should be skipped.
114      */
setMarkTestsSkipped(boolean skipTestCases)115     public void setMarkTestsSkipped(boolean skipTestCases) {
116         mSkipTestCases = skipTestCases;
117     }
118 
119     /**
120      * Set the {@link ModuleDefinition}'s runMetricCollector as a {@link
121      * GranularRetriableTestWrapper} attribute.
122      *
123      * @param runMetricCollectors A list of MetricCollector for the module.
124      */
setMetricCollectors(List<IMetricCollector> runMetricCollectors)125     public void setMetricCollectors(List<IMetricCollector> runMetricCollectors) {
126         mRunMetricCollectors = runMetricCollectors;
127     }
128 
129     /**
130      * Set the {@link ModuleDefinition}'s ModuleConfig as a {@link GranularRetriableTestWrapper}
131      * attribute.
132      *
133      * @param moduleConfiguration Provide the module metrics.
134      */
setModuleConfig(IConfiguration moduleConfiguration)135     public void setModuleConfig(IConfiguration moduleConfiguration) {
136         mModuleConfiguration = moduleConfiguration;
137     }
138 
139     /**
140      * Set the {@link IInvocationContext} as a {@link GranularRetriableTestWrapper} attribute.
141      *
142      * @param moduleInvocationContext The wrapper uses the InvocationContext to initialize the
143      *     MetricCollector when necessary.
144      */
setInvocationContext(IInvocationContext moduleInvocationContext)145     public void setInvocationContext(IInvocationContext moduleInvocationContext) {
146         mModuleInvocationContext = moduleInvocationContext;
147     }
148 
149     /**
150      * Set the Module's {@link ILogSaver} as a {@link GranularRetriableTestWrapper} attribute.
151      *
152      * @param logSaver The listeners for each test run should save the logs.
153      */
setLogSaver(ILogSaver logSaver)154     public void setLogSaver(ILogSaver logSaver) {
155         mLogSaver = logSaver;
156     }
157 
158     @VisibleForTesting
createModuleListener()159     ModuleListener createModuleListener() {
160         return new ModuleListener(mMainListener);
161     }
162 
163     /**
164      * Initialize a new {@link ModuleListener} for each test run.
165      *
166      * @return a {@link ITestInvocationListener} listener which contains the new {@link
167      *     ModuleListener}, the main {@link ITestInvocationListener} and main {@link
168      *     TestFailureListener}, and wrapped by RunMetricsCollector and Module MetricCollector (if
169      *     not initialized).
170      */
prepareRunListener()171     private ITestInvocationListener prepareRunListener() {
172         ModuleListener moduleListener = createModuleListener();
173         mModuleListenerCollector.add(moduleListener);
174         moduleListener.setMarkTestsSkipped(mSkipTestCases);
175         List<ITestInvocationListener> currentTestListener = new ArrayList<>();
176         // Add all the module level listeners, including TestFailureListener
177         if (mModuleLevelListeners != null) {
178             currentTestListener.addAll(mModuleLevelListeners);
179         }
180         currentTestListener.add(moduleListener);
181 
182         ITestInvocationListener runListener =
183                 new LogSaverResultForwarder(mLogSaver, currentTestListener);
184         if (mFailureListener != null) {
185             mFailureListener.setLogger(runListener);
186             currentTestListener.add(mFailureListener);
187         }
188 
189         // TODO: For RunMetricCollector and moduleMetricCollector, we only gather the
190         // metrics in the first run. This part can be improved if we want to gather metrics for
191         // every run.
192         if (!mIsMetricCollectorInitialized) {
193             if (mRunMetricCollectors != null) {
194                 // Module only init the collectors here to avoid triggering the collectors when
195                 // replaying the cached events at the end. This ensure metrics are capture at
196                 // the proper time in the invocation.
197                 for (IMetricCollector collector : mRunMetricCollectors) {
198                     runListener = collector.init(mModuleInvocationContext, runListener);
199                 }
200             }
201             // The module collectors itself are added: this list will be very limited.
202             for (IMetricCollector collector : mModuleConfiguration.getMetricCollectors()) {
203                 runListener = collector.init(mModuleInvocationContext, runListener);
204             }
205             mIsMetricCollectorInitialized = true;
206         }
207         return runListener;
208     }
209 
210     /**
211      * Schedule a series of {@link IRemoteTest} "run". TODO: Customize the retry strategy; Each run
212      * is granularized to a subset of the whole testcases.
213      *
214      * @param listener The ResultForwarder listener which contains a new moduleListener for each
215      *     run.
216      */
217     @Override
run(ITestInvocationListener listener)218     public void run(ITestInvocationListener listener) throws DeviceNotAvailableException {
219         mMainListener = listener;
220         int count = 0;
221         while (count < mMaxRunLimit) {
222             intraModuleRun();
223             count += 1;
224             ModuleListener previousTestRunListener =
225                     mModuleListenerCollector.get(mModuleListenerCollector.size() - 1);
226             if (!previousTestRunListener.hasFailedTests()) {
227                 CLog.d("The test has no failed testcases. No need to retry.");
228                 break;
229             }
230             if (count == mMaxRunLimit) {
231                 CLog.d(
232                         "The test has reached its max number of run attempt: %d time(s)",
233                         mMaxRunLimit);
234                 break;
235             }
236             if (mIsGranulatedTestCaseRetriable) {
237                 Set<TestDescription> failedTests =
238                         previousTestRunListener.getCurrentRunResults().getFailedTests();
239                 addRetriedTestsToIncludeFilters(failedTests);
240             } else {
241                 // If the IRemoteTest can't support running a single testcase, we retry the whole
242                 // testcase list.
243                 CLog.d(
244                         "The test is not supported to run testcases in intra-module level. Trying "
245                                 + "to run the whole test again...");
246             }
247         }
248     }
249 
250     /**
251      * Update the arguments of {@link IRemoteTest} to only run failed tests. This arguments/logic is
252      * implemented differently for each IRemoteTest testtype in the overridden
253      * ITestFilterReceiver.addIncludeFilter method.
254      *
255      * @param testDescriptions The set of failed testDescriptions to retry.
256      */
addRetriedTestsToIncludeFilters(Set<TestDescription> testDescriptions)257     private void addRetriedTestsToIncludeFilters(Set<TestDescription> testDescriptions) {
258         // TODO(b/77548917): Right now we only support ITestFilterReciever. We should expect to
259         // support ITestFile*Filter*Receiver in the future.
260         if (mTest instanceof ITestFilterReceiver) {
261             for (TestDescription testCase : testDescriptions) {
262                 String filter = testCase.toString();
263                 ((ITestFilterReceiver) mTest).addIncludeFilter(filter);
264             }
265         }
266     }
267 
268     /**
269      * The workflow for each individual {@link IRemoteTest} run. TODO: When this function is called,
270      * the IRemoteTest should already has the subset of testcases identified.
271      */
272     @VisibleForTesting
intraModuleRun()273     final void intraModuleRun() throws DeviceNotAvailableException, DeviceUnresponsiveException {
274         ITestInvocationListener runListener = prepareRunListener();
275         try {
276             mTest.run(runListener);
277         } catch (RuntimeException re) {
278             CLog.e("Module '%s' - test '%s' threw exception:", mModuleId, mTest.getClass());
279             CLog.e(re);
280             CLog.e("Proceeding to the next test.");
281             runListener.testRunFailed(re.getMessage());
282         } catch (DeviceUnresponsiveException due) {
283             // being able to catch a DeviceUnresponsiveException here implies that recovery was
284             // successful, and test execution should proceed to next module.
285             CLog.w(
286                     "Ignored DeviceUnresponsiveException because recovery was "
287                             + "successful, proceeding with next module. Stack trace:");
288             CLog.w(due);
289             CLog.w("Proceeding to the next test.");
290             runListener.testRunFailed(due.getMessage());
291         } catch (DeviceNotAvailableException dnae) {
292             throw dnae;
293         } finally {
294             ModuleListener currentModuleListener =
295                     mModuleListenerCollector.get(mModuleListenerCollector.size() - 1);
296             mTestRunResultCollector.addAll(currentModuleListener.getRunResults());
297         }
298     }
299 
300     /** Get the merged TestRunResults from each {@link IRemoteTest} run. */
getFinalTestRunResult()301     public TestRunResult getFinalTestRunResult() {
302         return TestRunResult.merge(mTestRunResultCollector);
303     }
304 
305     @VisibleForTesting
getTestRunResultCollector()306     List<TestRunResult> getTestRunResultCollector() {
307         return mTestRunResultCollector;
308     }
309 
310     /** Check if any testRunResult has ever failed. */
hasFailed()311     public boolean hasFailed() {
312         for (ModuleListener listener : mModuleListenerCollector) {
313             if (listener.hasFailed()) {
314                 return true;
315             }
316         }
317         return false;
318     }
319 
320     /**
321      * Calculate the number of testcases in the {@link IRemoteTest}. This value distincts the same
322      * testcases that are rescheduled multiple times.
323      */
getNumIndividualTests()324     public int getNumIndividualTests() {
325         if (mModuleListenerCollector.isEmpty()) {
326             return 0;
327         }
328         return mModuleListenerCollector.get(0).getNumTotalTests();
329     }
330 }
331