1 /*
2  * Copyright (C) 2019 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 package com.android.tradefed.retry;
17 
18 import com.android.tradefed.invoker.IInvocationContext;
19 import com.android.tradefed.invoker.logger.InvocationMetricLogger;
20 import com.android.tradefed.invoker.logger.InvocationMetricLogger.InvocationMetricKey;
21 import com.android.tradefed.metrics.proto.MetricMeasurement.Metric;
22 import com.android.tradefed.result.CollectingTestListener;
23 import com.android.tradefed.result.FailureDescription;
24 import com.android.tradefed.result.ILogSaver;
25 import com.android.tradefed.result.ILogSaverListener;
26 import com.android.tradefed.result.ITestInvocationListener;
27 import com.android.tradefed.result.InputStreamSource;
28 import com.android.tradefed.result.LogDataType;
29 import com.android.tradefed.result.LogFile;
30 import com.android.tradefed.result.MultiFailureDescription;
31 import com.android.tradefed.result.ResultAndLogForwarder;
32 import com.android.tradefed.result.TestDescription;
33 import com.android.tradefed.result.TestResult;
34 import com.android.tradefed.result.TestRunResult;
35 import com.android.tradefed.result.retry.ISupportGranularResults;
36 
37 import com.google.common.annotations.VisibleForTesting;
38 
39 import java.util.ArrayList;
40 import java.util.HashMap;
41 import java.util.HashSet;
42 import java.util.LinkedHashMap;
43 import java.util.List;
44 import java.util.Map;
45 import java.util.Map.Entry;
46 import java.util.Set;
47 import java.util.stream.Collectors;
48 
49 /**
50  * Special forwarder that aggregates the results when needed, based on the retry strategy that was
51  * taken.
52  */
53 public class ResultAggregator extends CollectingTestListener {
54 
55     /* Forwarder to ALL result reporters */
56     private ResultAndLogForwarder mAllForwarder;
57     /* Forwarder to result reporters that only support aggregated results */
58     private ResultAndLogForwarder mAggregatedForwarder;
59     /* Forwarder to result reporters that support the attempt reporting */
60     private ResultAndLogForwarder mDetailedForwarder;
61     private RetryStrategy mRetryStrategy;
62     // Track whether or not a module was started.
63     private boolean mModuleInProgress = false;
64 
65     // Holders for results in progress
66     private TestRunResult mDetailedRunResults = null;
67     private boolean mShouldReportFailure = true;
68     private List<FailureDescription> mAllDetailedFailures = new ArrayList<>();
69     // Since we store some of the module level events, ensure the logs order is maintained.
70     private Map<String, LogFile> mDetailedModuleLogs = new LinkedHashMap<>();
71 
72     // In some configuration of non-module retry, all attempts of runs might not be adjacent. We
73     // track that a special handling needs to be applied for this case.
74     private boolean mUnorderedRetry = true;
75     // Stores the results from non-module test runs until they are ready to be replayed.
76     private final Map<String, List<TestRunResult>> mPureRunResultForAgg = new LinkedHashMap<>();
77 
ResultAggregator(List<ITestInvocationListener> listeners, RetryStrategy strategy)78     public ResultAggregator(List<ITestInvocationListener> listeners, RetryStrategy strategy) {
79         mAllForwarder = new ResultAndLogForwarder(listeners);
80 
81         List<ITestInvocationListener> supportDetails =
82                 listeners
83                         .stream()
84                         .filter(
85                                 i ->
86                                         ((i instanceof ISupportGranularResults)
87                                                 && ((ISupportGranularResults) i)
88                                                         .supportGranularResults()))
89                         .collect(Collectors.toList());
90         List<ITestInvocationListener> noSupportDetails =
91                 listeners
92                         .stream()
93                         .filter(
94                                 i ->
95                                         !(i instanceof ISupportGranularResults)
96                                                 || !((ISupportGranularResults) i)
97                                                         .supportGranularResults())
98                         .collect(Collectors.toList());
99 
100         mAggregatedForwarder = new ResultAndLogForwarder(noSupportDetails);
101         mDetailedForwarder = new ResultAndLogForwarder(supportDetails);
102 
103         mRetryStrategy = strategy;
104         MergeStrategy mergeStrategy = MergeStrategy.getMergeStrategy(mRetryStrategy);
105         setMergeStrategy(mergeStrategy);
106     }
107 
108     /** {@inheritDoc} */
109     @Override
invocationStarted(IInvocationContext context)110     public void invocationStarted(IInvocationContext context) {
111         super.invocationStarted(context);
112         mAllForwarder.invocationStarted(context);
113     }
114 
115     /** {@inheritDoc} */
116     @Override
invocationFailed(Throwable cause)117     public void invocationFailed(Throwable cause) {
118         super.invocationFailed(cause);
119         mAllForwarder.invocationFailed(cause);
120     }
121 
122     /** {@inheritDoc} */
123     @Override
invocationFailed(FailureDescription failure)124     public void invocationFailed(FailureDescription failure) {
125         super.invocationFailed(failure);
126         mAllForwarder.invocationFailed(failure);
127     }
128 
129     /** {@inheritDoc} */
130     @Override
invocationEnded(long elapsedTime)131     public void invocationEnded(long elapsedTime) {
132         if (!mPureRunResultForAgg.isEmpty()) {
133             for (String name : mPureRunResultForAgg.keySet()) {
134                 forwardTestRunResults(mPureRunResultForAgg.get(name), mAggregatedForwarder);
135             }
136             mPureRunResultForAgg.clear();
137         }
138 
139         forwardDetailedFailure();
140         for (Entry<String, LogFile> assos : mDetailedModuleLogs.entrySet()) {
141             mDetailedForwarder.logAssociation(assos.getKey(), assos.getValue());
142         }
143         mDetailedModuleLogs.clear();
144         super.invocationEnded(elapsedTime);
145         // Make sure to forward the logs for the invocation.
146         forwardAggregatedInvocationLogs();
147         mAllForwarder.invocationEnded(elapsedTime);
148     }
149 
150     /**
151      * Forward all the invocation level logs to the result reporters that don't support the granular
152      * results.
153      */
forwardAggregatedInvocationLogs()154     public final void forwardAggregatedInvocationLogs() {
155         for (Entry<String, LogFile> invocLog : getNonAssociatedLogFiles().entrySet()) {
156             mAggregatedForwarder.logAssociation(invocLog.getKey(), invocLog.getValue());
157         }
158     }
159 
160     /** {@inheritDoc} */
161     @Override
testModuleStarted(IInvocationContext moduleContext)162     public void testModuleStarted(IInvocationContext moduleContext) {
163         mUnorderedRetry = false;
164         if (!mPureRunResultForAgg.isEmpty()) {
165             for (String name : mPureRunResultForAgg.keySet()) {
166                 forwardTestRunResults(mPureRunResultForAgg.get(name), mAggregatedForwarder);
167             }
168             mPureRunResultForAgg.clear();
169         }
170 
171         // Reset the reporting since we start a new module
172         mShouldReportFailure = true;
173         if (mDetailedRunResults != null) {
174             forwardDetailedFailure();
175         }
176 
177         mModuleInProgress = true;
178         super.testModuleStarted(moduleContext);
179         mAllForwarder.testModuleStarted(moduleContext);
180     }
181 
182     /** {@inheritDoc} */
183     @Override
setLogSaver(ILogSaver logSaver)184     public void setLogSaver(ILogSaver logSaver) {
185         super.setLogSaver(logSaver);
186         mAllForwarder.setLogSaver(logSaver);
187     }
188 
189     /** {@inheritDoc} */
190     @Override
testLog(String dataName, LogDataType dataType, InputStreamSource dataStream)191     public void testLog(String dataName, LogDataType dataType, InputStreamSource dataStream) {
192         super.testLog(dataName, dataType, dataStream);
193         mAllForwarder.testLog(dataName, dataType, dataStream);
194     }
195 
196     // ====== Forwarders to the detailed result reporters
197 
198     @Override
testRunStarted(String name, int testCount, int attemptNumber, long startTime)199     public void testRunStarted(String name, int testCount, int attemptNumber, long startTime) {
200         // Due to retries happening after several other testRunStart, we need to wait before making
201         // the forwarding.
202         if (!mUnorderedRetry) {
203             if (!mPureRunResultForAgg.isEmpty() && mPureRunResultForAgg.get(name) != null) {
204                 forwardTestRunResults(mPureRunResultForAgg.get(name), mAggregatedForwarder);
205                 mPureRunResultForAgg.remove(name);
206             }
207         }
208 
209         if (mDetailedRunResults != null) {
210             if (mDetailedRunResults.getName().equals(name)) {
211                 if (!mDetailedRunResults.isRunFailure()) {
212                     if (RetryStrategy.RETRY_ANY_FAILURE.equals(mRetryStrategy)) {
213                         mShouldReportFailure = false;
214                     }
215                 }
216                 mDetailedForwarder.testRunEnded(
217                         mDetailedRunResults.getElapsedTime(),
218                         mDetailedRunResults.getRunProtoMetrics());
219                 mDetailedRunResults = null;
220             } else {
221                 mShouldReportFailure = true;
222                 forwardDetailedFailure();
223             }
224         }
225         super.testRunStarted(name, testCount, attemptNumber, startTime);
226         mDetailedForwarder.testRunStarted(name, testCount, attemptNumber, startTime);
227     }
228 
229     @Override
testRunFailed(String errorMessage)230     public void testRunFailed(String errorMessage) {
231         super.testRunFailed(errorMessage);
232         // Don't forward here to the detailed forwarder in case we need to clear it.
233     }
234 
235     @Override
testRunFailed(FailureDescription failure)236     public void testRunFailed(FailureDescription failure) {
237         super.testRunFailed(failure);
238         // Don't forward here to the detailed forwarder in case we need to clear it.
239     }
240 
241     @Override
testStarted(TestDescription test, long startTime)242     public void testStarted(TestDescription test, long startTime) {
243         super.testStarted(test, startTime);
244         mDetailedForwarder.testStarted(test, startTime);
245     }
246 
247     @Override
testIgnored(TestDescription test)248     public void testIgnored(TestDescription test) {
249         super.testIgnored(test);
250         mDetailedForwarder.testIgnored(test);
251     }
252 
253     @Override
testAssumptionFailure(TestDescription test, String trace)254     public void testAssumptionFailure(TestDescription test, String trace) {
255         super.testAssumptionFailure(test, trace);
256         mDetailedForwarder.testAssumptionFailure(test, trace);
257     }
258 
259     @Override
testFailed(TestDescription test, String trace)260     public void testFailed(TestDescription test, String trace) {
261         super.testFailed(test, trace);
262         mDetailedForwarder.testFailed(test, trace);
263     }
264 
265     @Override
testFailed(TestDescription test, FailureDescription failure)266     public void testFailed(TestDescription test, FailureDescription failure) {
267         super.testFailed(test, failure);
268         mDetailedForwarder.testFailed(test, failure);
269     }
270 
271     @Override
testEnded(TestDescription test, long endTime, HashMap<String, Metric> testMetrics)272     public void testEnded(TestDescription test, long endTime, HashMap<String, Metric> testMetrics) {
273         super.testEnded(test, endTime, testMetrics);
274         mDetailedForwarder.testEnded(test, endTime, testMetrics);
275     }
276 
277     @Override
logAssociation(String dataName, LogFile logFile)278     public void logAssociation(String dataName, LogFile logFile) {
279         super.logAssociation(dataName, logFile);
280         if (mDetailedRunResults != null) {
281             mDetailedModuleLogs.put(dataName, logFile);
282         } else {
283             mDetailedForwarder.logAssociation(dataName, logFile);
284         }
285     }
286 
287     @Override
testLogSaved( String dataName, LogDataType dataType, InputStreamSource dataStream, LogFile logFile)288     public void testLogSaved(
289             String dataName, LogDataType dataType, InputStreamSource dataStream, LogFile logFile) {
290         super.testLogSaved(dataName, dataType, dataStream, logFile);
291         mDetailedForwarder.testLogSaved(dataName, dataType, dataStream, logFile);
292     }
293 
294     // ===== Forwarders to the aggregated reporters.
295 
296     @Override
testRunEnded(long elapsedTime, HashMap<String, Metric> runMetrics)297     public void testRunEnded(long elapsedTime, HashMap<String, Metric> runMetrics) {
298         super.testRunEnded(elapsedTime, runMetrics);
299         mDetailedRunResults = getCurrentRunResults();
300         if (mDetailedRunResults.isRunFailure()) {
301             FailureDescription currentFailure = mDetailedRunResults.getRunFailureDescription();
302             if (currentFailure instanceof MultiFailureDescription) {
303                 mAllDetailedFailures.addAll(
304                         ((MultiFailureDescription) currentFailure).getFailures());
305             } else {
306                 mAllDetailedFailures.add(currentFailure);
307             }
308         }
309 
310         // If we are not a module and we reach here. This allows to support non-suite scenarios
311         if (!mModuleInProgress) {
312             List<TestRunResult> results =
313                     mPureRunResultForAgg.getOrDefault(
314                             getCurrentRunResults().getName(), new ArrayList<>());
315             results.add(getCurrentRunResults());
316             mPureRunResultForAgg.put(getCurrentRunResults().getName(), results);
317         }
318     }
319 
320     @Override
testModuleEnded()321     public void testModuleEnded() {
322         forwardDetailedFailure();
323         for (Entry<String, LogFile> assos : mDetailedModuleLogs.entrySet()) {
324             mDetailedForwarder.logAssociation(assos.getKey(), assos.getValue());
325         }
326         mDetailedModuleLogs.clear();
327         mModuleInProgress = false;
328         super.testModuleEnded();
329         // We still forward the testModuleEnd to the detailed reporters
330         mDetailedForwarder.testModuleEnded();
331 
332         List<TestRunResult> mergedResults = getMergedTestRunResults();
333         Set<String> resultNames = new HashSet<>();
334         int expectedTestCount = 0;
335         for (TestRunResult result : mergedResults) {
336             expectedTestCount += result.getExpectedTestCount();
337             resultNames.add(result.getName());
338         }
339         // Forward all the results aggregated
340         mAggregatedForwarder.testRunStarted(
341                 getCurrentRunResults().getName(),
342                 expectedTestCount,
343                 /* Attempt*/ 0,
344                 /* Start Time */ getCurrentRunResults().getStartTime());
345         for (TestRunResult runResult : mergedResults) {
346             forwardTestResults(runResult.getTestResults(), mAggregatedForwarder);
347             if (runResult.isRunFailure()) {
348                 mAggregatedForwarder.testRunFailed(runResult.getRunFailureDescription());
349             }
350             // Provide a strong association of the run to its logs.
351             for (String key : runResult.getRunLoggedFiles().keySet()) {
352                 for (LogFile logFile : runResult.getRunLoggedFiles().get(key)) {
353                     mAggregatedForwarder.logAssociation(key, logFile);
354                 }
355             }
356         }
357 
358         mAggregatedForwarder.testRunEnded(
359                 getCurrentRunResults().getElapsedTime(),
360                 getCurrentRunResults().getRunProtoMetrics());
361         mAggregatedForwarder.testModuleEnded();
362         // Ensure we don't carry results from one module to another.
363         for (String name : resultNames) {
364             clearResultsForName(name);
365         }
366         mUnorderedRetry = true;
367     }
368 
369     @VisibleForTesting
getInvocationMetricRunError()370     String getInvocationMetricRunError() {
371         return InvocationMetricLogger.getInvocationMetrics()
372                 .get(InvocationMetricKey.CLEARED_RUN_ERROR.toString());
373     }
374 
375     @VisibleForTesting
addInvocationMetricRunError(String errors)376     void addInvocationMetricRunError(String errors) {
377         InvocationMetricLogger.addInvocationMetrics(InvocationMetricKey.CLEARED_RUN_ERROR, errors);
378     }
379 
forwardTestResults( Map<TestDescription, TestResult> testResults, ITestInvocationListener listener)380     private void forwardTestResults(
381             Map<TestDescription, TestResult> testResults, ITestInvocationListener listener) {
382         for (Map.Entry<TestDescription, TestResult> testEntry : testResults.entrySet()) {
383             listener.testStarted(testEntry.getKey(), testEntry.getValue().getStartTime());
384             switch (testEntry.getValue().getStatus()) {
385                 case FAILURE:
386                     listener.testFailed(testEntry.getKey(), testEntry.getValue().getFailure());
387                     break;
388                 case ASSUMPTION_FAILURE:
389                     listener.testAssumptionFailure(
390                             testEntry.getKey(), testEntry.getValue().getFailure());
391                     break;
392                 case IGNORED:
393                     listener.testIgnored(testEntry.getKey());
394                     break;
395                 case INCOMPLETE:
396                     listener.testFailed(
397                             testEntry.getKey(),
398                             FailureDescription.create("Test did not complete due to exception."));
399                     break;
400                 default:
401                     break;
402             }
403             // Provide a strong association of the test to its logs.
404             for (Entry<String, LogFile> logFile :
405                     testEntry.getValue().getLoggedFiles().entrySet()) {
406                 if (listener instanceof ILogSaverListener) {
407                     ((ILogSaverListener) listener)
408                             .logAssociation(logFile.getKey(), logFile.getValue());
409                 }
410             }
411             listener.testEnded(
412                     testEntry.getKey(),
413                     testEntry.getValue().getEndTime(),
414                     testEntry.getValue().getProtoMetrics());
415         }
416     }
417 
418     /**
419      * Helper method to forward the results from multiple attempts of the same Test Run (same name).
420      */
forwardTestRunResults(List<TestRunResult> results, ILogSaverListener listener)421     private void forwardTestRunResults(List<TestRunResult> results, ILogSaverListener listener) {
422         TestRunResult result =
423                 TestRunResult.merge(results, MergeStrategy.getMergeStrategy(mRetryStrategy));
424 
425         listener.testRunStarted(
426                 result.getName(), result.getExpectedTestCount(), 0, result.getStartTime());
427         forwardTestResults(result.getTestResults(), listener);
428         if (result.isRunFailure()) {
429             listener.testRunFailed(result.getRunFailureDescription());
430         }
431         // Provide a strong association of the run to its logs.
432         for (String key : result.getRunLoggedFiles().keySet()) {
433             for (LogFile logFile : result.getRunLoggedFiles().get(key)) {
434                 listener.logAssociation(key, logFile);
435             }
436         }
437         listener.testRunEnded(result.getElapsedTime(), result.getRunProtoMetrics());
438         // Ensure we don't keep track of the results we just forwarded
439         clearResultsForName(result.getName());
440     }
441 
forwardDetailedFailure()442     private void forwardDetailedFailure() {
443         if (mDetailedRunResults != null) {
444             if (mDetailedRunResults.isRunFailure() && mShouldReportFailure) {
445                 if (mAllDetailedFailures.size() == 1) {
446                     mDetailedForwarder.testRunFailed(mAllDetailedFailures.get(0));
447                 } else {
448                     mDetailedForwarder.testRunFailed(
449                             new MultiFailureDescription(mAllDetailedFailures));
450                 }
451             } else {
452                 // Log the run failure that was cleared
453                 List<String> invocationFailures = new ArrayList<>();
454                 String value = getInvocationMetricRunError();
455                 if (value != null) {
456                     invocationFailures.add(value);
457                 }
458                 // If there are failure, track them
459                 if (!mAllDetailedFailures.isEmpty()) {
460                     invocationFailures.add(
461                             new MultiFailureDescription(mAllDetailedFailures).toString());
462                     addInvocationMetricRunError(String.join("\n\n", invocationFailures));
463                 }
464             }
465             mAllDetailedFailures.clear();
466             mDetailedForwarder.testRunEnded(
467                     mDetailedRunResults.getElapsedTime(), mDetailedRunResults.getRunProtoMetrics());
468             mDetailedRunResults = null;
469         }
470     }
471 }
472