1 /*
2  * Copyright (C) 2010 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 package com.android.tradefed.result;
17 
18 import com.android.ddmlib.testrunner.TestResult.TestStatus;
19 import com.android.tradefed.build.IBuildInfo;
20 import com.android.tradefed.config.Option;
21 import com.android.tradefed.invoker.IInvocationContext;
22 import com.android.tradefed.metrics.proto.MetricMeasurement.Metric;
23 
24 import com.google.common.annotations.VisibleForTesting;
25 
26 import java.util.Collection;
27 import java.util.Collections;
28 import java.util.HashMap;
29 import java.util.LinkedHashMap;
30 import java.util.Map;
31 
32 /**
33  * A {@link ITestInvocationListener} that will collect all test results.
34  *
35  * <p>Although the data structures used in this object are thread-safe, the {@link
36  * ITestInvocationListener} callbacks must be called in the correct order.
37  */
38 public class CollectingTestListener implements ITestInvocationListener, ILogSaverListener {
39 
40     // Stores the test results
41     // Uses a synchronized map to make thread safe.
42     // Uses a LinkedHashmap to have predictable iteration order
43     private Map<String, TestRunResult> mRunResultsMap =
44         Collections.synchronizedMap(new LinkedHashMap<String, TestRunResult>());
45     private Map<TestRunResult, IInvocationContext> mModuleContextMap =
46             Collections.synchronizedMap(new LinkedHashMap<TestRunResult, IInvocationContext>());
47     private TestRunResult mCurrentResults =  new TestRunResult();
48     private IInvocationContext mCurrentModuleContext = null;
49 
50     /** represents sums of tests in each TestStatus state for all runs.
51      * Indexed by TestStatus.ordinal() */
52     private int[] mStatusCounts = new int[TestStatus.values().length];
53     /** tracks if mStatusCounts is accurate, or if it needs to be recalculated */
54     private boolean mIsCountDirty = true;
55 
56     @Option(name = "aggregate-metrics", description =
57         "attempt to add test metrics values for test runs with the same name." )
58     private boolean mIsAggregateMetrics = false;
59 
60     private IBuildInfo mBuildInfo;
61     private IInvocationContext mContext;
62 
63     /** Toggle the 'aggregate metrics' option */
setIsAggregrateMetrics(boolean aggregate)64     protected void setIsAggregrateMetrics(boolean aggregate) {
65         mIsAggregateMetrics = aggregate;
66     }
67 
68     /**
69      * {@inheritDoc}
70      */
71     @Override
invocationStarted(IInvocationContext context)72     public void invocationStarted(IInvocationContext context) {
73         mContext = context;
74         if (context != null) {
75             mBuildInfo = context.getBuildInfos().get(0);
76         }
77     }
78 
79     /**
80      * Return the primary build info that was reported via {@link
81      * #invocationStarted(IInvocationContext)}. Primary build is the build returned by the first
82      * build provider of the running configuration. Returns null if there is no context (no build to
83      * test case).
84      */
getPrimaryBuildInfo()85     public IBuildInfo getPrimaryBuildInfo() {
86         if (mContext == null) {
87             return null;
88         } else {
89             return mContext.getBuildInfos().get(0);
90         }
91     }
92 
93     /**
94      * Return the invocation context that was reported via
95      * {@link #invocationStarted(IInvocationContext)}
96      */
getInvocationContext()97     public IInvocationContext getInvocationContext() {
98         return mContext;
99     }
100 
101     /**
102      * Returns the build info.
103      *
104      * @deprecated rely on the {@link IBuildInfo} from {@link #getInvocationContext()}.
105      */
106     @Deprecated
getBuildInfo()107     public IBuildInfo getBuildInfo() {
108         return mBuildInfo;
109     }
110 
111     /**
112      * Set the build info. Should only be used for testing.
113      *
114      * @deprecated Not necessary for testing anymore.
115      */
116     @VisibleForTesting
117     @Deprecated
setBuildInfo(IBuildInfo buildInfo)118     public void setBuildInfo(IBuildInfo buildInfo) {
119         mBuildInfo = buildInfo;
120     }
121 
122     @Override
testModuleStarted(IInvocationContext moduleContext)123     public void testModuleStarted(IInvocationContext moduleContext) {
124         mCurrentModuleContext = moduleContext;
125     }
126 
127     @Override
testModuleEnded()128     public void testModuleEnded() {
129         mCurrentModuleContext = null;
130     }
131 
132     /**
133      * {@inheritDoc}
134      */
135     @Override
testRunStarted(String name, int numTests)136     public void testRunStarted(String name, int numTests) {
137         if (mRunResultsMap.containsKey(name)) {
138             // rerun of previous run. Add test results to it
139             mCurrentResults = mRunResultsMap.get(name);
140         } else {
141             // new run
142             mCurrentResults = new TestRunResult();
143             mCurrentResults.setAggregateMetrics(mIsAggregateMetrics);
144 
145             mRunResultsMap.put(name, mCurrentResults);
146             // track the module context associated with the results.
147             if (mCurrentModuleContext != null) {
148                 mModuleContextMap.put(mCurrentResults, mCurrentModuleContext);
149             }
150         }
151         mCurrentResults.testRunStarted(name, numTests);
152         mIsCountDirty = true;
153     }
154 
155     /** {@inheritDoc} */
156     @Override
testStarted(TestDescription test)157     public void testStarted(TestDescription test) {
158         testStarted(test, System.currentTimeMillis());
159     }
160 
161     /** {@inheritDoc} */
162     @Override
testStarted(TestDescription test, long startTime)163     public void testStarted(TestDescription test, long startTime) {
164         mIsCountDirty = true;
165         mCurrentResults.testStarted(test, startTime);
166     }
167 
168     /** {@inheritDoc} */
169     @Override
testEnded(TestDescription test, HashMap<String, Metric> testMetrics)170     public void testEnded(TestDescription test, HashMap<String, Metric> testMetrics) {
171         testEnded(test, System.currentTimeMillis(), testMetrics);
172     }
173 
174     /** {@inheritDoc} */
175     @Override
testEnded(TestDescription test, long endTime, HashMap<String, Metric> testMetrics)176     public void testEnded(TestDescription test, long endTime, HashMap<String, Metric> testMetrics) {
177         mIsCountDirty = true;
178         mCurrentResults.testEnded(test, endTime, testMetrics);
179     }
180 
181     /** {@inheritDoc} */
182     @Override
testFailed(TestDescription test, String trace)183     public void testFailed(TestDescription test, String trace) {
184         mIsCountDirty = true;
185         mCurrentResults.testFailed(test, trace);
186     }
187 
188     @Override
testAssumptionFailure(TestDescription test, String trace)189     public void testAssumptionFailure(TestDescription test, String trace) {
190         mIsCountDirty = true;
191         mCurrentResults.testAssumptionFailure(test, trace);
192 
193     }
194 
195     @Override
testIgnored(TestDescription test)196     public void testIgnored(TestDescription test) {
197         mIsCountDirty = true;
198         mCurrentResults.testIgnored(test);
199     }
200 
201     /** {@inheritDoc} */
202     @Override
testRunEnded(long elapsedTime, HashMap<String, Metric> runMetrics)203     public void testRunEnded(long elapsedTime, HashMap<String, Metric> runMetrics) {
204         mIsCountDirty = true;
205         mCurrentResults.testRunEnded(elapsedTime, runMetrics);
206     }
207 
208     /**
209      * {@inheritDoc}
210      */
211     @Override
testRunFailed(String errorMessage)212     public void testRunFailed(String errorMessage) {
213         mIsCountDirty = true;
214         mCurrentResults.testRunFailed(errorMessage);
215     }
216 
217     /**
218      * {@inheritDoc}
219      */
220     @Override
testRunStopped(long elapsedTime)221     public void testRunStopped(long elapsedTime) {
222         mIsCountDirty = true;
223         mCurrentResults.testRunStopped(elapsedTime);
224     }
225 
226     /**
227      * Gets the results for the current test run.
228      * <p/>
229      * Note the results may not be complete. It is recommended to test the value of {@link
230      * TestRunResult#isRunComplete()} and/or (@link TestRunResult#isRunFailure()} as appropriate
231      * before processing the results.
232      *
233      * @return the {@link TestRunResult} representing data collected during last test run
234      */
getCurrentRunResults()235     public TestRunResult getCurrentRunResults() {
236         return mCurrentResults;
237     }
238 
239     /**
240      * Gets the results for all test runs.
241      */
getRunResults()242     public Collection<TestRunResult> getRunResults() {
243         return mRunResultsMap.values();
244     }
245 
246     /**
247      * Returns the {@link IInvocationContext} of the module associated with the results or null if
248      * it was not associated with any module.
249      */
getModuleContextForRunResult(TestRunResult res)250     public IInvocationContext getModuleContextForRunResult(TestRunResult res) {
251         return mModuleContextMap.get(res);
252     }
253 
254     /** Returns True if the result map already has an entry for the run name. */
hasResultFor(String runName)255     public boolean hasResultFor(String runName) {
256         return mRunResultsMap.containsKey(runName);
257     }
258 
259     /** Gets the total number of complete tests for all runs. */
getNumTotalTests()260     public int getNumTotalTests() {
261         int total = 0;
262         // force test count
263         getNumTestsInState(TestStatus.PASSED);
264         for (TestStatus s : TestStatus.values()) {
265             total += mStatusCounts[s.ordinal()];
266         }
267         return total;
268     }
269 
270     /**
271      * Gets the number of tests in given state for this run.
272      */
getNumTestsInState(TestStatus status)273     public int getNumTestsInState(TestStatus status) {
274         if (mIsCountDirty) {
275             for (TestStatus s : TestStatus.values()) {
276                 mStatusCounts[s.ordinal()] = 0;
277                 for (TestRunResult result : mRunResultsMap.values()) {
278                     mStatusCounts[s.ordinal()] += result.getNumTestsInState(s);
279                 }
280             }
281             mIsCountDirty = false;
282         }
283         return mStatusCounts[status.ordinal()];
284     }
285 
286     /**
287      * @return true if invocation had any failed or assumption failed tests.
288      */
hasFailedTests()289     public boolean hasFailedTests() {
290         return getNumAllFailedTests() > 0;
291     }
292 
293     /**
294      * {@inheritDoc}
295      */
296     @Override
invocationEnded(long elapsedTime)297     public void invocationEnded(long elapsedTime) {
298         // ignore
299     }
300 
301     /**
302      * {@inheritDoc}
303      */
304     @Override
invocationFailed(Throwable cause)305     public void invocationFailed(Throwable cause) {
306         // ignore
307     }
308 
309     /**
310      * {@inheritDoc}
311      */
312     @Override
getSummary()313     public TestSummary getSummary() {
314         // ignore
315         return null;
316     }
317 
318     /** {@inheritDoc} */
319     @Override
testLog(String dataName, LogDataType dataType, InputStreamSource dataStream)320     public void testLog(String dataName, LogDataType dataType, InputStreamSource dataStream) {
321         // ignore, logAssociation is implemented.
322     }
323 
324     /** {@inheritDoc} */
325     @Override
testLogSaved( String dataName, LogDataType dataType, InputStreamSource dataStream, LogFile logFile)326     public void testLogSaved(
327             String dataName, LogDataType dataType, InputStreamSource dataStream, LogFile logFile) {
328         // ignore, logAssociation is used to save the files
329     }
330 
331     /** {@inheritDoc} */
332     @Override
setLogSaver(ILogSaver logSaver)333     public void setLogSaver(ILogSaver logSaver) {
334         // CollectingTestListener does not need the logSaver
335     }
336 
337     /** {@inheritDoc} */
338     @Override
logAssociation(String dataName, LogFile logFile)339     public void logAssociation(String dataName, LogFile logFile) {
340         mCurrentResults.testLogSaved(dataName, logFile);
341     }
342 
343     /**
344      * Return total number of tests in a failure state (only failed, assumption failures do not
345      * count toward it).
346      */
getNumAllFailedTests()347     public int getNumAllFailedTests() {
348         return getNumTestsInState(TestStatus.FAILURE);
349     }
350 
351     /**
352      * Return total number of test runs in a failure state
353      */
getNumAllFailedTestRuns()354     public int getNumAllFailedTestRuns() {
355         int count = 0;
356         for (Map.Entry<String, TestRunResult> e : mRunResultsMap.entrySet()) {
357             if (e.getValue().isRunFailure()) {
358                 count++;
359             }
360         }
361         return count;
362     }
363 }
364