1 /*
2  * Copyright (C) 2020 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 package com.android.tradefed.testtype;
17 
18 import com.android.tradefed.device.DeviceNotAvailableException;
19 import com.android.tradefed.device.ITestDevice;
20 import com.android.tradefed.log.LogUtil.CLog;
21 import com.android.tradefed.metrics.proto.MetricMeasurement.Metric;
22 import com.android.tradefed.result.FailureDescription;
23 import com.android.tradefed.result.ITestInvocationListener;
24 import com.android.tradefed.result.LogcatCrashResultForwarder;
25 import com.android.tradefed.result.TestDescription;
26 import com.android.tradefed.result.proto.TestRecordProto.FailureStatus;
27 import com.android.tradefed.util.ProcessInfo;
28 
29 import java.util.Collection;
30 import java.util.HashMap;
31 import java.util.HashSet;
32 import java.util.LinkedHashSet;
33 import java.util.Set;
34 
35 /**
36  * Internal listener to Trade Federation for {@link InstrumentationTest}. It allows to collect extra
37  * information needed for easier debugging.
38  */
39 final class InstrumentationListener extends LogcatCrashResultForwarder {
40 
41     // Message from ddmlib InstrumentationResultParser for interrupted instrumentation.
42     private static final String DDMLIB_INSTRU_FAILURE_MSG = "Test run failed to complete";
43 
44     private Set<TestDescription> mTests = new HashSet<>();
45     private Set<TestDescription> mDuplicateTests = new HashSet<>();
46     private final Collection<TestDescription> mExpectedTests;
47     private boolean mDisableDuplicateCheck = false;
48     private boolean mReportUnexecutedTests = false;
49     private ProcessInfo mSystemServerProcess = null;
50 
51     /**
52      * @param device
53      * @param listeners
54      */
InstrumentationListener( ITestDevice device, Collection<TestDescription> expectedTests, ITestInvocationListener... listeners)55     public InstrumentationListener(
56             ITestDevice device,
57             Collection<TestDescription> expectedTests,
58             ITestInvocationListener... listeners) {
59         super(device, listeners);
60         mExpectedTests = expectedTests;
61     }
62 
63     /** Whether or not to disable the duplicate test method check. */
setDisableDuplicateCheck(boolean disable)64     public void setDisableDuplicateCheck(boolean disable) {
65         mDisableDuplicateCheck = disable;
66     }
67 
setOriginalSystemServer(ProcessInfo info)68     public void setOriginalSystemServer(ProcessInfo info) {
69         mSystemServerProcess = info;
70     }
71 
setReportUnexecutedTests(boolean enable)72     public void setReportUnexecutedTests(boolean enable) {
73         mReportUnexecutedTests = enable;
74     }
75 
76     @Override
testRunStarted(String runName, int testCount)77     public void testRunStarted(String runName, int testCount) {
78         // In case of crash, run will attempt to report with 0
79         if (testCount == 0 && !mExpectedTests.isEmpty()) {
80             CLog.e("Run reported 0 tests while we collected %s", mExpectedTests.size());
81             super.testRunStarted(runName, mExpectedTests.size());
82         } else {
83             super.testRunStarted(runName, testCount);
84         }
85     }
86 
87     @Override
testStarted(TestDescription test, long startTime)88     public void testStarted(TestDescription test, long startTime) {
89         super.testStarted(test, startTime);
90         if (!mTests.add(test)) {
91             mDuplicateTests.add(test);
92         }
93     }
94 
95     @Override
testRunFailed(FailureDescription error)96     public void testRunFailed(FailureDescription error) {
97         if (error.getErrorMessage().startsWith(DDMLIB_INSTRU_FAILURE_MSG)) {
98             Set<TestDescription> expected = new LinkedHashSet<>(mExpectedTests);
99             expected.removeAll(mTests);
100             String helpMessage = String.format("The following tests didn't run: %s", expected);
101             error.setDebugHelpMessage(helpMessage);
102             error.setFailureStatus(FailureStatus.TEST_FAILURE);
103             if (mSystemServerProcess != null) {
104                 boolean restarted = false;
105                 try {
106                     restarted = getDevice().deviceSoftRestarted(mSystemServerProcess);
107                 } catch (DeviceNotAvailableException e) {
108                     // Ignore
109                 }
110                 if (restarted) {
111                     error.setFailureStatus(FailureStatus.SYSTEM_UNDER_TEST_CRASHED);
112                 }
113             }
114         }
115         super.testRunFailed(error);
116     }
117 
118     @Override
testRunEnded(long elapsedTime, HashMap<String, Metric> runMetrics)119     public void testRunEnded(long elapsedTime, HashMap<String, Metric> runMetrics) {
120         if (!mDuplicateTests.isEmpty() && !mDisableDuplicateCheck) {
121             FailureDescription error =
122                     FailureDescription.create(
123                             String.format(
124                                     "The following tests ran more than once: %s. Check "
125                                             + "your run configuration, you might be "
126                                             + "including the same test class several "
127                                             + "times.",
128                                     mDuplicateTests));
129             error.setFailureStatus(FailureStatus.TEST_FAILURE);
130             super.testRunFailed(error);
131         } else if (mReportUnexecutedTests && mExpectedTests.size() > mTests.size()) {
132             Set<TestDescription> missingTests = new LinkedHashSet<>(mExpectedTests);
133             missingTests.removeAll(mTests);
134             for (TestDescription miss : missingTests) {
135                 super.testStarted(miss);
136                 FailureDescription failure =
137                         FailureDescription.create(
138                                 "test did not run due to instrumentation issue.",
139                                 FailureStatus.NOT_EXECUTED);
140                 super.testFailed(miss, failure);
141                 super.testEnded(miss, new HashMap<String, Metric>());
142             }
143         }
144         super.testRunEnded(elapsedTime, runMetrics);
145     }
146 }
147