1 /*
2  * Copyright (C) 2019 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 package com.android.tradefed.testtype.binary;
17 
18 import com.android.tradefed.config.Option;
19 import com.android.tradefed.config.OptionCopier;
20 import com.android.tradefed.device.DeviceNotAvailableException;
21 import com.android.tradefed.invoker.TestInformation;
22 import com.android.tradefed.metrics.proto.MetricMeasurement.Metric;
23 import com.android.tradefed.result.FailureDescription;
24 import com.android.tradefed.result.ITestInvocationListener;
25 import com.android.tradefed.result.TestDescription;
26 import com.android.tradefed.testtype.IAbi;
27 import com.android.tradefed.testtype.IAbiReceiver;
28 import com.android.tradefed.testtype.IRemoteTest;
29 import com.android.tradefed.testtype.IRuntimeHintProvider;
30 import com.android.tradefed.testtype.IShardableTest;
31 import com.android.tradefed.testtype.ITestCollector;
32 import com.android.tradefed.testtype.ITestFilterReceiver;
33 import com.android.tradefed.util.StreamUtil;
34 
35 import java.io.File;
36 import java.io.IOException;
37 import java.lang.reflect.InvocationTargetException;
38 import java.util.ArrayList;
39 import java.util.Collection;
40 import java.util.HashMap;
41 import java.util.LinkedHashMap;
42 import java.util.LinkedHashSet;
43 import java.util.List;
44 import java.util.Map;
45 import java.util.Set;
46 
47 /** Base class for executable style of tests. For example: binaries, shell scripts. */
48 public abstract class ExecutableBaseTest
49         implements IRemoteTest,
50                 IRuntimeHintProvider,
51                 ITestCollector,
52                 IShardableTest,
53                 IAbiReceiver,
54                 ITestFilterReceiver {
55 
56     public static final String NO_BINARY_ERROR = "Binary %s does not exist.";
57 
58     @Option(
59             name = "per-binary-timeout",
60             isTimeVal = true,
61             description = "Timeout applied to each binary for their execution.")
62     private long mTimeoutPerBinaryMs = 5 * 60 * 1000L;
63 
64     @Option(name = "binary", description = "Path to the binary to be run. Can be repeated.")
65     private List<String> mBinaryPaths = new ArrayList<>();
66 
67     @Option(
68             name = "test-command-line",
69             description = "The test commands of each test names.",
70             requiredForRerun = true)
71     private Map<String, String> mTestCommands = new LinkedHashMap<>();
72 
73     @Option(
74             name = "collect-tests-only",
75             description = "Only dry-run through the tests, do not actually run them.")
76     private boolean mCollectTestsOnly = false;
77 
78     @Option(
79         name = "runtime-hint",
80         description = "The hint about the test's runtime.",
81         isTimeVal = true
82     )
83     private long mRuntimeHintMs = 60000L; // 1 minute
84 
85     private IAbi mAbi;
86     private TestInformation mTestInfo;
87     private Set<String> mIncludeFilters = new LinkedHashSet<>();
88     private Set<String> mExcludeFilters = new LinkedHashSet<>();
89 
90     /** @return the timeout applied to each binary for their execution. */
getTimeoutPerBinaryMs()91     protected long getTimeoutPerBinaryMs() {
92         return mTimeoutPerBinaryMs;
93     }
94 
95     /** {@inheritDoc} */
96     @Override
addIncludeFilter(String filter)97     public void addIncludeFilter(String filter) {
98         mIncludeFilters.add(filter);
99     }
100 
101     /** {@inheritDoc} */
102     @Override
addExcludeFilter(String filter)103     public void addExcludeFilter(String filter) {
104         mExcludeFilters.add(filter);
105     }
106 
107     /** {@inheritDoc} */
108     @Override
addAllIncludeFilters(Set<String> filters)109     public void addAllIncludeFilters(Set<String> filters) {
110         mIncludeFilters.addAll(filters);
111     }
112 
113     /** {@inheritDoc} */
114     @Override
addAllExcludeFilters(Set<String> filters)115     public void addAllExcludeFilters(Set<String> filters) {
116         mExcludeFilters.addAll(filters);
117     }
118 
119     /** {@inheritDoc} */
120     @Override
clearIncludeFilters()121     public void clearIncludeFilters() {
122         mIncludeFilters.clear();
123     }
124 
125     /** {@inheritDoc} */
126     @Override
clearExcludeFilters()127     public void clearExcludeFilters() {
128         mExcludeFilters.clear();
129     }
130 
131     /** {@inheritDoc} */
132     @Override
getIncludeFilters()133     public Set<String> getIncludeFilters() {
134         return mIncludeFilters;
135     }
136 
137     /** {@inheritDoc} */
138     @Override
getExcludeFilters()139     public Set<String> getExcludeFilters() {
140         return mExcludeFilters;
141     }
142 
143     @Override
run(TestInformation testInfo, ITestInvocationListener listener)144     public void run(TestInformation testInfo, ITestInvocationListener listener)
145             throws DeviceNotAvailableException {
146         mTestInfo = testInfo;
147         Map<String, String> testCommands = getAllTestCommands();
148         for (String testName : testCommands.keySet()) {
149             String cmd = testCommands.get(testName);
150             String path = findBinary(cmd);
151             TestDescription description = new TestDescription(testName, testName);
152             if (shouldSkipCurrentTest(description)) continue;
153             if (path == null) {
154                 listener.testRunStarted(testName, 0);
155                 listener.testRunFailed(String.format(NO_BINARY_ERROR, cmd));
156                 listener.testRunEnded(0L, new HashMap<String, Metric>());
157             } else {
158                 listener.testRunStarted(testName, 1);
159                 long startTimeMs = System.currentTimeMillis();
160                 listener.testStarted(description);
161                 try {
162                     if (!mCollectTestsOnly) {
163                         // Do not actually run the test if we are dry running it.
164                         runBinary(path, listener, description);
165                     }
166                 } catch (IOException e) {
167                     listener.testFailed(
168                             description, FailureDescription.create(StreamUtil.getStackTrace(e)));
169                 } finally {
170                     listener.testEnded(description, new HashMap<String, Metric>());
171                     listener.testRunEnded(
172                             System.currentTimeMillis() - startTimeMs,
173                             new HashMap<String, Metric>());
174                 }
175             }
176         }
177     }
178 
179     /**
180      * Check if current test should be skipped.
181      *
182      * @param description The test in progress.
183      * @return true if the test should be skipped.
184      */
shouldSkipCurrentTest(TestDescription description)185     private boolean shouldSkipCurrentTest(TestDescription description) {
186         // Force to skip any test not listed in include filters, or listed in exclude filters.
187         // exclude filters have highest priority.
188         String testName = description.getTestName();
189         if (mExcludeFilters.contains(testName)
190                 || mExcludeFilters.contains(description.toString())) {
191             return true;
192         }
193         if (!mIncludeFilters.isEmpty()) {
194             return !mIncludeFilters.contains(testName)
195                     && !mIncludeFilters.contains(description.toString());
196         }
197         return false;
198     }
199 
200     /**
201      * Search for the binary to be able to run it.
202      *
203      * @param binary the path of the binary or simply the binary name.
204      * @return The path to the binary, or null if not found.
205      */
findBinary(String binary)206     public abstract String findBinary(String binary) throws DeviceNotAvailableException;
207 
208     /**
209      * Actually run the binary at the given path.
210      *
211      * @param binaryPath The path of the binary.
212      * @param listener The listener where to report the results.
213      * @param description The test in progress.
214      */
runBinary( String binaryPath, ITestInvocationListener listener, TestDescription description)215     public abstract void runBinary(
216             String binaryPath, ITestInvocationListener listener, TestDescription description)
217             throws DeviceNotAvailableException, IOException;
218 
219     /** {@inheritDoc} */
220     @Override
setCollectTestsOnly(boolean shouldCollectTest)221     public final void setCollectTestsOnly(boolean shouldCollectTest) {
222         mCollectTestsOnly = shouldCollectTest;
223     }
224 
225     /** {@inheritDoc} */
226     @Override
getRuntimeHint()227     public final long getRuntimeHint() {
228         return mRuntimeHintMs;
229     }
230 
231     /** {@inheritDoc} */
232     @Override
setAbi(IAbi abi)233     public final void setAbi(IAbi abi) {
234         mAbi = abi;
235     }
236 
237     /** {@inheritDoc} */
238     @Override
getAbi()239     public IAbi getAbi() {
240         return mAbi;
241     }
242 
getTestInfo()243     TestInformation getTestInfo() {
244         return mTestInfo;
245     }
246 
247     /** {@inheritDoc} */
248     @Override
split()249     public final Collection<IRemoteTest> split() {
250         if (mBinaryPaths.size() <= 2) {
251             return null;
252         }
253         Collection<IRemoteTest> tests = new ArrayList<>();
254         for (String path : mBinaryPaths) {
255             tests.add(getTestShard(path));
256         }
257         return tests;
258     }
259 
getTestShard(String path)260     private IRemoteTest getTestShard(String path) {
261         ExecutableBaseTest shard = null;
262         try {
263             shard = this.getClass().getDeclaredConstructor().newInstance();
264             OptionCopier.copyOptionsNoThrow(this, shard);
265             // We approximate the runtime of each shard to be equal since we can't know.
266             shard.mRuntimeHintMs = mRuntimeHintMs / shard.mBinaryPaths.size();
267             // Set one binary per shard
268             shard.mBinaryPaths.clear();
269             shard.mBinaryPaths.add(path);
270         } catch (InstantiationException
271                 | IllegalAccessException
272                 | InvocationTargetException
273                 | NoSuchMethodException e) {
274             // This cannot happen because the class was already created once at that point.
275             throw new RuntimeException(
276                     String.format(
277                             "%s (%s) when attempting to create shard object",
278                             e.getClass().getSimpleName(), e.getMessage()));
279         }
280         return shard;
281     }
282 
283     /**
284      * Convert mBinaryPaths to mTestCommands for consistency.
285      *
286      * @return a Map{@link LinkedHashMap}<String, String> of testCommands.
287      */
getAllTestCommands()288     private Map<String, String> getAllTestCommands() {
289         Map<String, String> testCommands = new LinkedHashMap<>(mTestCommands);
290         for (String binary : mBinaryPaths) {
291             testCommands.put(new File(binary).getName(), binary);
292         }
293         return testCommands;
294     }
295 }
296