1 /*
2  * Copyright (C) 2017 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 package com.android.tradefed.testtype.python;
17 
18 import com.android.annotations.VisibleForTesting;
19 import com.android.tradefed.build.IDeviceBuildInfo;
20 import com.android.tradefed.config.GlobalConfiguration;
21 import com.android.tradefed.config.Option;
22 import com.android.tradefed.config.OptionClass;
23 import com.android.tradefed.device.DeviceNotAvailableException;
24 import com.android.tradefed.device.StubDevice;
25 import com.android.tradefed.invoker.ExecutionFiles.FilesKey;
26 import com.android.tradefed.invoker.TestInformation;
27 import com.android.tradefed.log.LogUtil.CLog;
28 import com.android.tradefed.metrics.proto.MetricMeasurement.Metric;
29 import com.android.tradefed.result.FailureDescription;
30 import com.android.tradefed.result.FileInputStreamSource;
31 import com.android.tradefed.result.ITestInvocationListener;
32 import com.android.tradefed.result.LogDataType;
33 import com.android.tradefed.result.ResultForwarder;
34 import com.android.tradefed.result.proto.TestRecordProto.FailureStatus;
35 import com.android.tradefed.testtype.IRemoteTest;
36 import com.android.tradefed.testtype.ITestFilterReceiver;
37 import com.android.tradefed.testtype.PythonUnitTestResultParser;
38 import com.android.tradefed.util.CommandResult;
39 import com.android.tradefed.util.CommandStatus;
40 import com.android.tradefed.util.FileUtil;
41 import com.android.tradefed.util.IRunUtil;
42 import com.android.tradefed.util.RunUtil;
43 import com.android.tradefed.util.SubprocessTestResultsParser;
44 
45 import com.google.common.base.Joiner;
46 
47 import java.io.File;
48 import java.io.IOException;
49 import java.util.ArrayList;
50 import java.util.Arrays;
51 import java.util.Formatter;
52 import java.util.HashMap;
53 import java.util.HashSet;
54 import java.util.LinkedHashSet;
55 import java.util.List;
56 import java.util.Set;
57 
58 /**
59  * Host test meant to run a python binary file from the Android Build system (Soong)
60  *
61  * <p>The test runner supports include-filter and exclude-filter. Note that exclude-filter works by
62  * ignoring the test result, instead of skipping the actual test. The tests specified in the
63  * exclude-filter will still be executed.
64  */
65 @OptionClass(alias = "python-host")
66 public class PythonBinaryHostTest implements IRemoteTest, ITestFilterReceiver {
67 
68     protected static final String ANDROID_SERIAL_VAR = "ANDROID_SERIAL";
69     protected static final String LD_LIBRARY_PATH = "LD_LIBRARY_PATH";
70     protected static final String PATH_VAR = "PATH";
71     protected static final long PATH_TIMEOUT_MS = 60000L;
72 
73     @VisibleForTesting static final String USE_TEST_OUTPUT_FILE_OPTION = "use-test-output-file";
74     static final String TEST_OUTPUT_FILE_FLAG = "test-output-file";
75 
76     private static final String PYTHON_LOG_STDERR_FORMAT = "%s-stderr";
77     private static final String PYTHON_LOG_TEST_OUTPUT_FORMAT = "%s-test-output";
78 
79     private Set<String> mIncludeFilters = new LinkedHashSet<>();
80     private Set<String> mExcludeFilters = new LinkedHashSet<>();
81     private String mLdLibraryPath = null;
82 
83     @Option(name = "par-file-name", description = "The binary names inside the build info to run.")
84     private Set<String> mBinaryNames = new HashSet<>();
85 
86     @Option(
87         name = "python-binaries",
88         description = "The full path to a runnable python binary. Can be repeated."
89     )
90     private Set<File> mBinaries = new HashSet<>();
91 
92     @Option(
93         name = "test-timeout",
94         description = "Timeout for a single par file to terminate.",
95         isTimeVal = true
96     )
97     private long mTestTimeout = 20 * 1000L;
98 
99     @Option(
100             name = "inject-serial-option",
101             description = "Whether or not to pass a -s <serialnumber> option to the binary")
102     private boolean mInjectSerial = false;
103 
104     @Option(
105             name = "inject-android-serial",
106             description = "Whether or not to pass a ANDROID_SERIAL variable to the process.")
107     private boolean mInjectAndroidSerialVar = true;
108 
109     @Option(
110         name = "python-options",
111         description = "Option string to be passed to the binary when running"
112     )
113     private List<String> mTestOptions = new ArrayList<>();
114 
115     @Option(
116             name = USE_TEST_OUTPUT_FILE_OPTION,
117             description =
118                     "Whether the test should write results to the file specified via the --"
119                             + TEST_OUTPUT_FILE_FLAG
120                             + " flag instead of stderr which could contain spurious messages that "
121                             + "break result parsing. Using this option requires that the Python "
122                             + "test have the necessary logic to accept the flag and write results "
123                             + "in the expected format.")
124     private boolean mUseTestOutputFile = false;
125 
126     private TestInformation mTestInfo;
127     private IRunUtil mRunUtil;
128 
129     /** {@inheritDoc} */
130     @Override
addIncludeFilter(String filter)131     public void addIncludeFilter(String filter) {
132         mIncludeFilters.add(filter);
133     }
134 
135     /** {@inheritDoc} */
136     @Override
addExcludeFilter(String filter)137     public void addExcludeFilter(String filter) {
138         mExcludeFilters.add(filter);
139     }
140 
141     /** {@inheritDoc} */
142     @Override
addAllIncludeFilters(Set<String> filters)143     public void addAllIncludeFilters(Set<String> filters) {
144         mIncludeFilters.addAll(filters);
145     }
146 
147     /** {@inheritDoc} */
148     @Override
addAllExcludeFilters(Set<String> filters)149     public void addAllExcludeFilters(Set<String> filters) {
150         mExcludeFilters.addAll(filters);
151     }
152 
153     /** {@inheritDoc} */
154     @Override
clearIncludeFilters()155     public void clearIncludeFilters() {
156         mIncludeFilters.clear();
157     }
158 
159     /** {@inheritDoc} */
160     @Override
clearExcludeFilters()161     public void clearExcludeFilters() {
162         mExcludeFilters.clear();
163     }
164 
165     /** {@inheritDoc} */
166     @Override
getIncludeFilters()167     public Set<String> getIncludeFilters() {
168         return mIncludeFilters;
169     }
170 
171     /** {@inheritDoc} */
172     @Override
getExcludeFilters()173     public Set<String> getExcludeFilters() {
174         return mExcludeFilters;
175     }
176 
177     @Override
run(TestInformation testInfo, ITestInvocationListener listener)178     public final void run(TestInformation testInfo, ITestInvocationListener listener)
179             throws DeviceNotAvailableException {
180         mTestInfo = testInfo;
181         File testDir = mTestInfo.executionFiles().get(FilesKey.HOST_TESTS_DIRECTORY);
182         if (testDir == null || !testDir.exists()) {
183             testDir = mTestInfo.executionFiles().get(FilesKey.TESTS_DIRECTORY);
184         }
185         if (testDir != null && testDir.exists()) {
186             File libDir = new File(testDir, "lib");
187             List<String> ldLibraryPath = new ArrayList<>();
188             if (libDir.exists()) {
189                 ldLibraryPath.add(libDir.getAbsolutePath());
190             }
191 
192             File lib64Dir = new File(testDir, "lib64");
193             if (lib64Dir.exists()) {
194                 ldLibraryPath.add(lib64Dir.getAbsolutePath());
195             }
196             if (!ldLibraryPath.isEmpty()) {
197                 mLdLibraryPath = Joiner.on(":").join(ldLibraryPath);
198             }
199         }
200         List<File> pythonFilesList = findParFiles();
201         for (File pyFile : pythonFilesList) {
202             if (!pyFile.exists()) {
203                 CLog.d(
204                         "ignoring %s which doesn't look like a test file.",
205                         pyFile.getAbsolutePath());
206                 continue;
207             }
208             pyFile.setExecutable(true);
209             runSinglePythonFile(listener, testInfo, pyFile);
210         }
211     }
212 
findParFiles()213     private List<File> findParFiles() {
214         File testsDir = null;
215         if (mTestInfo.getBuildInfo() instanceof IDeviceBuildInfo) {
216             testsDir = ((IDeviceBuildInfo) mTestInfo.getBuildInfo()).getTestsDir();
217         }
218         List<File> files = new ArrayList<>();
219         for (String parFileName : mBinaryNames) {
220             File res = null;
221             // search tests dir
222             if (testsDir != null) {
223                 res = FileUtil.findFile(testsDir, parFileName);
224             }
225 
226             // TODO: is there other places to search?
227             if (res == null) {
228                 throw new RuntimeException(
229                         String.format("Couldn't find a par file %s", parFileName));
230             }
231             files.add(res);
232         }
233         files.addAll(mBinaries);
234         return files;
235     }
236 
runSinglePythonFile( ITestInvocationListener listener, TestInformation testInfo, File pyFile)237     private void runSinglePythonFile(
238             ITestInvocationListener listener, TestInformation testInfo, File pyFile) {
239         List<String> commandLine = new ArrayList<>();
240         commandLine.add(pyFile.getAbsolutePath());
241         // If we have a physical device, pass it to the python test by serial
242         if (!(mTestInfo.getDevice().getIDevice() instanceof StubDevice) && mInjectSerial) {
243             // TODO: support multi-device python tests?
244             commandLine.add("-s");
245             commandLine.add(mTestInfo.getDevice().getSerialNumber());
246         }
247 
248         if (mLdLibraryPath != null) {
249             getRunUtil().setEnvVariable(LD_LIBRARY_PATH, mLdLibraryPath);
250         }
251         if (mInjectAndroidSerialVar) {
252             getRunUtil()
253                     .setEnvVariable(ANDROID_SERIAL_VAR, mTestInfo.getDevice().getSerialNumber());
254         }
255 
256         File tempTestOutputFile = null;
257         if (mUseTestOutputFile) {
258             try {
259                 tempTestOutputFile = FileUtil.createTempFile("python-test-output", ".txt");
260             } catch (IOException e) {
261                 throw new RuntimeException(e);
262             }
263 
264             commandLine.add("--" + TEST_OUTPUT_FILE_FLAG);
265             commandLine.add(tempTestOutputFile.getAbsolutePath());
266         }
267 
268         File updatedAdb = testInfo.executionFiles().get(FilesKey.ADB_BINARY);
269         if (updatedAdb == null) {
270             String adbPath = getAdbPath();
271             // Don't check if it's the adb on the $PATH
272             if (!adbPath.equals("adb")) {
273                 updatedAdb = new File(adbPath);
274                 if (!updatedAdb.exists()) {
275                     updatedAdb = null;
276                 }
277             }
278         }
279         if (updatedAdb != null) {
280             CLog.d("Testing with adb binary at: %s", updatedAdb);
281             // If a special adb version is used, pass it to the PATH
282             CommandResult pathResult =
283                     getRunUtil()
284                             .runTimedCmd(PATH_TIMEOUT_MS, "/bin/bash", "-c", "echo $" + PATH_VAR);
285             if (!CommandStatus.SUCCESS.equals(pathResult.getStatus())) {
286                 throw new RuntimeException(
287                         String.format(
288                                 "Failed to get the $PATH. status: %s, stdout: %s, stderr: %s",
289                                 pathResult.getStatus(),
290                                 pathResult.getStdout(),
291                                 pathResult.getStderr()));
292             }
293             // Include the directory of the adb on the PATH to be used.
294             String path =
295                     String.format(
296                             "%s:%s",
297                             updatedAdb.getParentFile().getAbsolutePath(),
298                             pathResult.getStdout().trim());
299             CLog.d("Using $PATH with updated adb: %s", path);
300             getRunUtil().setEnvVariable(PATH_VAR, path);
301             // Log the version of adb seen
302             CommandResult versionRes = getRunUtil().runTimedCmd(PATH_TIMEOUT_MS, "adb", "version");
303             CLog.d("%s", versionRes.getStdout());
304             CLog.d("%s", versionRes.getStderr());
305         }
306         // Add all the other options
307         commandLine.addAll(mTestOptions);
308 
309         CommandResult result =
310                 getRunUtil().runTimedCmd(mTestTimeout, commandLine.toArray(new String[0]));
311         String runName = pyFile.getName();
312         PythonForwarder forwarder = new PythonForwarder(listener, runName);
313         if (!CommandStatus.SUCCESS.equals(result.getStatus())) {
314             CLog.e(
315                     "Something went wrong when running the python binary:\nstdout: "
316                             + "%s\nstderr:%s",
317                     result.getStdout(), result.getStderr());
318         }
319 
320         File stderrFile = null;
321         try {
322             // Note that we still log stderr when parsing results from a test-written output file
323             // since it most likely contains useful debugging information.
324             stderrFile = FileUtil.createTempFile("python-res", ".txt");
325             FileUtil.writeToFile(result.getStderr(), stderrFile);
326             testLogFile(listener, String.format(PYTHON_LOG_STDERR_FORMAT, runName), stderrFile);
327 
328             File testOutputFile = stderrFile;
329             String testOutput = result.getStderr();
330 
331             if (mUseTestOutputFile) {
332                 testOutputFile = tempTestOutputFile;
333                 // This assumes that the output file is encoded using the same charset as the
334                 // currently configured default.
335                 testOutput = FileUtil.readStringFromFile(testOutputFile);
336                 testLogFile(
337                         listener,
338                         String.format(PYTHON_LOG_TEST_OUTPUT_FORMAT, runName),
339                         testOutputFile);
340             }
341 
342             // If it doesn't have the std output TEST_RUN_STARTED, use regular parser.
343             if (!testOutput.contains("TEST_RUN_STARTED")) {
344                 // Attempt to parse the pure python output
345                 PythonUnitTestResultParser pythonParser =
346                         new PythonUnitTestResultParser(
347                                 Arrays.asList(forwarder),
348                                 "python-run",
349                                 mIncludeFilters,
350                                 mExcludeFilters);
351                 pythonParser.processNewLines(testOutput.split("\n"));
352             } else {
353                 if (!mIncludeFilters.isEmpty() || !mExcludeFilters.isEmpty()) {
354                     throw new RuntimeException(
355                             "Non-unittest python test does not support using filters in "
356                                     + "PythonBinaryHostTest. Please use test runner "
357                                     + "ExecutableHostTest instead.");
358                 }
359                 try (SubprocessTestResultsParser parser =
360                         new SubprocessTestResultsParser(forwarder, mTestInfo.getContext())) {
361                     parser.parseFile(testOutputFile);
362                 }
363             }
364         } catch (RuntimeException e) {
365             StringBuilder message = new StringBuilder();
366             Formatter formatter = new Formatter(message);
367 
368             formatter.format(
369                     "Failed to parse the python logs: %s. Please ensure that verbosity of "
370                             + "output is high enough to be parsed.",
371                     e.getMessage());
372 
373             if (mUseTestOutputFile) {
374                 formatter.format(
375                         " Make sure that your test writes its output to the file specified "
376                                 + "by the --%s flag and that its contents (%s) are in the format "
377                                 + "expected by the test runner.",
378                         TEST_OUTPUT_FILE_FLAG,
379                         String.format(PYTHON_LOG_TEST_OUTPUT_FORMAT, runName));
380             }
381 
382             reportFailure(listener, runName, message.toString());
383             CLog.e(e);
384         } catch (IOException e) {
385             throw new RuntimeException(e);
386         } finally {
387             FileUtil.deleteFile(stderrFile);
388             FileUtil.deleteFile(tempTestOutputFile);
389         }
390     }
391 
392     @VisibleForTesting
getRunUtil()393     IRunUtil getRunUtil() {
394         if (mRunUtil == null) {
395             mRunUtil = new RunUtil();
396         }
397         return mRunUtil;
398     }
399 
400     @VisibleForTesting
getAdbPath()401     String getAdbPath() {
402         return GlobalConfiguration.getDeviceManagerInstance().getAdbPath();
403     }
404 
reportFailure( ITestInvocationListener listener, String runName, String errorMessage)405     private void reportFailure(
406             ITestInvocationListener listener, String runName, String errorMessage) {
407         listener.testRunStarted(runName, 0);
408         FailureDescription description =
409                 FailureDescription.create(errorMessage, FailureStatus.TEST_FAILURE);
410         listener.testRunFailed(description);
411         listener.testRunEnded(0L, new HashMap<String, Metric>());
412     }
413 
testLogFile(ITestInvocationListener listener, String dataName, File f)414     private static void testLogFile(ITestInvocationListener listener, String dataName, File f) {
415         try (FileInputStreamSource data = new FileInputStreamSource(f)) {
416             listener.testLog(dataName, LogDataType.TEXT, data);
417         }
418     }
419 
420     /** Result forwarder to replace the run name by the binary name. */
421     public static class PythonForwarder extends ResultForwarder {
422 
423         private String mRunName;
424 
425         /** Ctor with the run name using the binary name. */
PythonForwarder(ITestInvocationListener listener, String name)426         public PythonForwarder(ITestInvocationListener listener, String name) {
427             super(listener);
428             mRunName = name;
429         }
430 
431         @Override
testRunStarted(String runName, int testCount)432         public void testRunStarted(String runName, int testCount) {
433             // Replace run name
434             testRunStarted(runName, testCount, 0);
435         }
436 
437         @Override
testRunStarted(String runName, int testCount, int attempt)438         public void testRunStarted(String runName, int testCount, int attempt) {
439             // Replace run name
440             testRunStarted(runName, testCount, attempt, System.currentTimeMillis());
441         }
442 
443         @Override
testRunStarted(String runName, int testCount, int attempt, long startTime)444         public void testRunStarted(String runName, int testCount, int attempt, long startTime) {
445             // Replace run name
446             super.testRunStarted(mRunName, testCount, attempt, startTime);
447         }
448     }
449 }
450