1 /* 2 * Copyright (C) 2017 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.tradefed.testtype.python; 17 18 import com.android.annotations.VisibleForTesting; 19 import com.android.tradefed.build.IDeviceBuildInfo; 20 import com.android.tradefed.config.GlobalConfiguration; 21 import com.android.tradefed.config.Option; 22 import com.android.tradefed.config.OptionClass; 23 import com.android.tradefed.device.DeviceNotAvailableException; 24 import com.android.tradefed.device.StubDevice; 25 import com.android.tradefed.invoker.ExecutionFiles.FilesKey; 26 import com.android.tradefed.invoker.TestInformation; 27 import com.android.tradefed.log.LogUtil.CLog; 28 import com.android.tradefed.metrics.proto.MetricMeasurement.Metric; 29 import com.android.tradefed.result.FailureDescription; 30 import com.android.tradefed.result.FileInputStreamSource; 31 import com.android.tradefed.result.ITestInvocationListener; 32 import com.android.tradefed.result.LogDataType; 33 import com.android.tradefed.result.ResultForwarder; 34 import com.android.tradefed.result.proto.TestRecordProto.FailureStatus; 35 import com.android.tradefed.testtype.IRemoteTest; 36 import com.android.tradefed.testtype.ITestFilterReceiver; 37 import com.android.tradefed.testtype.PythonUnitTestResultParser; 38 import com.android.tradefed.util.CommandResult; 39 import com.android.tradefed.util.CommandStatus; 40 import com.android.tradefed.util.FileUtil; 41 import com.android.tradefed.util.IRunUtil; 42 import com.android.tradefed.util.RunUtil; 43 import com.android.tradefed.util.SubprocessTestResultsParser; 44 45 import com.google.common.base.Joiner; 46 47 import java.io.File; 48 import java.io.IOException; 49 import java.util.ArrayList; 50 import java.util.Arrays; 51 import java.util.Formatter; 52 import java.util.HashMap; 53 import java.util.HashSet; 54 import java.util.LinkedHashSet; 55 import java.util.List; 56 import java.util.Set; 57 58 /** 59 * Host test meant to run a python binary file from the Android Build system (Soong) 60 * 61 * <p>The test runner supports include-filter and exclude-filter. Note that exclude-filter works by 62 * ignoring the test result, instead of skipping the actual test. The tests specified in the 63 * exclude-filter will still be executed. 64 */ 65 @OptionClass(alias = "python-host") 66 public class PythonBinaryHostTest implements IRemoteTest, ITestFilterReceiver { 67 68 protected static final String ANDROID_SERIAL_VAR = "ANDROID_SERIAL"; 69 protected static final String LD_LIBRARY_PATH = "LD_LIBRARY_PATH"; 70 protected static final String PATH_VAR = "PATH"; 71 protected static final long PATH_TIMEOUT_MS = 60000L; 72 73 @VisibleForTesting static final String USE_TEST_OUTPUT_FILE_OPTION = "use-test-output-file"; 74 static final String TEST_OUTPUT_FILE_FLAG = "test-output-file"; 75 76 private static final String PYTHON_LOG_STDERR_FORMAT = "%s-stderr"; 77 private static final String PYTHON_LOG_TEST_OUTPUT_FORMAT = "%s-test-output"; 78 79 private Set<String> mIncludeFilters = new LinkedHashSet<>(); 80 private Set<String> mExcludeFilters = new LinkedHashSet<>(); 81 private String mLdLibraryPath = null; 82 83 @Option(name = "par-file-name", description = "The binary names inside the build info to run.") 84 private Set<String> mBinaryNames = new HashSet<>(); 85 86 @Option( 87 name = "python-binaries", 88 description = "The full path to a runnable python binary. Can be repeated." 89 ) 90 private Set<File> mBinaries = new HashSet<>(); 91 92 @Option( 93 name = "test-timeout", 94 description = "Timeout for a single par file to terminate.", 95 isTimeVal = true 96 ) 97 private long mTestTimeout = 20 * 1000L; 98 99 @Option( 100 name = "inject-serial-option", 101 description = "Whether or not to pass a -s <serialnumber> option to the binary") 102 private boolean mInjectSerial = false; 103 104 @Option( 105 name = "inject-android-serial", 106 description = "Whether or not to pass a ANDROID_SERIAL variable to the process.") 107 private boolean mInjectAndroidSerialVar = true; 108 109 @Option( 110 name = "python-options", 111 description = "Option string to be passed to the binary when running" 112 ) 113 private List<String> mTestOptions = new ArrayList<>(); 114 115 @Option( 116 name = USE_TEST_OUTPUT_FILE_OPTION, 117 description = 118 "Whether the test should write results to the file specified via the --" 119 + TEST_OUTPUT_FILE_FLAG 120 + " flag instead of stderr which could contain spurious messages that " 121 + "break result parsing. Using this option requires that the Python " 122 + "test have the necessary logic to accept the flag and write results " 123 + "in the expected format.") 124 private boolean mUseTestOutputFile = false; 125 126 private TestInformation mTestInfo; 127 private IRunUtil mRunUtil; 128 129 /** {@inheritDoc} */ 130 @Override addIncludeFilter(String filter)131 public void addIncludeFilter(String filter) { 132 mIncludeFilters.add(filter); 133 } 134 135 /** {@inheritDoc} */ 136 @Override addExcludeFilter(String filter)137 public void addExcludeFilter(String filter) { 138 mExcludeFilters.add(filter); 139 } 140 141 /** {@inheritDoc} */ 142 @Override addAllIncludeFilters(Set<String> filters)143 public void addAllIncludeFilters(Set<String> filters) { 144 mIncludeFilters.addAll(filters); 145 } 146 147 /** {@inheritDoc} */ 148 @Override addAllExcludeFilters(Set<String> filters)149 public void addAllExcludeFilters(Set<String> filters) { 150 mExcludeFilters.addAll(filters); 151 } 152 153 /** {@inheritDoc} */ 154 @Override clearIncludeFilters()155 public void clearIncludeFilters() { 156 mIncludeFilters.clear(); 157 } 158 159 /** {@inheritDoc} */ 160 @Override clearExcludeFilters()161 public void clearExcludeFilters() { 162 mExcludeFilters.clear(); 163 } 164 165 /** {@inheritDoc} */ 166 @Override getIncludeFilters()167 public Set<String> getIncludeFilters() { 168 return mIncludeFilters; 169 } 170 171 /** {@inheritDoc} */ 172 @Override getExcludeFilters()173 public Set<String> getExcludeFilters() { 174 return mExcludeFilters; 175 } 176 177 @Override run(TestInformation testInfo, ITestInvocationListener listener)178 public final void run(TestInformation testInfo, ITestInvocationListener listener) 179 throws DeviceNotAvailableException { 180 mTestInfo = testInfo; 181 File testDir = mTestInfo.executionFiles().get(FilesKey.HOST_TESTS_DIRECTORY); 182 if (testDir == null || !testDir.exists()) { 183 testDir = mTestInfo.executionFiles().get(FilesKey.TESTS_DIRECTORY); 184 } 185 if (testDir != null && testDir.exists()) { 186 File libDir = new File(testDir, "lib"); 187 List<String> ldLibraryPath = new ArrayList<>(); 188 if (libDir.exists()) { 189 ldLibraryPath.add(libDir.getAbsolutePath()); 190 } 191 192 File lib64Dir = new File(testDir, "lib64"); 193 if (lib64Dir.exists()) { 194 ldLibraryPath.add(lib64Dir.getAbsolutePath()); 195 } 196 if (!ldLibraryPath.isEmpty()) { 197 mLdLibraryPath = Joiner.on(":").join(ldLibraryPath); 198 } 199 } 200 List<File> pythonFilesList = findParFiles(); 201 for (File pyFile : pythonFilesList) { 202 if (!pyFile.exists()) { 203 CLog.d( 204 "ignoring %s which doesn't look like a test file.", 205 pyFile.getAbsolutePath()); 206 continue; 207 } 208 pyFile.setExecutable(true); 209 runSinglePythonFile(listener, testInfo, pyFile); 210 } 211 } 212 findParFiles()213 private List<File> findParFiles() { 214 File testsDir = null; 215 if (mTestInfo.getBuildInfo() instanceof IDeviceBuildInfo) { 216 testsDir = ((IDeviceBuildInfo) mTestInfo.getBuildInfo()).getTestsDir(); 217 } 218 List<File> files = new ArrayList<>(); 219 for (String parFileName : mBinaryNames) { 220 File res = null; 221 // search tests dir 222 if (testsDir != null) { 223 res = FileUtil.findFile(testsDir, parFileName); 224 } 225 226 // TODO: is there other places to search? 227 if (res == null) { 228 throw new RuntimeException( 229 String.format("Couldn't find a par file %s", parFileName)); 230 } 231 files.add(res); 232 } 233 files.addAll(mBinaries); 234 return files; 235 } 236 runSinglePythonFile( ITestInvocationListener listener, TestInformation testInfo, File pyFile)237 private void runSinglePythonFile( 238 ITestInvocationListener listener, TestInformation testInfo, File pyFile) { 239 List<String> commandLine = new ArrayList<>(); 240 commandLine.add(pyFile.getAbsolutePath()); 241 // If we have a physical device, pass it to the python test by serial 242 if (!(mTestInfo.getDevice().getIDevice() instanceof StubDevice) && mInjectSerial) { 243 // TODO: support multi-device python tests? 244 commandLine.add("-s"); 245 commandLine.add(mTestInfo.getDevice().getSerialNumber()); 246 } 247 248 if (mLdLibraryPath != null) { 249 getRunUtil().setEnvVariable(LD_LIBRARY_PATH, mLdLibraryPath); 250 } 251 if (mInjectAndroidSerialVar) { 252 getRunUtil() 253 .setEnvVariable(ANDROID_SERIAL_VAR, mTestInfo.getDevice().getSerialNumber()); 254 } 255 256 File tempTestOutputFile = null; 257 if (mUseTestOutputFile) { 258 try { 259 tempTestOutputFile = FileUtil.createTempFile("python-test-output", ".txt"); 260 } catch (IOException e) { 261 throw new RuntimeException(e); 262 } 263 264 commandLine.add("--" + TEST_OUTPUT_FILE_FLAG); 265 commandLine.add(tempTestOutputFile.getAbsolutePath()); 266 } 267 268 File updatedAdb = testInfo.executionFiles().get(FilesKey.ADB_BINARY); 269 if (updatedAdb == null) { 270 String adbPath = getAdbPath(); 271 // Don't check if it's the adb on the $PATH 272 if (!adbPath.equals("adb")) { 273 updatedAdb = new File(adbPath); 274 if (!updatedAdb.exists()) { 275 updatedAdb = null; 276 } 277 } 278 } 279 if (updatedAdb != null) { 280 CLog.d("Testing with adb binary at: %s", updatedAdb); 281 // If a special adb version is used, pass it to the PATH 282 CommandResult pathResult = 283 getRunUtil() 284 .runTimedCmd(PATH_TIMEOUT_MS, "/bin/bash", "-c", "echo $" + PATH_VAR); 285 if (!CommandStatus.SUCCESS.equals(pathResult.getStatus())) { 286 throw new RuntimeException( 287 String.format( 288 "Failed to get the $PATH. status: %s, stdout: %s, stderr: %s", 289 pathResult.getStatus(), 290 pathResult.getStdout(), 291 pathResult.getStderr())); 292 } 293 // Include the directory of the adb on the PATH to be used. 294 String path = 295 String.format( 296 "%s:%s", 297 updatedAdb.getParentFile().getAbsolutePath(), 298 pathResult.getStdout().trim()); 299 CLog.d("Using $PATH with updated adb: %s", path); 300 getRunUtil().setEnvVariable(PATH_VAR, path); 301 // Log the version of adb seen 302 CommandResult versionRes = getRunUtil().runTimedCmd(PATH_TIMEOUT_MS, "adb", "version"); 303 CLog.d("%s", versionRes.getStdout()); 304 CLog.d("%s", versionRes.getStderr()); 305 } 306 // Add all the other options 307 commandLine.addAll(mTestOptions); 308 309 CommandResult result = 310 getRunUtil().runTimedCmd(mTestTimeout, commandLine.toArray(new String[0])); 311 String runName = pyFile.getName(); 312 PythonForwarder forwarder = new PythonForwarder(listener, runName); 313 if (!CommandStatus.SUCCESS.equals(result.getStatus())) { 314 CLog.e( 315 "Something went wrong when running the python binary:\nstdout: " 316 + "%s\nstderr:%s", 317 result.getStdout(), result.getStderr()); 318 } 319 320 File stderrFile = null; 321 try { 322 // Note that we still log stderr when parsing results from a test-written output file 323 // since it most likely contains useful debugging information. 324 stderrFile = FileUtil.createTempFile("python-res", ".txt"); 325 FileUtil.writeToFile(result.getStderr(), stderrFile); 326 testLogFile(listener, String.format(PYTHON_LOG_STDERR_FORMAT, runName), stderrFile); 327 328 File testOutputFile = stderrFile; 329 String testOutput = result.getStderr(); 330 331 if (mUseTestOutputFile) { 332 testOutputFile = tempTestOutputFile; 333 // This assumes that the output file is encoded using the same charset as the 334 // currently configured default. 335 testOutput = FileUtil.readStringFromFile(testOutputFile); 336 testLogFile( 337 listener, 338 String.format(PYTHON_LOG_TEST_OUTPUT_FORMAT, runName), 339 testOutputFile); 340 } 341 342 // If it doesn't have the std output TEST_RUN_STARTED, use regular parser. 343 if (!testOutput.contains("TEST_RUN_STARTED")) { 344 // Attempt to parse the pure python output 345 PythonUnitTestResultParser pythonParser = 346 new PythonUnitTestResultParser( 347 Arrays.asList(forwarder), 348 "python-run", 349 mIncludeFilters, 350 mExcludeFilters); 351 pythonParser.processNewLines(testOutput.split("\n")); 352 } else { 353 if (!mIncludeFilters.isEmpty() || !mExcludeFilters.isEmpty()) { 354 throw new RuntimeException( 355 "Non-unittest python test does not support using filters in " 356 + "PythonBinaryHostTest. Please use test runner " 357 + "ExecutableHostTest instead."); 358 } 359 try (SubprocessTestResultsParser parser = 360 new SubprocessTestResultsParser(forwarder, mTestInfo.getContext())) { 361 parser.parseFile(testOutputFile); 362 } 363 } 364 } catch (RuntimeException e) { 365 StringBuilder message = new StringBuilder(); 366 Formatter formatter = new Formatter(message); 367 368 formatter.format( 369 "Failed to parse the python logs: %s. Please ensure that verbosity of " 370 + "output is high enough to be parsed.", 371 e.getMessage()); 372 373 if (mUseTestOutputFile) { 374 formatter.format( 375 " Make sure that your test writes its output to the file specified " 376 + "by the --%s flag and that its contents (%s) are in the format " 377 + "expected by the test runner.", 378 TEST_OUTPUT_FILE_FLAG, 379 String.format(PYTHON_LOG_TEST_OUTPUT_FORMAT, runName)); 380 } 381 382 reportFailure(listener, runName, message.toString()); 383 CLog.e(e); 384 } catch (IOException e) { 385 throw new RuntimeException(e); 386 } finally { 387 FileUtil.deleteFile(stderrFile); 388 FileUtil.deleteFile(tempTestOutputFile); 389 } 390 } 391 392 @VisibleForTesting getRunUtil()393 IRunUtil getRunUtil() { 394 if (mRunUtil == null) { 395 mRunUtil = new RunUtil(); 396 } 397 return mRunUtil; 398 } 399 400 @VisibleForTesting getAdbPath()401 String getAdbPath() { 402 return GlobalConfiguration.getDeviceManagerInstance().getAdbPath(); 403 } 404 reportFailure( ITestInvocationListener listener, String runName, String errorMessage)405 private void reportFailure( 406 ITestInvocationListener listener, String runName, String errorMessage) { 407 listener.testRunStarted(runName, 0); 408 FailureDescription description = 409 FailureDescription.create(errorMessage, FailureStatus.TEST_FAILURE); 410 listener.testRunFailed(description); 411 listener.testRunEnded(0L, new HashMap<String, Metric>()); 412 } 413 testLogFile(ITestInvocationListener listener, String dataName, File f)414 private static void testLogFile(ITestInvocationListener listener, String dataName, File f) { 415 try (FileInputStreamSource data = new FileInputStreamSource(f)) { 416 listener.testLog(dataName, LogDataType.TEXT, data); 417 } 418 } 419 420 /** Result forwarder to replace the run name by the binary name. */ 421 public static class PythonForwarder extends ResultForwarder { 422 423 private String mRunName; 424 425 /** Ctor with the run name using the binary name. */ PythonForwarder(ITestInvocationListener listener, String name)426 public PythonForwarder(ITestInvocationListener listener, String name) { 427 super(listener); 428 mRunName = name; 429 } 430 431 @Override testRunStarted(String runName, int testCount)432 public void testRunStarted(String runName, int testCount) { 433 // Replace run name 434 testRunStarted(runName, testCount, 0); 435 } 436 437 @Override testRunStarted(String runName, int testCount, int attempt)438 public void testRunStarted(String runName, int testCount, int attempt) { 439 // Replace run name 440 testRunStarted(runName, testCount, attempt, System.currentTimeMillis()); 441 } 442 443 @Override testRunStarted(String runName, int testCount, int attempt, long startTime)444 public void testRunStarted(String runName, int testCount, int attempt, long startTime) { 445 // Replace run name 446 super.testRunStarted(mRunName, testCount, attempt, startTime); 447 } 448 } 449 } 450