1 /* 2 * Copyright (C) 2010 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.cts.tradefed.testtype; 18 19 import com.android.cts.tradefed.build.CtsBuildHelper; 20 import com.android.cts.tradefed.device.DeviceInfoCollector; 21 import com.android.cts.tradefed.result.CtsTestStatus; 22 import com.android.cts.tradefed.result.PlanCreator; 23 import com.android.cts.util.AbiUtils; 24 import com.android.ddmlib.Log; 25 import com.android.ddmlib.Log.LogLevel; 26 import com.android.ddmlib.testrunner.TestIdentifier; 27 import com.android.tradefed.build.IBuildInfo; 28 import com.android.tradefed.config.ConfigurationException; 29 import com.android.tradefed.config.Option; 30 import com.android.tradefed.config.Option.Importance; 31 import com.android.tradefed.config.OptionCopier; 32 import com.android.tradefed.device.DeviceNotAvailableException; 33 import com.android.tradefed.device.ITestDevice; 34 import com.android.tradefed.device.TestDeviceOptions; 35 import com.android.tradefed.log.LogUtil.CLog; 36 import com.android.tradefed.result.ITestInvocationListener; 37 import com.android.tradefed.result.InputStreamSource; 38 import com.android.tradefed.result.LogDataType; 39 import com.android.tradefed.result.ResultForwarder; 40 import com.android.tradefed.targetprep.BuildError; 41 import com.android.tradefed.targetprep.ITargetCleaner; 42 import com.android.tradefed.targetprep.ITargetPreparer; 43 import com.android.tradefed.targetprep.TargetSetupError; 44 import com.android.tradefed.testtype.IAbi; 45 import com.android.tradefed.testtype.IAbiReceiver; 46 import com.android.tradefed.testtype.IBuildReceiver; 47 import com.android.tradefed.testtype.IDeviceTest; 48 import com.android.tradefed.testtype.IRemoteTest; 49 import com.android.tradefed.testtype.IResumableTest; 50 import com.android.tradefed.testtype.IShardableTest; 51 import com.android.tradefed.testtype.InstrumentationTest; 52 import com.android.tradefed.util.AbiFormatter; 53 import com.android.tradefed.util.RunUtil; 54 import com.android.tradefed.util.xml.AbstractXmlParser.ParseException; 55 56 import junit.framework.Test; 57 58 import java.io.BufferedInputStream; 59 import java.io.File; 60 import java.io.FileInputStream; 61 import java.io.FileNotFoundException; 62 import java.io.InputStream; 63 import java.util.ArrayList; 64 import java.util.Arrays; 65 import java.util.Collection; 66 import java.util.Collections; 67 import java.util.HashMap; 68 import java.util.HashSet; 69 import java.util.LinkedHashSet; 70 import java.util.LinkedList; 71 import java.util.List; 72 import java.util.ListIterator; 73 import java.util.Map; 74 import java.util.Set; 75 76 77 /** 78 * A {@link Test} for running CTS tests. 79 * <p/> 80 * Supports running all the tests contained in a CTS plan, or individual test packages. 81 */ 82 public class CtsTest implements IDeviceTest, IResumableTest, IShardableTest, IBuildReceiver { 83 private static final String LOG_TAG = "CtsTest"; 84 85 public static final String PLAN_OPTION = "plan"; 86 private static final String PACKAGE_OPTION = "package"; 87 private static final String CLASS_OPTION = "class"; 88 private static final String METHOD_OPTION = "method"; 89 private static final String TEST_OPTION = "test"; 90 public static final String CONTINUE_OPTION = "continue-session"; 91 public static final String RUN_KNOWN_FAILURES_OPTION = "run-known-failures"; 92 private static final String INCLUDE_FILTERS_OPTION = "include"; 93 private static final String EXCLUDE_FILTERS_OPTION = "exclude"; 94 95 public static final String PACKAGE_NAME_METRIC = "packageName"; 96 public static final String PACKAGE_ABI_METRIC = "packageAbi"; 97 public static final String PACKAGE_DIGEST_METRIC = "packageDigest"; 98 99 @Option(name = PLAN_OPTION, description = "the test plan to run.", 100 importance = Importance.IF_UNSET) 101 private String mPlanName = null; 102 103 @Option(name = PACKAGE_OPTION, shortName = 'p', description = "the test packages(s) to run.", 104 importance = Importance.IF_UNSET) 105 private Collection<String> mPackageNames = new ArrayList<String>(); 106 107 @Option(name = "exclude-package", description = "the test packages(s) to exclude from the run.") 108 private Collection<String> mExcludedPackageNames = new ArrayList<String>(); 109 110 @Option(name = CLASS_OPTION, shortName = 'c', description = "run a specific test class.", 111 importance = Importance.IF_UNSET) 112 private String mClassName = null; 113 114 @Option(name = METHOD_OPTION, shortName = 'm', 115 description = "run a specific test method, from given --class.", 116 importance = Importance.IF_UNSET) 117 private String mMethodName = null; 118 119 @Option(name = TEST_OPTION, shortName = 't', description = "run a specific test", 120 importance = Importance.IF_UNSET) 121 private String mTestName = null; 122 123 @Option(name = CONTINUE_OPTION, 124 description = "continue a previous test session.", 125 importance = Importance.IF_UNSET) 126 private Integer mContinueSessionId = null; 127 128 @Option(name = "skip-device-info", shortName = 'd', description = 129 "flag to control whether to collect info from device. Providing this flag will speed up " + 130 "test execution for short test runs but will result in required data being omitted from " + 131 "the test report.") 132 private boolean mSkipDeviceInfo = false; 133 134 @Option(name = "resume", description = 135 "flag to attempt to automatically resume aborted test run on another connected device. ") 136 private boolean mResume = false; 137 138 @Option(name = "shards", description = 139 "shard the tests to run into separately runnable chunks to execute on multiple devices " + 140 "concurrently.") 141 private int mShards = 1; 142 143 @Option(name = "screenshot", description = 144 "flag for taking a screenshot of the device when test execution is complete.") 145 private boolean mScreenshot = false; 146 147 @Option(name = "bugreport", shortName = 'b', description = 148 "take a bugreport after each failed test. " + 149 "Warning: can potentially use a lot of disk space.") 150 private boolean mBugreport = false; 151 152 @Option(name = RUN_KNOWN_FAILURES_OPTION, shortName = 'k', description = 153 "run tests including known failures") 154 private boolean mIncludeKnownFailures; 155 156 @Option(name = "disable-reboot", description = 157 "Do not reboot device after running some amount of tests. Default behavior is to reboot.") 158 private boolean mDisableReboot = false; 159 160 @Option(name = "reboot-wait-time", description = 161 "Additional wait time in ms after boot complete.") 162 private int mRebootWaitTimeMSec = 2 * 60 * 1000; 163 164 @Option(name = "reboot-interval", description = 165 "Interval between each reboot in min.") 166 private int mRebootIntervalMin = 30; 167 168 @Option(name = "screenshot-on-failure", description = 169 "take a screenshot on every test failure.") 170 private boolean mScreenshotOnFailures = false; 171 172 @Option(name = "logcat-on-failure", description = 173 "take a logcat snapshot on every test failure. Unlike --bugreport, this can capture" + 174 "logs even if connection with device has been lost, as well as being much more " + 175 "performant.") 176 private boolean mLogcatOnFailures = false; 177 178 @Option(name = AbiFormatter.FORCE_ABI_STRING, 179 description = AbiFormatter.FORCE_ABI_DESCRIPTION, 180 importance = Importance.IF_UNSET) 181 private String mForceAbi = null; 182 183 @Option(name = "logcat-on-failure-size", description = 184 "The max number of logcat data in bytes to capture when --logcat-on-failure is on. " + 185 "Should be an amount that can comfortably fit in memory.") 186 private int mMaxLogcatBytes = 500 * 1024; // 500K 187 188 @Option(name = "collect-deqp-logs", description = 189 "Collect dEQP logs from the device.") 190 private boolean mCollectDeqpLogs = false; 191 192 @Option(name = INCLUDE_FILTERS_OPTION, description = "Positive filters to pass to tests.") 193 private List<String> mPositiveFilters = new ArrayList<> (); 194 195 @Option(name = EXCLUDE_FILTERS_OPTION, description = "Negative filters to pass to tests.") 196 private List<String> mNegativeFilters = new ArrayList<> (); 197 198 @Option(name = "min-pre-reboot-package-count", description = 199 "The minimum number of packages to require a pre test reboot") 200 private int mMinPreRebootPackageCount = 2; 201 private final int mShardAssignment; 202 private final int mTotalShards; 203 private ITestDevice mDevice = null; 204 private CtsBuildHelper mCtsBuild = null; 205 private IBuildInfo mBuildInfo = null; 206 // last reboot time 207 private long mPrevRebootTime; 208 // The list of packages to run. populated in {@code setupTestPackageList} 209 // This is a member variable so that run can be called more than once 210 // and the test run is resumed. 211 private List<TestPackage> mTestPackageList = new ArrayList<>(); 212 // The index in the pacakge list of the last test to complete 213 private int mLastTestPackageIndex = 0; 214 215 /** data structure for a {@link IRemoteTest} and its known tests */ 216 static class TestPackage { 217 private final IRemoteTest mTestForPackage; 218 private final ITestPackageDef mPackageDef; 219 private final Collection<TestIdentifier> mKnownTests; 220 TestPackage(ITestPackageDef packageDef, IRemoteTest testForPackage)221 TestPackage(ITestPackageDef packageDef, IRemoteTest testForPackage) { 222 mPackageDef = packageDef; 223 mTestForPackage = testForPackage; 224 mKnownTests = packageDef.getTests(); 225 } 226 getTestForPackage()227 IRemoteTest getTestForPackage() { 228 return mTestForPackage; 229 } 230 getKnownTests()231 Collection<TestIdentifier> getKnownTests() { 232 return mKnownTests; 233 } 234 getPackageDef()235 ITestPackageDef getPackageDef() { 236 return mPackageDef; 237 } 238 239 /** 240 * @return the test run name that should be used for the TestPackage. 241 */ getTestRunName()242 String getTestRunName() { 243 return mPackageDef.getId(); 244 } 245 246 /** 247 * @return the ABI on which the test will run. 248 */ getAbi()249 IAbi getAbi() { 250 return mPackageDef.getAbi(); 251 } 252 } 253 254 /** 255 * A {@link ResultForwarder} that will forward a bugreport on each failed test. 256 */ 257 private static class FailedTestBugreportGenerator extends ResultForwarder { 258 private ITestDevice mDevice; 259 FailedTestBugreportGenerator(ITestInvocationListener listener, ITestDevice device)260 public FailedTestBugreportGenerator(ITestInvocationListener listener, ITestDevice device) { 261 super(listener); 262 mDevice = device; 263 } 264 265 @Override testFailed(TestIdentifier test, String trace)266 public void testFailed(TestIdentifier test, String trace) { 267 super.testFailed(test, trace); 268 InputStreamSource bugSource = mDevice.getBugreport(); 269 super.testLog(String.format("bug-%s_%s", test.getClassName(), test.getTestName()), 270 LogDataType.TEXT, bugSource); 271 bugSource.cancel(); 272 } 273 } 274 275 /** 276 * A {@link ResultForwarder} that will forward a logcat snapshot on each failed test. 277 */ 278 private static class FailedTestLogcatGenerator extends ResultForwarder { 279 private ITestDevice mDevice; 280 private int mNumLogcatBytes; 281 FailedTestLogcatGenerator(ITestInvocationListener listener, ITestDevice device, int maxLogcatBytes)282 public FailedTestLogcatGenerator(ITestInvocationListener listener, ITestDevice device, 283 int maxLogcatBytes) { 284 super(listener); 285 mDevice = device; 286 mNumLogcatBytes = maxLogcatBytes; 287 } 288 289 @Override testFailed(TestIdentifier test, String trace)290 public void testFailed(TestIdentifier test, String trace) { 291 super.testFailed(test, trace); 292 // sleep 2s to ensure test failure stack trace makes it into logcat capture 293 RunUtil.getDefault().sleep(2 * 1000); 294 InputStreamSource logSource = mDevice.getLogcat(mNumLogcatBytes); 295 super.testLog(String.format("logcat-%s_%s", test.getClassName(), test.getTestName()), 296 LogDataType.TEXT, logSource); 297 logSource.cancel(); 298 } 299 } 300 301 /** 302 * A {@link ResultForwarder} that will forward a screenshot on test failures. 303 */ 304 private static class FailedTestScreenshotGenerator extends ResultForwarder { 305 private ITestDevice mDevice; 306 FailedTestScreenshotGenerator(ITestInvocationListener listener, ITestDevice device)307 public FailedTestScreenshotGenerator(ITestInvocationListener listener, 308 ITestDevice device) { 309 super(listener); 310 mDevice = device; 311 } 312 313 @Override testFailed(TestIdentifier test, String trace)314 public void testFailed(TestIdentifier test, String trace) { 315 super.testFailed(test, trace); 316 317 try { 318 InputStreamSource screenSource = mDevice.getScreenshot(); 319 super.testLog(String.format("screenshot-%s_%s", test.getClassName(), 320 test.getTestName()), LogDataType.PNG, screenSource); 321 screenSource.cancel(); 322 } catch (DeviceNotAvailableException e) { 323 // TODO: rethrow this somehow 324 CLog.e("Device %s became unavailable while capturing screenshot, %s", 325 mDevice.getSerialNumber(), e.toString()); 326 } 327 } 328 } 329 330 /** 331 * Create a new {@link CtsTest} that will run the default list of {@link TestPackage}s. 332 */ CtsTest()333 public CtsTest() { 334 this(0 /*shardAssignment*/, 1 /*totalShards*/); 335 } 336 337 /** 338 * Create a new {@link CtsTest} that will run the given {@link List} of {@link TestPackage}s. 339 */ CtsTest(int shardAssignment, int totalShards)340 public CtsTest(int shardAssignment, int totalShards) { 341 if (shardAssignment < 0) { 342 throw new IllegalArgumentException( 343 "shardAssignment cannot be negative. found:" + shardAssignment); 344 } 345 if (totalShards < 1) { 346 throw new IllegalArgumentException( 347 "shardAssignment must be at least 1. found:" + totalShards); 348 } 349 this.mShardAssignment = shardAssignment; 350 this.mTotalShards = totalShards; 351 } 352 353 /** 354 * {@inheritDoc} 355 */ 356 @Override getDevice()357 public ITestDevice getDevice() { 358 return mDevice; 359 } 360 361 /** 362 * {@inheritDoc} 363 */ 364 @Override setDevice(ITestDevice device)365 public void setDevice(ITestDevice device) { 366 mDevice = device; 367 } 368 369 /** 370 * Set the plan name to run. 371 * <p/> 372 * Exposed for unit testing 373 */ setPlanName(String planName)374 void setPlanName(String planName) { 375 mPlanName = planName; 376 } 377 378 /** 379 * Set the skip collect device info flag. 380 * <p/> 381 * Exposed for unit testing 382 */ setSkipDeviceInfo(boolean skipDeviceInfo)383 void setSkipDeviceInfo(boolean skipDeviceInfo) { 384 mSkipDeviceInfo = skipDeviceInfo; 385 } 386 387 /** 388 * Adds a package name to the list of test packages to run. 389 * <p/> 390 * Exposed for unit testing 391 */ addPackageName(String packageName)392 void addPackageName(String packageName) { 393 mPackageNames.add(packageName); 394 } 395 396 /** 397 * Adds a package name to the list of test packages to exclude. 398 * <p/> 399 * Exposed for unit testing 400 */ addExcludedPackageName(String packageName)401 void addExcludedPackageName(String packageName) { 402 mExcludedPackageNames.add(packageName); 403 } 404 405 /** 406 * Set the test class name to run. 407 * <p/> 408 * Exposed for unit testing 409 */ setClassName(String className)410 void setClassName(String className) { 411 mClassName = className; 412 } 413 414 /** 415 * Set the test method name to run. 416 * <p/> 417 * Exposed for unit testing 418 */ setMethodName(String methodName)419 void setMethodName(String methodName) { 420 mMethodName = methodName; 421 } 422 423 /** 424 * Set the test name to run e.g. android.test.cts.SampleTest#testSample 425 * <p/> 426 * Exposed for unit testing 427 */ setTestName(String testName)428 void setTestName(String testName) { 429 mTestName = testName; 430 } 431 432 /** 433 * Sets the test session id to continue. 434 * <p/> 435 * Exposed for unit testing 436 */ setContinueSessionId(int sessionId)437 void setContinueSessionId(int sessionId) { 438 mContinueSessionId = sessionId; 439 } 440 441 /** 442 * {@inheritDoc} 443 */ 444 @Override isResumable()445 public boolean isResumable() { 446 return mResume; 447 } 448 449 /** 450 * {@inheritDoc} 451 */ 452 @Override setBuild(IBuildInfo build)453 public void setBuild(IBuildInfo build) { 454 mCtsBuild = CtsBuildHelper.createBuildHelper(build); 455 mBuildInfo = build; 456 } 457 458 /** 459 * Set the CTS build container. 460 * <p/> 461 * Exposed so unit tests can mock the provided build. 462 */ setBuildHelper(CtsBuildHelper buildHelper)463 void setBuildHelper(CtsBuildHelper buildHelper) { 464 mCtsBuild = buildHelper; 465 } 466 467 /** 468 * {@inheritDoc} 469 */ 470 @Override run(ITestInvocationListener listener)471 public void run(ITestInvocationListener listener) throws DeviceNotAvailableException { 472 if (getDevice() == null) { 473 throw new IllegalArgumentException("missing device"); 474 } 475 476 Set<String> abiSet = getAbis(); 477 if (abiSet == null || abiSet.isEmpty()) { 478 throw new IllegalArgumentException("could not get device's ABIs"); 479 } 480 Log.logAndDisplay(LogLevel.INFO, LOG_TAG, "ABIs: " + abiSet); 481 482 checkFields(); 483 setupTestPackageList(abiSet); 484 if (mBugreport) { 485 listener = new FailedTestBugreportGenerator(listener, getDevice()); 486 } 487 if (mScreenshotOnFailures) { 488 listener = new FailedTestScreenshotGenerator(listener, getDevice()); 489 } 490 if (mLogcatOnFailures) { 491 listener = new FailedTestLogcatGenerator(listener, getDevice(), mMaxLogcatBytes); 492 } 493 494 // Setup the a map of Test id to ResultFilter 495 Map<String, ResultFilter> filterMap = new HashMap<>(); 496 int totalTestCount = 0; 497 for (TestPackage testPackage : mTestPackageList) { 498 ResultFilter resultFilter = new ResultFilter(listener, testPackage); 499 totalTestCount += resultFilter.getKnownTestCount(); 500 filterMap.put(testPackage.getPackageDef().getId(), resultFilter); 501 } 502 503 // collect and install the prerequisiteApks first, to save time when multiple test 504 // packages are using the same prerequisite apk 505 Map<String, Set<String>> prerequisiteApks = getPrerequisiteApks(mTestPackageList, abiSet); 506 Collection<String> uninstallPackages = getPrerequisitePackageNames(mTestPackageList); 507 508 try { 509 // always collect the device info, even for resumed runs, since test will likely be 510 // running on a different device 511 collectDeviceInfo(getDevice(), mCtsBuild, listener); 512 preRebootIfNecessary(mTestPackageList); 513 514 mPrevRebootTime = System.currentTimeMillis(); 515 int remainingPackageCount = mTestPackageList.size(); 516 Log.logAndDisplay(LogLevel.INFO, LOG_TAG, 517 String.format("Start test run of %,d packages, containing %,d tests", 518 remainingPackageCount, totalTestCount)); 519 IAbi currentAbi = null; 520 521 for (int i = mLastTestPackageIndex; i < mTestPackageList.size(); i++) { 522 TestPackage testPackage = mTestPackageList.get(i); 523 524 if (currentAbi == null || 525 !currentAbi.getName().equals(testPackage.getAbi().getName())) { 526 currentAbi = testPackage.getAbi(); 527 installPrerequisiteApks( 528 prerequisiteApks.get(currentAbi.getName()), currentAbi); 529 } 530 531 IRemoteTest test = testPackage.getTestForPackage(); 532 if (test instanceof IBuildReceiver) { 533 ((IBuildReceiver) test).setBuild(mBuildInfo); 534 } 535 if (test instanceof IDeviceTest) { 536 ((IDeviceTest) test).setDevice(getDevice()); 537 } 538 if (test instanceof DeqpTestRunner) { 539 ((DeqpTestRunner)test).setCollectLogs(mCollectDeqpLogs); 540 } 541 if (test instanceof GeeTest) { 542 if (!mPositiveFilters.isEmpty()) { 543 String positivePatterns = join(mPositiveFilters, ":"); 544 ((GeeTest)test).setPositiveFilters(positivePatterns); 545 } 546 if (!mNegativeFilters.isEmpty()) { 547 String negativePatterns = join(mNegativeFilters, ":"); 548 ((GeeTest)test).setPositiveFilters(negativePatterns); 549 } 550 } 551 if (test instanceof InstrumentationTest) { 552 if (!mPositiveFilters.isEmpty()) { 553 String annotation = join(mPositiveFilters, ","); 554 ((InstrumentationTest)test).addInstrumentationArg( 555 "annotation", annotation); 556 } 557 if (!mNegativeFilters.isEmpty()) { 558 String notAnnotation = join(mNegativeFilters, ","); 559 ((InstrumentationTest)test).addInstrumentationArg( 560 "notAnnotation", notAnnotation); 561 } 562 } 563 564 forwardPackageDetails(testPackage.getPackageDef(), listener); 565 performPackagePrepareSetup(testPackage.getPackageDef()); 566 test.run(filterMap.get(testPackage.getPackageDef().getId())); 567 performPackagePreparerTearDown(testPackage.getPackageDef()); 568 if (i < mTestPackageList.size() - 1) { 569 TestPackage nextPackage = mTestPackageList.get(i + 1); 570 rebootIfNecessary(testPackage, nextPackage); 571 changeToHomeScreen(); 572 } 573 // Track of the last complete test package index for resume 574 mLastTestPackageIndex = i; 575 } 576 577 if (mScreenshot) { 578 InputStreamSource screenshotSource = getDevice().getScreenshot(); 579 try { 580 listener.testLog("screenshot", LogDataType.PNG, screenshotSource); 581 } finally { 582 screenshotSource.cancel(); 583 } 584 } 585 586 uninstallPrequisiteApks(uninstallPackages); 587 588 } catch (RuntimeException e) { 589 CLog.e(e); 590 throw e; 591 } catch (Error e) { 592 CLog.e(e); 593 throw e; 594 } finally { 595 for (ResultFilter filter : filterMap.values()) { 596 filter.reportUnexecutedTests(); 597 } 598 } 599 } 600 601 /** 602 * Invokes {@link ITargetPreparer}s configured for the test package. {@link TargetSetupError}s 603 * thrown by any preparer will be rethrown as {@link RuntimeException} so that the entire test 604 * package will be skipped for execution. Note that preparers will be invoked in the same order 605 * as they are defined in the module test config. 606 * @param packageDef definition for the test package 607 * @throws DeviceNotAvailableException 608 */ performPackagePrepareSetup(ITestPackageDef packageDef)609 private void performPackagePrepareSetup(ITestPackageDef packageDef) 610 throws DeviceNotAvailableException { 611 List<ITargetPreparer> preparers = packageDef.getPackagePreparers(); 612 if (preparers != null) { 613 for (ITargetPreparer preparer : preparers) { 614 if (preparer instanceof IAbiReceiver) { 615 ((IAbiReceiver) preparer).setAbi(packageDef.getAbi()); 616 } 617 try { 618 preparer.setUp(getDevice(), mBuildInfo); 619 } catch (BuildError e) { 620 // This should only happen for flashing new build 621 CLog.e("Unexpected BuildError from preparer: %s", 622 preparer.getClass().getCanonicalName()); 623 } catch (TargetSetupError e) { 624 // log preparer class then rethrow & let caller handle 625 CLog.e("TargetSetupError in preparer: %s", 626 preparer.getClass().getCanonicalName()); 627 throw new RuntimeException(e); 628 } 629 } 630 } 631 } 632 633 /** 634 * Invokes clean up step for {@link ITargetCleaner}s configured for the test package. Note that 635 * the cleaners will be invoked in the reverse order as they are defined in module test config. 636 * @param packageDef definition for the test package 637 * @throws DeviceNotAvailableException 638 */ performPackagePreparerTearDown(ITestPackageDef packageDef)639 private void performPackagePreparerTearDown(ITestPackageDef packageDef) 640 throws DeviceNotAvailableException { 641 List<ITargetPreparer> preparers = packageDef.getPackagePreparers(); 642 if (preparers != null) { 643 ListIterator<ITargetPreparer> itr = preparers.listIterator(preparers.size()); 644 // do teardown in reverse order 645 while (itr.hasPrevious()) { 646 ITargetPreparer preparer = itr.previous(); 647 if (preparer instanceof ITargetCleaner) { 648 ((ITargetCleaner) preparer).tearDown(getDevice(), mBuildInfo, null); 649 } 650 } 651 } 652 } 653 654 /** 655 * Helper method to join strings. Exposed for unit tests 656 * @param input 657 * @param conjunction 658 * @return string with elements of the input list with interleaved conjunction. 659 */ join(List<String> input, String conjunction)660 protected static String join(List<String> input, String conjunction) { 661 StringBuilder sb = new StringBuilder(); 662 boolean first = true; 663 for (String item : input) { 664 if (first) { 665 first = false; 666 } else { 667 sb.append(conjunction); 668 } 669 sb.append(item); 670 } 671 return sb.toString(); 672 } 673 674 /** 675 * @param allTestPackageDefList The package list to filter 676 * @param deviceAbiSet The ABIs supported by the device being tested 677 * @return A {@link List} of {@link ITestPackageDef}s that should be tested 678 */ filterByAbi( List<ITestPackageDef> allTestPackageDefList, Set<String> deviceAbiSet)679 private static List<ITestPackageDef> filterByAbi( 680 List<ITestPackageDef> allTestPackageDefList, Set<String> deviceAbiSet) { 681 List<ITestPackageDef> filteredTestPackageDefList = new LinkedList<>(); 682 for (ITestPackageDef testPackageDef : allTestPackageDefList) { 683 if (deviceAbiSet.contains(testPackageDef.getAbi().getName())) { 684 // We only need test packages that are not empty and of matching ABIs 685 filteredTestPackageDefList.add(testPackageDef); 686 } 687 } 688 return filteredTestPackageDefList; 689 } 690 691 /** Reboot then the device iff the list of packages exceeds the minimum */ preRebootIfNecessary(List<TestPackage> testPackageList)692 private void preRebootIfNecessary(List<TestPackage> testPackageList) 693 throws DeviceNotAvailableException { 694 if (mDisableReboot) { 695 return; 696 } 697 698 Set<String> packageNameSet = new HashSet<>(); 699 for (TestPackage testPackage : testPackageList) { 700 // Parse the package name 701 packageNameSet.add(AbiUtils.parseTestName(testPackage.getPackageDef().getId())); 702 } 703 if (packageNameSet.size() < mMinPreRebootPackageCount) { 704 // There is actually only one unique package name. No need to reboot. 705 return; 706 } 707 708 // Reboot is needed 709 Log.logAndDisplay(LogLevel.INFO, LOG_TAG, 710 String.format("Pre-test reboot (%,d packages). Use --disable-reboot to skip", 711 packageNameSet.size())); 712 713 rebootDevice(); 714 } 715 rebootIfNecessary(TestPackage testFinished, TestPackage testToRun)716 private void rebootIfNecessary(TestPackage testFinished, TestPackage testToRun) 717 throws DeviceNotAvailableException { 718 // If there comes spurious failure like INJECT_EVENTS for a package, 719 // reboot it before running it. 720 // Also reboot after package which is know to leave pop-up behind 721 final List<String> rebootAfterList = Arrays.asList( 722 "CtsMediaTestCases", 723 "CtsAccessibilityTestCases"); 724 final List<String> rebootBeforeList = Arrays.asList( 725 "CtsAnimationTestCases", 726 "CtsGraphicsTestCases", 727 "CtsViewTestCases", 728 "CtsWidgetTestCases" ); 729 long intervalInMSec = mRebootIntervalMin * 60 * 1000; 730 if (mDisableReboot || mDevice.getSerialNumber().startsWith("emulator-")) { 731 return; 732 } 733 long currentTime = System.currentTimeMillis(); 734 if (((currentTime - mPrevRebootTime) > intervalInMSec) || 735 rebootAfterList.contains(testFinished.getPackageDef().getName()) || 736 rebootBeforeList.contains(testToRun.getPackageDef().getName()) ) { 737 Log.i(LOG_TAG, 738 String.format("Rebooting after running package %s, before package %s", 739 testFinished.getPackageDef().getName(), 740 testToRun.getPackageDef().getName())); 741 rebootDevice(); 742 mPrevRebootTime = System.currentTimeMillis(); 743 } 744 } 745 rebootDevice()746 private void rebootDevice() throws DeviceNotAvailableException { 747 final int TIMEOUT_MS = 10 * 60 * 1000; 748 TestDeviceOptions options = mDevice.getOptions(); 749 // store default value and increase time-out for reboot 750 int rebootTimeout = options.getRebootTimeout(); 751 long onlineTimeout = options.getOnlineTimeout(); 752 options.setRebootTimeout(TIMEOUT_MS); 753 options.setOnlineTimeout(TIMEOUT_MS); 754 mDevice.setOptions(options); 755 756 mDevice.reboot(); 757 758 // restore default values 759 options.setRebootTimeout(rebootTimeout); 760 options.setOnlineTimeout(onlineTimeout); 761 mDevice.setOptions(options); 762 Log.i(LOG_TAG, "Rebooting done"); 763 try { 764 Thread.sleep(mRebootWaitTimeMSec); 765 } catch (InterruptedException e) { 766 Log.i(LOG_TAG, "Boot wait interrupted"); 767 } 768 } 769 770 /** 771 * Remove artifacts like status bar from the previous test. 772 * But this cannot dismiss dialog popped-up. 773 */ changeToHomeScreen()774 private void changeToHomeScreen() throws DeviceNotAvailableException { 775 final String homeCmd = "input keyevent 3"; 776 777 mDevice.executeShellCommand(homeCmd); 778 try { 779 Thread.sleep(1000); 780 } catch (InterruptedException e) { 781 //ignore 782 } 783 } 784 785 /** 786 * Set {@code mTestPackageList} to the list of test packages to run filtered by ABI. 787 */ setupTestPackageList(Set<String> abis)788 private void setupTestPackageList(Set<String> abis) throws DeviceNotAvailableException { 789 if (!mTestPackageList.isEmpty()) { 790 Log.logAndDisplay(LogLevel.INFO, LOG_TAG, "Resume tests using existing package list"); 791 return; 792 } 793 try { 794 // Collect ALL tests 795 ITestPackageRepo testRepo = createTestCaseRepo(); 796 List<ITestPackageDef> testPkgDefs = new ArrayList<>(getAvailableTestPackages(testRepo)); 797 testPkgDefs = filterByAbi(testPkgDefs, abis); 798 // Note: run() relies on the fact that the list is reliably sorted for sharding purposes 799 Collections.sort(testPkgDefs); 800 // Create test package list. 801 List<TestPackage> testPackageList = new ArrayList<>(); 802 for (ITestPackageDef testPackageDef : testPkgDefs) { 803 // Note: createTest filters the test list inside of testPackageDef by exclusion list 804 IRemoteTest testForPackage = testPackageDef.createTest(mCtsBuild.getTestCasesDir()); 805 if (testPackageDef.getTests().size() > 0) { 806 testPackageList.add(new TestPackage(testPackageDef, testForPackage)); 807 } 808 } 809 810 // Filter by shard 811 int numTestPackages = testPackageList.size(); 812 int totalShards = Math.min(mTotalShards, numTestPackages); 813 814 List<TestPackage> shardTestPackageList = new ArrayList<>(); 815 for (int i = mShardAssignment; i < numTestPackages; i += totalShards) { 816 shardTestPackageList.add(testPackageList.get(i)); 817 } 818 mTestPackageList.addAll(shardTestPackageList); 819 } catch (FileNotFoundException e) { 820 throw new IllegalArgumentException("failed to find test plan file", e); 821 } catch (ParseException e) { 822 throw new IllegalArgumentException("failed to parse test plan file", e); 823 } catch (ConfigurationException e) { 824 throw new IllegalArgumentException("failed to process arguments", e); 825 } 826 } 827 828 /** 829 * Return the {@link Set} of {@link ITestPackageDef}s to run unfiltered by ABI 830 * 831 * @return the {@link Set} of {@link ITestPackageDef}s to run 832 * @throws ParseException 833 * @throws FileNotFoundException 834 * @throws ConfigurationException 835 */ getAvailableTestPackages(ITestPackageRepo testRepo)836 private Set<ITestPackageDef> getAvailableTestPackages(ITestPackageRepo testRepo) 837 throws ParseException, FileNotFoundException, ConfigurationException { 838 // use LinkedHashSet to have predictable iteration order 839 Set<ITestPackageDef> testPkgDefs = new LinkedHashSet<>(); 840 if (mPlanName != null) { 841 Log.i(LOG_TAG, String.format("Executing CTS test plan %s", mPlanName)); 842 File ctsPlanFile = mCtsBuild.getTestPlanFile(mPlanName); 843 ITestPlan plan = createPlan(mPlanName); 844 plan.parse(createXmlStream(ctsPlanFile)); 845 846 for (String testId : plan.getTestIds()) { 847 if (mExcludedPackageNames.contains(AbiUtils.parseTestName(testId))) { 848 continue; 849 } 850 ITestPackageDef testPackageDef = testRepo.getTestPackage(testId); 851 if (testPackageDef == null) { 852 CLog.e("Could not find test id %s referenced in plan %s", testId, mPlanName); 853 continue; 854 } 855 856 testPackageDef.setTestFilter(plan.getTestFilter(testId)); 857 testPkgDefs.add(testPackageDef); 858 } 859 } else if (mPackageNames.size() > 0){ 860 Log.i(LOG_TAG, String.format("Executing test packages %s", mPackageNames)); 861 862 Map<String, List<ITestPackageDef>> testPackageDefMap = 863 testRepo.getTestPackageDefsByName(); 864 865 for (String name : mPackageNames) { 866 if (!testPackageDefMap.containsKey(name)) { 867 throw new IllegalArgumentException(String.format( 868 "Could not find test package %s. " + 869 "Use 'list packages' to see available packages.", name)); 870 } 871 testPkgDefs.addAll(testPackageDefMap.get(name)); 872 } 873 } else if (mClassName != null) { 874 Log.i(LOG_TAG, String.format("Executing CTS test class %s", mClassName)); 875 testPkgDefs.addAll(buildTestPackageDefSet(testRepo, mClassName, mMethodName)); 876 } else if (mTestName != null) { 877 Log.i(LOG_TAG, String.format("Executing CTS test %s", mTestName)); 878 String [] split = mTestName.split("#"); 879 if (split.length != 2) { 880 Log.logAndDisplay(LogLevel.WARN, LOG_TAG, String.format( 881 "Could not parse class and method from test %s", mTestName)); 882 } else { 883 String className = split[0]; 884 String methodName = split[1]; 885 testPkgDefs.addAll(buildTestPackageDefSet(testRepo, className, methodName)); 886 } 887 } else if (mContinueSessionId != null) { 888 // create an in-memory derived plan that contains the notExecuted tests from previous 889 // session use timestamp as plan name so it will hopefully be unique 890 String uniquePlanName = Long.toString(System.currentTimeMillis()); 891 PlanCreator planCreator = new PlanCreator(uniquePlanName, mContinueSessionId, 892 CtsTestStatus.NOT_EXECUTED); 893 ITestPlan plan = createPlan(planCreator); 894 for (String testId : plan.getTestIds()) { 895 if (mExcludedPackageNames.contains(AbiUtils.parseTestName(testId))) { 896 continue; 897 } 898 ITestPackageDef testPackageDef = testRepo.getTestPackage(testId); 899 if (testPackageDef == null) { 900 CLog.e("Could not find test id %s referenced in plan %s", testId, mPlanName); 901 continue; 902 } 903 904 testPackageDef.setTestFilter(plan.getTestFilter(testId)); 905 testPkgDefs.add(testPackageDef); 906 } 907 } else { 908 // should never get here - was checkFields() not called? 909 throw new IllegalStateException("nothing to run?"); 910 } 911 return testPkgDefs; 912 } 913 914 /** 915 * Return the list of unique prerequisite Android package names 916 * 917 * @param testPackages The {@link TestPackage}s that contain prerequisites 918 */ getPrerequisitePackageNames(List<TestPackage> testPackages)919 private Collection<String> getPrerequisitePackageNames(List<TestPackage> testPackages) { 920 Set<String> pkgNames = new HashSet<>(); 921 for (TestPackage testPkg : testPackages) { 922 String pkgName = testPkg.mPackageDef.getTargetPackageName(); 923 if (pkgName != null) { 924 pkgNames.add(pkgName); 925 } 926 } 927 return pkgNames; 928 } 929 930 /** 931 * @return a {@link Set} containing {@link ITestPackageDef}s pertaining to the given 932 * {@code className} and {@code methodName}. 933 */ buildTestPackageDefSet( ITestPackageRepo testRepo, String className, String methodName)934 private static Set<ITestPackageDef> buildTestPackageDefSet( 935 ITestPackageRepo testRepo, String className, String methodName) { 936 Set<ITestPackageDef> testPkgDefs = new LinkedHashSet<>(); 937 // try to find packages to run from class name 938 List<String> packageIds = testRepo.findPackageIdsForTest(className); 939 if (packageIds.isEmpty()) { 940 Log.logAndDisplay(LogLevel.WARN, LOG_TAG, String.format( 941 "Could not find package for test class %s", className)); 942 } 943 for (String packageId: packageIds) { 944 ITestPackageDef testPackageDef = testRepo.getTestPackage(packageId); 945 if (testPackageDef != null) { 946 testPackageDef.setClassName(className, methodName); 947 testPkgDefs.add(testPackageDef); 948 } 949 } 950 return testPkgDefs; 951 } 952 953 /** 954 * Return the list (by abi) of unique prerequisite apks to install 955 * 956 * @param testPackages The {@link List} of {@link TestPackage} that contain prerequisite APKs 957 */ getPrerequisiteApks( List<TestPackage> testPackages, Set<String> abiSet)958 private Map<String, Set<String>> getPrerequisiteApks( 959 List<TestPackage> testPackages, Set<String> abiSet) { 960 Map<String, Set<String>> abiToApkMap = new HashMap<>(); 961 for (TestPackage testPkg : testPackages) { 962 if (testPkg.getKnownTests().size() == 0) { 963 // No tests, no point in installing pre-reqs 964 continue; 965 } 966 String apkName = testPkg.mPackageDef.getTargetApkName(); 967 if (apkName == null) { 968 continue; 969 } 970 String abiName = testPkg.getAbi().getName(); 971 if (!abiSet.contains(abiName)) { 972 continue; 973 } 974 975 if (!abiToApkMap.containsKey(abiName)) { 976 abiToApkMap.put(abiName, new HashSet<String>()); 977 } 978 abiToApkMap.get(abiName).add(apkName); 979 } 980 return abiToApkMap; 981 } 982 983 /** 984 * FIXME eventually this should be removed once we get rid of CtsTestStubs, any other 985 * prerequisite apks should be installed by the test runner 986 * 987 * Install the collection of test apk file names 988 * 989 * @param prerequisiteApks The APKs that must be installed 990 * @throws DeviceNotAvailableException 991 */ installPrerequisiteApks(Collection<String> prerequisiteApks, IAbi abi)992 private void installPrerequisiteApks(Collection<String> prerequisiteApks, IAbi abi) 993 throws DeviceNotAvailableException { 994 if (prerequisiteApks == null) { 995 return; 996 } 997 Log.logAndDisplay(LogLevel.INFO, LOG_TAG, "Installing prerequisites"); 998 for (String apkName : prerequisiteApks) { 999 try { 1000 File apkFile = mCtsBuild.getTestApp(apkName); 1001 String[] options = {AbiUtils.createAbiFlag(abi.getName())}; 1002 String errorCode = getDevice().installPackage(apkFile, true, options); 1003 if (errorCode != null) { 1004 CLog.e("Failed to install %s. Reason: %s", apkName, errorCode); 1005 } 1006 } catch (FileNotFoundException e) { 1007 CLog.e("Could not find test apk %s", apkName); 1008 } 1009 } 1010 } 1011 1012 /** 1013 * Uninstalls the collection of android package names from device. 1014 * 1015 * @param uninstallPackages The packages that must be uninstalled 1016 */ uninstallPrequisiteApks(Collection<String> uninstallPackages)1017 private void uninstallPrequisiteApks(Collection<String> uninstallPackages) 1018 throws DeviceNotAvailableException { 1019 for (String pkgName : uninstallPackages) { 1020 getDevice().uninstallPackage(pkgName); 1021 } 1022 } 1023 1024 /** 1025 * {@inheritDoc} 1026 */ 1027 @Override split()1028 public Collection<IRemoteTest> split() { 1029 if (mShards <= 1) { 1030 return null; 1031 } 1032 checkFields(); 1033 1034 List<IRemoteTest> shardQueue = new LinkedList<>(); 1035 for (int shardAssignment = 0; shardAssignment < mShards; shardAssignment++) { 1036 CtsTest ctsTest = new CtsTest(shardAssignment, mShards /* totalShards */); 1037 OptionCopier.copyOptionsNoThrow(this, ctsTest); 1038 // Set the shard count because the copy option on the previous line copies 1039 // over the mShard value 1040 ctsTest.mShards = 0; 1041 shardQueue.add(ctsTest); 1042 } 1043 1044 return shardQueue; 1045 } 1046 1047 /** 1048 * Runs the device info collector instrumentation on device, and forwards it to test listeners 1049 * as run metrics. 1050 * <p/> 1051 * Exposed so unit tests can mock. 1052 * 1053 * @throws DeviceNotAvailableException 1054 */ collectDeviceInfo(ITestDevice device, CtsBuildHelper ctsBuild, ITestInvocationListener listener)1055 void collectDeviceInfo(ITestDevice device, CtsBuildHelper ctsBuild, 1056 ITestInvocationListener listener) throws DeviceNotAvailableException { 1057 if (!mSkipDeviceInfo) { 1058 String abi = AbiFormatter.getDefaultAbi(device, ""); 1059 DeviceInfoCollector.collectDeviceInfo(device, abi, ctsBuild.getTestCasesDir(), listener); 1060 DeviceInfoCollector.collectExtendedDeviceInfo( 1061 device, abi, ctsBuild.getTestCasesDir(), listener, mBuildInfo); 1062 } 1063 } 1064 1065 /** 1066 * Factory method for creating a {@link ITestPackageRepo}. 1067 * <p/> 1068 * Exposed for unit testing 1069 */ createTestCaseRepo()1070 ITestPackageRepo createTestCaseRepo() { 1071 return new TestPackageRepo(mCtsBuild.getTestCasesDir(), mIncludeKnownFailures); 1072 } 1073 1074 /** 1075 * Factory method for creating a {@link TestPlan}. 1076 * <p/> 1077 * Exposed for unit testing 1078 */ createPlan(String planName)1079 ITestPlan createPlan(String planName) { 1080 return new TestPlan(planName, AbiUtils.getAbisSupportedByCts()); 1081 } 1082 1083 /** 1084 * Gets the set of ABIs supported by both CTS and the device under test 1085 * <p/> 1086 * Exposed for unit testing 1087 * @return The set of ABIs to run the tests on 1088 * @throws DeviceNotAvailableException 1089 */ getAbis()1090 Set<String> getAbis() throws DeviceNotAvailableException { 1091 String bitness = (mForceAbi == null) ? "" : mForceAbi; 1092 Set<String> abis = new HashSet<>(); 1093 for (String abi : AbiFormatter.getSupportedAbis(mDevice, bitness)) { 1094 if (AbiUtils.isAbiSupportedByCts(abi)) { 1095 abis.add(abi); 1096 } 1097 } 1098 return abis; 1099 } 1100 1101 /** 1102 * Factory method for creating a {@link TestPlan} from a {@link PlanCreator}. 1103 * <p/> 1104 * Exposed for unit testing 1105 * @throws ConfigurationException 1106 */ createPlan(PlanCreator planCreator)1107 ITestPlan createPlan(PlanCreator planCreator) 1108 throws ConfigurationException { 1109 return planCreator.createDerivedPlan(mCtsBuild, AbiUtils.getAbisSupportedByCts()); 1110 } 1111 1112 /** 1113 * Factory method for creating a {@link InputStream} from a plan xml file. 1114 * <p/> 1115 * Exposed for unit testing 1116 */ createXmlStream(File xmlFile)1117 InputStream createXmlStream(File xmlFile) throws FileNotFoundException { 1118 return new BufferedInputStream(new FileInputStream(xmlFile)); 1119 } 1120 checkFields()1121 private void checkFields() { 1122 // for simplicity of command line usage, make --plan, --package, --test and --class mutually 1123 // exclusive 1124 boolean mutualExclusiveArgs = xor(mPlanName != null, mPackageNames.size() > 0, 1125 mClassName != null, mContinueSessionId != null, mTestName != null); 1126 1127 if (!mutualExclusiveArgs) { 1128 throw new IllegalArgumentException(String.format( 1129 "Ambiguous or missing arguments. " + 1130 "One and only one of --%s --%s(s), --%s or --%s to run can be specified", 1131 PLAN_OPTION, PACKAGE_OPTION, CLASS_OPTION, CONTINUE_OPTION)); 1132 } 1133 if (mMethodName != null && mClassName == null) { 1134 throw new IllegalArgumentException(String.format( 1135 "Must specify --%s when --%s is used", CLASS_OPTION, METHOD_OPTION)); 1136 } 1137 if (mCtsBuild == null) { 1138 throw new IllegalArgumentException("missing CTS build"); 1139 } 1140 } 1141 1142 /** 1143 * Helper method to perform exclusive or on list of boolean arguments 1144 * 1145 * @param args set of booleans on which to perform exclusive or 1146 * @return <code>true</code> if one and only one of <var>args</code> is <code>true</code>. 1147 * Otherwise return <code>false</code>. 1148 */ xor(boolean... args)1149 private static boolean xor(boolean... args) { 1150 boolean currentVal = args[0]; 1151 for (int i=1; i < args.length; i++) { 1152 if (currentVal && args[i]) { 1153 return false; 1154 } 1155 currentVal |= args[i]; 1156 } 1157 return currentVal; 1158 } 1159 1160 /** 1161 * Forward the digest and package name to the listener as a metric 1162 * 1163 * @param listener Handles test results 1164 */ forwardPackageDetails(ITestPackageDef def, ITestInvocationListener listener)1165 private static void forwardPackageDetails(ITestPackageDef def, ITestInvocationListener listener) { 1166 Map<String, String> metrics = new HashMap<>(3); 1167 metrics.put(PACKAGE_NAME_METRIC, def.getName()); 1168 metrics.put(PACKAGE_ABI_METRIC, def.getAbi().getName()); 1169 metrics.put(PACKAGE_DIGEST_METRIC, def.getDigest()); 1170 listener.testRunStarted(def.getId(), 0); 1171 listener.testRunEnded(0, metrics); 1172 } 1173 } 1174