1 /* 2 * Copyright (C) 2007 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.test; 18 19 import android.test.suitebuilder.annotation.MediumTest; 20 import android.test.suitebuilder.annotation.SmallTest; 21 import com.android.internal.util.Predicate; 22 23 import android.app.Activity; 24 import android.app.Instrumentation; 25 import android.os.Bundle; 26 import android.os.Debug; 27 import android.os.Looper; 28 import android.test.suitebuilder.TestMethod; 29 import android.test.suitebuilder.TestPredicates; 30 import android.test.suitebuilder.TestSuiteBuilder; 31 import android.test.suitebuilder.annotation.LargeTest; 32 import android.util.Log; 33 34 import java.io.ByteArrayOutputStream; 35 import java.io.File; 36 import java.io.PrintStream; 37 import java.lang.annotation.Annotation; 38 import java.lang.reflect.InvocationTargetException; 39 import java.lang.reflect.Method; 40 import java.util.ArrayList; 41 import java.util.List; 42 43 import junit.framework.AssertionFailedError; 44 import junit.framework.Test; 45 import junit.framework.TestCase; 46 import junit.framework.TestListener; 47 import junit.framework.TestResult; 48 import junit.framework.TestSuite; 49 import junit.runner.BaseTestRunner; 50 import junit.textui.ResultPrinter; 51 52 import static android.test.suitebuilder.TestPredicates.hasAnnotation; 53 54 /** 55 * An {@link Instrumentation} that runs various types of {@link junit.framework.TestCase}s against 56 * an Android package (application). 57 * 58 * <div class="special reference"> 59 * <h3>Developer Guides</h3> 60 * <p>For more information about application testing, read the 61 * <a href="{@docRoot}guide/topics/testing/index.html">Testing</a> developer guide.</p> 62 * </div> 63 * 64 * <h3>Typical Usage</h3> 65 * <ol> 66 * <li>Write {@link junit.framework.TestCase}s that perform unit, functional, or performance tests 67 * against the classes in your package. Typically these are subclassed from: 68 * <ul><li>{@link android.test.ActivityInstrumentationTestCase2}</li> 69 * <li>{@link android.test.ActivityUnitTestCase}</li> 70 * <li>{@link android.test.AndroidTestCase}</li> 71 * <li>{@link android.test.ApplicationTestCase}</li> 72 * <li>{@link android.test.InstrumentationTestCase}</li> 73 * <li>{@link android.test.ProviderTestCase}</li> 74 * <li>{@link android.test.ServiceTestCase}</li> 75 * <li>{@link android.test.SingleLaunchActivityTestCase}</li></ul> 76 * <li>Set the <code>android:targetPackage</code> attribute of the <code><instrumentation></code> 77 * element in the test package's manifest. You should set the attribute value 78 * to the package name of the target application under test. 79 * <li>Run the instrumentation using "adb shell am instrument -w", 80 * with no optional arguments, to run all tests (except performance tests). 81 * <li>Run the instrumentation using "adb shell am instrument -w", 82 * with the argument '-e func true' to run all functional tests. These are tests that derive from 83 * {@link android.test.InstrumentationTestCase}. 84 * <li>Run the instrumentation using "adb shell am instrument -w", 85 * with the argument '-e unit true' to run all unit tests. These are tests that <i>do not</i>derive 86 * from {@link android.test.InstrumentationTestCase} (and are not performance tests). 87 * <li>Run the instrumentation using "adb shell am instrument -w", 88 * with the argument '-e class' set to run an individual {@link junit.framework.TestCase}. 89 * </ol> 90 * <p/> 91 * <b>Running all tests:</b> adb shell am instrument -w 92 * com.android.foo/android.test.InstrumentationTestRunner 93 * <p/> 94 * <b>Running all small tests:</b> adb shell am instrument -w 95 * -e size small 96 * com.android.foo/android.test.InstrumentationTestRunner 97 * <p/> 98 * <b>Running all medium tests:</b> adb shell am instrument -w 99 * -e size medium 100 * com.android.foo/android.test.InstrumentationTestRunner 101 * <p/> 102 * <b>Running all large tests:</b> adb shell am instrument -w 103 * -e size large 104 * com.android.foo/android.test.InstrumentationTestRunner 105 * <p/> 106 * <b>Filter test run to tests with given annotation:</b> adb shell am instrument -w 107 * -e annotation com.android.foo.MyAnnotation 108 * com.android.foo/android.test.InstrumentationTestRunner 109 * <p/> 110 * If used with other options, the resulting test run will contain the union of the two options. 111 * e.g. "-e size large -e annotation com.android.foo.MyAnnotation" will run only tests with both 112 * the {@link LargeTest} and "com.android.foo.MyAnnotation" annotations. 113 * <p/> 114 * <b>Filter test run to tests <i>without</i> given annotation:</b> adb shell am instrument -w 115 * -e notAnnotation com.android.foo.MyAnnotation 116 * com.android.foo/android.test.InstrumentationTestRunner 117 * <p/> 118 * <b>Running a single testcase:</b> adb shell am instrument -w 119 * -e class com.android.foo.FooTest 120 * com.android.foo/android.test.InstrumentationTestRunner 121 * <p/> 122 * <b>Running a single test:</b> adb shell am instrument -w 123 * -e class com.android.foo.FooTest#testFoo 124 * com.android.foo/android.test.InstrumentationTestRunner 125 * <p/> 126 * <b>Running multiple tests:</b> adb shell am instrument -w 127 * -e class com.android.foo.FooTest,com.android.foo.TooTest 128 * com.android.foo/android.test.InstrumentationTestRunner 129 * <p/> 130 * <b>Running all tests in a java package:</b> adb shell am instrument -w 131 * -e package com.android.foo.subpkg 132 * com.android.foo/android.test.InstrumentationTestRunner 133 * <p/> 134 * <b>Including performance tests:</b> adb shell am instrument -w 135 * -e perf true 136 * com.android.foo/android.test.InstrumentationTestRunner 137 * <p/> 138 * <b>To debug your tests, set a break point in your code and pass:</b> 139 * -e debug true 140 * <p/> 141 * <b>To run in 'log only' mode</b> 142 * -e log true 143 * This option will load and iterate through all test classes and methods, but will bypass actual 144 * test execution. Useful for quickly obtaining info on the tests to be executed by an 145 * instrumentation command. 146 * <p/> 147 * <b>To generate EMMA code coverage:</b> 148 * -e coverage true 149 * Note: this requires an emma instrumented build. By default, the code coverage results file 150 * will be saved in a /data/<app>/coverage.ec file, unless overridden by coverageFile flag (see 151 * below) 152 * <p/> 153 * <b> To specify EMMA code coverage results file path:</b> 154 * -e coverageFile /sdcard/myFile.ec 155 * <br/> 156 * in addition to the other arguments. 157 * @deprecated Use 158 * <a href="{@docRoot}reference/android/support/test/runner/AndroidJUnitRunner.html"> 159 * AndroidJUnitRunner</a> instead. New tests should be written using the 160 * <a href="{@docRoot}tools/testing-support-library/index.html">Android Testing Support Library</a>. 161 */ 162 163 /* (not JavaDoc) 164 * Although not necessary in most case, another way to use this class is to extend it and have the 165 * derived class return the desired test suite from the {@link #getTestSuite()} method. The test 166 * suite returned from this method will be used if no target class is defined in the meta-data or 167 * command line argument parameters. If a derived class is used it needs to be added as an 168 * instrumentation to the AndroidManifest.xml and the command to run it would look like: 169 * <p/> 170 * adb shell am instrument -w com.android.foo/<i>com.android.FooInstrumentationTestRunner</i> 171 * <p/> 172 * Where <i>com.android.FooInstrumentationTestRunner</i> is the derived class. 173 * 174 * This model is used by many existing app tests, but can probably be deprecated. 175 */ 176 @Deprecated 177 public class InstrumentationTestRunner extends Instrumentation implements TestSuiteProvider { 178 179 /** @hide */ 180 static final String ARGUMENT_TEST_CLASS = "class"; 181 /** @hide */ 182 private static final String ARGUMENT_TEST_PACKAGE = "package"; 183 /** @hide */ 184 private static final String ARGUMENT_TEST_SIZE_PREDICATE = "size"; 185 /** @hide */ 186 static final String ARGUMENT_DELAY_MSEC = "delay_msec"; 187 188 private static final String SMALL_SUITE = "small"; 189 private static final String MEDIUM_SUITE = "medium"; 190 private static final String LARGE_SUITE = "large"; 191 192 private static final String ARGUMENT_LOG_ONLY = "log"; 193 /** @hide */ 194 static final String ARGUMENT_ANNOTATION = "annotation"; 195 /** @hide */ 196 static final String ARGUMENT_NOT_ANNOTATION = "notAnnotation"; 197 198 private static final Predicate<TestMethod> SELECT_SMALL = hasAnnotation(SmallTest.class); 199 200 private static final Predicate<TestMethod> SELECT_MEDIUM = hasAnnotation(MediumTest.class); 201 202 private static final Predicate<TestMethod> SELECT_LARGE = hasAnnotation(LargeTest.class); 203 204 /** 205 * This constant defines the maximum allowed runtime (in ms) for a test included in the "small" 206 * suite. It is used to make an educated guess at what suite an unlabeled test belongs. 207 */ 208 private static final float SMALL_SUITE_MAX_RUNTIME = 100; 209 210 /** 211 * This constant defines the maximum allowed runtime (in ms) for a test included in the 212 * "medium" suite. It is used to make an educated guess at what suite an unlabeled test belongs. 213 */ 214 private static final float MEDIUM_SUITE_MAX_RUNTIME = 1000; 215 216 /* 217 * The following keys are used in the status bundle to provide structured reports to 218 * an IInstrumentationWatcher. 219 */ 220 221 /** 222 * This value, if stored with key {@link android.app.Instrumentation#REPORT_KEY_IDENTIFIER}, 223 * identifies InstrumentationTestRunner as the source of the report. This is sent with all 224 * status messages. 225 */ 226 public static final String REPORT_VALUE_ID = "InstrumentationTestRunner"; 227 /** 228 * If included in the status or final bundle sent to an IInstrumentationWatcher, this key 229 * identifies the total number of tests that are being run. This is sent with all status 230 * messages. 231 */ 232 public static final String REPORT_KEY_NUM_TOTAL = "numtests"; 233 /** 234 * If included in the status or final bundle sent to an IInstrumentationWatcher, this key 235 * identifies the sequence number of the current test. This is sent with any status message 236 * describing a specific test being started or completed. 237 */ 238 public static final String REPORT_KEY_NUM_CURRENT = "current"; 239 /** 240 * If included in the status or final bundle sent to an IInstrumentationWatcher, this key 241 * identifies the name of the current test class. This is sent with any status message 242 * describing a specific test being started or completed. 243 */ 244 public static final String REPORT_KEY_NAME_CLASS = "class"; 245 /** 246 * If included in the status or final bundle sent to an IInstrumentationWatcher, this key 247 * identifies the name of the current test. This is sent with any status message 248 * describing a specific test being started or completed. 249 */ 250 public static final String REPORT_KEY_NAME_TEST = "test"; 251 /** 252 * If included in the status or final bundle sent to an IInstrumentationWatcher, this key 253 * reports the run time in seconds of the current test. 254 */ 255 private static final String REPORT_KEY_RUN_TIME = "runtime"; 256 /** 257 * If included in the status or final bundle sent to an IInstrumentationWatcher, this key 258 * reports the number of total iterations of the current test. 259 */ 260 private static final String REPORT_KEY_NUM_ITERATIONS = "numiterations"; 261 /** 262 * If included in the status or final bundle sent to an IInstrumentationWatcher, this key 263 * reports the guessed suite assignment for the current test. 264 */ 265 private static final String REPORT_KEY_SUITE_ASSIGNMENT = "suiteassignment"; 266 /** 267 * If included in the status or final bundle sent to an IInstrumentationWatcher, this key 268 * identifies the path to the generated code coverage file. 269 */ 270 private static final String REPORT_KEY_COVERAGE_PATH = "coverageFilePath"; 271 272 /** 273 * The test is starting. 274 */ 275 public static final int REPORT_VALUE_RESULT_START = 1; 276 /** 277 * The test completed successfully. 278 */ 279 public static final int REPORT_VALUE_RESULT_OK = 0; 280 /** 281 * The test completed with an error. 282 */ 283 public static final int REPORT_VALUE_RESULT_ERROR = -1; 284 /** 285 * The test completed with a failure. 286 */ 287 public static final int REPORT_VALUE_RESULT_FAILURE = -2; 288 /** 289 * If included in the status bundle sent to an IInstrumentationWatcher, this key 290 * identifies a stack trace describing an error or failure. This is sent with any status 291 * message describing a specific test being completed. 292 */ 293 public static final String REPORT_KEY_STACK = "stack"; 294 295 // Default file name for code coverage 296 private static final String DEFAULT_COVERAGE_FILE_NAME = "coverage.ec"; 297 298 private static final String LOG_TAG = "InstrumentationTestRunner"; 299 300 private final Bundle mResults = new Bundle(); 301 private Bundle mArguments; 302 private AndroidTestRunner mTestRunner; 303 private boolean mDebug; 304 private boolean mJustCount; 305 private boolean mSuiteAssignmentMode; 306 private int mTestCount; 307 private String mPackageOfTests; 308 private boolean mCoverage; 309 private String mCoverageFilePath; 310 private int mDelayMsec; 311 312 @Override onCreate(Bundle arguments)313 public void onCreate(Bundle arguments) { 314 super.onCreate(arguments); 315 mArguments = arguments; 316 317 // Apk paths used to search for test classes when using TestSuiteBuilders. 318 String[] apkPaths = 319 {getTargetContext().getPackageCodePath(), getContext().getPackageCodePath()}; 320 ClassPathPackageInfoSource.setApkPaths(apkPaths); 321 322 Predicate<TestMethod> testSizePredicate = null; 323 Predicate<TestMethod> testAnnotationPredicate = null; 324 Predicate<TestMethod> testNotAnnotationPredicate = null; 325 String testClassesArg = null; 326 boolean logOnly = false; 327 328 if (arguments != null) { 329 // Test class name passed as an argument should override any meta-data declaration. 330 testClassesArg = arguments.getString(ARGUMENT_TEST_CLASS); 331 mDebug = getBooleanArgument(arguments, "debug"); 332 mJustCount = getBooleanArgument(arguments, "count"); 333 mSuiteAssignmentMode = getBooleanArgument(arguments, "suiteAssignment"); 334 mPackageOfTests = arguments.getString(ARGUMENT_TEST_PACKAGE); 335 testSizePredicate = getSizePredicateFromArg( 336 arguments.getString(ARGUMENT_TEST_SIZE_PREDICATE)); 337 testAnnotationPredicate = getAnnotationPredicate( 338 arguments.getString(ARGUMENT_ANNOTATION)); 339 testNotAnnotationPredicate = getNotAnnotationPredicate( 340 arguments.getString(ARGUMENT_NOT_ANNOTATION)); 341 342 logOnly = getBooleanArgument(arguments, ARGUMENT_LOG_ONLY); 343 mCoverage = getBooleanArgument(arguments, "coverage"); 344 mCoverageFilePath = arguments.getString("coverageFile"); 345 346 try { 347 Object delay = arguments.get(ARGUMENT_DELAY_MSEC); // Accept either string or int 348 if (delay != null) mDelayMsec = Integer.parseInt(delay.toString()); 349 } catch (NumberFormatException e) { 350 Log.e(LOG_TAG, "Invalid delay_msec parameter", e); 351 } 352 } 353 354 TestSuiteBuilder testSuiteBuilder = new TestSuiteBuilder(getClass().getName(), 355 getTargetContext().getClassLoader()); 356 357 if (testSizePredicate != null) { 358 testSuiteBuilder.addRequirements(testSizePredicate); 359 } 360 if (testAnnotationPredicate != null) { 361 testSuiteBuilder.addRequirements(testAnnotationPredicate); 362 } 363 if (testNotAnnotationPredicate != null) { 364 testSuiteBuilder.addRequirements(testNotAnnotationPredicate); 365 } 366 367 if (testClassesArg == null) { 368 if (mPackageOfTests != null) { 369 testSuiteBuilder.includePackages(mPackageOfTests); 370 } else { 371 TestSuite testSuite = getTestSuite(); 372 if (testSuite != null) { 373 testSuiteBuilder.addTestSuite(testSuite); 374 } else { 375 // no package or class bundle arguments were supplied, and no test suite 376 // provided so add all tests in application 377 testSuiteBuilder.includePackages(""); 378 } 379 } 380 } else { 381 parseTestClasses(testClassesArg, testSuiteBuilder); 382 } 383 384 testSuiteBuilder.addRequirements(getBuilderRequirements()); 385 386 mTestRunner = getAndroidTestRunner(); 387 mTestRunner.setContext(getTargetContext()); 388 mTestRunner.setInstrumentation(this); 389 mTestRunner.setSkipExecution(logOnly); 390 mTestRunner.setTest(testSuiteBuilder.build()); 391 mTestCount = mTestRunner.getTestCases().size(); 392 if (mSuiteAssignmentMode) { 393 mTestRunner.addTestListener(new SuiteAssignmentPrinter()); 394 } else { 395 WatcherResultPrinter resultPrinter = new WatcherResultPrinter(mTestCount); 396 mTestRunner.addTestListener(new TestPrinter("TestRunner", false)); 397 mTestRunner.addTestListener(resultPrinter); 398 } 399 start(); 400 } 401 402 /** 403 * Get the arguments passed to this instrumentation. 404 * 405 * @return the Bundle object 406 */ getArguments()407 public Bundle getArguments() { 408 return mArguments; 409 } 410 411 /** 412 * Add a {@link TestListener} 413 * @hide 414 */ addTestListener(TestListener listener)415 protected void addTestListener(TestListener listener){ 416 if(mTestRunner!=null && listener!=null){ 417 mTestRunner.addTestListener(listener); 418 } 419 } 420 getBuilderRequirements()421 List<Predicate<TestMethod>> getBuilderRequirements() { 422 return new ArrayList<Predicate<TestMethod>>(); 423 } 424 425 /** 426 * Parses and loads the specified set of test classes 427 * 428 * @param testClassArg - comma-separated list of test classes and methods 429 * @param testSuiteBuilder - builder to add tests to 430 */ parseTestClasses(String testClassArg, TestSuiteBuilder testSuiteBuilder)431 private void parseTestClasses(String testClassArg, TestSuiteBuilder testSuiteBuilder) { 432 String[] testClasses = testClassArg.split(","); 433 for (String testClass : testClasses) { 434 parseTestClass(testClass, testSuiteBuilder); 435 } 436 } 437 438 /** 439 * Parse and load the given test class and, optionally, method 440 * 441 * @param testClassName - full package name of test class and optionally method to add. 442 * Expected format: com.android.TestClass#testMethod 443 * @param testSuiteBuilder - builder to add tests to 444 */ parseTestClass(String testClassName, TestSuiteBuilder testSuiteBuilder)445 private void parseTestClass(String testClassName, TestSuiteBuilder testSuiteBuilder) { 446 int methodSeparatorIndex = testClassName.indexOf('#'); 447 String testMethodName = null; 448 449 if (methodSeparatorIndex > 0) { 450 testMethodName = testClassName.substring(methodSeparatorIndex + 1); 451 testClassName = testClassName.substring(0, methodSeparatorIndex); 452 } 453 testSuiteBuilder.addTestClassByName(testClassName, testMethodName, getTargetContext()); 454 } 455 getAndroidTestRunner()456 protected AndroidTestRunner getAndroidTestRunner() { 457 return new AndroidTestRunner(); 458 } 459 getBooleanArgument(Bundle arguments, String tag)460 private boolean getBooleanArgument(Bundle arguments, String tag) { 461 String tagString = arguments.getString(tag); 462 return tagString != null && Boolean.parseBoolean(tagString); 463 } 464 465 /* 466 * Returns the size predicate object, corresponding to the "size" argument value. 467 */ getSizePredicateFromArg(String sizeArg)468 private Predicate<TestMethod> getSizePredicateFromArg(String sizeArg) { 469 470 if (SMALL_SUITE.equals(sizeArg)) { 471 return SELECT_SMALL; 472 } else if (MEDIUM_SUITE.equals(sizeArg)) { 473 return SELECT_MEDIUM; 474 } else if (LARGE_SUITE.equals(sizeArg)) { 475 return SELECT_LARGE; 476 } else { 477 return null; 478 } 479 } 480 481 /** 482 * Returns the test predicate object, corresponding to the annotation class value provided via 483 * the {@link #ARGUMENT_ANNOTATION} argument. 484 * 485 * @return the predicate or <code>null</code> 486 */ getAnnotationPredicate(String annotationClassName)487 private Predicate<TestMethod> getAnnotationPredicate(String annotationClassName) { 488 Class<? extends Annotation> annotationClass = getAnnotationClass(annotationClassName); 489 if (annotationClass != null) { 490 return hasAnnotation(annotationClass); 491 } 492 return null; 493 } 494 495 /** 496 * Returns the negative test predicate object, corresponding to the annotation class value 497 * provided via the {@link #ARGUMENT_NOT_ANNOTATION} argument. 498 * 499 * @return the predicate or <code>null</code> 500 */ getNotAnnotationPredicate(String annotationClassName)501 private Predicate<TestMethod> getNotAnnotationPredicate(String annotationClassName) { 502 Class<? extends Annotation> annotationClass = getAnnotationClass(annotationClassName); 503 if (annotationClass != null) { 504 return TestPredicates.not(hasAnnotation(annotationClass)); 505 } 506 return null; 507 } 508 509 /** 510 * Helper method to return the annotation class with specified name 511 * 512 * @param annotationClassName the fully qualified name of the class 513 * @return the annotation class or <code>null</code> 514 */ getAnnotationClass(String annotationClassName)515 private Class<? extends Annotation> getAnnotationClass(String annotationClassName) { 516 if (annotationClassName == null) { 517 return null; 518 } 519 try { 520 Class<?> annotationClass = Class.forName(annotationClassName); 521 if (annotationClass.isAnnotation()) { 522 return (Class<? extends Annotation>)annotationClass; 523 } else { 524 Log.e(LOG_TAG, String.format("Provided annotation value %s is not an Annotation", 525 annotationClassName)); 526 } 527 } catch (ClassNotFoundException e) { 528 Log.e(LOG_TAG, String.format("Could not find class for specified annotation %s", 529 annotationClassName)); 530 } 531 return null; 532 } 533 534 /** 535 * Initialize the current thread as a looper. 536 * <p/> 537 * Exposed for unit testing. 538 */ prepareLooper()539 void prepareLooper() { 540 Looper.prepare(); 541 } 542 543 @Override onStart()544 public void onStart() { 545 prepareLooper(); 546 547 if (mJustCount) { 548 mResults.putString(Instrumentation.REPORT_KEY_IDENTIFIER, REPORT_VALUE_ID); 549 mResults.putInt(REPORT_KEY_NUM_TOTAL, mTestCount); 550 finish(Activity.RESULT_OK, mResults); 551 } else { 552 if (mDebug) { 553 Debug.waitForDebugger(); 554 } 555 556 ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); 557 PrintStream writer = new PrintStream(byteArrayOutputStream); 558 try { 559 StringResultPrinter resultPrinter = new StringResultPrinter(writer); 560 561 mTestRunner.addTestListener(resultPrinter); 562 563 long startTime = System.currentTimeMillis(); 564 mTestRunner.runTest(); 565 long runTime = System.currentTimeMillis() - startTime; 566 567 resultPrinter.printResult(mTestRunner.getTestResult(), runTime); 568 } catch (Throwable t) { 569 // catch all exceptions so a more verbose error message can be outputted 570 writer.println(String.format("Test run aborted due to unexpected exception: %s", 571 t.getMessage())); 572 t.printStackTrace(writer); 573 } finally { 574 mResults.putString(Instrumentation.REPORT_KEY_STREAMRESULT, 575 String.format("\nTest results for %s=%s", 576 mTestRunner.getTestClassName(), 577 byteArrayOutputStream.toString())); 578 579 if (mCoverage) { 580 generateCoverageReport(); 581 } 582 writer.close(); 583 584 finish(Activity.RESULT_OK, mResults); 585 } 586 } 587 } 588 getTestSuite()589 public TestSuite getTestSuite() { 590 return getAllTests(); 591 } 592 593 /** 594 * Override this to define all of the tests to run in your package. 595 */ getAllTests()596 public TestSuite getAllTests() { 597 return null; 598 } 599 600 /** 601 * Override this to provide access to the class loader of your package. 602 */ getLoader()603 public ClassLoader getLoader() { 604 return null; 605 } 606 generateCoverageReport()607 private void generateCoverageReport() { 608 // use reflection to call emma dump coverage method, to avoid 609 // always statically compiling against emma jar 610 String coverageFilePath = getCoverageFilePath(); 611 java.io.File coverageFile = new java.io.File(coverageFilePath); 612 try { 613 Class<?> emmaRTClass = Class.forName("com.vladium.emma.rt.RT"); 614 Method dumpCoverageMethod = emmaRTClass.getMethod("dumpCoverageData", 615 coverageFile.getClass(), boolean.class, boolean.class); 616 617 dumpCoverageMethod.invoke(null, coverageFile, false, false); 618 // output path to generated coverage file so it can be parsed by a test harness if 619 // needed 620 mResults.putString(REPORT_KEY_COVERAGE_PATH, coverageFilePath); 621 // also output a more user friendly msg 622 final String currentStream = mResults.getString( 623 Instrumentation.REPORT_KEY_STREAMRESULT); 624 mResults.putString(Instrumentation.REPORT_KEY_STREAMRESULT, 625 String.format("%s\nGenerated code coverage data to %s", currentStream, 626 coverageFilePath)); 627 } catch (ClassNotFoundException e) { 628 reportEmmaError("Is emma jar on classpath?", e); 629 } catch (SecurityException e) { 630 reportEmmaError(e); 631 } catch (NoSuchMethodException e) { 632 reportEmmaError(e); 633 } catch (IllegalArgumentException e) { 634 reportEmmaError(e); 635 } catch (IllegalAccessException e) { 636 reportEmmaError(e); 637 } catch (InvocationTargetException e) { 638 reportEmmaError(e); 639 } 640 } 641 getCoverageFilePath()642 private String getCoverageFilePath() { 643 if (mCoverageFilePath == null) { 644 return getTargetContext().getFilesDir().getAbsolutePath() + File.separator + 645 DEFAULT_COVERAGE_FILE_NAME; 646 } else { 647 return mCoverageFilePath; 648 } 649 } 650 reportEmmaError(Exception e)651 private void reportEmmaError(Exception e) { 652 reportEmmaError("", e); 653 } 654 reportEmmaError(String hint, Exception e)655 private void reportEmmaError(String hint, Exception e) { 656 String msg = "Failed to generate emma coverage. " + hint; 657 Log.e(LOG_TAG, msg, e); 658 mResults.putString(Instrumentation.REPORT_KEY_STREAMRESULT, "\nError: " + msg); 659 } 660 661 // TODO kill this, use status() and prettyprint model for better output 662 private class StringResultPrinter extends ResultPrinter { 663 StringResultPrinter(PrintStream writer)664 public StringResultPrinter(PrintStream writer) { 665 super(writer); 666 } 667 printResult(TestResult result, long runTime)668 public synchronized void printResult(TestResult result, long runTime) { 669 printHeader(runTime); 670 printFooter(result); 671 } 672 } 673 674 /** 675 * This class sends status reports back to the IInstrumentationWatcher about 676 * which suite each test belongs. 677 */ 678 private class SuiteAssignmentPrinter implements TestListener { 679 680 private Bundle mTestResult; 681 private long mStartTime; 682 private long mEndTime; 683 private boolean mTimingValid; 684 SuiteAssignmentPrinter()685 public SuiteAssignmentPrinter() { 686 } 687 688 /** 689 * send a status for the start of a each test, so long tests can be seen as "running" 690 */ startTest(Test test)691 public void startTest(Test test) { 692 mTimingValid = true; 693 mStartTime = System.currentTimeMillis(); 694 } 695 696 /** 697 * @see junit.framework.TestListener#addError(Test, Throwable) 698 */ addError(Test test, Throwable t)699 public void addError(Test test, Throwable t) { 700 mTimingValid = false; 701 } 702 703 /** 704 * @see junit.framework.TestListener#addFailure(Test, AssertionFailedError) 705 */ addFailure(Test test, AssertionFailedError t)706 public void addFailure(Test test, AssertionFailedError t) { 707 mTimingValid = false; 708 } 709 710 /** 711 * @see junit.framework.TestListener#endTest(Test) 712 */ endTest(Test test)713 public void endTest(Test test) { 714 float runTime; 715 String assignmentSuite; 716 mEndTime = System.currentTimeMillis(); 717 mTestResult = new Bundle(); 718 719 if (!mTimingValid || mStartTime < 0) { 720 assignmentSuite = "NA"; 721 runTime = -1; 722 } else { 723 runTime = mEndTime - mStartTime; 724 if (runTime < SMALL_SUITE_MAX_RUNTIME 725 && !InstrumentationTestCase.class.isAssignableFrom(test.getClass())) { 726 assignmentSuite = SMALL_SUITE; 727 } else if (runTime < MEDIUM_SUITE_MAX_RUNTIME) { 728 assignmentSuite = MEDIUM_SUITE; 729 } else { 730 assignmentSuite = LARGE_SUITE; 731 } 732 } 733 // Clear mStartTime so that we can verify that it gets set next time. 734 mStartTime = -1; 735 736 mTestResult.putString(Instrumentation.REPORT_KEY_STREAMRESULT, 737 test.getClass().getName() + "#" + ((TestCase) test).getName() 738 + "\nin " + assignmentSuite + " suite\nrunTime: " 739 + String.valueOf(runTime) + "\n"); 740 mTestResult.putFloat(REPORT_KEY_RUN_TIME, runTime); 741 mTestResult.putString(REPORT_KEY_SUITE_ASSIGNMENT, assignmentSuite); 742 743 sendStatus(0, mTestResult); 744 } 745 } 746 747 /** 748 * This class sends status reports back to the IInstrumentationWatcher 749 */ 750 private class WatcherResultPrinter implements TestListener { 751 private final Bundle mResultTemplate; 752 Bundle mTestResult; 753 int mTestNum = 0; 754 int mTestResultCode = 0; 755 String mTestClass = null; 756 WatcherResultPrinter(int numTests)757 public WatcherResultPrinter(int numTests) { 758 mResultTemplate = new Bundle(); 759 mResultTemplate.putString(Instrumentation.REPORT_KEY_IDENTIFIER, REPORT_VALUE_ID); 760 mResultTemplate.putInt(REPORT_KEY_NUM_TOTAL, numTests); 761 } 762 763 /** 764 * send a status for the start of a each test, so long tests can be seen 765 * as "running" 766 */ startTest(Test test)767 public void startTest(Test test) { 768 String testClass = test.getClass().getName(); 769 String testName = ((TestCase)test).getName(); 770 mTestResult = new Bundle(mResultTemplate); 771 mTestResult.putString(REPORT_KEY_NAME_CLASS, testClass); 772 mTestResult.putString(REPORT_KEY_NAME_TEST, testName); 773 mTestResult.putInt(REPORT_KEY_NUM_CURRENT, ++mTestNum); 774 // pretty printing 775 if (testClass != null && !testClass.equals(mTestClass)) { 776 mTestResult.putString(Instrumentation.REPORT_KEY_STREAMRESULT, 777 String.format("\n%s:", testClass)); 778 mTestClass = testClass; 779 } else { 780 mTestResult.putString(Instrumentation.REPORT_KEY_STREAMRESULT, ""); 781 } 782 783 Method testMethod = null; 784 try { 785 testMethod = test.getClass().getMethod(testName); 786 // Report total number of iterations, if test is repetitive 787 if (testMethod.isAnnotationPresent(RepetitiveTest.class)) { 788 int numIterations = testMethod.getAnnotation( 789 RepetitiveTest.class).numIterations(); 790 mTestResult.putInt(REPORT_KEY_NUM_ITERATIONS, numIterations); 791 } 792 } catch (NoSuchMethodException e) { 793 // ignore- the test with given name does not exist. Will be handled during test 794 // execution 795 } 796 797 // The delay_msec parameter is normally used to provide buffers of idle time 798 // for power measurement purposes. To make sure there is a delay before and after 799 // every test in a suite, we delay *after* every test (see endTest below) and also 800 // delay *before* the first test. So, delay test1 delay test2 delay. 801 802 try { 803 if (mTestNum == 1) Thread.sleep(mDelayMsec); 804 } catch (InterruptedException e) { 805 throw new IllegalStateException(e); 806 } 807 808 sendStatus(REPORT_VALUE_RESULT_START, mTestResult); 809 mTestResultCode = 0; 810 } 811 812 /** 813 * @see junit.framework.TestListener#addError(Test, Throwable) 814 */ addError(Test test, Throwable t)815 public void addError(Test test, Throwable t) { 816 mTestResult.putString(REPORT_KEY_STACK, BaseTestRunner.getFilteredTrace(t)); 817 mTestResultCode = REPORT_VALUE_RESULT_ERROR; 818 // pretty printing 819 mTestResult.putString(Instrumentation.REPORT_KEY_STREAMRESULT, 820 String.format("\nError in %s:\n%s", 821 ((TestCase)test).getName(), BaseTestRunner.getFilteredTrace(t))); 822 } 823 824 /** 825 * @see junit.framework.TestListener#addFailure(Test, AssertionFailedError) 826 */ addFailure(Test test, AssertionFailedError t)827 public void addFailure(Test test, AssertionFailedError t) { 828 mTestResult.putString(REPORT_KEY_STACK, BaseTestRunner.getFilteredTrace(t)); 829 mTestResultCode = REPORT_VALUE_RESULT_FAILURE; 830 // pretty printing 831 mTestResult.putString(Instrumentation.REPORT_KEY_STREAMRESULT, 832 String.format("\nFailure in %s:\n%s", 833 ((TestCase)test).getName(), BaseTestRunner.getFilteredTrace(t))); 834 } 835 836 /** 837 * @see junit.framework.TestListener#endTest(Test) 838 */ endTest(Test test)839 public void endTest(Test test) { 840 if (mTestResultCode == 0) { 841 mTestResult.putString(Instrumentation.REPORT_KEY_STREAMRESULT, "."); 842 } 843 sendStatus(mTestResultCode, mTestResult); 844 845 try { // Sleep after every test, if specified 846 Thread.sleep(mDelayMsec); 847 } catch (InterruptedException e) { 848 throw new IllegalStateException(e); 849 } 850 } 851 852 // TODO report the end of the cycle 853 } 854 } 855