1 /* 2 * Copyright (C) 2010 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.tradefed.testtype; 17 18 import static com.google.common.truth.Truth.assertThat; 19 20 import static org.junit.Assert.fail; 21 import static org.mockito.ArgumentMatchers.any; 22 import static org.mockito.ArgumentMatchers.anyCollectionOf; 23 import static org.mockito.ArgumentMatchers.anyLong; 24 import static org.mockito.ArgumentMatchers.eq; 25 import static org.mockito.Mockito.doAnswer; 26 import static org.mockito.Mockito.doReturn; 27 import static org.mockito.Mockito.mock; 28 import static org.mockito.Mockito.times; 29 import static org.mockito.Mockito.verify; 30 31 import com.android.ddmlib.IDevice; 32 import com.android.ddmlib.testrunner.IRemoteAndroidTestRunner; 33 import com.android.ddmlib.testrunner.InstrumentationResultParser; 34 import com.android.ddmlib.testrunner.RemoteAndroidTestRunner; 35 import com.android.tradefed.config.ConfigurationException; 36 import com.android.tradefed.config.OptionSetter; 37 import com.android.tradefed.device.DeviceNotAvailableException; 38 import com.android.tradefed.device.ITestDevice; 39 import com.android.tradefed.metrics.proto.MetricMeasurement.Metric; 40 import com.android.tradefed.result.ITestInvocationListener; 41 import com.android.tradefed.result.ITestLifeCycleReceiver; 42 import com.android.tradefed.result.TestDescription; 43 import com.android.tradefed.util.ListInstrumentationParser; 44 import com.android.tradefed.util.ListInstrumentationParser.InstrumentationTarget; 45 46 import com.google.common.collect.ImmutableList; 47 import com.google.common.collect.ImmutableMap; 48 49 import org.easymock.IAnswer; 50 import org.junit.Before; 51 import org.junit.Test; 52 import org.junit.runner.RunWith; 53 import org.junit.runners.JUnit4; 54 import org.mockito.ArgumentCaptor; 55 import org.mockito.Captor; 56 import org.mockito.InOrder; 57 import org.mockito.Mock; 58 import org.mockito.Mockito; 59 import org.mockito.MockitoAnnotations; 60 import org.mockito.invocation.InvocationOnMock; 61 import org.mockito.stubbing.Answer; 62 63 import java.util.Collection; 64 import java.util.HashMap; 65 import java.util.Map; 66 67 /** Unit tests for {@link InstrumentationTest} */ 68 @RunWith(JUnit4.class) 69 public class InstrumentationTestTest { 70 71 private static final String TEST_PACKAGE_VALUE = "com.foo"; 72 private static final String TEST_RUNNER_VALUE = ".FooRunner"; 73 private static final TestDescription TEST1 = new TestDescription("Test", "test1"); 74 private static final TestDescription TEST2 = new TestDescription("Test", "test2"); 75 private static final String RUN_ERROR_MSG = "error"; 76 private static final HashMap<String, Metric> EMPTY_STRING_MAP = new HashMap<>(); 77 78 /** The {@link InstrumentationTest} under test, with all dependencies mocked out */ 79 private InstrumentationTest mInstrumentationTest; 80 81 // The mock objects. 82 @Mock IDevice mMockIDevice; 83 @Mock ITestDevice mMockTestDevice; 84 @Mock ITestInvocationListener mMockListener; 85 @Mock ListInstrumentationParser mMockListInstrumentationParser; 86 87 @Captor private ArgumentCaptor<Collection<TestDescription>> testCaptor; 88 89 /** 90 * Helper class for providing an {@link IAnswer} to a {@link 91 * ITestDevice#runInstrumentationTests(IRemoteAndroidTestRunner, ITestInvocationListener...)} 92 * call. 93 */ 94 @FunctionalInterface 95 interface RunInstrumentationTestsAnswer extends Answer<Boolean> { 96 @Override answer(InvocationOnMock invocation)97 default Boolean answer(InvocationOnMock invocation) throws Exception { 98 Object[] args = invocation.getArguments(); 99 return answer((IRemoteAndroidTestRunner) args[0], (ITestLifeCycleReceiver) args[1]); 100 } 101 answer(IRemoteAndroidTestRunner runner, ITestLifeCycleReceiver listener)102 Boolean answer(IRemoteAndroidTestRunner runner, ITestLifeCycleReceiver listener) 103 throws Exception; 104 } 105 106 @Before setUp()107 public void setUp() { 108 MockitoAnnotations.initMocks(this); 109 110 doReturn(mMockIDevice).when(mMockTestDevice).getIDevice(); 111 doReturn("serial").when(mMockTestDevice).getSerialNumber(); 112 113 InstrumentationTarget target1 = 114 new InstrumentationTarget(TEST_PACKAGE_VALUE, "runner1", "target1"); 115 InstrumentationTarget target2 = new InstrumentationTarget("package2", "runner2", "target2"); 116 doReturn(ImmutableList.of(target1, target2)) 117 .when(mMockListInstrumentationParser) 118 .getInstrumentationTargets(); 119 120 mInstrumentationTest = Mockito.spy(new InstrumentationTest()); 121 mInstrumentationTest.setPackageName(TEST_PACKAGE_VALUE); 122 mInstrumentationTest.setRunnerName(TEST_RUNNER_VALUE); 123 mInstrumentationTest.setDevice(mMockTestDevice); 124 mInstrumentationTest.setListInstrumentationParser(mMockListInstrumentationParser); 125 } 126 127 /** Test normal run scenario. */ 128 @Test testRun()129 public void testRun() throws DeviceNotAvailableException { 130 // verify the mock listener is passed through to the runner 131 RunInstrumentationTestsAnswer runTests = 132 (runner, listener) -> { 133 // perform call back on listener to show run of two tests 134 listener.testRunStarted(TEST_PACKAGE_VALUE, 2); 135 listener.testStarted(TEST1); 136 listener.testEnded(TEST1, EMPTY_STRING_MAP); 137 listener.testStarted(TEST2); 138 listener.testEnded(TEST2, EMPTY_STRING_MAP); 139 listener.testRunEnded(1, EMPTY_STRING_MAP); 140 return true; 141 }; 142 doAnswer(runTests) 143 .when(mMockTestDevice) 144 .runInstrumentationTests( 145 any(IRemoteAndroidTestRunner.class), any(ITestInvocationListener.class)); 146 147 mInstrumentationTest.run(mMockListener); 148 149 InOrder inOrder = Mockito.inOrder(mInstrumentationTest, mMockTestDevice, mMockListener); 150 ArgumentCaptor<IRemoteAndroidTestRunner> runner = 151 ArgumentCaptor.forClass(IRemoteAndroidTestRunner.class); 152 inOrder.verify(mInstrumentationTest).setRunnerArgs(runner.capture()); 153 inOrder.verify(mMockTestDevice, times(2)) 154 .runInstrumentationTests(eq(runner.getValue()), any(ITestLifeCycleReceiver.class)); 155 156 inOrder.verify(mMockListener).testRunStarted(TEST_PACKAGE_VALUE, 2); 157 inOrder.verify(mMockListener).testStarted(eq(TEST1), anyLong()); 158 inOrder.verify(mMockListener).testEnded(eq(TEST1), anyLong(), eq(EMPTY_STRING_MAP)); 159 inOrder.verify(mMockListener).testStarted(eq(TEST2), anyLong()); 160 inOrder.verify(mMockListener).testEnded(eq(TEST2), anyLong(), eq(EMPTY_STRING_MAP)); 161 inOrder.verify(mMockListener).testRunEnded(1, EMPTY_STRING_MAP); 162 } 163 164 @Test testRun_bothAbi()165 public void testRun_bothAbi() throws DeviceNotAvailableException { 166 mInstrumentationTest.setAbi(mock(IAbi.class)); 167 mInstrumentationTest.setForceAbi("test"); 168 try { 169 mInstrumentationTest.run(mMockListener); 170 fail("Should have thrown an exception"); 171 } catch (IllegalArgumentException e) { 172 // expected 173 } 174 } 175 176 /** Test normal run scenario with --no-hidden-api-check specified */ 177 @Test testRun_hiddenApiCheck()178 public void testRun_hiddenApiCheck() throws Exception { 179 doReturn(28).when(mMockTestDevice).getApiLevel(); 180 OptionSetter setter = new OptionSetter(mInstrumentationTest); 181 setter.setOptionValue("hidden-api-checks", "false"); 182 RemoteAndroidTestRunner runner = 183 (RemoteAndroidTestRunner) 184 mInstrumentationTest.createRemoteAndroidTestRunner("", "", mMockIDevice); 185 assertThat(runner.getRunOptions()).contains("--no-hidden-api-checks"); 186 } 187 188 /** Test normal run scenario with a test class specified. */ 189 @Test testRun_class()190 public void testRun_class() { 191 String className = "FooTest"; 192 FakeTestRunner runner = new FakeTestRunner("unused", "unused"); 193 194 mInstrumentationTest.setClassName(className); 195 mInstrumentationTest.setRunnerArgs(runner); 196 197 assertThat(runner.getArgs()).containsEntry("class", "'" + className + "'"); 198 } 199 200 /** Test normal run scenario with a test class and method specified. */ 201 @Test testRun_classMethod()202 public void testRun_classMethod() { 203 String className = "FooTest"; 204 String methodName = "testFoo"; 205 FakeTestRunner runner = new FakeTestRunner("unused", "unused"); 206 207 mInstrumentationTest.setClassName(className); 208 mInstrumentationTest.setMethodName(methodName); 209 mInstrumentationTest.setRunnerArgs(runner); 210 211 assertThat(runner.getArgs()).containsEntry("class", "'FooTest#testFoo'"); 212 } 213 214 /** Test normal run scenario with a test package specified. */ 215 @Test testRun_testPackage()216 public void testRun_testPackage() { 217 String testPackageName = "com.foo"; 218 FakeTestRunner runner = new FakeTestRunner("unused", "unused"); 219 220 mInstrumentationTest.setTestPackageName(testPackageName); 221 mInstrumentationTest.setRunnerArgs(runner); 222 223 assertThat(runner.getArgs()).containsEntry("package", testPackageName); 224 } 225 226 /** Verify test package name is not passed to the runner if class name is set */ 227 @Test testRun_testPackageAndClass()228 public void testRun_testPackageAndClass() { 229 String testClassName = "FooTest"; 230 FakeTestRunner runner = new FakeTestRunner("unused", "unused"); 231 232 mInstrumentationTest.setTestPackageName("com.foo"); 233 mInstrumentationTest.setClassName(testClassName); 234 mInstrumentationTest.setRunnerArgs(runner); 235 assertThat(runner.getArgs()).containsEntry("class", "'" + testClassName + "'"); 236 assertThat(runner.getArgs()).doesNotContainKey("package"); 237 } 238 239 /** Test that IllegalArgumentException is thrown when attempting run without setting device. */ 240 @Test testRun_noDevice()241 public void testRun_noDevice() throws DeviceNotAvailableException { 242 mInstrumentationTest.setDevice(null); 243 244 try { 245 mInstrumentationTest.run(mMockListener); 246 fail("IllegalArgumentException not thrown"); 247 } catch (IllegalArgumentException e) { 248 // expected 249 } 250 } 251 252 /** Test the rerun mode when test run has no tests. */ 253 @Test testRun_rerunEmpty()254 public void testRun_rerunEmpty() throws DeviceNotAvailableException { 255 mInstrumentationTest.setRerunMode(true); 256 257 // collect tests run 258 RunInstrumentationTestsAnswer collectTest = 259 (runner, listener) -> { 260 listener.testRunStarted(TEST_PACKAGE_VALUE, 0); 261 listener.testRunEnded(1, EMPTY_STRING_MAP); 262 return true; 263 }; 264 doAnswer(collectTest) 265 .when(mMockTestDevice) 266 .runInstrumentationTests( 267 any(IRemoteAndroidTestRunner.class), any(ITestLifeCycleReceiver.class)); 268 269 mInstrumentationTest.run(mMockListener); 270 271 // note: expect run to not be reported 272 Mockito.verifyNoMoreInteractions(mMockListener); 273 } 274 275 /** Test the rerun mode when first test run fails. */ 276 @Test testRun_rerun()277 public void testRun_rerun() throws Exception { 278 OptionSetter setter = new OptionSetter(mInstrumentationTest); 279 setter.setOptionValue("bugreport-on-run-failure", "true"); 280 mInstrumentationTest.setRerunMode(true); 281 282 // Mock collected tests 283 RunInstrumentationTestsAnswer collected = 284 (runner, listener) -> { 285 // perform call back on listener to show run of two tests 286 listener.testRunStarted(TEST_PACKAGE_VALUE, 2); 287 listener.testStarted(TEST1); 288 listener.testEnded(TEST1, EMPTY_STRING_MAP); 289 listener.testStarted(TEST2); 290 listener.testEnded(TEST2, EMPTY_STRING_MAP); 291 listener.testRunEnded(1, EMPTY_STRING_MAP); 292 return true; 293 }; 294 RunInstrumentationTestsAnswer partialRun = 295 (runner, listener) -> { 296 // perform call back on listener to show run failed - only one test 297 listener.testRunStarted(TEST_PACKAGE_VALUE, 2); 298 listener.testStarted(TEST1); 299 listener.testEnded(TEST1, EMPTY_STRING_MAP); 300 listener.testRunFailed(RUN_ERROR_MSG); 301 listener.testRunEnded(1, EMPTY_STRING_MAP); 302 return true; 303 }; 304 RunInstrumentationTestsAnswer rerun = 305 (runner, listener) -> { 306 // perform call back on listeners to show run remaining test was run 307 listener.testRunStarted(TEST_PACKAGE_VALUE, 1); 308 listener.testStarted(TEST2); 309 listener.testEnded(TEST2, EMPTY_STRING_MAP); 310 listener.testRunEnded(1, EMPTY_STRING_MAP); 311 return true; 312 }; 313 314 doAnswer(collected) 315 .doAnswer(partialRun) 316 .doAnswer(rerun) 317 .when(mMockTestDevice) 318 .runInstrumentationTests( 319 any(IRemoteAndroidTestRunner.class), any(ITestLifeCycleReceiver.class)); 320 321 mInstrumentationTest.run(mMockListener); 322 323 InOrder inOrder = Mockito.inOrder(mMockListener, mMockTestDevice); 324 inOrder.verify(mMockListener).testRunStarted(TEST_PACKAGE_VALUE, 2); 325 inOrder.verify(mMockListener).testStarted(eq(TEST1), anyLong()); 326 inOrder.verify(mMockListener).testEnded(eq(TEST1), anyLong(), eq(EMPTY_STRING_MAP)); 327 inOrder.verify(mMockListener).testRunFailed(RUN_ERROR_MSG); 328 inOrder.verify(mMockListener).testRunEnded(1, EMPTY_STRING_MAP); 329 // Expect a bugreport since there was a failure. 330 inOrder.verify(mMockTestDevice) 331 .logBugreport("bugreport-on-run-failure-com.foo", mMockListener); 332 inOrder.verify(mMockListener).testRunStarted(TEST_PACKAGE_VALUE, 1); 333 inOrder.verify(mMockListener).testStarted(eq(TEST2), anyLong()); 334 inOrder.verify(mMockListener).testEnded(eq(TEST2), anyLong(), eq(EMPTY_STRING_MAP)); 335 inOrder.verify(mMockListener).testRunEnded(1, EMPTY_STRING_MAP); 336 inOrder.verifyNoMoreInteractions(); 337 } 338 339 /** Verify that all tests are re-run when there is a failure during a coverage run. */ 340 @Test testRun_rerunCoverage()341 public void testRun_rerunCoverage() throws ConfigurationException, DeviceNotAvailableException { 342 mInstrumentationTest.setRerunMode(true); 343 mInstrumentationTest.setCoverage(true); 344 345 Collection<TestDescription> expectedTests = ImmutableList.of(TEST1, TEST2); 346 347 // Mock collected tests 348 RunInstrumentationTestsAnswer collected = 349 (runner, listener) -> { 350 // perform call back on listener to show run of two tests 351 listener.testRunStarted(TEST_PACKAGE_VALUE, 2); 352 listener.testStarted(TEST1); 353 listener.testEnded(TEST1, EMPTY_STRING_MAP); 354 listener.testStarted(TEST2); 355 listener.testEnded(TEST2, EMPTY_STRING_MAP); 356 listener.testRunEnded(1, EMPTY_STRING_MAP); 357 return true; 358 }; 359 RunInstrumentationTestsAnswer partialRun = 360 (runner, listener) -> { 361 // perform call back on listener to show run failed - only one test 362 listener.testRunStarted(TEST_PACKAGE_VALUE, 2); 363 listener.testStarted(TEST1); 364 listener.testEnded(TEST1, EMPTY_STRING_MAP); 365 listener.testRunFailed(RUN_ERROR_MSG); 366 listener.testRunEnded(1, EMPTY_STRING_MAP); 367 return true; 368 }; 369 RunInstrumentationTestsAnswer rerun1 = 370 (runner, listener) -> { 371 // perform call back on listeners to show run remaining test was run 372 listener.testRunStarted(TEST_PACKAGE_VALUE, 1); 373 listener.testStarted(TEST1); 374 listener.testEnded(TEST1, EMPTY_STRING_MAP); 375 listener.testRunEnded(1, EMPTY_STRING_MAP); 376 return true; 377 }; 378 RunInstrumentationTestsAnswer rerun2 = 379 (runner, listener) -> { 380 // perform call back on listeners to show run remaining test was run 381 listener.testRunStarted(TEST_PACKAGE_VALUE, 1); 382 listener.testStarted(TEST2); 383 listener.testEnded(TEST2, EMPTY_STRING_MAP); 384 listener.testRunEnded(1, EMPTY_STRING_MAP); 385 return true; 386 }; 387 388 doAnswer(collected) 389 .doAnswer(partialRun) 390 .doAnswer(rerun1) 391 .doAnswer(rerun2) 392 .when(mMockTestDevice) 393 .runInstrumentationTests( 394 any(IRemoteAndroidTestRunner.class), any(ITestLifeCycleReceiver.class)); 395 396 mInstrumentationTest.run(mMockListener); 397 398 verify(mInstrumentationTest).getTestReRunner(testCaptor.capture()); 399 assertThat(testCaptor.getValue()).containsExactlyElementsIn(expectedTests); 400 401 InOrder inOrder = Mockito.inOrder(mMockListener); 402 inOrder.verify(mMockListener).testRunStarted(TEST_PACKAGE_VALUE, 2); 403 inOrder.verify(mMockListener).testStarted(eq(TEST1), anyLong()); 404 inOrder.verify(mMockListener).testEnded(eq(TEST1), anyLong(), eq(EMPTY_STRING_MAP)); 405 inOrder.verify(mMockListener).testRunFailed(RUN_ERROR_MSG); 406 inOrder.verify(mMockListener).testRunEnded(1, EMPTY_STRING_MAP); 407 inOrder.verify(mMockListener).testRunStarted(TEST_PACKAGE_VALUE, 1); 408 inOrder.verify(mMockListener).testStarted(eq(TEST1), anyLong()); 409 inOrder.verify(mMockListener).testEnded(eq(TEST1), anyLong(), eq(EMPTY_STRING_MAP)); 410 inOrder.verify(mMockListener).testRunEnded(1, EMPTY_STRING_MAP); 411 inOrder.verify(mMockListener).testStarted(eq(TEST2), anyLong()); 412 inOrder.verify(mMockListener).testEnded(eq(TEST2), anyLong(), eq(EMPTY_STRING_MAP)); 413 inOrder.verify(mMockListener).testRunEnded(1, EMPTY_STRING_MAP); 414 inOrder.verifyNoMoreInteractions(); 415 } 416 417 /** Test the reboot before re-run option. */ 418 @Test testRun_rebootBeforeReRun()419 public void testRun_rebootBeforeReRun() throws DeviceNotAvailableException { 420 mInstrumentationTest.setRerunMode(true); 421 mInstrumentationTest.setRebootBeforeReRun(true); 422 423 RunInstrumentationTestsAnswer collected = 424 (runner, listener) -> { 425 // perform call back on listener to show run of two tests 426 listener.testRunStarted(TEST_PACKAGE_VALUE, 2); 427 listener.testStarted(TEST1); 428 listener.testEnded(TEST1, EMPTY_STRING_MAP); 429 listener.testStarted(TEST2); 430 listener.testEnded(TEST2, EMPTY_STRING_MAP); 431 listener.testRunEnded(1, EMPTY_STRING_MAP); 432 return true; 433 }; 434 RunInstrumentationTestsAnswer partialRun = 435 (runner, listener) -> { 436 // perform call back on listener to show run failed - only one test 437 listener.testRunStarted(TEST_PACKAGE_VALUE, 2); 438 listener.testStarted(TEST1); 439 listener.testEnded(TEST1, EMPTY_STRING_MAP); 440 listener.testRunFailed(RUN_ERROR_MSG); 441 listener.testRunEnded(1, EMPTY_STRING_MAP); 442 return true; 443 }; 444 RunInstrumentationTestsAnswer rerun = 445 (runner, listener) -> { 446 // perform call back on listeners to show run remaining test was run 447 listener.testRunStarted(TEST_PACKAGE_VALUE, 1); 448 listener.testStarted(TEST2); 449 listener.testEnded(TEST2, EMPTY_STRING_MAP); 450 listener.testRunEnded(1, EMPTY_STRING_MAP); 451 return true; 452 }; 453 454 doAnswer(collected) 455 .doAnswer(partialRun) 456 .doAnswer(rerun) 457 .when(mMockTestDevice) 458 .runInstrumentationTests( 459 any(IRemoteAndroidTestRunner.class), any(ITestInvocationListener.class)); 460 461 mInstrumentationTest.run(mMockListener); 462 463 InOrder inOrder = Mockito.inOrder(mMockListener, mMockTestDevice); 464 inOrder.verify(mMockListener).testRunStarted(TEST_PACKAGE_VALUE, 2); 465 inOrder.verify(mMockListener).testStarted(eq(TEST1), anyLong()); 466 inOrder.verify(mMockListener).testEnded(eq(TEST1), anyLong(), eq(EMPTY_STRING_MAP)); 467 inOrder.verify(mMockListener).testRunFailed(RUN_ERROR_MSG); 468 inOrder.verify(mMockListener).testRunEnded(1, EMPTY_STRING_MAP); 469 inOrder.verify(mMockTestDevice).reboot(); 470 inOrder.verify(mMockListener).testRunStarted(TEST_PACKAGE_VALUE, 1); 471 inOrder.verify(mMockListener).testStarted(eq(TEST2), anyLong()); 472 inOrder.verify(mMockListener).testEnded(eq(TEST2), anyLong(), eq(EMPTY_STRING_MAP)); 473 inOrder.verify(mMockListener).testRunEnded(1, EMPTY_STRING_MAP); 474 } 475 476 /** 477 * Test resuming a test run when first run is aborted due to {@link DeviceNotAvailableException} 478 */ 479 @Test testRun_resume()480 public void testRun_resume() throws DeviceNotAvailableException { 481 RunInstrumentationTestsAnswer collected = 482 (runner, listener) -> { 483 // perform call back on listener to show run of two tests 484 listener.testRunStarted(TEST_PACKAGE_VALUE, 2); 485 listener.testStarted(TEST1); 486 listener.testEnded(TEST1, EMPTY_STRING_MAP); 487 listener.testStarted(TEST2); 488 listener.testEnded(TEST2, EMPTY_STRING_MAP); 489 listener.testRunEnded(1, EMPTY_STRING_MAP); 490 return true; 491 }; 492 RunInstrumentationTestsAnswer partialRun = 493 (runner, listener) -> { 494 // perform call back on listener to show run failed - only one test 495 listener.testRunStarted(TEST_PACKAGE_VALUE, 2); 496 listener.testStarted(TEST1); 497 listener.testEnded(TEST1, EMPTY_STRING_MAP); 498 listener.testRunFailed(RUN_ERROR_MSG); 499 listener.testRunEnded(1, EMPTY_STRING_MAP); 500 throw new DeviceNotAvailableException(); 501 }; 502 RunInstrumentationTestsAnswer rerun = 503 (runner, listener) -> { 504 // perform call back on listeners to show run remaining test was run 505 listener.testRunStarted(TEST_PACKAGE_VALUE, 1); 506 listener.testStarted(TEST2); 507 listener.testEnded(TEST2, EMPTY_STRING_MAP); 508 listener.testRunEnded(1, EMPTY_STRING_MAP); 509 return true; 510 }; 511 512 doAnswer(collected) 513 .doAnswer(partialRun) 514 .doAnswer(rerun) 515 .when(mMockTestDevice) 516 .runInstrumentationTests( 517 any(IRemoteAndroidTestRunner.class), any(ITestLifeCycleReceiver.class)); 518 519 try { 520 mInstrumentationTest.run(mMockListener); 521 fail("DeviceNotAvailableException not thrown"); 522 } catch (DeviceNotAvailableException e) { 523 // expected 524 } 525 mInstrumentationTest.run(mMockListener); 526 527 InOrder inOrder = Mockito.inOrder(mMockListener); 528 inOrder.verify(mMockListener).testRunStarted(TEST_PACKAGE_VALUE, 2); 529 inOrder.verify(mMockListener).testStarted(eq(TEST1), anyLong()); 530 inOrder.verify(mMockListener).testEnded(eq(TEST1), anyLong(), eq(EMPTY_STRING_MAP)); 531 inOrder.verify(mMockListener).testRunFailed(RUN_ERROR_MSG); 532 inOrder.verify(mMockListener).testRunEnded(1, EMPTY_STRING_MAP); 533 inOrder.verify(mMockListener).testRunStarted(TEST_PACKAGE_VALUE, 1); 534 inOrder.verify(mMockListener).testStarted(eq(TEST2), anyLong()); 535 inOrder.verify(mMockListener).testEnded(eq(TEST2), anyLong(), eq(EMPTY_STRING_MAP)); 536 inOrder.verify(mMockListener).testRunEnded(1, EMPTY_STRING_MAP); 537 } 538 539 /** 540 * Test that IllegalArgumentException is thrown when attempting run with negative timeout args. 541 */ 542 @Test testRun_negativeTimeouts()543 public void testRun_negativeTimeouts() throws DeviceNotAvailableException { 544 mInstrumentationTest.setShellTimeout(-1); 545 mInstrumentationTest.setTestTimeout(-2); 546 547 try { 548 mInstrumentationTest.run(mMockListener); 549 fail("IllegalArgumentException not thrown"); 550 } catch (IllegalArgumentException e) { 551 // expected 552 } 553 } 554 555 /** Test that IllegalArgumentException is thrown if an invalid test size is provided. */ 556 @Test testRun_badTestSize()557 public void testRun_badTestSize() throws DeviceNotAvailableException { 558 mInstrumentationTest.setTestSize("foo"); 559 560 try { 561 mInstrumentationTest.run(mMockListener); 562 fail("IllegalArgumentException not thrown"); 563 } catch (IllegalArgumentException e) { 564 // expected 565 } 566 } 567 568 @Test testQueryRunnerName()569 public void testQueryRunnerName() throws DeviceNotAvailableException { 570 String queriedRunner = mInstrumentationTest.queryRunnerName(); 571 assertThat(queriedRunner).isEqualTo("runner1"); 572 } 573 574 @Test testQueryRunnerName_noMatch()575 public void testQueryRunnerName_noMatch() throws DeviceNotAvailableException { 576 mInstrumentationTest.setPackageName("noMatchPackage"); 577 String queriedRunner = mInstrumentationTest.queryRunnerName(); 578 assertThat(queriedRunner).isNull(); 579 } 580 581 @Test testRun_noMatchingRunner()582 public void testRun_noMatchingRunner() throws DeviceNotAvailableException { 583 mInstrumentationTest.setPackageName("noMatchPackage"); 584 mInstrumentationTest.setRunnerName(null); 585 try { 586 mInstrumentationTest.run(mMockListener); 587 fail("Should have thrown an exception."); 588 } catch (IllegalArgumentException e) { 589 // expected 590 } 591 } 592 593 /** 594 * Test for {@link InstrumentationTest#collectTestsAndRetry(IRemoteAndroidTestRunner, 595 * ITestInvocationListener)} when the collection fails. 596 */ 597 @Test testCollectTestsAndRetry_Failure()598 public void testCollectTestsAndRetry_Failure() throws DeviceNotAvailableException { 599 RunInstrumentationTestsAnswer collected = 600 (runner, listener) -> { 601 listener.testRunStarted(TEST_PACKAGE_VALUE, 0); 602 listener.testRunFailed(InstrumentationResultParser.INVALID_OUTPUT_ERR_MSG); 603 listener.testRunEnded(1, EMPTY_STRING_MAP); 604 return true; 605 }; 606 607 doAnswer(collected) 608 .when(mMockTestDevice) 609 .runInstrumentationTests( 610 any(IRemoteAndroidTestRunner.class), any(ITestLifeCycleReceiver.class)); 611 612 mInstrumentationTest.setEnforceFormat(true); 613 614 try { 615 mInstrumentationTest.collectTestsAndRetry(mock(IRemoteAndroidTestRunner.class), null); 616 fail("Should have thrown an exception"); 617 } catch (RuntimeException e) { 618 // expected. 619 } 620 } 621 622 /** 623 * Test for {@link InstrumentationTest#collectTestsAndRetry(IRemoteAndroidTestRunner, 624 * ITestInvocationListener)} when the tests collection fails but we do not enforce the format, 625 * so we don't throw an exception. 626 */ 627 @Test testCollectTestsAndRetry_notEnforced()628 public void testCollectTestsAndRetry_notEnforced() throws DeviceNotAvailableException { 629 RunInstrumentationTestsAnswer collected = 630 (runner, listener) -> { 631 listener.testRunStarted(TEST_PACKAGE_VALUE, 0); 632 listener.testRunFailed(InstrumentationResultParser.INVALID_OUTPUT_ERR_MSG); 633 listener.testRunEnded(1, EMPTY_STRING_MAP); 634 return true; 635 }; 636 637 doAnswer(collected) 638 .when(mMockTestDevice) 639 .runInstrumentationTests( 640 any(IRemoteAndroidTestRunner.class), any(ITestLifeCycleReceiver.class)); 641 642 mInstrumentationTest.setEnforceFormat(false); 643 644 Collection<TestDescription> result = 645 mInstrumentationTest.collectTestsAndRetry( 646 mock(IRemoteAndroidTestRunner.class), null); 647 assertThat(result).isNull(); 648 } 649 650 /** 651 * Test that if we successfully collect a list of tests and the run crash and try to report 0 we 652 * instead do report the number of collected test to get an appropriate count. 653 */ 654 @Test testCollectWorks_RunCrash()655 public void testCollectWorks_RunCrash() throws Exception { 656 doReturn(mock(IRemoteTest.class)) 657 .when(mInstrumentationTest) 658 .getTestReRunner(anyCollectionOf(TestDescription.class)); 659 660 // We collect successfully 5 tests 661 RunInstrumentationTestsAnswer collected = 662 (runner, listener) -> { 663 listener.testRunStarted("fakeName", 5); 664 for (int i = 0; i < 5; i++) { 665 TestDescription tid = new TestDescription("fakeclass", "fakemethod" + i); 666 listener.testStarted(tid, 5); 667 listener.testEnded(tid, 15, EMPTY_STRING_MAP); 668 } 669 listener.testRunEnded(500, EMPTY_STRING_MAP); 670 return true; 671 }; 672 673 // We attempt to run and crash 674 RunInstrumentationTestsAnswer partialRun = 675 (runner, listener) -> { 676 listener.testRunStarted("fakeName", 0); 677 listener.testRunFailed("Instrumentation run failed due to 'Process crashed.'"); 678 listener.testRunEnded(1, EMPTY_STRING_MAP); 679 return true; 680 }; 681 682 doAnswer(collected) 683 .doAnswer(partialRun) 684 .when(mMockTestDevice) 685 .runInstrumentationTests( 686 any(IRemoteAndroidTestRunner.class), any(ITestLifeCycleReceiver.class)); 687 688 mInstrumentationTest.run(mMockListener); 689 690 // The reported number of tests is the one from the collected output 691 InOrder inOrder = Mockito.inOrder(mMockListener); 692 inOrder.verify(mMockListener).testRunStarted("fakeName", 5); 693 inOrder.verify(mMockListener) 694 .testRunFailed("Instrumentation run failed due to 'Process crashed.'"); 695 inOrder.verify(mMockListener).testRunEnded(1, EMPTY_STRING_MAP); 696 } 697 698 @Test testAddScreenshotListener_enabled()699 public void testAddScreenshotListener_enabled() { 700 mInstrumentationTest.setScreenshotOnFailure(true); 701 702 ITestInvocationListener listener = 703 mInstrumentationTest.addScreenshotListenerIfEnabled(mMockListener); 704 assertThat(listener).isInstanceOf(InstrumentationTest.FailedTestScreenshotGenerator.class); 705 } 706 707 @Test testAddScreenshotListener_disabled()708 public void testAddScreenshotListener_disabled() { 709 mInstrumentationTest.setScreenshotOnFailure(false); 710 711 ITestInvocationListener listener = 712 mInstrumentationTest.addScreenshotListenerIfEnabled(mMockListener); 713 assertThat(listener).isSameAs(mMockListener); 714 } 715 716 @Test testAddLogcatListener_enabled()717 public void testAddLogcatListener_enabled() { 718 mInstrumentationTest.setLogcatOnFailure(true); 719 720 ITestInvocationListener listener = 721 mInstrumentationTest.addLogcatListenerIfEnabled(mMockListener); 722 assertThat(listener).isInstanceOf(InstrumentationTest.FailedTestLogcatGenerator.class); 723 } 724 725 @Test testAddLogcatListener_setMaxSize()726 public void testAddLogcatListener_setMaxSize() { 727 int maxSize = 1234; 728 mInstrumentationTest.setLogcatOnFailure(true); 729 mInstrumentationTest.setLogcatOnFailureSize(maxSize); 730 731 ITestInvocationListener listener = 732 mInstrumentationTest.addLogcatListenerIfEnabled(mMockListener); 733 assertThat(listener).isInstanceOf(InstrumentationTest.FailedTestLogcatGenerator.class); 734 735 InstrumentationTest.FailedTestLogcatGenerator logcatGenerator = 736 (InstrumentationTest.FailedTestLogcatGenerator) listener; 737 assertThat(logcatGenerator.getMaxSize()).isEqualTo(maxSize); 738 } 739 740 @Test testAddLogcatListener_disabled()741 public void testAddLogcatListener_disabled() { 742 mInstrumentationTest.setLogcatOnFailure(false); 743 744 ITestInvocationListener listener = 745 mInstrumentationTest.addLogcatListenerIfEnabled(mMockListener); 746 assertThat(listener).isSameAs(mMockListener); 747 } 748 749 @Test testAddCoverageListener_enabled()750 public void testAddCoverageListener_enabled() { 751 mInstrumentationTest.setCoverage(true); 752 753 ITestInvocationListener listener = 754 mInstrumentationTest.addCoverageListenerIfEnabled(mMockListener); 755 assertThat(listener).isInstanceOf(CodeCoverageListener.class); 756 } 757 758 @Test testAddCoverageListener_disabled()759 public void testAddCoverageListener_disabled() { 760 mInstrumentationTest.setCoverage(false); 761 762 ITestInvocationListener listener = 763 mInstrumentationTest.addCoverageListenerIfEnabled(mMockListener); 764 assertThat(listener).isSameAs(mMockListener); 765 } 766 767 private static class FakeTestRunner extends RemoteAndroidTestRunner { 768 769 private Map<String, String> mArgs = new HashMap<>(); 770 private String mRunOptions; 771 FakeTestRunner(String packageName, String runnerName)772 FakeTestRunner(String packageName, String runnerName) { 773 super(packageName, runnerName, null); 774 } 775 776 @Override addInstrumentationArg(String name, String value)777 public void addInstrumentationArg(String name, String value) { 778 mArgs.put(name, value); 779 } 780 getArgs()781 Map<String, String> getArgs() { 782 return ImmutableMap.copyOf(mArgs); 783 } 784 } 785 } 786