1 /* 2 * Copyright (C) 2016 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.tradefed.testtype.suite; 17 18 import static org.junit.Assert.assertEquals; 19 import static org.junit.Assert.fail; 20 21 import com.android.ddmlib.IDevice; 22 import com.android.tradefed.build.IBuildInfo; 23 import com.android.tradefed.command.remote.DeviceDescriptor; 24 import com.android.tradefed.config.Configuration; 25 import com.android.tradefed.config.ConfigurationDescriptor; 26 import com.android.tradefed.config.IConfiguration; 27 import com.android.tradefed.config.OptionSetter; 28 import com.android.tradefed.device.DeviceNotAvailableException; 29 import com.android.tradefed.device.DeviceUnresponsiveException; 30 import com.android.tradefed.device.ITestDevice; 31 import com.android.tradefed.invoker.IInvocationContext; 32 import com.android.tradefed.invoker.TestInvocation; 33 import com.android.tradefed.metrics.proto.MetricMeasurement.Metric; 34 import com.android.tradefed.result.ByteArrayInputStreamSource; 35 import com.android.tradefed.result.ILogSaver; 36 import com.android.tradefed.result.ILogSaverListener; 37 import com.android.tradefed.result.ITestInvocationListener; 38 import com.android.tradefed.result.LogDataType; 39 import com.android.tradefed.result.LogFile; 40 import com.android.tradefed.result.LogSaverResultForwarder; 41 import com.android.tradefed.result.TestDescription; 42 import com.android.tradefed.targetprep.BaseTargetPreparer; 43 import com.android.tradefed.targetprep.BuildError; 44 import com.android.tradefed.targetprep.ITargetCleaner; 45 import com.android.tradefed.targetprep.ITargetPreparer; 46 import com.android.tradefed.targetprep.TargetSetupError; 47 import com.android.tradefed.targetprep.multi.IMultiTargetPreparer; 48 import com.android.tradefed.testtype.Abi; 49 import com.android.tradefed.testtype.IBuildReceiver; 50 import com.android.tradefed.testtype.IDeviceTest; 51 import com.android.tradefed.testtype.IRemoteTest; 52 import com.android.tradefed.testtype.suite.module.BaseModuleController; 53 import com.android.tradefed.testtype.suite.module.IModuleController; 54 import com.android.tradefed.testtype.suite.module.TestFailureModuleController; 55 56 import org.easymock.EasyMock; 57 import org.junit.Before; 58 import org.junit.Test; 59 import org.junit.runner.RunWith; 60 import org.junit.runners.JUnit4; 61 62 import java.util.ArrayList; 63 import java.util.Arrays; 64 import java.util.HashMap; 65 import java.util.LinkedHashMap; 66 import java.util.List; 67 import java.util.Map; 68 69 70 /** Unit tests for {@link ModuleDefinition} */ 71 @RunWith(JUnit4.class) 72 public class ModuleDefinitionTest { 73 74 private static final String MODULE_NAME = "fakeName"; 75 private static final String DEFAULT_DEVICE_NAME = "DEFAULT_DEVICE"; 76 private ModuleDefinition mModule; 77 private List<IRemoteTest> mTestList; 78 private ITestInterface mMockTest; 79 private ITargetPreparer mMockPrep; 80 private ITargetCleaner mMockCleaner; 81 private List<ITargetPreparer> mTargetPrepList; 82 private Map<String, List<ITargetPreparer>> mMapDeviceTargetPreparer; 83 private List<IMultiTargetPreparer> mMultiTargetPrepList; 84 private ITestInvocationListener mMockListener; 85 private IBuildInfo mMockBuildInfo; 86 private ITestDevice mMockDevice; 87 // Extra mock for log saving testing 88 private ILogSaver mMockLogSaver; 89 private ILogSaverListener mMockLogSaverListener; 90 91 private interface ITestInterface extends IRemoteTest, IBuildReceiver, IDeviceTest {} 92 93 /** Test implementation that allows us to exercise different use cases * */ 94 private class TestObject implements ITestInterface { 95 96 private ITestDevice mDevice; 97 private String mRunName; 98 private int mNumTest; 99 private boolean mShouldThrow; 100 private boolean mDeviceUnresponsive = false; 101 TestObject(String runName, int numTest, boolean shouldThrow)102 public TestObject(String runName, int numTest, boolean shouldThrow) { 103 mRunName = runName; 104 mNumTest = numTest; 105 mShouldThrow = shouldThrow; 106 } 107 TestObject( String runName, int numTest, boolean shouldThrow, boolean deviceUnresponsive)108 public TestObject( 109 String runName, int numTest, boolean shouldThrow, boolean deviceUnresponsive) { 110 mRunName = runName; 111 mNumTest = numTest; 112 mShouldThrow = shouldThrow; 113 mDeviceUnresponsive = deviceUnresponsive; 114 } 115 116 @Override run(ITestInvocationListener listener)117 public void run(ITestInvocationListener listener) throws DeviceNotAvailableException { 118 listener.testRunStarted(mRunName, mNumTest); 119 for (int i = 0; i < mNumTest; i++) { 120 TestDescription test = new TestDescription(mRunName + "class", "test" + i); 121 listener.testStarted(test); 122 if (mShouldThrow && i == mNumTest / 2) { 123 throw new DeviceNotAvailableException(); 124 } 125 if (mDeviceUnresponsive) { 126 throw new DeviceUnresponsiveException(); 127 } 128 listener.testEnded(test, new HashMap<String, Metric>()); 129 } 130 listener.testRunEnded(0, new HashMap<String, Metric>()); 131 } 132 133 @Override setBuild(IBuildInfo buildInfo)134 public void setBuild(IBuildInfo buildInfo) { 135 // ignore 136 } 137 138 @Override setDevice(ITestDevice device)139 public void setDevice(ITestDevice device) { 140 mDevice = device; 141 } 142 143 @Override getDevice()144 public ITestDevice getDevice() { 145 return mDevice; 146 } 147 } 148 149 @Before setUp()150 public void setUp() { 151 mMockLogSaver = EasyMock.createMock(ILogSaver.class); 152 mMockLogSaverListener = EasyMock.createStrictMock(ILogSaverListener.class); 153 154 mMockListener = EasyMock.createMock(ITestInvocationListener.class); 155 mTestList = new ArrayList<>(); 156 mMockTest = EasyMock.createMock(ITestInterface.class); 157 mTestList.add(mMockTest); 158 mTargetPrepList = new ArrayList<>(); 159 mMockPrep = EasyMock.createMock(ITargetPreparer.class); 160 mMockCleaner = EasyMock.createMock(ITargetCleaner.class); 161 mTargetPrepList.add(mMockPrep); 162 mTargetPrepList.add(mMockCleaner); 163 mMapDeviceTargetPreparer = new LinkedHashMap<>(); 164 mMapDeviceTargetPreparer.put(DEFAULT_DEVICE_NAME, mTargetPrepList); 165 166 mMultiTargetPrepList = new ArrayList<>(); 167 mMockBuildInfo = EasyMock.createMock(IBuildInfo.class); 168 mMockDevice = EasyMock.createMock(ITestDevice.class); 169 mModule = 170 new ModuleDefinition( 171 MODULE_NAME, 172 mTestList, 173 mMapDeviceTargetPreparer, 174 mMultiTargetPrepList, 175 new Configuration("", "")); 176 177 mModule.getModuleInvocationContext().addAllocatedDevice(DEFAULT_DEVICE_NAME, mMockDevice); 178 mModule.getModuleInvocationContext() 179 .addDeviceBuildInfo(DEFAULT_DEVICE_NAME, mMockBuildInfo); 180 } 181 182 /** Helper for replaying mocks. */ replayMocks()183 private void replayMocks() { 184 EasyMock.replay(mMockListener, mMockLogSaver, mMockLogSaverListener, mMockDevice); 185 for (IRemoteTest test : mTestList) { 186 EasyMock.replay(test); 187 } 188 for (ITargetPreparer prep : mTargetPrepList) { 189 try { 190 EasyMock.replay(prep); 191 } catch (IllegalArgumentException e) { 192 // ignore 193 } 194 } 195 } 196 197 /** Helper for verifying mocks. */ verifyMocks()198 private void verifyMocks() { 199 EasyMock.verify(mMockListener, mMockLogSaver, mMockLogSaverListener, mMockDevice); 200 for (IRemoteTest test : mTestList) { 201 EasyMock.verify(test); 202 } 203 for (ITargetPreparer prep : mTargetPrepList) { 204 try { 205 EasyMock.verify(prep); 206 } catch (IllegalArgumentException e) { 207 // ignore 208 } 209 } 210 } 211 212 /** 213 * Test that {@link ModuleDefinition#run(ITestInvocationListener)} is properly going through the 214 * execution flow. 215 */ 216 @Test testRun()217 public void testRun() throws Exception { 218 mModule.setBuild(mMockBuildInfo); 219 mModule.setDevice(mMockDevice); 220 EasyMock.expect(mMockPrep.isDisabled()).andReturn(false); 221 mMockPrep.setUp(EasyMock.eq(mMockDevice), EasyMock.eq(mMockBuildInfo)); 222 EasyMock.expect(mMockCleaner.isDisabled()).andStubReturn(false); 223 mMockCleaner.setUp(EasyMock.eq(mMockDevice), EasyMock.eq(mMockBuildInfo)); 224 mMockTest.setBuild(EasyMock.eq(mMockBuildInfo)); 225 mMockTest.setDevice(EasyMock.eq(mMockDevice)); 226 mMockTest.run((ITestInvocationListener) EasyMock.anyObject()); 227 EasyMock.expect(mMockCleaner.isTearDownDisabled()).andStubReturn(false); 228 mMockCleaner.tearDown(EasyMock.eq(mMockDevice), EasyMock.eq(mMockBuildInfo), 229 EasyMock.isNull()); 230 mMockListener.testRunStarted(MODULE_NAME, 0); 231 mMockListener.testRunEnded( 232 EasyMock.anyLong(), (HashMap<String, Metric>) EasyMock.anyObject()); 233 replayMocks(); 234 mModule.run(mMockListener); 235 verifyMocks(); 236 } 237 238 /** 239 * Test that {@link ModuleDefinition#run(ITestInvocationListener)} is properly going through the 240 * execution flow and skip target preparers if disabled. 241 */ 242 @Test testRun_disabledPreparation()243 public void testRun_disabledPreparation() throws Exception { 244 mModule.setBuild(mMockBuildInfo); 245 mModule.setDevice(mMockDevice); 246 // No setup and teardown expected from preparers. 247 EasyMock.expect(mMockPrep.isDisabled()).andReturn(true); 248 EasyMock.expect(mMockCleaner.isDisabled()).andStubReturn(true); 249 mMockTest.setBuild(EasyMock.eq(mMockBuildInfo)); 250 mMockTest.setDevice(EasyMock.eq(mMockDevice)); 251 mMockTest.run((ITestInvocationListener) EasyMock.anyObject()); 252 mMockListener.testRunStarted(MODULE_NAME, 0); 253 mMockListener.testRunEnded( 254 EasyMock.anyLong(), (HashMap<String, Metric>) EasyMock.anyObject()); 255 replayMocks(); 256 mModule.run(mMockListener); 257 verifyMocks(); 258 } 259 260 /** 261 * Test that {@link ModuleDefinition#run(ITestInvocationListener)} is properly going through the 262 * execution flow and skip target cleanup if teardown is disabled. 263 */ 264 @Test testRun_disabledTearDown()265 public void testRun_disabledTearDown() throws Exception { 266 mModule.setBuild(mMockBuildInfo); 267 mModule.setDevice(mMockDevice); 268 // Setup expected from preparers. 269 EasyMock.expect(mMockPrep.isDisabled()).andReturn(false); 270 mMockPrep.setUp(EasyMock.eq(mMockDevice), EasyMock.eq(mMockBuildInfo)); 271 EasyMock.expect(mMockCleaner.isDisabled()).andStubReturn(false); 272 mMockCleaner.setUp(EasyMock.eq(mMockDevice), EasyMock.eq(mMockBuildInfo)); 273 mMockTest.setBuild(EasyMock.eq(mMockBuildInfo)); 274 mMockTest.setDevice(EasyMock.eq(mMockDevice)); 275 mMockTest.run((ITestInvocationListener) EasyMock.anyObject()); 276 EasyMock.expect(mMockCleaner.isTearDownDisabled()).andStubReturn(true); 277 // But no teardown expected from Cleaner. 278 mMockListener.testRunStarted(MODULE_NAME, 0); 279 mMockListener.testRunEnded( 280 EasyMock.anyLong(), (HashMap<String, Metric>) EasyMock.anyObject()); 281 replayMocks(); 282 mModule.run(mMockListener); 283 verifyMocks(); 284 } 285 286 /** 287 * Test that {@link ModuleDefinition#run(ITestInvocationListener)} 288 */ 289 @Test testRun_failPreparation()290 public void testRun_failPreparation() throws Exception { 291 final String exceptionMessage = "ouch I failed"; 292 mTargetPrepList.clear(); 293 mTargetPrepList.add( 294 new BaseTargetPreparer() { 295 @Override 296 public void setUp(ITestDevice device, IBuildInfo buildInfo) 297 throws TargetSetupError, BuildError, DeviceNotAvailableException { 298 DeviceDescriptor nullDescriptor = null; 299 throw new TargetSetupError(exceptionMessage, nullDescriptor); 300 } 301 }); 302 mModule = 303 new ModuleDefinition( 304 MODULE_NAME, 305 mTestList, 306 mMapDeviceTargetPreparer, 307 mMultiTargetPrepList, 308 new Configuration("", "")); 309 mModule.getModuleInvocationContext().addAllocatedDevice(DEFAULT_DEVICE_NAME, mMockDevice); 310 mModule.getModuleInvocationContext() 311 .addDeviceBuildInfo(DEFAULT_DEVICE_NAME, mMockBuildInfo); 312 mMockCleaner.tearDown(EasyMock.eq(mMockDevice), EasyMock.eq(mMockBuildInfo), 313 EasyMock.isNull()); 314 mMockListener.testRunStarted(EasyMock.eq(MODULE_NAME), EasyMock.eq(1)); 315 mMockListener.testRunFailed(EasyMock.contains(exceptionMessage)); 316 mMockListener.testRunEnded( 317 EasyMock.anyLong(), (HashMap<String, Metric>) EasyMock.anyObject()); 318 replayMocks(); 319 mModule.run(mMockListener); 320 verifyMocks(); 321 } 322 323 /** 324 * Test that {@link ModuleDefinition#run(ITestInvocationListener)} is properly going through the 325 * execution flow with actual test callbacks. 326 */ 327 @Test testRun_fullPass()328 public void testRun_fullPass() throws Exception { 329 final int testCount = 5; 330 List<IRemoteTest> testList = new ArrayList<>(); 331 testList.add(new TestObject("run1", testCount, false)); 332 mModule = 333 new ModuleDefinition( 334 MODULE_NAME, 335 testList, 336 mMapDeviceTargetPreparer, 337 mMultiTargetPrepList, 338 new Configuration("", "")); 339 mModule.getModuleInvocationContext().addAllocatedDevice(DEFAULT_DEVICE_NAME, mMockDevice); 340 mModule.getModuleInvocationContext() 341 .addDeviceBuildInfo(DEFAULT_DEVICE_NAME, mMockBuildInfo); 342 mModule.setBuild(mMockBuildInfo); 343 mModule.setDevice(mMockDevice); 344 EasyMock.expect(mMockPrep.isDisabled()).andReturn(false); 345 // no isTearDownDisabled() expected for setup 346 mMockPrep.setUp(EasyMock.eq(mMockDevice), EasyMock.eq(mMockBuildInfo)); 347 EasyMock.expect(mMockCleaner.isDisabled()).andStubReturn(false); 348 mMockCleaner.setUp(EasyMock.eq(mMockDevice), EasyMock.eq(mMockBuildInfo)); 349 EasyMock.expect(mMockCleaner.isTearDownDisabled()).andStubReturn(false); 350 mMockCleaner.tearDown( 351 EasyMock.eq(mMockDevice), EasyMock.eq(mMockBuildInfo), EasyMock.isNull()); 352 mMockListener.testRunStarted(MODULE_NAME, testCount); 353 for (int i = 0; i < testCount; i++) { 354 mMockListener.testStarted((TestDescription) EasyMock.anyObject(), EasyMock.anyLong()); 355 mMockListener.testEnded( 356 (TestDescription) EasyMock.anyObject(), 357 EasyMock.anyLong(), 358 (HashMap<String, Metric>) EasyMock.anyObject()); 359 } 360 mMockListener.testRunEnded( 361 EasyMock.anyLong(), (HashMap<String, Metric>) EasyMock.anyObject()); 362 replayMocks(); 363 mModule.run(mMockListener); 364 verifyMocks(); 365 } 366 367 /** 368 * Test that {@link ModuleDefinition#run(ITestInvocationListener)} is properly going through the 369 * execution flow with actual test callbacks. 370 */ 371 @Test testRun_partialRun()372 public void testRun_partialRun() throws Exception { 373 final int testCount = 4; 374 List<IRemoteTest> testList = new ArrayList<>(); 375 testList.add(new TestObject("run1", testCount, true)); 376 mModule = 377 new ModuleDefinition( 378 MODULE_NAME, 379 testList, 380 mMapDeviceTargetPreparer, 381 mMultiTargetPrepList, 382 new Configuration("", "")); 383 mModule.getModuleInvocationContext().addAllocatedDevice(DEFAULT_DEVICE_NAME, mMockDevice); 384 mModule.getModuleInvocationContext() 385 .addDeviceBuildInfo(DEFAULT_DEVICE_NAME, mMockBuildInfo); 386 mModule.setBuild(mMockBuildInfo); 387 mModule.setDevice(mMockDevice); 388 EasyMock.expect(mMockPrep.isDisabled()).andReturn(false); 389 // no isTearDownDisabled() expected for setup 390 mMockPrep.setUp(EasyMock.eq(mMockDevice), EasyMock.eq(mMockBuildInfo)); 391 EasyMock.expect(mMockCleaner.isDisabled()).andStubReturn(false); 392 mMockCleaner.setUp(EasyMock.eq(mMockDevice), EasyMock.eq(mMockBuildInfo)); 393 EasyMock.expect(mMockCleaner.isTearDownDisabled()).andStubReturn(false); 394 mMockCleaner.tearDown( 395 EasyMock.eq(mMockDevice), EasyMock.eq(mMockBuildInfo), EasyMock.isNull()); 396 EasyMock.expectLastCall().andThrow(new DeviceNotAvailableException()); 397 mMockListener.testRunStarted(MODULE_NAME, testCount); 398 for (int i = 0; i < 3; i++) { 399 mMockListener.testStarted((TestDescription) EasyMock.anyObject(), EasyMock.anyLong()); 400 mMockListener.testEnded( 401 (TestDescription) EasyMock.anyObject(), 402 EasyMock.anyLong(), 403 (HashMap<String, Metric>) EasyMock.anyObject()); 404 } 405 mMockListener.testFailed(EasyMock.anyObject(), EasyMock.anyObject()); 406 mMockListener.testRunFailed(EasyMock.anyObject()); 407 mMockListener.testRunEnded( 408 EasyMock.anyLong(), (HashMap<String, Metric>) EasyMock.anyObject()); 409 replayMocks(); 410 try { 411 mModule.run(mMockListener); 412 fail("Should have thrown an exception."); 413 } catch (DeviceNotAvailableException expected) { 414 // expected 415 } 416 // Only one module 417 assertEquals(1, mModule.getTestsResults().size()); 418 assertEquals(2, mModule.getTestsResults().get(0).getNumCompleteTests()); 419 verifyMocks(); 420 } 421 422 /** 423 * Test that when a module is created with some particular informations, the resulting {@link 424 * IInvocationContext} of the module is properly populated. 425 */ 426 @Test testAbiSetting()427 public void testAbiSetting() throws Exception { 428 final int testCount = 5; 429 IConfiguration config = new Configuration("", ""); 430 ConfigurationDescriptor descriptor = new ConfigurationDescriptor(); 431 descriptor.setAbi(new Abi("arm", "32")); 432 descriptor.setModuleName(MODULE_NAME); 433 config.setConfigurationObject( 434 Configuration.CONFIGURATION_DESCRIPTION_TYPE_NAME, descriptor); 435 List<IRemoteTest> testList = new ArrayList<>(); 436 testList.add(new TestObject("run1", testCount, false)); 437 mModule = 438 new ModuleDefinition( 439 "arm32 " + MODULE_NAME, 440 testList, 441 mMapDeviceTargetPreparer, 442 mMultiTargetPrepList, 443 config); 444 // Check that the invocation module created has expected informations 445 IInvocationContext moduleContext = mModule.getModuleInvocationContext(); 446 assertEquals( 447 MODULE_NAME, 448 moduleContext.getAttributes().get(ModuleDefinition.MODULE_NAME).get(0)); 449 assertEquals("arm", moduleContext.getAttributes().get(ModuleDefinition.MODULE_ABI).get(0)); 450 assertEquals( 451 "arm32 " + MODULE_NAME, 452 moduleContext.getAttributes().get(ModuleDefinition.MODULE_ID).get(0)); 453 } 454 455 /** 456 * Test running a module when the configuration has a module controller object that force a full 457 * bypass of the module. 458 */ 459 @Test testModuleController_fullBypass()460 public void testModuleController_fullBypass() throws Exception { 461 IConfiguration config = new Configuration("", ""); 462 BaseModuleController moduleConfig = 463 new BaseModuleController() { 464 @Override 465 public RunStrategy shouldRun(IInvocationContext context) { 466 return RunStrategy.FULL_MODULE_BYPASS; 467 } 468 }; 469 config.setConfigurationObject(ModuleDefinition.MODULE_CONTROLLER, moduleConfig); 470 List<IRemoteTest> testList = new ArrayList<>(); 471 testList.add( 472 new IRemoteTest() { 473 @Override 474 public void run(ITestInvocationListener listener) 475 throws DeviceNotAvailableException { 476 listener.testRunStarted("test", 1); 477 listener.testFailed( 478 new TestDescription("failedclass", "failedmethod"), "trace"); 479 } 480 }); 481 mTargetPrepList.clear(); 482 mModule = 483 new ModuleDefinition( 484 MODULE_NAME, 485 testList, 486 mMapDeviceTargetPreparer, 487 mMultiTargetPrepList, 488 config); 489 mModule.getModuleInvocationContext().addAllocatedDevice(DEFAULT_DEVICE_NAME, mMockDevice); 490 mModule.getModuleInvocationContext() 491 .addDeviceBuildInfo(DEFAULT_DEVICE_NAME, mMockBuildInfo); 492 // module is completely skipped, no tests is recorded. 493 replayMocks(); 494 mModule.run(mMockListener, null, null); 495 verifyMocks(); 496 } 497 498 /** 499 * Test running a module when the configuration has a module controller object that force to 500 * skip all the module test cases. 501 */ 502 @Test testModuleController_skipTestCases()503 public void testModuleController_skipTestCases() throws Exception { 504 IConfiguration config = new Configuration("", ""); 505 BaseModuleController moduleConfig = 506 new BaseModuleController() { 507 @Override 508 public RunStrategy shouldRun(IInvocationContext context) { 509 return RunStrategy.SKIP_MODULE_TESTCASES; 510 } 511 }; 512 config.setConfigurationObject(ModuleDefinition.MODULE_CONTROLLER, moduleConfig); 513 List<IRemoteTest> testList = new ArrayList<>(); 514 testList.add( 515 new IRemoteTest() { 516 @Override 517 public void run(ITestInvocationListener listener) 518 throws DeviceNotAvailableException { 519 TestDescription tid = new TestDescription("class", "method"); 520 listener.testRunStarted("test", 1); 521 listener.testStarted(tid); 522 listener.testFailed(tid, "I failed"); 523 listener.testEnded(tid, new HashMap<String, Metric>()); 524 listener.testRunEnded(0, new HashMap<String, Metric>()); 525 } 526 }); 527 mTargetPrepList.clear(); 528 mModule = 529 new ModuleDefinition( 530 MODULE_NAME, 531 testList, 532 mMapDeviceTargetPreparer, 533 mMultiTargetPrepList, 534 config); 535 mModule.getModuleInvocationContext().addAllocatedDevice(DEFAULT_DEVICE_NAME, mMockDevice); 536 mModule.getModuleInvocationContext() 537 .addDeviceBuildInfo(DEFAULT_DEVICE_NAME, mMockBuildInfo); 538 // expect the module to run but tests to be ignored 539 mMockListener.testRunStarted(EasyMock.anyObject(), EasyMock.anyInt()); 540 mMockListener.testStarted(EasyMock.anyObject(), EasyMock.anyLong()); 541 mMockListener.testIgnored(EasyMock.anyObject()); 542 mMockListener.testEnded( 543 EasyMock.anyObject(), 544 EasyMock.anyLong(), 545 (HashMap<String, Metric>) EasyMock.anyObject()); 546 mMockListener.testRunEnded( 547 EasyMock.anyLong(), (HashMap<String, Metric>) EasyMock.anyObject()); 548 replayMocks(); 549 mModule.run(mMockListener, null, null); 550 verifyMocks(); 551 } 552 553 /** Test {@link IRemoteTest} that log a file during its run. */ 554 public class TestLogClass implements ITestInterface { 555 556 @Override run(ITestInvocationListener listener)557 public void run(ITestInvocationListener listener) throws DeviceNotAvailableException { 558 listener.testLog( 559 "testlogclass", 560 LogDataType.TEXT, 561 new ByteArrayInputStreamSource("".getBytes())); 562 } 563 564 @Override setBuild(IBuildInfo buildInfo)565 public void setBuild(IBuildInfo buildInfo) {} 566 567 @Override setDevice(ITestDevice device)568 public void setDevice(ITestDevice device) {} 569 570 @Override getDevice()571 public ITestDevice getDevice() { 572 return null; 573 } 574 } 575 576 /** 577 * Test that the invocation level result_reporter receive the testLogSaved information from the 578 * modules. 579 * 580 * <p>The {@link LogSaverResultForwarder} from the module is expected to log the file and ensure 581 * that it passes the information to the {@link LogSaverResultForwarder} from the {@link 582 * TestInvocation} in order for final result_reporter to know about logged files. 583 */ 584 @Test testModule_LogSaverResultForwarder()585 public void testModule_LogSaverResultForwarder() throws Exception { 586 List<IRemoteTest> testList = new ArrayList<>(); 587 testList.add(new TestLogClass()); 588 mModule = 589 new ModuleDefinition( 590 MODULE_NAME, 591 testList, 592 mMapDeviceTargetPreparer, 593 mMultiTargetPrepList, 594 new Configuration("", "")); 595 mModule.setLogSaver(mMockLogSaver); 596 mModule.getModuleInvocationContext().addAllocatedDevice(DEFAULT_DEVICE_NAME, mMockDevice); 597 mModule.getModuleInvocationContext() 598 .addDeviceBuildInfo(DEFAULT_DEVICE_NAME, mMockBuildInfo); 599 mModule.setBuild(mMockBuildInfo); 600 mModule.setDevice(mMockDevice); 601 EasyMock.expect(mMockPrep.isDisabled()).andReturn(false); 602 // no isTearDownDisabled() expected for setup 603 mMockPrep.setUp(EasyMock.eq(mMockDevice), EasyMock.eq(mMockBuildInfo)); 604 EasyMock.expect(mMockCleaner.isDisabled()).andStubReturn(false); 605 mMockCleaner.setUp(EasyMock.eq(mMockDevice), EasyMock.eq(mMockBuildInfo)); 606 EasyMock.expect(mMockCleaner.isTearDownDisabled()).andStubReturn(false); 607 mMockCleaner.tearDown( 608 EasyMock.eq(mMockDevice), EasyMock.eq(mMockBuildInfo), EasyMock.isNull()); 609 mMockLogSaverListener.setLogSaver(mMockLogSaver); 610 // The final reporter still receive the testLog signal 611 mMockLogSaverListener.testLog( 612 EasyMock.eq("testlogclass"), EasyMock.eq(LogDataType.TEXT), EasyMock.anyObject()); 613 614 LogFile loggedFile = new LogFile("path", "url", LogDataType.TEXT); 615 EasyMock.expect( 616 mMockLogSaver.saveLogData( 617 EasyMock.eq("testlogclass"), 618 EasyMock.eq(LogDataType.TEXT), 619 EasyMock.anyObject())) 620 .andReturn(loggedFile); 621 // mMockLogSaverListener should receive the testLogSaved call even from the module 622 mMockLogSaverListener.testLogSaved( 623 EasyMock.eq("testlogclass"), 624 EasyMock.eq(LogDataType.TEXT), 625 EasyMock.anyObject(), 626 EasyMock.eq(loggedFile)); 627 628 mMockLogSaverListener.testRunStarted(MODULE_NAME, 0); 629 mMockLogSaverListener.testRunEnded( 630 EasyMock.anyLong(), (HashMap<String, Metric>) EasyMock.anyObject()); 631 632 // Simulate how the invoker actually put the log saver 633 replayMocks(); 634 LogSaverResultForwarder forwarder = 635 new LogSaverResultForwarder(mMockLogSaver, Arrays.asList(mMockLogSaverListener)); 636 mModule.run(forwarder); 637 verifyMocks(); 638 } 639 640 /** 641 * Test that the {@link IModuleController} object can override the behavior of the capture of 642 * the failure. 643 */ 644 @Test testOverrideModuleConfig()645 public void testOverrideModuleConfig() throws Exception { 646 // failure listener with capture logcat on failure and screenshot on failure. 647 List<ITestDevice> listDevice = new ArrayList<>(); 648 listDevice.add(mMockDevice); 649 EasyMock.expect(mMockDevice.getSerialNumber()).andReturn("Serial"); 650 TestFailureListener failureListener = 651 new TestFailureListener(listDevice, false, true, true, false, 5); 652 failureListener.setLogger(mMockListener); 653 IConfiguration config = new Configuration("", ""); 654 TestFailureModuleController moduleConfig = new TestFailureModuleController(); 655 OptionSetter setter = new OptionSetter(moduleConfig); 656 // Module option should override the logcat on failure 657 setter.setOptionValue("logcat-on-failure", "false"); 658 config.setConfigurationObject(ModuleDefinition.MODULE_CONTROLLER, moduleConfig); 659 List<IRemoteTest> testList = new ArrayList<>(); 660 testList.add( 661 new IRemoteTest() { 662 @Override 663 public void run(ITestInvocationListener listener) 664 throws DeviceNotAvailableException { 665 listener.testFailed( 666 new TestDescription("failedclass", "failedmethod"), "trace"); 667 } 668 }); 669 mTargetPrepList.clear(); 670 mModule = 671 new ModuleDefinition( 672 MODULE_NAME, 673 testList, 674 mMapDeviceTargetPreparer, 675 mMultiTargetPrepList, 676 config); 677 mModule.setLogSaver(mMockLogSaver); 678 mMockListener.testRunStarted("fakeName", 0); 679 mMockListener.testRunEnded( 680 EasyMock.anyLong(), (HashMap<String, Metric>) EasyMock.anyObject()); 681 // Only screenshot is captured 682 EasyMock.expect(mMockDevice.getScreenshot()) 683 .andReturn(new ByteArrayInputStreamSource("".getBytes())); 684 // Only a screenshot is capture, logcat for that module was disabled. 685 LogFile loggedFile = new LogFile("path", "url", LogDataType.PNG); 686 EasyMock.expect( 687 mMockLogSaver.saveLogData( 688 EasyMock.eq("failedclass#failedmethod-Serial-screenshot"), 689 EasyMock.eq(LogDataType.PNG), 690 EasyMock.anyObject())) 691 .andReturn(loggedFile); 692 replayMocks(); 693 mModule.run(mMockListener, null, failureListener); 694 verifyMocks(); 695 } 696 697 /** Test when the test yields a DeviceUnresponsive exception. */ 698 @Test testRun_partialRun_deviceUnresponsive()699 public void testRun_partialRun_deviceUnresponsive() throws Exception { 700 final int testCount = 4; 701 List<IRemoteTest> testList = new ArrayList<>(); 702 testList.add(new TestObject("run1", testCount, false, true)); 703 mModule = 704 new ModuleDefinition( 705 MODULE_NAME, 706 testList, 707 mMapDeviceTargetPreparer, 708 mMultiTargetPrepList, 709 new Configuration("", "")); 710 mModule.getModuleInvocationContext().addAllocatedDevice(DEFAULT_DEVICE_NAME, mMockDevice); 711 mModule.getModuleInvocationContext() 712 .addDeviceBuildInfo(DEFAULT_DEVICE_NAME, mMockBuildInfo); 713 mModule.setBuild(mMockBuildInfo); 714 mModule.setDevice(mMockDevice); 715 EasyMock.expect(mMockPrep.isDisabled()).andReturn(false); 716 // no isTearDownDisabled() expected for setup 717 mMockPrep.setUp(EasyMock.eq(mMockDevice), EasyMock.eq(mMockBuildInfo)); 718 EasyMock.expect(mMockCleaner.isDisabled()).andStubReturn(false); 719 mMockCleaner.setUp(EasyMock.eq(mMockDevice), EasyMock.eq(mMockBuildInfo)); 720 EasyMock.expect(mMockCleaner.isTearDownDisabled()).andStubReturn(false); 721 mMockCleaner.tearDown( 722 EasyMock.eq(mMockDevice), EasyMock.eq(mMockBuildInfo), EasyMock.isNull()); 723 mMockListener.testRunStarted(MODULE_NAME, testCount); 724 for (int i = 0; i < 1; i++) { 725 mMockListener.testStarted((TestDescription) EasyMock.anyObject(), EasyMock.anyLong()); 726 mMockListener.testEnded( 727 (TestDescription) EasyMock.anyObject(), 728 EasyMock.anyLong(), 729 (HashMap<String, Metric>) EasyMock.anyObject()); 730 } 731 mMockListener.testFailed(EasyMock.anyObject(), EasyMock.anyObject()); 732 mMockListener.testRunFailed(EasyMock.anyObject()); 733 mMockListener.testRunEnded( 734 EasyMock.anyLong(), (HashMap<String, Metric>) EasyMock.anyObject()); 735 736 // There was a module failure so a bugreport should be captured. 737 EasyMock.expect(mMockDevice.getIDevice()).andStubReturn(EasyMock.createMock(IDevice.class)); 738 EasyMock.expect(mMockDevice.getSerialNumber()).andReturn("SERIAL"); 739 EasyMock.expect( 740 mMockDevice.logBugreport( 741 EasyMock.eq("module-fakeName-failure-SERIAL-bugreport"), 742 EasyMock.anyObject())) 743 .andReturn(true); 744 745 replayMocks(); 746 // DeviceUnresponsive should not throw since it indicates that the device was recovered. 747 mModule.run(mMockListener); 748 // Only one module 749 assertEquals(1, mModule.getTestsResults().size()); 750 assertEquals(0, mModule.getTestsResults().get(0).getNumCompleteTests()); 751 verifyMocks(); 752 } 753 754 /** 755 * Test that when a module level listener is specified it receives the events before the 756 * buffering and replay. 757 */ 758 @Test testRun_moduleLevelListeners()759 public void testRun_moduleLevelListeners() throws Exception { 760 mMockListener = EasyMock.createStrictMock(ITestInvocationListener.class); 761 final int testCount = 5; 762 List<IRemoteTest> testList = new ArrayList<>(); 763 testList.add(new TestObject("run1", testCount, false)); 764 mModule = 765 new ModuleDefinition( 766 MODULE_NAME, 767 testList, 768 mMapDeviceTargetPreparer, 769 mMultiTargetPrepList, 770 new Configuration("", "")); 771 mModule.setLogSaver(mMockLogSaver); 772 mModule.getModuleInvocationContext().addAllocatedDevice(DEFAULT_DEVICE_NAME, mMockDevice); 773 mModule.getModuleInvocationContext() 774 .addDeviceBuildInfo(DEFAULT_DEVICE_NAME, mMockBuildInfo); 775 mModule.setBuild(mMockBuildInfo); 776 mModule.setDevice(mMockDevice); 777 EasyMock.expect(mMockPrep.isDisabled()).andReturn(false); 778 // no isTearDownDisabled() expected for setup 779 mMockPrep.setUp(EasyMock.eq(mMockDevice), EasyMock.eq(mMockBuildInfo)); 780 EasyMock.expect(mMockCleaner.isDisabled()).andStubReturn(false); 781 mMockCleaner.setUp(EasyMock.eq(mMockDevice), EasyMock.eq(mMockBuildInfo)); 782 EasyMock.expect(mMockCleaner.isTearDownDisabled()).andStubReturn(false); 783 mMockCleaner.tearDown( 784 EasyMock.eq(mMockDevice), EasyMock.eq(mMockBuildInfo), EasyMock.isNull()); 785 786 mMockLogSaverListener.setLogSaver(mMockLogSaver); 787 788 mMockListener.testRunStarted("run1", testCount); 789 for (int i = 0; i < testCount; i++) { 790 mMockListener.testStarted((TestDescription) EasyMock.anyObject(), EasyMock.anyLong()); 791 mMockListener.testEnded( 792 (TestDescription) EasyMock.anyObject(), 793 EasyMock.anyLong(), 794 (HashMap<String, Metric>) EasyMock.anyObject()); 795 } 796 mMockListener.testRunEnded( 797 EasyMock.anyLong(), (HashMap<String, Metric>) EasyMock.anyObject()); 798 799 mMockLogSaverListener.testRunStarted(MODULE_NAME, testCount); 800 for (int i = 0; i < testCount; i++) { 801 mMockLogSaverListener.testStarted( 802 (TestDescription) EasyMock.anyObject(), EasyMock.anyLong()); 803 mMockLogSaverListener.testEnded( 804 (TestDescription) EasyMock.anyObject(), 805 EasyMock.anyLong(), 806 (HashMap<String, Metric>) EasyMock.anyObject()); 807 } 808 mMockLogSaverListener.testRunEnded( 809 EasyMock.anyLong(), (HashMap<String, Metric>) EasyMock.anyObject()); 810 811 // Simulate how the invoker actually put the log saver 812 replayMocks(); 813 LogSaverResultForwarder forwarder = 814 new LogSaverResultForwarder(mMockLogSaver, Arrays.asList(mMockLogSaverListener)); 815 mModule.run(forwarder, Arrays.asList(mMockListener), null); 816 verifyMocks(); 817 } 818 } 819