1 /* 2 * Copyright (C) 2010 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.tradefed.testtype; 18 19 import static org.junit.Assert.assertEquals; 20 import static org.junit.Assert.assertNotNull; 21 import static org.junit.Assert.assertTrue; 22 23 import com.android.ddmlib.Log; 24 import com.android.ddmlib.testrunner.TestResult.TestStatus; 25 import com.android.tradefed.TestAppConstants; 26 import com.android.tradefed.config.OptionSetter; 27 import com.android.tradefed.device.DeviceNotAvailableException; 28 import com.android.tradefed.device.DeviceUnresponsiveException; 29 import com.android.tradefed.device.ITestDevice; 30 import com.android.tradefed.device.ITestDevice.RecoveryMode; 31 import com.android.tradefed.metrics.proto.MetricMeasurement.Metric; 32 import com.android.tradefed.result.CollectingTestListener; 33 import com.android.tradefed.result.ITestInvocationListener; 34 import com.android.tradefed.result.TestDescription; 35 import com.android.tradefed.util.RunUtil; 36 37 import org.easymock.EasyMock; 38 import org.junit.Before; 39 import org.junit.Ignore; 40 import org.junit.Test; 41 import org.junit.runner.RunWith; 42 43 import java.io.IOException; 44 import java.util.HashMap; 45 46 /** Functional tests for {@link InstrumentationTest}. */ 47 @RunWith(DeviceJUnit4ClassRunner.class) 48 public class InstrumentationTestFuncTest implements IDeviceTest { 49 50 private static final String LOG_TAG = "InstrumentationTestFuncTest"; 51 private static final long SHELL_TIMEOUT = 2500; 52 private static final int TEST_TIMEOUT = 2000; 53 private static final long WAIT_FOR_DEVICE_AVAILABLE = 5 * 60 * 1000; 54 55 private ITestDevice mDevice; 56 57 /** The {@link InstrumentationTest} under test */ 58 private InstrumentationTest mInstrumentationTest; 59 60 private ITestInvocationListener mMockListener; 61 62 @Override setDevice(ITestDevice device)63 public void setDevice(ITestDevice device) { 64 mDevice = device; 65 } 66 67 @Override getDevice()68 public ITestDevice getDevice() { 69 return mDevice; 70 } 71 72 @Before setUp()73 public void setUp() throws Exception { 74 mInstrumentationTest = new InstrumentationTest(); 75 mInstrumentationTest.setPackageName(TestAppConstants.TESTAPP_PACKAGE); 76 mInstrumentationTest.setDevice(getDevice()); 77 // use no timeout by default 78 mInstrumentationTest.setShellTimeout(-1); 79 // set to no rerun by default 80 mInstrumentationTest.setRerunMode(false); 81 mMockListener = EasyMock.createStrictMock(ITestInvocationListener.class); 82 getDevice().disableKeyguard(); 83 } 84 85 /** Test normal run scenario with a single passed test result. */ 86 @Test testRun()87 public void testRun() throws DeviceNotAvailableException { 88 Log.i(LOG_TAG, "testRun"); 89 TestDescription expectedTest = 90 new TestDescription( 91 TestAppConstants.TESTAPP_CLASS, TestAppConstants.PASSED_TEST_METHOD); 92 mInstrumentationTest.setClassName(TestAppConstants.TESTAPP_CLASS); 93 mInstrumentationTest.setMethodName(TestAppConstants.PASSED_TEST_METHOD); 94 mInstrumentationTest.setTestTimeout(TEST_TIMEOUT); 95 mInstrumentationTest.setShellTimeout(SHELL_TIMEOUT); 96 mMockListener.testRunStarted(TestAppConstants.TESTAPP_PACKAGE, 1); 97 mMockListener.testStarted(EasyMock.eq(expectedTest)); 98 mMockListener.testEnded( 99 EasyMock.eq(expectedTest), (HashMap<String, Metric>) EasyMock.anyObject()); 100 mMockListener.testRunEnded( 101 EasyMock.anyLong(), (HashMap<String, Metric>) EasyMock.anyObject()); 102 EasyMock.replay(mMockListener); 103 mInstrumentationTest.run(mMockListener); 104 EasyMock.verify(mMockListener); 105 } 106 107 /** Test normal run scenario with a single failed test result. */ 108 @Test testRun_testFailed()109 public void testRun_testFailed() throws DeviceNotAvailableException { 110 Log.i(LOG_TAG, "testRun_testFailed"); 111 mInstrumentationTest.setClassName(TestAppConstants.TESTAPP_CLASS); 112 mInstrumentationTest.setMethodName(TestAppConstants.FAILED_TEST_METHOD); 113 mInstrumentationTest.setTestTimeout(TEST_TIMEOUT); 114 mInstrumentationTest.setShellTimeout(SHELL_TIMEOUT); 115 String[] error = new String[1]; 116 error[0] = null; 117 mInstrumentationTest.run( 118 new ITestInvocationListener() { 119 @Override 120 public void testFailed(TestDescription test, String trace) { 121 error[0] = trace; 122 } 123 }); 124 assertNotNull("testFailed was not called", error[0]); 125 assertTrue(error[0].contains("junit.framework.AssertionFailedError: test failed")); 126 } 127 128 /** Test run scenario where test process crashes. */ 129 @Test testRun_testCrash()130 public void testRun_testCrash() throws DeviceNotAvailableException { 131 Log.i(LOG_TAG, "testRun_testCrash"); 132 TestDescription expectedTest = 133 new TestDescription( 134 TestAppConstants.TESTAPP_CLASS, TestAppConstants.CRASH_TEST_METHOD); 135 mInstrumentationTest.setClassName(TestAppConstants.TESTAPP_CLASS); 136 mInstrumentationTest.setMethodName(TestAppConstants.CRASH_TEST_METHOD); 137 mInstrumentationTest.setTestTimeout(TEST_TIMEOUT); 138 mInstrumentationTest.setShellTimeout(SHELL_TIMEOUT); 139 mMockListener.testRunStarted(TestAppConstants.TESTAPP_PACKAGE, 1); 140 mMockListener.testStarted(EasyMock.eq(expectedTest)); 141 if (getDevice().getApiLevel() <= 23) { 142 // Before N handling of instrumentation crash is slightly different. 143 mMockListener.testFailed( 144 EasyMock.eq(expectedTest), EasyMock.contains("RuntimeException")); 145 mMockListener.testEnded( 146 EasyMock.eq(expectedTest), (HashMap<String, Metric>) EasyMock.anyObject()); 147 mMockListener.testRunFailed( 148 EasyMock.eq("Instrumentation run failed due to 'java.lang.RuntimeException'")); 149 } else { 150 mMockListener.testFailed( 151 EasyMock.eq(expectedTest), EasyMock.contains("Process crashed.")); 152 mMockListener.testEnded( 153 EasyMock.eq(expectedTest), (HashMap<String, Metric>) EasyMock.anyObject()); 154 mMockListener.testRunFailed( 155 EasyMock.eq("Instrumentation run failed due to 'Process crashed.'")); 156 } 157 mMockListener.testRunEnded( 158 EasyMock.anyLong(), (HashMap<String, Metric>) EasyMock.anyObject()); 159 try { 160 EasyMock.replay(mMockListener); 161 mInstrumentationTest.run(mMockListener); 162 EasyMock.verify(mMockListener); 163 } finally { 164 getDevice().waitForDeviceAvailable(); 165 } 166 } 167 168 /** Test run scenario where test run hangs indefinitely, and times out. */ 169 @Test testRun_testTimeout()170 public void testRun_testTimeout() throws DeviceNotAvailableException { 171 Log.i(LOG_TAG, "testRun_testTimeout"); 172 RecoveryMode initMode = getDevice().getRecoveryMode(); 173 getDevice().setRecoveryMode(RecoveryMode.NONE); 174 try { 175 mInstrumentationTest.setClassName(TestAppConstants.TESTAPP_CLASS); 176 mInstrumentationTest.setMethodName(TestAppConstants.TIMEOUT_TEST_METHOD); 177 mInstrumentationTest.setShellTimeout(SHELL_TIMEOUT); 178 mInstrumentationTest.setTestTimeout(TEST_TIMEOUT); 179 180 String[] error = new String[1]; 181 error[0] = null; 182 mInstrumentationTest.run( 183 new ITestInvocationListener() { 184 @Override 185 public void testFailed(TestDescription test, String trace) { 186 error[0] = trace; 187 } 188 }); 189 assertEquals( 190 "Test failed to run to completion. Reason: 'Failed to receive adb shell test " 191 + "output within 2500 ms. Test may have timed out, or adb connection to device " 192 + "became unresponsive'. Check device logcat for details", 193 error[0]); 194 } finally { 195 getDevice().setRecoveryMode(initMode); 196 RunUtil.getDefault().sleep(500); 197 } 198 } 199 200 /** Test run scenario where device reboots during test run. */ 201 @Test testRun_deviceReboot()202 public void testRun_deviceReboot() throws Exception { 203 Log.i(LOG_TAG, "testRun_deviceReboot"); 204 mInstrumentationTest.setClassName(TestAppConstants.TESTAPP_CLASS); 205 mInstrumentationTest.setMethodName(TestAppConstants.TIMEOUT_TEST_METHOD); 206 mInstrumentationTest.setShellTimeout(0); 207 mInstrumentationTest.setTestTimeout(0); 208 // Set a max timeout to avoid hanging forever for safety 209 //OptionSetter setter = new OptionSetter(mInstrumentationTest); 210 //setter.setOptionValue("max-timeout", "600000"); 211 212 // fork off a thread to do the reboot 213 Thread rebootThread = 214 new Thread() { 215 @Override 216 public void run() { 217 // wait for test run to begin 218 try { 219 // Give time to the instrumentation to start 220 Thread.sleep(2000); 221 getDevice().reboot(); 222 } catch (InterruptedException e) { 223 Log.w(LOG_TAG, "interrupted"); 224 } catch (DeviceNotAvailableException dnae) { 225 Log.w(LOG_TAG, "Device did not come back online after reboot"); 226 } 227 } 228 }; 229 rebootThread.setName("InstrumentationTestFuncTest#testRun_deviceReboot"); 230 rebootThread.start(); 231 try { 232 String[] error = new String[1]; 233 error[0] = null; 234 mInstrumentationTest.run( 235 new ITestInvocationListener() { 236 @Override 237 public void testRunFailed(String errorMessage) { 238 error[0] = errorMessage; 239 } 240 }); 241 assertEquals("Test run failed to complete. Expected 1 tests, received 0", error[0]); 242 } catch (DeviceUnresponsiveException expected) { 243 // expected 244 } finally { 245 rebootThread.join(WAIT_FOR_DEVICE_AVAILABLE); 246 getDevice().waitForDeviceAvailable(); 247 } 248 } 249 250 /** Test that when a max-timeout is set the instrumentation is stopped. */ 251 @Test testRun_maxTimeout()252 public void testRun_maxTimeout() throws Exception { 253 Log.i(LOG_TAG, "testRun_maxTimeout"); 254 mInstrumentationTest.setClassName(TestAppConstants.TESTAPP_CLASS); 255 mInstrumentationTest.setMethodName(TestAppConstants.TIMEOUT_TEST_METHOD); 256 mInstrumentationTest.setShellTimeout(0); 257 mInstrumentationTest.setTestTimeout(0); 258 OptionSetter setter = new OptionSetter(mInstrumentationTest); 259 setter.setOptionValue("max-timeout", "5000"); 260 final String[] called = new String[1]; 261 called[0] = null; 262 mInstrumentationTest.run( 263 new ITestInvocationListener() { 264 @Override 265 public void testRunFailed(String errorMessage) { 266 called[0] = errorMessage; 267 } 268 }); 269 assertEquals( 270 "com.android.ddmlib.TimeoutException: executeRemoteCommand timed out after 5000ms", 271 called[0]); 272 } 273 274 /** Test run scenario where device runtime resets during test run. */ 275 @Test 276 @Ignore testRun_deviceRuntimeReset()277 public void testRun_deviceRuntimeReset() throws Exception { 278 Log.i(LOG_TAG, "testRun_deviceRuntimeReset"); 279 mInstrumentationTest.setShellTimeout(SHELL_TIMEOUT); 280 mInstrumentationTest.setTestTimeout(TEST_TIMEOUT); 281 mInstrumentationTest.setClassName(TestAppConstants.TESTAPP_CLASS); 282 mInstrumentationTest.setMethodName(TestAppConstants.TIMEOUT_TEST_METHOD); 283 284 // fork off a thread to do the runtime reset 285 Thread resetThread = 286 new Thread() { 287 @Override 288 public void run() { 289 // wait for test run to begin 290 try { 291 Thread.sleep(1000); 292 Runtime.getRuntime() 293 .exec( 294 String.format( 295 "adb -s %s shell stop", 296 getDevice().getIDevice().getSerialNumber())); 297 Thread.sleep(500); 298 Runtime.getRuntime() 299 .exec( 300 String.format( 301 "adb -s %s shell start", 302 getDevice().getIDevice().getSerialNumber())); 303 } catch (InterruptedException e) { 304 Log.w(LOG_TAG, "interrupted"); 305 } catch (IOException e) { 306 Log.w(LOG_TAG, "IOException when rebooting"); 307 } 308 } 309 }; 310 resetThread.setName("InstrumentationTestFuncTest#testRun_deviceRuntimeReset"); 311 resetThread.start(); 312 try { 313 String[] error = new String[1]; 314 error[0] = null; 315 mInstrumentationTest.run( 316 new ITestInvocationListener() { 317 @Override 318 public void testRunFailed(String errorMessage) { 319 error[0] = errorMessage; 320 } 321 }); 322 assertEquals( 323 "Failed to receive adb shell test output within 120000 ms. Test may have " 324 + "timed out, or adb connection to device became unresponsive", 325 error[0]); 326 } finally { 327 resetThread.join(WAIT_FOR_DEVICE_AVAILABLE); 328 RunUtil.getDefault().sleep(5000); 329 getDevice().waitForDeviceAvailable(); 330 } 331 } 332 333 /** 334 * Test running all the tests with rerun on. At least one method will cause run to stop 335 * (currently TIMEOUT_TEST_METHOD and CRASH_TEST_METHOD). Verify that results are recorded for 336 * all tests in the suite. 337 */ 338 @Test testRun_rerun()339 public void testRun_rerun() throws Exception { 340 Log.i(LOG_TAG, "testRun_rerun"); 341 // run all tests in class 342 RecoveryMode initMode = getDevice().getRecoveryMode(); 343 getDevice().setRecoveryMode(RecoveryMode.NONE); 344 try { 345 OptionSetter setter = new OptionSetter(mInstrumentationTest); 346 setter.setOptionValue("collect-tests-timeout", Long.toString(SHELL_TIMEOUT)); 347 mInstrumentationTest.setClassName(TestAppConstants.TESTAPP_CLASS); 348 mInstrumentationTest.setRerunMode(true); 349 mInstrumentationTest.setShellTimeout(SHELL_TIMEOUT); 350 mInstrumentationTest.setTestTimeout(TEST_TIMEOUT); 351 CollectingTestListener listener = new CollectingTestListener(); 352 mInstrumentationTest.run(listener); 353 assertEquals(TestAppConstants.TOTAL_TEST_CLASS_TESTS, listener.getNumTotalTests()); 354 assertEquals( 355 TestAppConstants.TOTAL_TEST_CLASS_PASSED_TESTS, 356 listener.getNumTestsInState(TestStatus.PASSED)); 357 } finally { 358 getDevice().setRecoveryMode(initMode); 359 } 360 } 361 362 /** 363 * Test a run that crashes when collecting tests. 364 * 365 * <p>Expect run to proceed, but be reported as a run failure 366 */ 367 @Test testRun_rerunCrash()368 public void testRun_rerunCrash() throws Exception { 369 Log.i(LOG_TAG, "testRun_rerunCrash"); 370 mInstrumentationTest.setClassName(TestAppConstants.CRASH_ON_INIT_TEST_CLASS); 371 mInstrumentationTest.setMethodName(TestAppConstants.CRASH_ON_INIT_TEST_METHOD); 372 mInstrumentationTest.setRerunMode(false); 373 mInstrumentationTest.setShellTimeout(SHELL_TIMEOUT); 374 mInstrumentationTest.setTestTimeout(TEST_TIMEOUT); 375 CollectingTestListener listener = new CollectingTestListener(); 376 mInstrumentationTest.run(listener); 377 assertEquals(0, listener.getNumTotalTests()); 378 assertNotNull(listener.getCurrentRunResults()); 379 assertEquals(TestAppConstants.TESTAPP_PACKAGE, listener.getCurrentRunResults().getName()); 380 assertTrue(listener.getCurrentRunResults().isRunFailure()); 381 assertTrue(listener.getCurrentRunResults().isRunComplete()); 382 } 383 384 /** 385 * Test a run that hangs when collecting tests. 386 * 387 * <p>Expect a run failure to be reported 388 */ 389 @Test testRun_rerunHang()390 public void testRun_rerunHang() throws Exception { 391 Log.i(LOG_TAG, "testRun_rerunHang"); 392 RecoveryMode initMode = getDevice().getRecoveryMode(); 393 getDevice().setRecoveryMode(RecoveryMode.NONE); 394 try { 395 OptionSetter setter = new OptionSetter(mInstrumentationTest); 396 setter.setOptionValue("collect-tests-timeout", Long.toString(SHELL_TIMEOUT)); 397 mInstrumentationTest.setClassName(TestAppConstants.HANG_ON_INIT_TEST_CLASS); 398 mInstrumentationTest.setRerunMode(false); 399 mInstrumentationTest.setShellTimeout(SHELL_TIMEOUT); 400 mInstrumentationTest.setTestTimeout(TEST_TIMEOUT); 401 CollectingTestListener listener = new CollectingTestListener(); 402 mInstrumentationTest.run(listener); 403 assertEquals(0, listener.getNumTotalTests()); 404 assertEquals( 405 TestAppConstants.TESTAPP_PACKAGE, listener.getCurrentRunResults().getName()); 406 assertTrue(listener.getCurrentRunResults().isRunFailure()); 407 assertTrue(listener.getCurrentRunResults().isRunComplete()); 408 } finally { 409 getDevice().setRecoveryMode(initMode); 410 } 411 } 412 } 413