1 /* 2 * Copyright (C) 2012 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.compatibility.testtype; 18 19 import static com.google.common.base.Preconditions.checkArgument; 20 import static com.google.common.base.Preconditions.checkNotNull; 21 22 import com.android.compatibility.FailureCollectingListener; 23 import com.android.tradefed.config.IConfiguration; 24 import com.android.tradefed.config.IConfigurationReceiver; 25 import com.android.tradefed.config.Option; 26 import com.android.tradefed.device.DeviceNotAvailableException; 27 import com.android.tradefed.device.ITestDevice; 28 import com.android.tradefed.device.LogcatReceiver; 29 import com.android.tradefed.invoker.TestInformation; 30 import com.android.tradefed.log.LogUtil.CLog; 31 import com.android.tradefed.metrics.proto.MetricMeasurement.Metric; 32 import com.android.tradefed.result.ByteArrayInputStreamSource; 33 import com.android.tradefed.result.CompatibilityTestResult; 34 import com.android.tradefed.result.ITestInvocationListener; 35 import com.android.tradefed.result.InputStreamSource; 36 import com.android.tradefed.result.LogDataType; 37 import com.android.tradefed.result.TestDescription; 38 import com.android.tradefed.testtype.IDeviceTest; 39 import com.android.tradefed.testtype.IRemoteTest; 40 import com.android.tradefed.testtype.ITestFilterReceiver; 41 import com.android.tradefed.testtype.InstrumentationTest; 42 import com.android.tradefed.util.CommandResult; 43 import com.android.tradefed.util.CommandStatus; 44 import com.android.tradefed.util.StreamUtil; 45 46 import com.google.common.annotations.VisibleForTesting; 47 import com.google.common.base.Strings; 48 49 import org.json.JSONException; 50 import org.junit.Assert; 51 52 import java.io.ByteArrayOutputStream; 53 import java.io.IOException; 54 import java.util.Collections; 55 import java.util.HashMap; 56 import java.util.HashSet; 57 import java.util.Set; 58 59 /** A test that verifies that a single app can be successfully launched. */ 60 public class AppLaunchTest 61 implements IDeviceTest, IRemoteTest, IConfigurationReceiver, ITestFilterReceiver { 62 @VisibleForTesting static final String SCREENSHOT_AFTER_LAUNCH = "screenshot-after-launch"; 63 64 @Option( 65 name = SCREENSHOT_AFTER_LAUNCH, 66 description = "Whether to take a screenshost after a package is launched.") 67 private boolean mScreenshotAfterLaunch; 68 69 @Option(name = "package-name", description = "Package name of testing app.") 70 private String mPackageName; 71 72 @Option(name = "test-label", description = "Unique test identifier label.") 73 private String mTestLabel = "AppCompatibility"; 74 75 /** @deprecated */ 76 @Deprecated 77 @Option( 78 name = "retry-count", 79 description = "Number of times to retry a failed test case. 0 means no retry.") 80 private int mRetryCount = 0; 81 82 @Option(name = "include-filter", description = "The include filter of the test type.") 83 protected Set<String> mIncludeFilters = new HashSet<>(); 84 85 @Option(name = "exclude-filter", description = "The exclude filter of the test type.") 86 protected Set<String> mExcludeFilters = new HashSet<>(); 87 88 @Option(name = "dismiss-dialog", description = "Attempt to dismiss dialog from apps.") 89 protected boolean mDismissDialog = false; 90 91 @Option( 92 name = "app-launch-timeout-ms", 93 description = "Time to wait for app to launch in msecs.") 94 private int mAppLaunchTimeoutMs = 15000; 95 96 private static final String LAUNCH_TEST_RUNNER = 97 "com.android.compatibilitytest.AppCompatibilityRunner"; 98 private static final String LAUNCH_TEST_PACKAGE = "com.android.compatibilitytest"; 99 private static final String PACKAGE_TO_LAUNCH = "package_to_launch"; 100 private static final String ARG_DISMISS_DIALOG = "ARG_DISMISS_DIALOG"; 101 private static final String APP_LAUNCH_TIMEOUT_LABEL = "app_launch_timeout_ms"; 102 private static final int LOGCAT_SIZE_BYTES = 20 * 1024 * 1024; 103 private static final int BASE_INSTRUMENTATION_TEST_TIMEOUT_MS = 10 * 1000; 104 105 private ITestDevice mDevice; 106 private LogcatReceiver mLogcat; 107 private IConfiguration mConfiguration; 108 AppLaunchTest()109 public AppLaunchTest() { 110 this(null); 111 } 112 113 @VisibleForTesting AppLaunchTest(String packageName)114 public AppLaunchTest(String packageName) { 115 this(packageName, 0); 116 } 117 118 @VisibleForTesting AppLaunchTest(String packageName, int retryCount)119 public AppLaunchTest(String packageName, int retryCount) { 120 mPackageName = packageName; 121 mRetryCount = retryCount; 122 } 123 124 /** 125 * Creates and sets up an instrumentation test with information about the test runner as well as 126 * the package being tested (provided as a parameter). 127 */ createInstrumentationTest(String packageBeingTested)128 protected InstrumentationTest createInstrumentationTest(String packageBeingTested) { 129 InstrumentationTest instrumentationTest = new InstrumentationTest(); 130 131 instrumentationTest.setPackageName(LAUNCH_TEST_PACKAGE); 132 instrumentationTest.setConfiguration(mConfiguration); 133 instrumentationTest.addInstrumentationArg(PACKAGE_TO_LAUNCH, packageBeingTested); 134 instrumentationTest.setRunnerName(LAUNCH_TEST_RUNNER); 135 instrumentationTest.setDevice(mDevice); 136 instrumentationTest.addInstrumentationArg( 137 APP_LAUNCH_TIMEOUT_LABEL, Integer.toString(mAppLaunchTimeoutMs)); 138 instrumentationTest.addInstrumentationArg( 139 ARG_DISMISS_DIALOG, Boolean.toString(mDismissDialog)); 140 141 int testTimeoutMs = BASE_INSTRUMENTATION_TEST_TIMEOUT_MS + mAppLaunchTimeoutMs * 2; 142 instrumentationTest.setShellTimeout(testTimeoutMs); 143 instrumentationTest.setTestTimeout(testTimeoutMs); 144 145 return instrumentationTest; 146 } 147 148 /* 149 * {@inheritDoc} 150 */ 151 @Override run(final TestInformation testInfo, final ITestInvocationListener listener)152 public void run(final TestInformation testInfo, final ITestInvocationListener listener) 153 throws DeviceNotAvailableException { 154 CLog.d("Start of run method."); 155 CLog.d("Include filters: %s", mIncludeFilters); 156 CLog.d("Exclude filters: %s", mExcludeFilters); 157 158 Assert.assertNotNull("Package name cannot be null", mPackageName); 159 160 TestDescription testDescription = createTestDescription(); 161 162 if (!inFilter(testDescription.toString())) { 163 CLog.d("Test case %s doesn't match any filter", testDescription); 164 return; 165 } 166 CLog.d("Complete filtering test case: %s", testDescription); 167 168 long start = System.currentTimeMillis(); 169 listener.testRunStarted(mTestLabel, 1); 170 mLogcat = new LogcatReceiver(getDevice(), LOGCAT_SIZE_BYTES, 0); 171 mLogcat.start(); 172 173 try { 174 testPackage(testInfo, testDescription, listener); 175 } catch (InterruptedException e) { 176 CLog.e(e); 177 throw new RuntimeException(e); 178 } finally { 179 mLogcat.stop(); 180 listener.testRunEnded( 181 System.currentTimeMillis() - start, new HashMap<String, Metric>()); 182 } 183 } 184 185 /** 186 * Attempts to test a package and reports the results. 187 * 188 * @param listener The {@link ITestInvocationListener}. 189 * @throws DeviceNotAvailableException 190 */ testPackage( final TestInformation testInfo, TestDescription testDescription, ITestInvocationListener listener)191 private void testPackage( 192 final TestInformation testInfo, 193 TestDescription testDescription, 194 ITestInvocationListener listener) 195 throws DeviceNotAvailableException, InterruptedException { 196 CLog.d("Started testing package: %s.", mPackageName); 197 198 listener.testStarted(testDescription, System.currentTimeMillis()); 199 200 CompatibilityTestResult result = createCompatibilityTestResult(); 201 result.packageName = mPackageName; 202 203 try { 204 for (int i = 0; i <= mRetryCount; i++) { 205 result.status = null; 206 result.message = null; 207 // Clear test result between retries. 208 launchPackage(testInfo, result); 209 if (result.status == CompatibilityTestResult.STATUS_SUCCESS) { 210 break; 211 } 212 } 213 214 if (mScreenshotAfterLaunch) { 215 try (InputStreamSource screenSource = mDevice.getScreenshot()) { 216 listener.testLog( 217 mPackageName + "_screenshot_" + mDevice.getSerialNumber(), 218 LogDataType.PNG, 219 screenSource); 220 } catch (DeviceNotAvailableException e) { 221 CLog.e( 222 "Device %s became unavailable while capturing screenshot, %s", 223 mDevice.getSerialNumber(), e.toString()); 224 throw e; 225 } 226 } 227 } finally { 228 reportResult(listener, testDescription, result); 229 stopPackage(); 230 try { 231 postLogcat(result, listener); 232 } catch (JSONException e) { 233 CLog.w("Posting failed: %s.", e.getMessage()); 234 } 235 listener.testEnded( 236 testDescription, 237 System.currentTimeMillis(), 238 Collections.<String, String>emptyMap()); 239 240 CLog.d("Completed testing package: %s.", mPackageName); 241 } 242 } 243 244 /** 245 * Method which attempts to launch a package. 246 * 247 * <p>Will set the result status to success if the package could be launched. Otherwise the 248 * result status will be set to failure. 249 * 250 * @param result the {@link CompatibilityTestResult} containing the package info. 251 * @throws DeviceNotAvailableException 252 */ launchPackage(final TestInformation testInfo, CompatibilityTestResult result)253 private void launchPackage(final TestInformation testInfo, CompatibilityTestResult result) 254 throws DeviceNotAvailableException { 255 CLog.d("Launching package: %s.", result.packageName); 256 257 CommandResult resetResult = resetPackage(); 258 if (resetResult.getStatus() != CommandStatus.SUCCESS) { 259 result.status = CompatibilityTestResult.STATUS_ERROR; 260 result.message = resetResult.getStatus() + resetResult.getStderr(); 261 return; 262 } 263 264 InstrumentationTest instrTest = createInstrumentationTest(result.packageName); 265 266 FailureCollectingListener failureListener = createFailureListener(); 267 instrTest.run(testInfo, failureListener); 268 CLog.d("Stack Trace: %s", failureListener.getStackTrace()); 269 270 if (failureListener.getStackTrace() != null) { 271 CLog.w("Failed to launch package: %s.", result.packageName); 272 result.status = CompatibilityTestResult.STATUS_FAILURE; 273 result.message = failureListener.getStackTrace(); 274 } else { 275 result.status = CompatibilityTestResult.STATUS_SUCCESS; 276 } 277 278 CLog.d("Completed launching package: %s", result.packageName); 279 } 280 281 /** Helper method which reports a test failed if the status is either a failure or an error. */ reportResult( ITestInvocationListener listener, TestDescription id, CompatibilityTestResult result)282 private void reportResult( 283 ITestInvocationListener listener, TestDescription id, CompatibilityTestResult result) { 284 String message = result.message != null ? result.message : "unknown"; 285 String tag = errorStatusToTag(result.status); 286 if (tag != null) { 287 listener.testFailed(id, result.status + ":" + message); 288 } 289 } 290 errorStatusToTag(String status)291 private String errorStatusToTag(String status) { 292 if (status.equals(CompatibilityTestResult.STATUS_ERROR)) { 293 return "ERROR"; 294 } 295 if (status.equals(CompatibilityTestResult.STATUS_FAILURE)) { 296 return "FAILURE"; 297 } 298 return null; 299 } 300 301 /** Helper method which posts the logcat. */ postLogcat(CompatibilityTestResult result, ITestInvocationListener listener)302 private void postLogcat(CompatibilityTestResult result, ITestInvocationListener listener) 303 throws JSONException { 304 InputStreamSource stream = null; 305 String header = 306 String.format( 307 "%s%s%s\n", 308 CompatibilityTestResult.SEPARATOR, 309 result.toJsonString(), 310 CompatibilityTestResult.SEPARATOR); 311 312 try (InputStreamSource logcatData = mLogcat.getLogcatData()) { 313 try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) { 314 baos.write(header.getBytes()); 315 StreamUtil.copyStreams(logcatData.createInputStream(), baos); 316 stream = new ByteArrayInputStreamSource(baos.toByteArray()); 317 } catch (IOException e) { 318 CLog.e("error inserting compatibility test result into logcat"); 319 CLog.e(e); 320 // fallback to logcat data 321 stream = logcatData; 322 } 323 listener.testLog("logcat_" + result.packageName, LogDataType.LOGCAT, stream); 324 } finally { 325 StreamUtil.cancel(stream); 326 } 327 } 328 329 /** 330 * Return true if a test matches one or more of the include filters AND does not match any of 331 * the exclude filters. If no include filters are given all tests should return true as long as 332 * they do not match any of the exclude filters. 333 */ inFilter(String testName)334 protected boolean inFilter(String testName) { 335 if (mExcludeFilters.contains(testName)) { 336 return false; 337 } 338 if (mIncludeFilters.size() == 0 || mIncludeFilters.contains(testName)) { 339 return true; 340 } 341 return false; 342 } 343 resetPackage()344 protected CommandResult resetPackage() throws DeviceNotAvailableException { 345 return mDevice.executeShellV2Command(String.format("pm clear %s", mPackageName)); 346 } 347 stopPackage()348 private void stopPackage() throws DeviceNotAvailableException { 349 mDevice.executeShellCommand(String.format("am force-stop %s", mPackageName)); 350 } 351 352 @Override setConfiguration(IConfiguration configuration)353 public void setConfiguration(IConfiguration configuration) { 354 mConfiguration = configuration; 355 } 356 357 /* 358 * {@inheritDoc} 359 */ 360 @Override setDevice(ITestDevice device)361 public void setDevice(ITestDevice device) { 362 mDevice = device; 363 } 364 365 /* 366 * {@inheritDoc} 367 */ 368 @Override getDevice()369 public ITestDevice getDevice() { 370 return mDevice; 371 } 372 getmRetryCount()373 public int getmRetryCount() { 374 return mRetryCount; 375 } 376 377 /** 378 * Get a test description for use in logging. For compatibility with logs, this should be 379 * TestDescription(test class name, test type). 380 */ createTestDescription()381 private TestDescription createTestDescription() { 382 return new TestDescription(getClass().getSimpleName(), mPackageName); 383 } 384 385 /** Get a FailureCollectingListener for failure listening. */ createFailureListener()386 private FailureCollectingListener createFailureListener() { 387 return new FailureCollectingListener(); 388 } 389 390 /** 391 * Get a CompatibilityTestResult for encapsulating compatibility run results for a single app 392 * package tested. 393 */ createCompatibilityTestResult()394 private CompatibilityTestResult createCompatibilityTestResult() { 395 return new CompatibilityTestResult(); 396 } 397 398 /** {@inheritDoc} */ 399 @Override addIncludeFilter(String filter)400 public void addIncludeFilter(String filter) { 401 checkArgument(!Strings.isNullOrEmpty(filter), "Include filter cannot be null or empty."); 402 mIncludeFilters.add(filter); 403 } 404 405 /** {@inheritDoc} */ 406 @Override addAllIncludeFilters(Set<String> filters)407 public void addAllIncludeFilters(Set<String> filters) { 408 checkNotNull(filters, "Include filters cannot be null."); 409 mIncludeFilters.addAll(filters); 410 } 411 412 /** {@inheritDoc} */ 413 @Override clearIncludeFilters()414 public void clearIncludeFilters() { 415 mIncludeFilters.clear(); 416 } 417 418 /** {@inheritDoc} */ 419 @Override getIncludeFilters()420 public Set<String> getIncludeFilters() { 421 return Collections.unmodifiableSet(mIncludeFilters); 422 } 423 424 /** {@inheritDoc} */ 425 @Override addExcludeFilter(String filter)426 public void addExcludeFilter(String filter) { 427 checkArgument(!Strings.isNullOrEmpty(filter), "Exclude filter cannot be null or empty."); 428 mExcludeFilters.add(filter); 429 } 430 431 /** {@inheritDoc} */ 432 @Override addAllExcludeFilters(Set<String> filters)433 public void addAllExcludeFilters(Set<String> filters) { 434 checkNotNull(filters, "Exclude filters cannot be null."); 435 mExcludeFilters.addAll(filters); 436 } 437 438 /** {@inheritDoc} */ 439 @Override clearExcludeFilters()440 public void clearExcludeFilters() { 441 mExcludeFilters.clear(); 442 } 443 444 /** {@inheritDoc} */ 445 @Override getExcludeFilters()446 public Set<String> getExcludeFilters() { 447 return Collections.unmodifiableSet(mExcludeFilters); 448 } 449 } 450