1 /* 2 * Copyright (C) 2016 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.support.test.aupt; 18 19 import android.app.Service; 20 import android.content.Context; 21 import android.content.ContextWrapper; 22 import android.content.Intent; 23 import android.content.IntentFilter; 24 import android.os.Bundle; 25 import android.os.Environment; 26 import android.os.IBinder; 27 import android.os.SystemClock; 28 import android.support.test.uiautomator.UiDevice; 29 import android.test.AndroidTestRunner; 30 import android.test.InstrumentationTestCase; 31 import android.test.InstrumentationTestRunner; 32 import android.util.Log; 33 34 import junit.framework.AssertionFailedError; 35 import junit.framework.Test; 36 import junit.framework.TestCase; 37 import junit.framework.TestListener; 38 import junit.framework.TestResult; 39 import junit.framework.TestSuite; 40 41 import java.io.BufferedReader; 42 import java.io.File; 43 import java.io.FileInputStream; 44 import java.io.IOException; 45 import java.io.InputStreamReader; 46 import java.text.SimpleDateFormat; 47 import java.util.ArrayList; 48 import java.util.Arrays; 49 import java.util.Comparator; 50 import java.util.Date; 51 import java.util.HashMap; 52 import java.util.List; 53 import java.util.Map; 54 import java.util.Random; 55 import java.util.concurrent.TimeUnit; 56 import java.util.concurrent.TimeoutException; 57 58 /** 59 * Ultra-fancy TestRunner to use when running AUPT: supports 60 * 61 * - Picking up tests from dexed JARs 62 * - Running tests for multiple iterations or in a custom order 63 * - Terminating tests after UI errors, timeouts, or when dependent processes die 64 * - Injecting additional information into custom TestCase subclasses 65 * - Passing through continuous metric-collection to a DataCollector instance 66 * - Collecting bugreports and heapdumps 67 * 68 */ 69 public class AuptTestRunner extends InstrumentationTestRunner { 70 /* Constants */ 71 private static final String LOG_TAG = AuptTestRunner.class.getSimpleName(); 72 private static final Long ANR_DELAY = 30000L; 73 private static final Long DEFAULT_SUITE_TIMEOUT = 0L; 74 private static final Long DEFAULT_TEST_TIMEOUT = 10L; 75 private static final SimpleDateFormat SCREENSHOT_DATE_FORMAT = 76 new SimpleDateFormat("dd-mm-yy:HH:mm:ss:SSS"); 77 78 /* Keep a pointer to our argument bundle around for testing */ 79 private Bundle mParams; 80 81 /* Primitive Parameters */ 82 private boolean mDeleteOldFiles; 83 private long mFileRetainCount; 84 private boolean mGenerateAnr; 85 private boolean mRecordMeminfo; 86 private long mIterations; 87 private long mSeed; 88 89 /* Dumpheap Parameters */ 90 private boolean mDumpheapEnabled; 91 private long mDumpheapInterval; 92 private long mDumpheapThreshold; 93 private long mMaxDumpheaps; 94 95 /* String Parameters */ 96 private List<String> mJars = new ArrayList<>(); 97 private List<String> mMemoryTrackedProcesses = new ArrayList<>(); 98 private List<String> mFinishCommands; 99 100 /* Other Parameters */ 101 private File mResultsDirectory; 102 103 /* Helpers */ 104 private Scheduler mScheduler; 105 private DataCollector mDataCollector; 106 private DexTestRunner mRunner; 107 108 /* Logging */ 109 private ProcessStatusTracker mProcessTracker; 110 private List<MemHealthRecord> mMemHealthRecords = new ArrayList<>(); 111 private Map<String, Long> mDumpheapCount = new HashMap<>(); 112 private Map<String, Long> mLastDumpheap = new HashMap<>(); 113 114 /* Test Initialization */ 115 @Override onCreate(Bundle params)116 public void onCreate(Bundle params) { 117 mParams = params; 118 119 // Parse out primitive parameters 120 mIterations = parseLongParam("iterations", 1); 121 mRecordMeminfo = parseBoolParam("record_meminfo", false); 122 mDumpheapEnabled = parseBoolParam("enableDumpheap", false); 123 mDumpheapThreshold = parseLongParam("dumpheapThreshold", 200 * 1024 * 1024); 124 mDumpheapInterval = parseLongParam("dumpheapInterval", 60 * 60 * 1000); 125 mMaxDumpheaps = parseLongParam("maxDumpheaps", 5); 126 mSeed = parseLongParam("seed", new Random().nextLong()); 127 128 // Option: -e finishCommand 'a;b;c;d' 129 String finishCommandArg = parseStringParam("finishCommand", null); 130 mFinishCommands = 131 finishCommandArg == null 132 ? Arrays.<String>asList() 133 : Arrays.asList(finishCommandArg.split("\\s*;\\s*")); 134 135 // Option: -e shuffle true 136 mScheduler = parseBoolParam("shuffle", false) 137 ? Scheduler.shuffled(new Random(mSeed), mIterations) 138 : Scheduler.sequential(mIterations); 139 140 // Option: -e jars aupt-app-tests.jar:... 141 mJars.addAll(DexTestRunner.parseDexedJarPaths(parseStringParam("jars", ""))); 142 143 // Option: -e trackMemory com.pkg1,com.pkg2,... 144 String memoryTrackedProcesses = parseStringParam("trackMemory", null); 145 146 if (memoryTrackedProcesses != null) { 147 mMemoryTrackedProcesses = Arrays.asList(memoryTrackedProcesses.split(",")); 148 } else { 149 try { 150 // Deprecated approach: get tracked processes from a file. 151 String trackMemoryFileName = 152 Environment.getExternalStorageDirectory() + "/track_memory.txt"; 153 154 BufferedReader reader = new BufferedReader(new InputStreamReader( 155 new FileInputStream(new File(trackMemoryFileName)))); 156 157 mMemoryTrackedProcesses = Arrays.asList(reader.readLine().split(",")); 158 reader.close(); 159 } catch (NullPointerException | IOException ex) { 160 mMemoryTrackedProcesses = Arrays.asList(); 161 } 162 } 163 164 // Option: -e detectKill com.pkg1,...,com.pkg8 165 String processes = parseStringParam("detectKill", null); 166 167 if (processes != null) { 168 mProcessTracker = new ProcessStatusTracker(processes.split(",")); 169 } else { 170 mProcessTracker = new ProcessStatusTracker(new String[] {}); 171 } 172 173 // Option: -e outputLocation aupt_results 174 mResultsDirectory = new File(Environment.getExternalStorageDirectory(), 175 parseStringParam("outputLocation", "aupt_results")); 176 if (!mResultsDirectory.exists() && !mResultsDirectory.mkdirs()) { 177 Log.w(LOG_TAG, "Could not find or create output directory " + mResultsDirectory); 178 } 179 180 // Option: -e fileRetainCount 1 181 mFileRetainCount = parseLongParam("fileRetainCount", -1); 182 mDeleteOldFiles = (mFileRetainCount != -1); 183 184 // Primary logging infrastructure 185 mDataCollector = new DataCollector( 186 TimeUnit.MINUTES.toMillis(parseLongParam("bugreportInterval", 0)), 187 TimeUnit.MINUTES.toMillis(parseLongParam("jankInterval", 0)), 188 TimeUnit.MINUTES.toMillis(parseLongParam("meminfoInterval", 0)), 189 TimeUnit.MINUTES.toMillis(parseLongParam("cpuinfoInterval", 0)), 190 TimeUnit.MINUTES.toMillis(parseLongParam("fragmentationInterval", 0)), 191 TimeUnit.MINUTES.toMillis(parseLongParam("ionInterval", 0)), 192 TimeUnit.MINUTES.toMillis(parseLongParam("pagetypeinfoInterval", 0)), 193 TimeUnit.MINUTES.toMillis(parseLongParam("traceInterval", 0)), 194 TimeUnit.MINUTES.toMillis(parseLongParam("bugreportzInterval", 0)), 195 mResultsDirectory, this); 196 197 // Make our TestRunner and make sure we injectInstrumentation. 198 mRunner = new DexTestRunner(this, mScheduler, mJars, 199 TimeUnit.MINUTES.toMillis(parseLongParam("testCaseTimeout", DEFAULT_TEST_TIMEOUT)), 200 TimeUnit.MINUTES.toMillis(parseLongParam("suiteTimeout", DEFAULT_SUITE_TIMEOUT))) { 201 @Override 202 public void runTest(TestResult result) { 203 for (TestCase test: mTestCases) { 204 injectInstrumentation(test); 205 } 206 207 super.runTest(result); 208 } 209 }; 210 211 // Aupt's TestListeners 212 mRunner.addTestListener(new PeriodicHeapDumper()); 213 mRunner.addTestListener(new MemHealthRecorder()); 214 mRunner.addTestListener(new DcimCleaner()); 215 mRunner.addTestListener(new PidChecker()); 216 mRunner.addTestListener(new TimeoutStackDumper()); 217 mRunner.addTestListener(new MemInfoDumper()); 218 mRunner.addTestListener(new FinishCommandRunner()); 219 mRunner.addTestListenerIf(parseBoolParam("generateANR", false), new ANRTrigger()); 220 mRunner.addTestListenerIf(parseBoolParam("quitOnError", false), new QuitOnErrorListener()); 221 mRunner.addTestListenerIf(parseBoolParam("checkBattery", false), new BatteryChecker()); 222 mRunner.addTestListenerIf(parseBoolParam("screenshots", false), new Screenshotter()); 223 224 // Start our loggers 225 mDataCollector.start(); 226 227 // Start the test 228 super.onCreate(params); 229 } 230 231 @Override onDestroy()232 public void onDestroy() { 233 mDataCollector.stop(); 234 } 235 236 /* Option-parsing helpers */ 237 parseLongParam(String key, long alternative)238 private long parseLongParam(String key, long alternative) throws NumberFormatException { 239 if (mParams.containsKey(key)) { 240 return Long.parseLong(mParams.getString(key)); 241 } else { 242 return alternative; 243 } 244 } 245 parseBoolParam(String key, boolean alternative)246 private boolean parseBoolParam(String key, boolean alternative) 247 throws NumberFormatException { 248 if (mParams.containsKey(key)) { 249 return Boolean.parseBoolean(mParams.getString(key)); 250 } else { 251 return alternative; 252 } 253 } 254 parseStringParam(String key, String alternative)255 private String parseStringParam(String key, String alternative) { 256 if (mParams.containsKey(key)) { 257 return mParams.getString(key); 258 } else { 259 return alternative; 260 } 261 } 262 263 /* Utility methods */ 264 265 /** 266 * Injects instrumentation into InstrumentationTestCase and AuptTestCase instances 267 */ injectInstrumentation(Test test)268 private void injectInstrumentation(Test test) { 269 if (InstrumentationTestCase.class.isAssignableFrom(test.getClass())) { 270 InstrumentationTestCase instrTest = (InstrumentationTestCase) test; 271 272 instrTest.injectInstrumentation(AuptTestRunner.this); 273 } 274 } 275 276 /* Passthrough to our DexTestRunner */ 277 @Override getAndroidTestRunner()278 protected AndroidTestRunner getAndroidTestRunner() { 279 return mRunner; 280 } 281 282 @Override getTargetContext()283 public Context getTargetContext() { 284 return new ContextWrapper(super.getTargetContext()) { 285 @Override 286 public ClassLoader getClassLoader() { 287 if(mRunner != null) { 288 return mRunner.getDexClassLoader(); 289 } else { 290 throw new RuntimeException("DexTestRunner not initialized!"); 291 } 292 } 293 }; 294 } 295 296 /** 297 * A simple abstract instantiation of TestListener 298 * 299 * Primarily meant to work around Java 7's lack of interface-default methods. 300 */ 301 abstract static class AuptListener implements TestListener { 302 /** Called when a test throws an exception. */ 303 public void addError(Test test, Throwable t) {} 304 305 /** Called when a test fails. */ 306 public void addFailure(Test test, AssertionFailedError t) {} 307 308 /** Called whenever a test ends. */ 309 public void endTest(Test test) {} 310 311 /** Called whenever a test begins. */ 312 public void startTest(Test test) {} 313 } 314 315 /** 316 * Periodically Heap-dump to assist with memory-leaks. 317 */ 318 private class PeriodicHeapDumper extends AuptListener { 319 private Thread mHeapDumpThread; 320 321 private class InternalHeapDumper implements Runnable { 322 private void recordDumpheap(String proc, long pss) throws IOException { 323 if (!mDumpheapEnabled) { 324 return; 325 } 326 Long count = mDumpheapCount.get(proc); 327 if (count == null) { 328 count = 0L; 329 } 330 Long lastDumpheap = mLastDumpheap.get(proc); 331 if (lastDumpheap == null) { 332 lastDumpheap = 0L; 333 } 334 long currentTime = SystemClock.uptimeMillis(); 335 if (pss > mDumpheapThreshold && count < mMaxDumpheaps && 336 currentTime - lastDumpheap > mDumpheapInterval) { 337 recordDumpheap(proc); 338 mDumpheapCount.put(proc, count + 1); 339 mLastDumpheap.put(proc, currentTime); 340 } 341 } 342 343 private void recordDumpheap(String proc) throws IOException { 344 long count = mDumpheapCount.get(proc); 345 346 String filename = String.format("dumpheap-%s-%d", proc, count); 347 String tempFilename = "/data/local/tmp/" + filename; 348 String finalFilename = mResultsDirectory + "/" + filename; 349 350 AuptTestRunner.this.getUiAutomation().executeShellCommand( 351 String.format("am dumpheap %s %s", proc, tempFilename)); 352 353 SystemClock.sleep(3000); 354 355 AuptTestRunner.this.getUiAutomation().executeShellCommand( 356 String.format("cp %s %s", tempFilename, finalFilename)); 357 } 358 359 public void run() { 360 try { 361 while (true) { 362 Thread.sleep(mDumpheapInterval); 363 364 for(String proc : mMemoryTrackedProcesses) { 365 recordDumpheap(proc); 366 } 367 } 368 } catch (InterruptedException iex) { 369 } catch (IOException ioex) { 370 Log.e(LOG_TAG, "Failed to write heap dump!", ioex); 371 } 372 } 373 } 374 375 @Override 376 public void startTest(Test test) { 377 mHeapDumpThread = new Thread(new InternalHeapDumper()); 378 mHeapDumpThread.start(); 379 } 380 381 @Override 382 public void endTest(Test test) { 383 try { 384 mHeapDumpThread.interrupt(); 385 mHeapDumpThread.join(); 386 } catch (InterruptedException iex) { } 387 } 388 } 389 390 /** 391 * Dump memory info on test start/stop 392 */ 393 private class MemInfoDumper extends AuptListener { 394 private void dumpMemInfo() { 395 if (mRecordMeminfo) { 396 FilesystemUtil.dumpMeminfo(AuptTestRunner.this, "MemInfoDumper"); 397 } 398 } 399 400 @Override 401 public void startTest(Test test) { 402 dumpMemInfo(); 403 } 404 405 @Override 406 public void endTest(Test test) { 407 dumpMemInfo(); 408 } 409 } 410 411 /** 412 * Record all of our MemHealthRecords 413 */ 414 private class MemHealthRecorder extends AuptListener { 415 @Override 416 public void startTest(Test test) { 417 recordMemHealth(); 418 } 419 420 @Override 421 public void endTest(Test test) { 422 recordMemHealth(); 423 424 try { 425 MemHealthRecord.saveVerbose(mMemHealthRecords, 426 mResultsDirectory + "memory-health.txt"); 427 428 MemHealthRecord.saveCsv(mMemHealthRecords, 429 mResultsDirectory + "memory-health-details.txt"); 430 431 mMemHealthRecords.clear(); 432 } catch (IOException ioex) { 433 Log.e(LOG_TAG, "Error writing MemHealthRecords", ioex); 434 } 435 } 436 437 private void recordMemHealth() { 438 try { 439 mMemHealthRecords.addAll(MemHealthRecord.get( 440 AuptTestRunner.this, 441 mMemoryTrackedProcesses, 442 System.currentTimeMillis(), 443 getForegroundProcs())); 444 } catch (IOException ioex) { 445 Log.e(LOG_TAG, "Error collecting MemHealthRecords", ioex); 446 } 447 } 448 449 private List<String> getForegroundProcs() { 450 List<String> foregroundProcs = new ArrayList<String>(); 451 try { 452 String compactMeminfo = MemHealthRecord.getProcessOutput(AuptTestRunner.this, 453 "dumpsys meminfo -c"); 454 455 for (String line : compactMeminfo.split("\\r?\\n")) { 456 if (line.contains("proc,fore")) { 457 String proc = line.split(",")[2]; 458 foregroundProcs.add(proc); 459 } 460 } 461 } catch (IOException e) { 462 Log.e(LOG_TAG, "Error while getting foreground process", e); 463 } finally { 464 return foregroundProcs; 465 } 466 } 467 } 468 469 /** 470 * Kills application and dumps UI Hierarchy on test error 471 */ 472 private class QuitOnErrorListener extends AuptListener { 473 @Override 474 public void addError(Test test, Throwable t) { 475 Log.e(LOG_TAG, "Caught exception from a test", t); 476 477 if ((t instanceof AuptTerminator)) { 478 throw (AuptTerminator)t; 479 } else { 480 481 // Check if our exception is caused by process dependency 482 if (test instanceof AuptTestCase) { 483 mProcessTracker.setUiAutomation(getUiAutomation()); 484 mProcessTracker.verifyRunningProcess(); 485 } 486 487 // If that didn't throw, then dump our hierarchy 488 Log.v(LOG_TAG, "Dumping UI hierarchy"); 489 try { 490 UiDevice.getInstance(AuptTestRunner.this).dumpWindowHierarchy( 491 new File("/data/local/tmp/error_dump.xml")); 492 } catch (IOException e) { 493 Log.w(LOG_TAG, "Failed to create UI hierarchy dump for UI error", e); 494 } 495 } 496 497 // Quit on an error 498 throw new AuptTerminator(t.getMessage(), t); 499 } 500 501 @Override 502 public void addFailure(Test test, AssertionFailedError t) { 503 // Quit on an error 504 throw new AuptTerminator(t.getMessage(), t); 505 } 506 } 507 508 /** 509 * Makes sure the processes this test requires are all alive 510 */ 511 private class PidChecker extends AuptListener { 512 @Override 513 public void startTest(Test test) { 514 mProcessTracker.setUiAutomation(getUiAutomation()); 515 mProcessTracker.verifyRunningProcess(); 516 } 517 518 @Override 519 public void endTest(Test test) { 520 mProcessTracker.verifyRunningProcess(); 521 } 522 } 523 524 /** 525 * Initialization for tests that touch the camera 526 */ 527 private class DcimCleaner extends AuptListener { 528 @Override 529 public void startTest(Test test) { 530 if (!mDeleteOldFiles) { 531 return; 532 } 533 534 File dcimFolder = new File(Environment.getExternalStorageDirectory(), "DCIM"); 535 File cameraFolder = new File(dcimFolder, "Camera"); 536 537 if (dcimFolder.exists()) { 538 if (cameraFolder.exists()) { 539 File[] allMediaFiles = cameraFolder.listFiles(); 540 Arrays.sort(allMediaFiles, new Comparator<File>() { 541 public int compare(File f1, File f2) { 542 return Long.valueOf(f1.lastModified()).compareTo(f2.lastModified()); 543 } 544 }); 545 for (int i = 0; i < allMediaFiles.length - mFileRetainCount; i++) { 546 allMediaFiles[i].delete(); 547 } 548 } else { 549 Log.w(LOG_TAG, "No Camera folder found to delete from."); 550 } 551 } else { 552 Log.w(LOG_TAG, "No DCIM folder found to delete from."); 553 } 554 } 555 } 556 557 /** 558 * Makes sure the battery hasn't died before and after each test. 559 */ 560 private class BatteryChecker extends AuptListener { 561 private static final double BATTERY_THRESHOLD = 0.05; 562 563 private void checkBattery() { 564 Intent batteryIntent = getContext().registerReceiver(null, 565 new IntentFilter(Intent.ACTION_BATTERY_CHANGED)); 566 int rawLevel = batteryIntent.getIntExtra("level", -1); 567 int scale = batteryIntent.getIntExtra("scale", -1); 568 569 if (rawLevel < 0 || scale <= 0) { 570 return; 571 } 572 573 double level = (double) rawLevel / (double) scale; 574 if (level < BATTERY_THRESHOLD) { 575 throw new AuptTerminator(String.format("Current battery level %f lower than %f", 576 level, 577 BATTERY_THRESHOLD)); 578 } 579 } 580 581 @Override 582 public void startTest(Test test) { 583 checkBattery(); 584 } 585 } 586 587 /** 588 * Generates heap dumps when a test times out 589 */ 590 private class TimeoutStackDumper extends AuptListener { 591 private String getStackTraces() { 592 StringBuilder sb = new StringBuilder(); 593 Map<Thread, StackTraceElement[]> stacks = Thread.getAllStackTraces(); 594 for (Thread t : stacks.keySet()) { 595 sb.append(t.toString()).append('\n'); 596 for (StackTraceElement ste : t.getStackTrace()) { 597 sb.append("\tat ").append(ste.toString()).append('\n'); 598 } 599 sb.append('\n'); 600 } 601 return sb.toString(); 602 } 603 604 @Override 605 public void addError(Test test, Throwable t) { 606 if (t instanceof TimeoutException) { 607 Log.d("THREAD_DUMP", getStackTraces()); 608 } 609 } 610 } 611 612 /** Generates ANRs when a test takes too long. */ 613 private class ANRTrigger extends AuptListener { 614 @Override 615 public void addError(Test test, Throwable t) { 616 if (t instanceof TimeoutException) { 617 Context ctx = getTargetContext(); 618 Log.d(LOG_TAG, "About to induce artificial ANR for debugging"); 619 ctx.startService(new Intent(ctx, AnrGenerator.class)); 620 621 try { 622 Thread.sleep(ANR_DELAY); 623 } catch (InterruptedException e) { 624 throw new RuntimeException("Interrupted while waiting for AnrGenerator..."); 625 } 626 } 627 } 628 629 /** Service that hangs to trigger an ANR. */ 630 private class AnrGenerator extends Service { 631 @Override 632 public IBinder onBind(Intent intent) { 633 return null; 634 } 635 636 @Override 637 public int onStartCommand(Intent intent, int flags, int id) { 638 Log.i(LOG_TAG, "in service start -- about to hang"); 639 try { 640 Thread.sleep(ANR_DELAY); 641 } catch (InterruptedException e) { 642 Log.wtf(LOG_TAG, e); 643 } 644 Log.i(LOG_TAG, "service hang finished -- stopping and returning"); 645 stopSelf(); 646 return START_NOT_STICKY; 647 } 648 } 649 } 650 651 /** 652 * Collect a screenshot on test failure. 653 */ 654 private class Screenshotter extends AuptListener { 655 private void collectScreenshot(Test test, String suffix) { 656 UiDevice device = UiDevice.getInstance(AuptTestRunner.this); 657 658 if (device == null) { 659 Log.w(LOG_TAG, "Couldn't collect screenshot on test failure"); 660 return; 661 } 662 663 String testName = 664 test instanceof TestCase 665 ? ((TestCase) test).getName() 666 : (test instanceof TestSuite ? ((TestSuite) test).getName() : test.toString()); 667 668 String fileName = 669 mResultsDirectory.getPath() 670 + "/" + testName.replaceAll(".", "_") 671 + suffix + ".png"; 672 673 device.takeScreenshot(new File(fileName)); 674 } 675 676 @Override 677 public void addError(Test test, Throwable t) { 678 collectScreenshot(test, 679 "_failure_screenshot_" + SCREENSHOT_DATE_FORMAT.format(new Date())); 680 } 681 682 @Override 683 public void addFailure(Test test, AssertionFailedError t) { 684 collectScreenshot(test, 685 "_failure_screenshot_" + SCREENSHOT_DATE_FORMAT.format(new Date())); 686 } 687 } 688 689 /** Runs a command when a test finishes. */ 690 private class FinishCommandRunner extends AuptListener { 691 @Override 692 public void endTest(Test test) { 693 for (String command : mFinishCommands) { 694 AuptTestRunner.this.getUiAutomation().executeShellCommand(command); 695 } 696 } 697 } 698 } 699