1 /* 2 * Copyright (C) 2015 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.compatibility.common.tradefed.result; 17 18 import com.android.compatibility.common.tradefed.build.CompatibilityBuildHelper; 19 import com.android.compatibility.common.tradefed.testtype.retry.RetryFactoryTest; 20 import com.android.compatibility.common.tradefed.testtype.suite.CompatibilityTestSuite; 21 import com.android.compatibility.common.tradefed.util.FingerprintComparisonException; 22 import com.android.compatibility.common.tradefed.util.RetryType; 23 import com.android.compatibility.common.util.ChecksumReporter; 24 import com.android.compatibility.common.util.DeviceInfo; 25 import com.android.compatibility.common.util.ICaseResult; 26 import com.android.compatibility.common.util.IInvocationResult; 27 import com.android.compatibility.common.util.IModuleResult; 28 import com.android.compatibility.common.util.ITestResult; 29 import com.android.compatibility.common.util.InvocationResult; 30 import com.android.compatibility.common.util.InvocationResult.RunHistory; 31 import com.android.compatibility.common.util.MetricsStore; 32 import com.android.compatibility.common.util.ReportLog; 33 import com.android.compatibility.common.util.ResultHandler; 34 import com.android.compatibility.common.util.ResultUploader; 35 import com.android.compatibility.common.util.TestStatus; 36 import com.android.ddmlib.Log.LogLevel; 37 import com.android.tradefed.build.IBuildInfo; 38 import com.android.tradefed.config.IConfiguration; 39 import com.android.tradefed.config.IConfigurationReceiver; 40 import com.android.tradefed.config.Option; 41 import com.android.tradefed.config.Option.Importance; 42 import com.android.tradefed.config.OptionClass; 43 import com.android.tradefed.config.OptionCopier; 44 import com.android.tradefed.invoker.IInvocationContext; 45 import com.android.tradefed.log.LogUtil.CLog; 46 import com.android.tradefed.metrics.proto.MetricMeasurement.Metric; 47 import com.android.tradefed.result.FileInputStreamSource; 48 import com.android.tradefed.result.ILogSaver; 49 import com.android.tradefed.result.ILogSaverListener; 50 import com.android.tradefed.result.IShardableListener; 51 import com.android.tradefed.result.ITestInvocationListener; 52 import com.android.tradefed.result.ITestSummaryListener; 53 import com.android.tradefed.result.InputStreamSource; 54 import com.android.tradefed.result.LogDataType; 55 import com.android.tradefed.result.LogFile; 56 import com.android.tradefed.result.LogFileSaver; 57 import com.android.tradefed.result.TestDescription; 58 import com.android.tradefed.result.TestSummary; 59 import com.android.tradefed.result.suite.SuiteResultReporter; 60 import com.android.tradefed.util.FileUtil; 61 import com.android.tradefed.util.StreamUtil; 62 import com.android.tradefed.util.TimeUtil; 63 import com.android.tradefed.util.ZipUtil; 64 import com.android.tradefed.util.proto.TfMetricProtoUtil; 65 66 import com.google.common.annotations.VisibleForTesting; 67 import com.google.common.xml.XmlEscapers; 68 import com.google.gson.Gson; 69 70 import org.xmlpull.v1.XmlPullParserException; 71 72 import java.io.File; 73 import java.io.FileInputStream; 74 import java.io.FileNotFoundException; 75 import java.io.IOException; 76 import java.io.InputStream; 77 import java.nio.file.Files; 78 import java.nio.file.Path; 79 import java.util.Arrays; 80 import java.util.Collection; 81 import java.util.Collections; 82 import java.util.HashMap; 83 import java.util.HashSet; 84 import java.util.List; 85 import java.util.Map; 86 import java.util.Set; 87 import java.util.concurrent.CountDownLatch; 88 import java.util.concurrent.TimeUnit; 89 90 /** 91 * Collect test results for an entire invocation and output test results to disk. 92 */ 93 @OptionClass(alias="result-reporter") 94 public class ResultReporter implements ILogSaverListener, ITestInvocationListener, 95 ITestSummaryListener, IShardableListener, IConfigurationReceiver { 96 97 public static final String INCLUDE_HTML_IN_ZIP = "html-in-zip"; 98 private static final String UNKNOWN_DEVICE = "unknown_device"; 99 private static final String RESULT_KEY = "COMPATIBILITY_TEST_RESULT"; 100 private static final String CTS_PREFIX = "cts:"; 101 private static final String BUILD_INFO = CTS_PREFIX + "build_"; 102 private static final String LATEST_LINK_NAME = "latest"; 103 /** Used to get run history from the test result of last run. */ 104 private static final String RUN_HISTORY_KEY = "run_history"; 105 106 107 public static final String BUILD_BRAND = "build_brand"; 108 public static final String BUILD_DEVICE = "build_device"; 109 public static final String BUILD_FINGERPRINT = "build_fingerprint"; 110 public static final String BUILD_ID = "build_id"; 111 public static final String BUILD_MANUFACTURER = "build_manufacturer"; 112 public static final String BUILD_MODEL = "build_model"; 113 public static final String BUILD_PRODUCT = "build_product"; 114 public static final String BUILD_VERSION_RELEASE = "build_version_release"; 115 116 private static final List<String> NOT_RETRY_FILES = Arrays.asList( 117 ChecksumReporter.NAME, 118 ChecksumReporter.PREV_NAME, 119 ResultHandler.FAILURE_REPORT_NAME, 120 "diffs"); 121 122 @Option(name = RetryFactoryTest.RETRY_OPTION, 123 shortName = 'r', 124 description = "retry a previous session.", 125 importance = Importance.IF_UNSET) 126 private Integer mRetrySessionId = null; 127 128 @Option(name = RetryFactoryTest.RETRY_TYPE_OPTION, 129 description = "used with " + RetryFactoryTest.RETRY_OPTION 130 + ", retry tests of a certain status. Possible values include \"failed\", " 131 + "\"not_executed\", and \"custom\".", 132 importance = Importance.IF_UNSET) 133 private RetryType mRetryType = null; 134 135 @Option(name = "result-server", description = "Server to publish test results.") 136 private String mResultServer; 137 138 @Option(name = "disable-result-posting", description = "Disable result posting into report server.") 139 private boolean mDisableResultPosting = false; 140 141 @Option(name = "include-test-log-tags", description = "Include test log tags in report.") 142 private boolean mIncludeTestLogTags = false; 143 144 @Option(name = "use-log-saver", description = "Also saves generated result with log saver") 145 private boolean mUseLogSaver = false; 146 147 @Option(name = "compress-logs", description = "Whether logs will be saved with compression") 148 private boolean mCompressLogs = true; 149 150 @Option(name = INCLUDE_HTML_IN_ZIP, 151 description = "Whether failure summary report is included in the zip fie.") 152 private boolean mIncludeHtml = false; 153 154 @Option( 155 name = "result-attribute", 156 description = 157 "Extra key-value pairs to be added as attributes and corresponding" 158 + "values of the \"Result\" tag in the result XML.") 159 private Map<String, String> mResultAttributes = new HashMap<String, String>(); 160 161 private CompatibilityBuildHelper mBuildHelper; 162 private File mResultDir = null; 163 private File mLogDir = null; 164 private ResultUploader mUploader; 165 private String mReferenceUrl; 166 private ILogSaver mLogSaver; 167 private int invocationEndedCount = 0; 168 private CountDownLatch mFinalized = null; 169 170 protected IInvocationResult mResult = new InvocationResult(); 171 private IModuleResult mCurrentModuleResult; 172 private ICaseResult mCurrentCaseResult; 173 private ITestResult mCurrentResult; 174 private String mDeviceSerial = UNKNOWN_DEVICE; 175 private Set<String> mMasterDeviceSerials = new HashSet<>(); 176 private Set<IBuildInfo> mMasterBuildInfos = new HashSet<>(); 177 // Whether or not we failed the fingerprint check 178 private boolean mFingerprintFailure = false; 179 180 // mCurrentTestNum and mTotalTestsInModule track the progress within the module 181 // Note that this count is not necessarily equal to the count of tests contained 182 // in mCurrentModuleResult because of how special cases like ignored tests are reported. 183 private int mCurrentTestNum; 184 private int mTotalTestsInModule; 185 186 // Whether modules can be marked done for this invocation. Initialized in invocationStarted() 187 // Visible for unit testing 188 protected boolean mCanMarkDone; 189 // Whether the current test run has failed. If true, we will not mark the current module done 190 protected boolean mTestRunFailed; 191 // Whether the current module has previously been marked done 192 private boolean mModuleWasDone; 193 194 // Nullable. If null, "this" is considered the master and must handle 195 // result aggregation and reporting. When not null, it should forward events 196 // to the master. 197 private final ResultReporter mMasterResultReporter; 198 199 private LogFileSaver mTestLogSaver; 200 201 // Elapsed time from invocation started to ended. 202 private long mElapsedTime; 203 204 /** Invocation level configuration */ 205 private IConfiguration mConfiguration = null; 206 207 /** 208 * Default constructor. 209 */ ResultReporter()210 public ResultReporter() { 211 this(null); 212 mFinalized = new CountDownLatch(1); 213 } 214 215 /** 216 * Construct a shard ResultReporter that forwards module results to the 217 * masterResultReporter. 218 */ ResultReporter(ResultReporter masterResultReporter)219 public ResultReporter(ResultReporter masterResultReporter) { 220 mMasterResultReporter = masterResultReporter; 221 } 222 223 /** {@inheritDoc} */ 224 @Override setConfiguration(IConfiguration configuration)225 public void setConfiguration(IConfiguration configuration) { 226 mConfiguration = configuration; 227 } 228 229 /** 230 * {@inheritDoc} 231 */ 232 @Override invocationStarted(IInvocationContext context)233 public void invocationStarted(IInvocationContext context) { 234 IBuildInfo primaryBuild = context.getBuildInfos().get(0); 235 synchronized(this) { 236 if (mBuildHelper == null) { 237 mBuildHelper = new CompatibilityBuildHelper(primaryBuild); 238 } 239 if (mDeviceSerial == null && primaryBuild.getDeviceSerial() != null) { 240 mDeviceSerial = primaryBuild.getDeviceSerial(); 241 } 242 mCanMarkDone = canMarkDone(mBuildHelper.getRecentCommandLineArgs()); 243 } 244 245 if (isShardResultReporter()) { 246 // Shard ResultReporters forward invocationStarted to the mMasterResultReporter 247 mMasterResultReporter.invocationStarted(context); 248 return; 249 } 250 251 // NOTE: Everything after this line only applies to the master ResultReporter. 252 253 synchronized(this) { 254 if (primaryBuild.getDeviceSerial() != null) { 255 // The master ResultReporter collects all device serials being used 256 // for the current implementation. 257 mMasterDeviceSerials.add(primaryBuild.getDeviceSerial()); 258 } 259 260 // The master ResultReporter collects all buildInfos. 261 mMasterBuildInfos.add(primaryBuild); 262 263 if (mResultDir == null) { 264 // For the non-sharding case, invocationStarted is only called once, 265 // but for the sharding case, this might be called multiple times. 266 // Logic used to initialize the result directory should not be 267 // invoked twice during the same invocation. 268 initializeResultDirectories(); 269 } 270 } 271 } 272 273 /** 274 * Create directory structure where results and logs will be written. 275 */ initializeResultDirectories()276 private void initializeResultDirectories() { 277 debug("Initializing result directory"); 278 279 try { 280 // Initialize the result directory. Either a new directory or reusing 281 // an existing session. 282 if (mRetrySessionId != null) { 283 // Overwrite the mResult with the test results of the previous session 284 mResult = ResultHandler.findResult(mBuildHelper.getResultsDir(), mRetrySessionId); 285 } 286 mResult.setStartTime(mBuildHelper.getStartTime()); 287 mResultDir = mBuildHelper.getResultDir(); 288 if (mResultDir != null) { 289 mResultDir.mkdirs(); 290 } 291 } catch (FileNotFoundException e) { 292 throw new RuntimeException(e); 293 } 294 295 if (mResultDir == null) { 296 throw new RuntimeException("Result Directory was not created"); 297 } 298 if (!mResultDir.exists()) { 299 throw new RuntimeException("Result Directory was not created: " + 300 mResultDir.getAbsolutePath()); 301 } 302 303 debug("Results Directory: %s", mResultDir.getAbsolutePath()); 304 305 mUploader = new ResultUploader(mResultServer, mBuildHelper.getSuiteName()); 306 try { 307 mLogDir = new File(mBuildHelper.getLogsDir(), 308 CompatibilityBuildHelper.getDirSuffix(mBuildHelper.getStartTime())); 309 } catch (FileNotFoundException e) { 310 CLog.e(e); 311 } 312 if (mLogDir != null && mLogDir.mkdirs()) { 313 debug("Created log dir %s", mLogDir.getAbsolutePath()); 314 } 315 if (mLogDir == null || !mLogDir.exists()) { 316 throw new IllegalArgumentException(String.format("Could not create log dir %s", 317 mLogDir.getAbsolutePath())); 318 } 319 if (mTestLogSaver == null) { 320 mTestLogSaver = new LogFileSaver(mLogDir); 321 } 322 } 323 324 /** 325 * {@inheritDoc} 326 */ 327 @Override testRunStarted(String id, int numTests)328 public void testRunStarted(String id, int numTests) { 329 if (mCurrentModuleResult != null && mCurrentModuleResult.getId().equals(id) 330 && mCurrentModuleResult.isDone()) { 331 // Modules run with JarHostTest treat each test class as a separate module, 332 // resulting in additional unexpected test runs. 333 // This case exists only for N 334 mTotalTestsInModule += numTests; 335 } else { 336 // Handle non-JarHostTest case 337 mCurrentModuleResult = mResult.getOrCreateModule(id); 338 mModuleWasDone = mCurrentModuleResult.isDone(); 339 mTestRunFailed = false; 340 if (!mModuleWasDone) { 341 // we only want to update testRun variables if the IModuleResult is not yet done 342 // otherwise leave testRun variables alone so isDone evaluates to true. 343 if (mCurrentModuleResult.getExpectedTestRuns() == 0) { 344 mCurrentModuleResult.setExpectedTestRuns(TestRunHandler.getTestRuns( 345 mBuildHelper, mCurrentModuleResult.getId())); 346 } 347 mCurrentModuleResult.addTestRun(); 348 } 349 // Reset counters 350 mTotalTestsInModule = numTests; 351 mCurrentTestNum = 0; 352 } 353 mCurrentModuleResult.inProgress(true); 354 } 355 356 /** 357 * {@inheritDoc} 358 */ 359 @Override testStarted(TestDescription test)360 public void testStarted(TestDescription test) { 361 mCurrentCaseResult = mCurrentModuleResult.getOrCreateResult(test.getClassName()); 362 mCurrentResult = mCurrentCaseResult.getOrCreateResult(test.getTestName().trim()); 363 if (mCurrentResult.isRetry()) { 364 mCurrentResult.reset(); // clear result status for this invocation 365 } 366 mCurrentTestNum++; 367 } 368 369 /** 370 * {@inheritDoc} 371 */ 372 @Override testEnded(TestDescription test, HashMap<String, Metric> metrics)373 public void testEnded(TestDescription test, HashMap<String, Metric> metrics) { 374 if (mCurrentResult.getResultStatus() == TestStatus.FAIL) { 375 // Test has previously failed. 376 return; 377 } 378 // device test can have performance results in test metrics 379 Metric perfResult = metrics.get(RESULT_KEY); 380 ReportLog report = null; 381 if (perfResult != null) { 382 try { 383 report = ReportLog.parse(perfResult.getMeasurements().getSingleString()); 384 } catch (XmlPullParserException | IOException e) { 385 e.printStackTrace(); 386 } 387 } else { 388 // host test should be checked into MetricsStore. 389 report = MetricsStore.removeResult(mBuildHelper.getBuildInfo(), 390 mCurrentModuleResult.getAbi(), test.toString()); 391 } 392 if (mCurrentResult.getResultStatus() == null) { 393 // Only claim that we passed when we're certain our result was 394 // not any other state. 395 mCurrentResult.passed(report); 396 } 397 } 398 399 /** 400 * {@inheritDoc} 401 */ 402 @Override testIgnored(TestDescription test)403 public void testIgnored(TestDescription test) { 404 mCurrentResult.skipped(); 405 } 406 407 /** 408 * {@inheritDoc} 409 */ 410 @Override testFailed(TestDescription test, String trace)411 public void testFailed(TestDescription test, String trace) { 412 mCurrentResult.failed(sanitizeXmlContent(trace)); 413 } 414 415 /** 416 * {@inheritDoc} 417 */ 418 @Override testAssumptionFailure(TestDescription test, String trace)419 public void testAssumptionFailure(TestDescription test, String trace) { 420 mCurrentResult.skipped(); 421 } 422 423 /** 424 * {@inheritDoc} 425 */ 426 @Override testRunStopped(long elapsedTime)427 public void testRunStopped(long elapsedTime) { 428 // ignore 429 } 430 431 /** 432 * {@inheritDoc} 433 */ 434 @Override testRunEnded(long elapsedTime, Map<String, String> metrics)435 public void testRunEnded(long elapsedTime, Map<String, String> metrics) { 436 testRunEnded(elapsedTime, TfMetricProtoUtil.upgradeConvert(metrics)); 437 } 438 439 /** 440 * {@inheritDoc} 441 */ 442 @Override testRunEnded(long elapsedTime, HashMap<String, Metric> metrics)443 public void testRunEnded(long elapsedTime, HashMap<String, Metric> metrics) { 444 mCurrentModuleResult.inProgress(false); 445 mCurrentModuleResult.addRuntime(elapsedTime); 446 if (!mModuleWasDone && mCanMarkDone) { 447 if (mTestRunFailed) { 448 // set done to false for test run failures 449 mCurrentModuleResult.setDone(false); 450 } else { 451 // Only mark module done if: 452 // - status of the invocation allows it (mCanMarkDone), and 453 // - module has not already been marked done, and 454 // - no test run failure has been detected 455 mCurrentModuleResult.setDone(mCurrentTestNum >= mTotalTestsInModule); 456 } 457 } 458 if (isShardResultReporter()) { 459 // Forward module results to the master. 460 mMasterResultReporter.mergeModuleResult(mCurrentModuleResult); 461 mCurrentModuleResult.resetTestRuns(); 462 mCurrentModuleResult.resetRuntime(); 463 } 464 } 465 466 /** 467 * Directly add a module result. Note: this method is meant to be used by 468 * a shard ResultReporter. 469 */ mergeModuleResult(IModuleResult moduleResult)470 private void mergeModuleResult(IModuleResult moduleResult) { 471 // This merges the results in moduleResult to any existing results already 472 // contained in mResult. This is useful for retries and allows the final 473 // report from a retry to contain all test results. 474 synchronized(this) { 475 mResult.mergeModuleResult(moduleResult); 476 } 477 } 478 479 /** 480 * {@inheritDoc} 481 */ 482 @Override testRunFailed(String errorMessage)483 public void testRunFailed(String errorMessage) { 484 mTestRunFailed = true; 485 mCurrentModuleResult.setFailed(); 486 } 487 488 /** 489 * {@inheritDoc} 490 */ 491 @Override getSummary()492 public TestSummary getSummary() { 493 // ignore 494 return null; 495 } 496 497 /** 498 * {@inheritDoc} 499 */ 500 @Override putSummary(List<TestSummary> summaries)501 public void putSummary(List<TestSummary> summaries) { 502 for (TestSummary summary : summaries) { 503 // If one summary is from SuiteResultReporter, log it as an extra file. 504 if (SuiteResultReporter.SUITE_REPORTER_SOURCE.equals(summary.getSource())) { 505 File summaryFile = null; 506 try { 507 summaryFile = FileUtil.createTempFile("summary", ".txt"); 508 FileUtil.writeToFile(summary.getSummary().getString(), summaryFile); 509 try (InputStreamSource stream = new FileInputStreamSource(summaryFile)) { 510 testLog("summary", LogDataType.TEXT, stream); 511 } 512 } catch (IOException e) { 513 CLog.e(e); 514 } finally { 515 FileUtil.deleteFile(summaryFile); 516 } 517 } else if (mReferenceUrl == null && summary.getSummary().getString() != null) { 518 mReferenceUrl = summary.getSummary().getString(); 519 } 520 } 521 } 522 523 /** 524 * {@inheritDoc} 525 */ 526 @Override invocationEnded(long elapsedTime)527 public void invocationEnded(long elapsedTime) { 528 if (isShardResultReporter()) { 529 // Shard ResultReporters report 530 mMasterResultReporter.invocationEnded(elapsedTime); 531 return; 532 } 533 534 // NOTE: Everything after this line only applies to the master ResultReporter. 535 536 synchronized(this) { 537 // The master ResultReporter tracks the progress of all invocations across 538 // shard ResultReporters. Writing results should not proceed until all 539 // ResultReporters have completed. 540 if (++invocationEndedCount < mMasterBuildInfos.size()) { 541 return; 542 } 543 mElapsedTime = elapsedTime; 544 finalizeResults(); 545 mFinalized.countDown(); 546 } 547 } 548 549 /** 550 * Returns whether a report creation should be skipped. 551 */ shouldSkipReportCreation()552 protected boolean shouldSkipReportCreation() { 553 // This value is always false here for backwards compatibility. 554 // Extended classes have the option to override this. 555 return false; 556 } 557 finalizeResults()558 private void finalizeResults() { 559 if (mFingerprintFailure) { 560 CLog.w("Failed the fingerprint check. Skip result reporting."); 561 return; 562 } 563 // Add all device serials into the result to be serialized 564 for (String deviceSerial : mMasterDeviceSerials) { 565 mResult.addDeviceSerial(deviceSerial); 566 } 567 568 addDeviceBuildInfoToResult(); 569 570 Set<String> allExpectedModules = new HashSet<>(); 571 for (IBuildInfo buildInfo : mMasterBuildInfos) { 572 for (Map.Entry<String, String> entry : buildInfo.getBuildAttributes().entrySet()) { 573 String key = entry.getKey(); 574 String value = entry.getValue(); 575 if (key.equals(CompatibilityBuildHelper.MODULE_IDS) && value.length() > 0) { 576 Collections.addAll(allExpectedModules, value.split(",")); 577 } 578 } 579 } 580 581 // Include a record in the report of all expected modules ids, even if they weren't 582 // executed. 583 for (String moduleId : allExpectedModules) { 584 mResult.getOrCreateModule(moduleId); 585 } 586 587 String moduleProgress = String.format("%d of %d", 588 mResult.getModuleCompleteCount(), mResult.getModules().size()); 589 590 591 if (shouldSkipReportCreation()) { 592 return; 593 } 594 595 // Get run history from the test result of last run and add the run history of the current 596 // run to it. 597 // TODO(b/137973382): avoid casting by move the method to interface level. 598 Collection<RunHistory> runHistories = ((InvocationResult) mResult).getRunHistories(); 599 String runHistoryJSON = mResult.getInvocationInfo().get(RUN_HISTORY_KEY); 600 Gson gson = new Gson(); 601 if (runHistoryJSON != null) { 602 RunHistory[] runHistoryArray = gson.fromJson(runHistoryJSON, RunHistory[].class); 603 Collections.addAll(runHistories, runHistoryArray); 604 } 605 RunHistory newRun = new RunHistory(); 606 newRun.startTime = mResult.getStartTime(); 607 newRun.endTime = newRun.startTime + mElapsedTime; 608 runHistories.add(newRun); 609 mResult.addInvocationInfo(RUN_HISTORY_KEY, gson.toJson(runHistories)); 610 611 try { 612 // Zip the full test results directory. 613 copyDynamicConfigFiles(); 614 copyFormattingFiles(mResultDir, mBuildHelper.getSuiteName()); 615 616 File resultFile = generateResultXmlFile(); 617 if (mRetrySessionId != null) { 618 copyRetryFiles(ResultHandler.getResultDirectory( 619 mBuildHelper.getResultsDir(), mRetrySessionId), mResultDir); 620 } 621 File failureReport = null; 622 if (mIncludeHtml) { 623 // Create the html report before the zip file. 624 failureReport = ResultHandler.createFailureReport(resultFile); 625 } 626 File zippedResults = zipResults(mResultDir); 627 if (!mIncludeHtml) { 628 // Create failure report after zip file so extra data is not uploaded 629 failureReport = ResultHandler.createFailureReport(resultFile); 630 } 631 if (failureReport != null && failureReport.exists()) { 632 info("Test Result: %s", failureReport.getCanonicalPath()); 633 } else { 634 info("Test Result: %s", resultFile.getCanonicalPath()); 635 } 636 info("Test Logs: %s", mLogDir.getCanonicalPath()); 637 debug("Full Result: %s", zippedResults.getCanonicalPath()); 638 639 Path latestLink = createLatestLinkDirectory(mResultDir.toPath()); 640 if (latestLink != null) { 641 info("Latest results link: " + latestLink.toAbsolutePath()); 642 } 643 644 latestLink = createLatestLinkDirectory(mLogDir.toPath()); 645 if (latestLink != null) { 646 info("Latest logs link: " + latestLink.toAbsolutePath()); 647 } 648 649 saveLog(resultFile, zippedResults); 650 651 uploadResult(resultFile); 652 653 } catch (IOException | XmlPullParserException e) { 654 CLog.e("[%s] Exception while saving result XML.", mDeviceSerial); 655 CLog.e(e); 656 } 657 // print the run results last. 658 info("Invocation finished in %s. PASSED: %d, FAILED: %d, MODULES: %s", 659 TimeUtil.formatElapsedTime(mElapsedTime), 660 mResult.countResults(TestStatus.PASS), 661 mResult.countResults(TestStatus.FAIL), 662 moduleProgress); 663 } 664 createLatestLinkDirectory(Path directory)665 private Path createLatestLinkDirectory(Path directory) { 666 Path link = null; 667 668 Path parent = directory.getParent(); 669 670 if (parent != null) { 671 link = parent.resolve(LATEST_LINK_NAME); 672 try { 673 // if latest already exists, we have to remove it before creating 674 Files.deleteIfExists(link); 675 Files.createSymbolicLink(link, directory); 676 } catch (IOException ioe) { 677 CLog.e("Exception while attempting to create 'latest' link to: [%s]", 678 directory); 679 CLog.e(ioe); 680 return null; 681 } catch (UnsupportedOperationException uoe) { 682 CLog.e("Failed to create 'latest' symbolic link - unsupported operation"); 683 return null; 684 } 685 } 686 return link; 687 } 688 689 /** 690 * {@inheritDoc} 691 */ 692 @Override invocationFailed(Throwable cause)693 public void invocationFailed(Throwable cause) { 694 warn("Invocation failed: %s", cause); 695 InvocationFailureHandler.setFailed(mBuildHelper, cause); 696 if (cause instanceof FingerprintComparisonException) { 697 mFingerprintFailure = true; 698 } 699 } 700 701 /** 702 * {@inheritDoc} 703 */ 704 @Override testLog(String name, LogDataType type, InputStreamSource stream)705 public void testLog(String name, LogDataType type, InputStreamSource stream) { 706 // This is safe to be invoked on either the master or a shard ResultReporter 707 if (isShardResultReporter()) { 708 // Shard ResultReporters forward testLog to the mMasterResultReporter 709 mMasterResultReporter.testLog(name, type, stream); 710 return; 711 } 712 if (name.endsWith(DeviceInfo.FILE_SUFFIX)) { 713 // Handle device info file case 714 testLogDeviceInfo(name, stream); 715 } else { 716 // Handle default case 717 try { 718 File logFile = null; 719 if (mCompressLogs) { 720 try (InputStream inputStream = stream.createInputStream()) { 721 logFile = mTestLogSaver.saveAndGZipLogData(name, type, inputStream); 722 } 723 } else { 724 try (InputStream inputStream = stream.createInputStream()) { 725 logFile = mTestLogSaver.saveLogData(name, type, inputStream); 726 } 727 } 728 debug("Saved logs for %s in %s", name, logFile.getAbsolutePath()); 729 } catch (IOException e) { 730 warn("Failed to write log for %s", name); 731 CLog.e(e); 732 } 733 } 734 } 735 736 /* Write device-info files to the result, invoked only by the master result reporter */ testLogDeviceInfo(String name, InputStreamSource stream)737 private void testLogDeviceInfo(String name, InputStreamSource stream) { 738 try { 739 File ediDir = new File(mResultDir, DeviceInfo.RESULT_DIR_NAME); 740 ediDir.mkdirs(); 741 File ediFile = new File(ediDir, name); 742 if (!ediFile.exists()) { 743 // only write this file to the results if not already present 744 FileUtil.writeToFile(stream.createInputStream(), ediFile); 745 } 746 } catch (IOException e) { 747 warn("Failed to write device info %s to result", name); 748 CLog.e(e); 749 } 750 } 751 752 /** 753 * {@inheritDoc} 754 */ 755 @Override testLogSaved(String dataName, LogDataType dataType, InputStreamSource dataStream, LogFile logFile)756 public void testLogSaved(String dataName, LogDataType dataType, InputStreamSource dataStream, 757 LogFile logFile) { 758 // This is safe to be invoked on either the master or a shard ResultReporter 759 if (mIncludeTestLogTags && mCurrentResult != null 760 && dataName.startsWith(mCurrentResult.getFullName())) { 761 762 if (dataType == LogDataType.BUGREPORT) { 763 mCurrentResult.setBugReport(logFile.getUrl()); 764 } else if (dataType == LogDataType.LOGCAT) { 765 mCurrentResult.setLog(logFile.getUrl()); 766 } else if (dataType == LogDataType.PNG) { 767 mCurrentResult.setScreenshot(logFile.getUrl()); 768 } 769 } 770 } 771 772 /** 773 * {@inheritDoc} 774 */ 775 @Override setLogSaver(ILogSaver saver)776 public void setLogSaver(ILogSaver saver) { 777 // This is safe to be invoked on either the master or a shard ResultReporter 778 mLogSaver = saver; 779 } 780 781 /** 782 * When enabled, save log data using log saver 783 */ saveLog(File resultFile, File zippedResults)784 private void saveLog(File resultFile, File zippedResults) throws IOException { 785 if (!mUseLogSaver) { 786 return; 787 } 788 789 FileInputStream fis = null; 790 LogFile logFile = null; 791 try { 792 fis = new FileInputStream(resultFile); 793 logFile = mLogSaver.saveLogData("log-result", LogDataType.XML, fis); 794 debug("Result XML URL: %s", logFile.getUrl()); 795 logReportFiles(mConfiguration, resultFile, resultFile.getName(), LogDataType.XML); 796 } catch (IOException ioe) { 797 CLog.e("[%s] error saving XML with log saver", mDeviceSerial); 798 CLog.e(ioe); 799 } finally { 800 StreamUtil.close(fis); 801 } 802 // Save the full results folder. 803 if (zippedResults != null) { 804 FileInputStream zipResultStream = null; 805 try { 806 zipResultStream = new FileInputStream(zippedResults); 807 logFile = mLogSaver.saveLogData("results", LogDataType.ZIP, zipResultStream); 808 debug("Result zip URL: %s", logFile.getUrl()); 809 logReportFiles( 810 mConfiguration, zippedResults, "results", LogDataType.ZIP); 811 } finally { 812 StreamUtil.close(zipResultStream); 813 } 814 } 815 } 816 817 /** 818 * Return the path in which log saver persists log files or null if 819 * logSaver is not enabled. 820 */ getLogUrl()821 private String getLogUrl() { 822 if (!mUseLogSaver || mLogSaver == null) { 823 return null; 824 } 825 826 return mLogSaver.getLogReportDir().getUrl(); 827 } 828 829 @Override clone()830 public IShardableListener clone() { 831 ResultReporter clone = new ResultReporter(this); 832 OptionCopier.copyOptionsNoThrow(this, clone); 833 return clone; 834 } 835 836 /** 837 * Create results file compatible with CTSv2 (xml) report format. 838 */ generateResultXmlFile()839 protected File generateResultXmlFile() 840 throws IOException, XmlPullParserException { 841 return ResultHandler.writeResults( 842 mBuildHelper.getSuiteName(), 843 mBuildHelper.getSuiteVersion(), 844 getSuitePlan(mBuildHelper), 845 mBuildHelper.getSuiteBuild(), 846 mResult, 847 mResultDir, 848 mResult.getStartTime(), 849 mElapsedTime + mResult.getStartTime(), 850 mReferenceUrl, 851 getLogUrl(), 852 mBuildHelper.getCommandLineArgs(), 853 mResultAttributes); 854 } 855 856 /** 857 * Add build info collected from the device attributes to the results. 858 */ addDeviceBuildInfoToResult()859 protected void addDeviceBuildInfoToResult() { 860 // Add all build info to the result to be serialized 861 Map<String, String> buildProperties = mapBuildInfo(); 862 addBuildInfoToResult(buildProperties, mResult); 863 } 864 865 /** 866 * Override specific build properties so the report will be associated with the 867 * build fingerprint being certified. 868 */ addDeviceBuildInfoToResult(String buildFingerprintOverride, String manufactureOverride, String modelOverride)869 protected void addDeviceBuildInfoToResult(String buildFingerprintOverride, 870 String manufactureOverride, String modelOverride) { 871 872 Map<String, String> buildProperties = mapBuildInfo(); 873 874 // Extract and override values from build fingerprint. 875 // Build fingerprint format: brand/product/device:version/build_id/tags 876 String fingerprintPrefix = buildFingerprintOverride.split(":")[0]; 877 String fingerprintTail = buildFingerprintOverride.split(":")[1]; 878 String buildIdOverride = fingerprintTail.split("/")[1]; 879 buildProperties.put(BUILD_ID, buildIdOverride); 880 String brandOverride = fingerprintPrefix.split("/")[0]; 881 buildProperties.put(BUILD_BRAND, brandOverride); 882 String deviceOverride = fingerprintPrefix.split("/")[2]; 883 buildProperties.put(BUILD_DEVICE, deviceOverride); 884 String productOverride = fingerprintPrefix.split("/")[1]; 885 buildProperties.put(BUILD_PRODUCT, productOverride); 886 String versionOverride = fingerprintTail.split("/")[0]; 887 buildProperties.put(BUILD_VERSION_RELEASE, versionOverride); 888 buildProperties.put(BUILD_FINGERPRINT, buildFingerprintOverride); 889 buildProperties.put(BUILD_MANUFACTURER, manufactureOverride); 890 buildProperties.put(BUILD_MODEL, modelOverride); 891 892 // Add modified values to results. 893 addBuildInfoToResult(buildProperties, mResult); 894 mResult.setBuildFingerprint(buildFingerprintOverride); 895 } 896 /** Aggregate build info from member device info. */ mapBuildInfo()897 protected Map<String, String> mapBuildInfo() { 898 Map<String, String> buildProperties = new HashMap<>(); 899 for (IBuildInfo buildInfo : mMasterBuildInfos) { 900 for (Map.Entry<String, String> entry : buildInfo.getBuildAttributes().entrySet()) { 901 String key = entry.getKey(); 902 String value = entry.getValue(); 903 if (key.startsWith(BUILD_INFO)) { 904 buildProperties.put(key.substring(CTS_PREFIX.length()), value); 905 } 906 } 907 } 908 return buildProperties; 909 } 910 911 /** 912 * Add build info to results. 913 * @param buildProperties Build info to add. 914 */ addBuildInfoToResult(Map<String, String> buildProperties, IInvocationResult invocationResult)915 protected static void addBuildInfoToResult(Map<String, String> buildProperties, 916 IInvocationResult invocationResult) { 917 buildProperties.entrySet().stream().forEach(entry -> 918 invocationResult.addInvocationInfo(entry.getKey(), entry.getValue())); 919 } 920 921 /** 922 * Get the suite plan. This protected method was created for overrides. 923 * Extending classes can decide on the content of the output's suite_plan field. 924 * 925 * @param mBuildHelper Helper that contains build information. 926 * @return string Suite plan to use. 927 */ getSuitePlan(CompatibilityBuildHelper mBuildHelper)928 protected String getSuitePlan(CompatibilityBuildHelper mBuildHelper) { 929 return mBuildHelper.getSuitePlan(); 930 } 931 932 /** 933 * Return true if this instance is a shard ResultReporter and should propagate 934 * certain events to the master. 935 */ isShardResultReporter()936 private boolean isShardResultReporter() { 937 return mMasterResultReporter != null; 938 } 939 940 /** 941 * When enabled, upload the result to a server. 942 */ uploadResult(File resultFile)943 private void uploadResult(File resultFile) { 944 if (mResultServer != null && !mResultServer.trim().isEmpty() && !mDisableResultPosting) { 945 try { 946 debug("Result Server: %d", mUploader.uploadResult(resultFile, mReferenceUrl)); 947 } catch (IOException ioe) { 948 CLog.e("[%s] IOException while uploading result.", mDeviceSerial); 949 CLog.e(ioe); 950 } 951 } 952 } 953 954 /** 955 * Returns whether it is safe to mark modules as "done", given the invocation command-line 956 * arguments. Returns true unless this is a retry and specific filtering techniques are applied 957 * on the command-line, such as: 958 * --retry-type failed 959 * --include-filter 960 * --exclude-filter 961 * -t/--test 962 * --subplan 963 */ canMarkDone(String args)964 private boolean canMarkDone(String args) { 965 if (mRetrySessionId == null) { 966 return true; // always allow modules to be marked done if not retry 967 } 968 return !(RetryType.FAILED.equals(mRetryType) 969 || RetryType.CUSTOM.equals(mRetryType) 970 || args.contains(CompatibilityTestSuite.INCLUDE_FILTER_OPTION) 971 || args.contains(CompatibilityTestSuite.EXCLUDE_FILTER_OPTION) 972 || args.contains(CompatibilityTestSuite.SUBPLAN_OPTION) 973 || args.matches(String.format(".* (-%s|--%s) .*", 974 CompatibilityTestSuite.TEST_OPTION_SHORT_NAME, CompatibilityTestSuite.TEST_OPTION))); 975 } 976 977 /** 978 * Copy the xml formatting files stored in this jar to the results directory 979 * 980 * @param resultsDir 981 */ copyFormattingFiles(File resultsDir, String suiteName)982 static void copyFormattingFiles(File resultsDir, String suiteName) { 983 for (String resultFileName : ResultHandler.RESULT_RESOURCES) { 984 InputStream configStream = ResultHandler.class.getResourceAsStream( 985 String.format("/report/%s-%s", suiteName, resultFileName)); 986 if (configStream == null) { 987 // If suite specific files are not available, fallback to common. 988 configStream = ResultHandler.class.getResourceAsStream( 989 String.format("/report/%s", resultFileName)); 990 } 991 if (configStream != null) { 992 File resultFile = new File(resultsDir, resultFileName); 993 try { 994 FileUtil.writeToFile(configStream, resultFile); 995 } catch (IOException e) { 996 warn("Failed to write %s to file", resultFileName); 997 } 998 } else { 999 warn("Failed to load %s from jar", resultFileName); 1000 } 1001 } 1002 } 1003 1004 /** 1005 * move the dynamic config files to the results directory 1006 */ copyDynamicConfigFiles()1007 private void copyDynamicConfigFiles() { 1008 File configDir = new File(mResultDir, "config"); 1009 if (!configDir.mkdir()) { 1010 warn("Failed to make dynamic config directory \"%s\" in the result", 1011 configDir.getAbsolutePath()); 1012 } 1013 1014 Set<String> uniqueModules = new HashSet<>(); 1015 for (IBuildInfo buildInfo : mMasterBuildInfos) { 1016 CompatibilityBuildHelper helper = new CompatibilityBuildHelper(buildInfo); 1017 Map<String, File> dcFiles = helper.getDynamicConfigFiles(); 1018 for (String moduleName : dcFiles.keySet()) { 1019 File srcFile = dcFiles.get(moduleName); 1020 if (!uniqueModules.contains(moduleName)) { 1021 // have not seen config for this module yet, copy into result 1022 File destFile = new File(configDir, moduleName + ".dynamic"); 1023 try { 1024 FileUtil.copyFile(srcFile, destFile); 1025 uniqueModules.add(moduleName); // Add to uniqueModules if copy succeeds 1026 } catch (IOException e) { 1027 warn("Failure when copying config file \"%s\" to \"%s\" for module %s", 1028 srcFile.getAbsolutePath(), destFile.getAbsolutePath(), moduleName); 1029 CLog.e(e); 1030 } 1031 } 1032 FileUtil.deleteFile(srcFile); 1033 } 1034 } 1035 } 1036 1037 /** 1038 * Recursively copy any other files found in the previous session's result directory to the 1039 * new result directory, so long as they don't already exist. For example, a "screenshots" 1040 * directory generated in a previous session by a passing test will not be generated on retry 1041 * unless copied from the old result directory. 1042 * 1043 * @param oldDir 1044 * @param newDir 1045 */ copyRetryFiles(File oldDir, File newDir)1046 static void copyRetryFiles(File oldDir, File newDir) { 1047 File[] oldChildren = oldDir.listFiles(); 1048 for (File oldChild : oldChildren) { 1049 if (NOT_RETRY_FILES.contains(oldChild.getName())) { 1050 continue; // do not copy this file/directory or its children 1051 } 1052 File newChild = new File(newDir, oldChild.getName()); 1053 if (!newChild.exists()) { 1054 // If this old file or directory doesn't exist in new dir, simply copy it 1055 try { 1056 if (oldChild.isDirectory()) { 1057 FileUtil.recursiveCopy(oldChild, newChild); 1058 } else { 1059 FileUtil.copyFile(oldChild, newChild); 1060 } 1061 } catch (IOException e) { 1062 warn("Failed to copy file \"%s\" from previous session", oldChild.getName()); 1063 } 1064 } else if (oldChild.isDirectory() && newChild.isDirectory()) { 1065 // If both children exist as directories, make sure the children of the old child 1066 // directory exist in the new child directory. 1067 copyRetryFiles(oldChild, newChild); 1068 } 1069 } 1070 } 1071 1072 /** 1073 * Zip the contents of the given results directory. 1074 * 1075 * @param resultsDir 1076 */ zipResults(File resultsDir)1077 private static File zipResults(File resultsDir) { 1078 File zipResultFile = null; 1079 try { 1080 // create a file in parent directory, with same name as resultsDir 1081 zipResultFile = new File(resultsDir.getParent(), String.format("%s.zip", 1082 resultsDir.getName())); 1083 ZipUtil.createZip(resultsDir, zipResultFile); 1084 } catch (IOException e) { 1085 warn("Failed to create zip for %s", resultsDir.getName()); 1086 } 1087 return zipResultFile; 1088 } 1089 1090 /** 1091 * Log info to the console. 1092 */ info(String format, Object... args)1093 private static void info(String format, Object... args) { 1094 log(LogLevel.INFO, format, args); 1095 } 1096 1097 /** 1098 * Log debug to the console. 1099 */ debug(String format, Object... args)1100 private static void debug(String format, Object... args) { 1101 log(LogLevel.DEBUG, format, args); 1102 } 1103 1104 /** 1105 * Log a warning to the console. 1106 */ warn(String format, Object... args)1107 private static void warn(String format, Object... args) { 1108 log(LogLevel.WARN, format, args); 1109 } 1110 1111 /** 1112 * Log a message to the console 1113 */ log(LogLevel level, String format, Object... args)1114 private static void log(LogLevel level, String format, Object... args) { 1115 CLog.logAndDisplay(level, format, args); 1116 } 1117 1118 /** 1119 * For testing purpose. 1120 */ 1121 @VisibleForTesting getResult()1122 public IInvocationResult getResult() { 1123 return mResult; 1124 } 1125 1126 /** 1127 * Returns true if the reporter is finalized before the end of the timeout. False otherwise. 1128 */ 1129 @VisibleForTesting waitForFinalized(long timeout, TimeUnit unit)1130 public boolean waitForFinalized(long timeout, TimeUnit unit) throws InterruptedException { 1131 return mFinalized.await(timeout, unit); 1132 } 1133 sanitizeXmlContent(String s)1134 private static String sanitizeXmlContent(String s) { 1135 return XmlEscapers.xmlContentEscaper().escape(s); 1136 } 1137 1138 /** Re-log a result file to all reporters so they are aware of it. */ logReportFiles( IConfiguration configuration, File resultFile, String dataName, LogDataType type)1139 private void logReportFiles( 1140 IConfiguration configuration, File resultFile, String dataName, LogDataType type) { 1141 if (configuration == null) { 1142 return; 1143 } 1144 List<ITestInvocationListener> listeners = configuration.getTestInvocationListeners(); 1145 try (FileInputStreamSource source = new FileInputStreamSource(resultFile)) { 1146 for (ITestInvocationListener listener : listeners) { 1147 if (listener.equals(this)) { 1148 // Avoid logging agaisnt itself 1149 continue; 1150 } 1151 listener.testLog(dataName, type, source); 1152 } 1153 } 1154 } 1155 } 1156