1 /* 2 * Copyright (C) 2015 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License 15 */ 16 package com.android.compatibility.common.util; 17 18 import com.android.tradefed.util.FileUtil; 19 import com.android.tradefed.util.AbiUtils; 20 21 import org.w3c.dom.Element; 22 import org.w3c.dom.NodeList; 23 import org.xml.sax.InputSource; 24 25 import junit.framework.TestCase; 26 27 import java.io.File; 28 import java.io.FileWriter; 29 import java.io.IOException; 30 import java.io.StringReader; 31 import java.net.InetAddress; 32 import java.net.UnknownHostException; 33 import java.util.ArrayList; 34 import java.util.Arrays; 35 import java.util.Collection; 36 import java.util.HashMap; 37 import java.util.HashSet; 38 import java.util.List; 39 import java.util.Map; 40 import java.util.Set; 41 42 import javax.xml.xpath.XPath; 43 import javax.xml.xpath.XPathConstants; 44 import javax.xml.xpath.XPathExpressionException; 45 import javax.xml.xpath.XPathFactory; 46 47 /** 48 * Unit tests for {@link ResultHandler} 49 */ 50 public class ResultHandlerTest extends TestCase { 51 52 private static final String SUITE_NAME = "CTS"; 53 private static final String SUITE_VERSION = "5.0"; 54 private static final String SUITE_PLAN = "cts"; 55 private static final String SUITE_BUILD = "12345"; 56 private static final String REPORT_VERSION = "5.0"; 57 private static final String OS_NAME = System.getProperty("os.name"); 58 private static final String OS_VERSION = System.getProperty("os.version"); 59 private static final String OS_ARCH = System.getProperty("os.arch"); 60 private static final String JAVA_VENDOR = System.getProperty("java.vendor"); 61 private static final String JAVA_VERSION = System.getProperty("java.version"); 62 private static final String NAME_A = "ModuleA"; 63 private static final String NAME_B = "ModuleB"; 64 private static final String DONE_A = "false"; 65 private static final String DONE_B = "true"; 66 private static final String RUNTIME_A = "100"; 67 private static final String RUNTIME_B = "200"; 68 private static final String ABI = "mips64"; 69 private static final String ID_A = AbiUtils.createId(ABI, NAME_A); 70 private static final String ID_B = AbiUtils.createId(ABI, NAME_B); 71 72 private static final String BUILD_FINGERPRINT = "build_fingerprint"; 73 private static final String BUILD_FINGERPRINT_UNALTERED = "build_fingerprint_unaltered"; 74 private static final String BUILD_ID = "build_id"; 75 private static final String BUILD_PRODUCT = "build_product"; 76 private static final String RUN_HISTORY = "run_history"; 77 private static final String EXAMPLE_BUILD_ID = "XYZ"; 78 private static final String EXAMPLE_BUILD_PRODUCT = "wolverine"; 79 private static final String EXAMPLE_BUILD_FINGERPRINT = "example_build_fingerprint"; 80 private static final String EXAMPLE_BUILD_FINGERPRINT_UNALTERED = "example_build_fingerprint_unaltered"; 81 private static final String EXAMPLE_RUN_HISTORY = 82 "[{\"startTime\":10000000000000,\"endTime\":10000000000001}," 83 + "{\"startTime\":10000000000002,\"endTime\":10000000000003}]"; 84 85 private static final String DEVICE_A = "device123"; 86 private static final String DEVICE_B = "device456"; 87 private static final String DEVICES = "device456,device123"; 88 private static final String CLASS_A = "android.test.Foor"; 89 private static final String CLASS_B = "android.test.Bar"; 90 private static final String METHOD_1 = "testBlah1"; 91 private static final String METHOD_2 = "testBlah2"; 92 private static final String METHOD_3 = "testBlah3"; 93 private static final String METHOD_4 = "testBlah4"; 94 private static final String METHOD_5 = "testBlah5"; 95 private static final String SUMMARY_SOURCE = String.format("%s#%s:20", CLASS_B, METHOD_4); 96 private static final String SUMMARY_MESSAGE = "Headline"; 97 private static final double SUMMARY_VALUE = 9001; 98 private static final String MESSAGE = "Something small is not alright"; 99 private static final String STACK_TRACE = "Something small is not alright\n " + 100 "at four.big.insects.Marley.sing(Marley.java:10)"; 101 private static final String BUG_REPORT = "https://cnsviewer.corp.google.com/cns/bugreport.txt"; 102 private static final String LOGCAT = "https://cnsviewer.corp.google.com/cns/logcat.gz"; 103 private static final String SCREENSHOT = "https://cnsviewer.corp.google.com/screenshot.png"; 104 private static final long START_MS = 1431586801000L; 105 private static final long END_MS = 1431673199000L; 106 private static final String START_DISPLAY = "Fri Aug 20 15:13:03 PDT 2010"; 107 private static final String END_DISPLAY = "Fri Aug 20 15:13:04 PDT 2010"; 108 private static final long TEST_START_MS = 1000000000011L; 109 private static final long TEST_END_MS = 1000000000012L; 110 private static final boolean TEST_IS_AUTOMATED = false; 111 112 private static final String REFERENCE_URL = "http://android.com"; 113 private static final String LOG_URL = "file:///path/to/logs"; 114 private static final String COMMAND_LINE_ARGS = "cts -m CtsMyModuleTestCases"; 115 private static final String XML_BASE = 116 "<?xml version='1.0' encoding='UTF-8' standalone='no' ?>" + 117 "<?xml-stylesheet type=\"text/xsl\" href=\"compatibility_result.xsl\"?>\n" + 118 "<Result start=\"%d\" end=\"%d\" start_display=\"%s\"" + 119 "end_display=\"%s\" suite_name=\"%s\" suite_version=\"%s\" " + 120 "suite_plan=\"%s\" suite_build_number=\"%s\" report_version=\"%s\" " + 121 "devices=\"%s\" host_name=\"%s\"" + 122 "os_name=\"%s\" os_version=\"%s\" os_arch=\"%s\" java_vendor=\"%s\"" + 123 "java_version=\"%s\" reference_url=\"%s\" log_url=\"%s\"" + 124 "command_line_args=\"%s\">\n" + 125 "%s%s%s" + 126 "</Result>"; 127 private static final String XML_BUILD_INFO = 128 " <Build " + 129 BUILD_FINGERPRINT + "=\"%s\" " + 130 BUILD_ID + "=\"%s\" " + 131 BUILD_PRODUCT + "=\"%s\" " + 132 " />\n"; 133 private static final String XML_BUILD_INFO_WITH_UNALTERED_BUILD_FINGERPRINT = 134 " <Build " + 135 BUILD_FINGERPRINT + "=\"%s\" " + 136 BUILD_FINGERPRINT_UNALTERED + "=\"%s\" " + 137 BUILD_ID + "=\"%s\" " + 138 BUILD_PRODUCT + "=\"%s\" " + 139 " />\n"; 140 private static final String XML_SUMMARY = 141 " <Summary pass=\"%d\" failed=\"%d\" " + 142 "modules_done=\"1\" modules_total=\"1\" />\n"; 143 private static final String XML_MODULE = 144 " <Module name=\"%s\" abi=\"%s\" device=\"%s\" runtime=\"%s\" done=\"%s\">\n" + 145 "%s" + 146 " </Module>\n"; 147 private static final String XML_CASE = 148 " <TestCase name=\"%s\">\n" + 149 "%s" + 150 " </TestCase>\n"; 151 private static final String XML_TEST_PASS = 152 " <Test result=\"pass\" name=\"%s\"/>\n"; 153 private static final String XML_TEST_SKIP = 154 " <Test result=\"pass\" name=\"%s\" skipped=\"true\"/>\n"; 155 private static final String XML_TEST_FAIL = 156 " <Test result=\"fail\" name=\"%s\">\n" + 157 " <Failure message=\"%s\">\n" + 158 " <StackTrace>%s</StackTrace>\n" + 159 " </Failure>\n" + 160 " <BugReport>%s</BugReport>\n" + 161 " <Logcat>%s</Logcat>\n" + 162 " <Screenshot>%s</Screenshot>\n" + 163 " </Test>\n"; 164 private static final String XML_TEST_RESULT = 165 " <Test result=\"pass\" name=\"%s\">\n" + 166 " <Summary>\n" + 167 " <Metric source=\"%s\" message=\"%s\" score_type=\"%s\" score_unit=\"%s\">\n" + 168 " <Value>%s</Value>\n" + 169 " </Metric>\n" + 170 " </Summary>\n" + 171 " </Test>\n"; 172 private static final String NEW_XML_TEST_RESULT = 173 " <Test result=\"pass\" name=\"%s\">\n" 174 + " <Metric key=\"%s\">%s</Metric>\n" 175 + " </Test>\n"; 176 177 private File resultsDir = null; 178 private File resultDir = null; 179 180 @Override setUp()181 public void setUp() throws Exception { 182 resultsDir = FileUtil.createTempDir("results"); 183 resultDir = FileUtil.createTempDir("12345", resultsDir); 184 } 185 186 @Override tearDown()187 public void tearDown() throws Exception { 188 FileUtil.recursiveDelete(resultsDir); 189 } 190 testSerialization()191 public void testSerialization() throws Exception { 192 IInvocationResult result = new InvocationResult(); 193 result.setStartTime(START_MS); 194 result.setTestPlan(SUITE_PLAN); 195 result.addDeviceSerial(DEVICE_A); 196 result.addDeviceSerial(DEVICE_B); 197 result.addInvocationInfo(BUILD_FINGERPRINT, EXAMPLE_BUILD_FINGERPRINT); 198 result.addInvocationInfo(BUILD_ID, EXAMPLE_BUILD_ID); 199 result.addInvocationInfo(BUILD_PRODUCT, EXAMPLE_BUILD_PRODUCT); 200 result.addInvocationInfo(RUN_HISTORY, EXAMPLE_RUN_HISTORY); 201 Collection<InvocationResult.RunHistory> runHistories = 202 ((InvocationResult) result).getRunHistories(); 203 InvocationResult.RunHistory runHistory1 = new InvocationResult.RunHistory(); 204 runHistory1.startTime = 10000000000000L; 205 runHistory1.endTime = 10000000000001L; 206 runHistories.add(runHistory1); 207 InvocationResult.RunHistory runHistory2 = new InvocationResult.RunHistory(); 208 runHistory2.startTime = 10000000000002L; 209 runHistory2.endTime = 10000000000003L; 210 runHistories.add(runHistory2); 211 212 // Module A: test1 passes, test2 not executed 213 IModuleResult moduleA = result.getOrCreateModule(ID_A); 214 moduleA.setDone(false); 215 moduleA.addRuntime(Integer.parseInt(RUNTIME_A)); 216 ICaseResult moduleACase = moduleA.getOrCreateResult(CLASS_A); 217 ITestResult moduleATest1 = moduleACase.getOrCreateResult(METHOD_1); 218 moduleATest1.setResultStatus(TestStatus.PASS); 219 ITestResult moduleATest2 = moduleACase.getOrCreateResult(METHOD_2); 220 moduleATest2.setResultStatus(null); // not executed test 221 // Module B: test3 fails, test4 passes with report log, test5 passes with skip 222 IModuleResult moduleB = result.getOrCreateModule(ID_B); 223 moduleB.setDone(true); 224 moduleB.addRuntime(Integer.parseInt(RUNTIME_B)); 225 ICaseResult moduleBCase = moduleB.getOrCreateResult(CLASS_B); 226 ITestResult moduleBTest3 = moduleBCase.getOrCreateResult(METHOD_3); 227 moduleBTest3.setResultStatus(TestStatus.FAIL); 228 moduleBTest3.setMessage(MESSAGE); 229 moduleBTest3.setStackTrace(STACK_TRACE); 230 moduleBTest3.setBugReport(BUG_REPORT); 231 moduleBTest3.setLog(LOGCAT); 232 moduleBTest3.setScreenshot(SCREENSHOT); 233 ITestResult moduleBTest4 = moduleBCase.getOrCreateResult(METHOD_4); 234 moduleBTest4.setResultStatus(TestStatus.PASS); 235 ReportLog report = new ReportLog(); 236 ReportLog.Metric summary = new ReportLog.Metric(SUMMARY_SOURCE, SUMMARY_MESSAGE, 237 SUMMARY_VALUE, ResultType.HIGHER_BETTER, ResultUnit.SCORE); 238 report.setSummary(summary); 239 moduleBTest4.setReportLog(report); 240 ITestResult moduleBTest5 = moduleBCase.getOrCreateResult(METHOD_5); 241 moduleBTest5.skipped(); 242 243 Map<String, String> testAttributes = new HashMap<String, String>(); 244 testAttributes.put("foo1", "bar1"); 245 testAttributes.put("foo2", "bar2"); 246 // Serialize to file 247 File res = 248 ResultHandler.writeResults( 249 SUITE_NAME, 250 SUITE_VERSION, 251 SUITE_PLAN, 252 SUITE_BUILD, 253 result, 254 resultDir, 255 START_MS, 256 END_MS, 257 REFERENCE_URL, 258 LOG_URL, 259 COMMAND_LINE_ARGS, 260 testAttributes); 261 String content = FileUtil.readStringFromFile(res); 262 assertXmlContainsAttribute(content, "Result", "foo1", "bar1"); 263 assertXmlContainsAttribute(content, "Result", "foo2", "bar2"); 264 assertXmlContainsAttribute(content, "Result/Build", "run_history", EXAMPLE_RUN_HISTORY); 265 assertXmlContainsNode(content, "Result/RunHistory"); 266 assertXmlContainsAttribute(content, "Result/RunHistory/Run", "start", "10000000000000"); 267 assertXmlContainsAttribute(content, "Result/RunHistory/Run", "end", "10000000000001"); 268 assertXmlContainsAttribute(content, "Result/RunHistory/Run", "start", "10000000000002"); 269 assertXmlContainsAttribute(content, "Result/RunHistory/Run", "end", "10000000000003"); 270 271 // Parse the results and assert correctness 272 result = ResultHandler.getResultFromDir(resultDir); 273 checkResult(result, false); 274 checkRunHistory(result); 275 } 276 277 /* 278 * Test serialization for CTS Verifier since test results with test result history is only in 279 * CTS Verifier and was not parsed by suite harness. 280 */ testSerialization_whenTestResultWithTestResultHistoryWithoutParsing()281 public void testSerialization_whenTestResultWithTestResultHistoryWithoutParsing() 282 throws Exception { 283 IInvocationResult result = new InvocationResult(); 284 result.setStartTime(START_MS); 285 result.setTestPlan(SUITE_PLAN); 286 result.addDeviceSerial(DEVICE_A); 287 result.addDeviceSerial(DEVICE_B); 288 result.addInvocationInfo(BUILD_FINGERPRINT, EXAMPLE_BUILD_FINGERPRINT); 289 result.addInvocationInfo(BUILD_ID, EXAMPLE_BUILD_ID); 290 result.addInvocationInfo(BUILD_PRODUCT, EXAMPLE_BUILD_PRODUCT); 291 292 // Module A: test1 passes, test2 not executed 293 IModuleResult moduleA = result.getOrCreateModule(ID_A); 294 moduleA.setDone(false); 295 moduleA.addRuntime(Integer.parseInt(RUNTIME_A)); 296 ICaseResult moduleACase = moduleA.getOrCreateResult(CLASS_A); 297 ITestResult moduleATest1 = moduleACase.getOrCreateResult(METHOD_1); 298 moduleATest1.setResultStatus(TestStatus.PASS); 299 // Module B: test3 fails with test result history, test4 passes with report log, 300 // test5 passes with skip 301 IModuleResult moduleB = result.getOrCreateModule(ID_B); 302 moduleB.setDone(true); 303 moduleB.addRuntime(Integer.parseInt(RUNTIME_B)); 304 ICaseResult moduleBCase = moduleB.getOrCreateResult(CLASS_B); 305 Set<TestResultHistory.ExecutionRecord> executionRecords = 306 new HashSet<TestResultHistory.ExecutionRecord>(); 307 executionRecords.add( 308 new TestResultHistory.ExecutionRecord( 309 TEST_START_MS, TEST_END_MS, TEST_IS_AUTOMATED)); 310 311 ITestResult moduleBTest3 = moduleBCase.getOrCreateResult(METHOD_3); 312 moduleBTest3.setResultStatus(TestStatus.FAIL); 313 moduleBTest3.setMessage(MESSAGE); 314 moduleBTest3.setStackTrace(STACK_TRACE); 315 moduleBTest3.setBugReport(BUG_REPORT); 316 moduleBTest3.setLog(LOGCAT); 317 moduleBTest3.setScreenshot(SCREENSHOT); 318 List<TestResultHistory> resultHistories = new ArrayList<TestResultHistory>(); 319 TestResultHistory resultHistory = new TestResultHistory(METHOD_3, executionRecords); 320 resultHistories.add(resultHistory); 321 moduleBTest3.setTestResultHistories(resultHistories); 322 ITestResult moduleBTest4 = moduleBCase.getOrCreateResult(METHOD_4); 323 moduleBTest4.setResultStatus(TestStatus.PASS); 324 ReportLog report = new ReportLog(); 325 ReportLog.Metric summary = 326 new ReportLog.Metric( 327 SUMMARY_SOURCE, 328 SUMMARY_MESSAGE, 329 SUMMARY_VALUE, 330 ResultType.HIGHER_BETTER, 331 ResultUnit.SCORE); 332 report.setSummary(summary); 333 moduleBTest4.setReportLog(report); 334 ITestResult moduleBTest5 = moduleBCase.getOrCreateResult(METHOD_5); 335 moduleBTest5.skipped(); 336 337 // Serialize to file 338 File res = 339 ResultHandler.writeResults( 340 SUITE_NAME, 341 SUITE_VERSION, 342 SUITE_PLAN, 343 SUITE_BUILD, 344 result, 345 resultDir, 346 START_MS, 347 END_MS, 348 REFERENCE_URL, 349 LOG_URL, 350 COMMAND_LINE_ARGS, 351 null); 352 String content = FileUtil.readStringFromFile(res); 353 assertXmlContainsNode(content, "Result/Module/TestCase/Test/RunHistory"); 354 assertXmlContainsAttribute( 355 content, 356 "Result/Module/TestCase/Test/RunHistory/Run", 357 "start", 358 Long.toString(TEST_START_MS)); 359 assertXmlContainsAttribute( 360 content, 361 "Result/Module/TestCase/Test/RunHistory/Run", 362 "end", 363 Long.toString(TEST_END_MS)); 364 assertXmlContainsAttribute( 365 content, 366 "Result/Module/TestCase/Test/RunHistory/Run", 367 "isAutomated", 368 Boolean.toString(TEST_IS_AUTOMATED)); 369 checkResult(result, EXAMPLE_BUILD_FINGERPRINT, false, false); 370 } 371 testParsing()372 public void testParsing() throws Exception { 373 File resultDir = writeResultDir(resultsDir, false); 374 // Parse the results and assert correctness 375 checkResult(ResultHandler.getResultFromDir(resultDir), false); 376 } 377 testParsing_newTestFormat()378 public void testParsing_newTestFormat() throws Exception { 379 File resultDir = writeResultDir(resultsDir, true); 380 // Parse the results and assert correctness 381 checkResult(ResultHandler.getResultFromDir(resultDir), true); 382 } 383 testParsing_usesUnalteredBuildFingerprintWhenPresent()384 public void testParsing_usesUnalteredBuildFingerprintWhenPresent() throws Exception { 385 String buildInfo = String.format(XML_BUILD_INFO_WITH_UNALTERED_BUILD_FINGERPRINT, 386 EXAMPLE_BUILD_FINGERPRINT, EXAMPLE_BUILD_FINGERPRINT_UNALTERED, 387 EXAMPLE_BUILD_ID, EXAMPLE_BUILD_PRODUCT); 388 File resultDir = writeResultDir(resultsDir, buildInfo, false); 389 checkResult( 390 ResultHandler.getResultFromDir(resultDir), 391 EXAMPLE_BUILD_FINGERPRINT_UNALTERED, 392 false, 393 true); 394 } 395 testParsing_whenUnalteredBuildFingerprintIsEmpty_usesRegularBuildFingerprint()396 public void testParsing_whenUnalteredBuildFingerprintIsEmpty_usesRegularBuildFingerprint() throws Exception { 397 String buildInfo = String.format(XML_BUILD_INFO_WITH_UNALTERED_BUILD_FINGERPRINT, 398 EXAMPLE_BUILD_FINGERPRINT, "", EXAMPLE_BUILD_ID, EXAMPLE_BUILD_PRODUCT); 399 File resultDir = writeResultDir(resultsDir, buildInfo, false); 400 checkResult( 401 ResultHandler.getResultFromDir(resultDir), EXAMPLE_BUILD_FINGERPRINT, false, true); 402 } 403 testGetLightResults()404 public void testGetLightResults() throws Exception { 405 File resultDir = writeResultDir(resultsDir, false); 406 List<IInvocationResult> lightResults = ResultHandler.getLightResults(resultsDir); 407 assertEquals("Expected one result", 1, lightResults.size()); 408 IInvocationResult lightResult = lightResults.get(0); 409 checkLightResult(lightResult); 410 } 411 writeResultDir(File resultsDir, boolean newTestFormat)412 static File writeResultDir(File resultsDir, boolean newTestFormat) throws IOException { 413 String buildInfo = String.format(XML_BUILD_INFO, EXAMPLE_BUILD_FINGERPRINT, 414 EXAMPLE_BUILD_ID, EXAMPLE_BUILD_PRODUCT); 415 return writeResultDir(resultsDir, buildInfo, newTestFormat); 416 } 417 418 /* 419 * Helper to write a result to the results dir, for testing. 420 * @return the written resultDir 421 */ writeResultDir(File resultsDir, String buildInfo, boolean newTestFormat)422 static File writeResultDir(File resultsDir, String buildInfo, boolean newTestFormat) 423 throws IOException { 424 File resultDir = null; 425 FileWriter writer = null; 426 try { 427 resultDir = FileUtil.createTempDir("12345", resultsDir); 428 // Create the result file 429 File resultFile = new File(resultDir, ResultHandler.TEST_RESULT_FILE_NAME); 430 writer = new FileWriter(resultFile); 431 String summary = String.format(XML_SUMMARY, 2, 1); 432 String moduleATest = String.format(XML_TEST_PASS, METHOD_1); 433 String moduleACases = String.format(XML_CASE, CLASS_A, moduleATest); 434 String moduleA = String.format(XML_MODULE, NAME_A, ABI, DEVICE_A, RUNTIME_A, DONE_A, 435 moduleACases); 436 String moduleBTest3 = String.format(XML_TEST_FAIL, METHOD_3, MESSAGE, STACK_TRACE, 437 BUG_REPORT, LOGCAT, SCREENSHOT); 438 String moduleBTest4 = ""; 439 if (newTestFormat) { 440 moduleBTest4 = 441 String.format( 442 NEW_XML_TEST_RESULT, 443 METHOD_4, 444 SUMMARY_MESSAGE, 445 Double.toString(SUMMARY_VALUE)); 446 } else { 447 moduleBTest4 = 448 String.format( 449 XML_TEST_RESULT, 450 METHOD_4, 451 SUMMARY_SOURCE, 452 SUMMARY_MESSAGE, 453 ResultType.HIGHER_BETTER.toReportString(), 454 ResultUnit.SCORE.toReportString(), 455 Double.toString(SUMMARY_VALUE)); 456 } 457 458 String moduleBTest5 = String.format(XML_TEST_SKIP, METHOD_5); 459 String moduleBTests = String.join("", moduleBTest3, moduleBTest4, moduleBTest5); 460 String moduleBCases = String.format(XML_CASE, CLASS_B, moduleBTests); 461 String moduleB = String.format(XML_MODULE, NAME_B, ABI, DEVICE_B, RUNTIME_B, DONE_B, 462 moduleBCases); 463 String modules = String.join("", moduleA, moduleB); 464 String hostName = ""; 465 try { 466 hostName = InetAddress.getLocalHost().getHostName(); 467 } catch (UnknownHostException ignored) {} 468 String output = String.format(XML_BASE, START_MS, END_MS, START_DISPLAY, END_DISPLAY, 469 SUITE_NAME, SUITE_VERSION, SUITE_PLAN, SUITE_BUILD, REPORT_VERSION, DEVICES, 470 hostName, OS_NAME, OS_VERSION, OS_ARCH, JAVA_VENDOR, 471 JAVA_VERSION, REFERENCE_URL, LOG_URL, COMMAND_LINE_ARGS, 472 buildInfo, summary, modules); 473 writer.write(output); 474 writer.flush(); 475 } finally { 476 if (writer != null) { 477 writer.close(); 478 } 479 } 480 return resultDir; 481 } 482 checkLightResult(IInvocationResult lightResult)483 static void checkLightResult(IInvocationResult lightResult) throws Exception { 484 assertEquals("Expected 3 passes", 3, lightResult.countResults(TestStatus.PASS)); 485 assertEquals("Expected 1 failure", 1, lightResult.countResults(TestStatus.FAIL)); 486 487 Map<String, String> buildInfo = lightResult.getInvocationInfo(); 488 assertEquals("Incorrect Build ID", EXAMPLE_BUILD_ID, buildInfo.get(BUILD_ID)); 489 assertEquals("Incorrect Build Product", 490 EXAMPLE_BUILD_PRODUCT, buildInfo.get(BUILD_PRODUCT)); 491 492 Set<String> serials = lightResult.getDeviceSerials(); 493 assertTrue("Missing device", serials.contains(DEVICE_A)); 494 assertTrue("Missing device", serials.contains(DEVICE_B)); 495 assertEquals("Expected 2 devices", 2, serials.size()); 496 assertTrue("Incorrect devices", serials.contains(DEVICE_A) && serials.contains(DEVICE_B)); 497 assertEquals("Incorrect start time", START_MS, lightResult.getStartTime()); 498 assertEquals("Incorrect test plan", SUITE_PLAN, lightResult.getTestPlan()); 499 List<IModuleResult> modules = lightResult.getModules(); 500 assertEquals("Expected 1 completed module", 1, lightResult.getModuleCompleteCount()); 501 assertEquals("Expected 2 total modules", 2, modules.size()); 502 } 503 checkResult(IInvocationResult result, boolean newTestFormat)504 static void checkResult(IInvocationResult result, boolean newTestFormat) throws Exception { 505 checkResult(result, EXAMPLE_BUILD_FINGERPRINT, newTestFormat, true); 506 } 507 checkRunHistory(IInvocationResult result)508 static void checkRunHistory(IInvocationResult result) { 509 Map<String, String> buildInfo = result.getInvocationInfo(); 510 assertEquals("Incorrect run history", EXAMPLE_RUN_HISTORY, buildInfo.get(RUN_HISTORY)); 511 } 512 checkResult( IInvocationResult result, String expectedBuildFingerprint, boolean newTestFormat, boolean checkResultHistories)513 static void checkResult( 514 IInvocationResult result, String expectedBuildFingerprint, boolean newTestFormat, 515 boolean checkResultHistories) throws Exception { 516 assertEquals("Expected 3 passes", 3, result.countResults(TestStatus.PASS)); 517 assertEquals("Expected 1 failure", 1, result.countResults(TestStatus.FAIL)); 518 519 Map<String, String> buildInfo = result.getInvocationInfo(); 520 assertEquals("Incorrect Build Fingerprint", expectedBuildFingerprint, result.getBuildFingerprint()); 521 assertEquals("Incorrect Build ID", EXAMPLE_BUILD_ID, buildInfo.get(BUILD_ID)); 522 assertEquals("Incorrect Build Product", 523 EXAMPLE_BUILD_PRODUCT, buildInfo.get(BUILD_PRODUCT)); 524 525 Set<String> serials = result.getDeviceSerials(); 526 assertTrue("Missing device", serials.contains(DEVICE_A)); 527 assertTrue("Missing device", serials.contains(DEVICE_B)); 528 assertEquals("Expected 2 devices", 2, serials.size()); 529 assertTrue("Incorrect devices", serials.contains(DEVICE_A) && serials.contains(DEVICE_B)); 530 assertEquals("Incorrect start time", START_MS, result.getStartTime()); 531 assertEquals("Incorrect test plan", SUITE_PLAN, result.getTestPlan()); 532 533 List<IModuleResult> modules = result.getModules(); 534 assertEquals("Expected 2 modules", 2, modules.size()); 535 536 IModuleResult moduleA = modules.get(0); 537 assertEquals("Expected 1 pass", 1, moduleA.countResults(TestStatus.PASS)); 538 assertEquals("Expected 0 failures", 0, moduleA.countResults(TestStatus.FAIL)); 539 assertEquals("Incorrect ABI", ABI, moduleA.getAbi()); 540 assertEquals("Incorrect name", NAME_A, moduleA.getName()); 541 assertEquals("Incorrect ID", ID_A, moduleA.getId()); 542 assertEquals("Incorrect runtime", Integer.parseInt(RUNTIME_A), moduleA.getRuntime()); 543 List<ICaseResult> moduleACases = moduleA.getResults(); 544 assertEquals("Expected 1 test case", 1, moduleACases.size()); 545 ICaseResult moduleACase = moduleACases.get(0); 546 assertEquals("Incorrect name", CLASS_A, moduleACase.getName()); 547 List<ITestResult> moduleAResults = moduleACase.getResults(); 548 assertEquals("Expected 1 result", 1, moduleAResults.size()); 549 ITestResult moduleATest1 = moduleAResults.get(0); 550 assertEquals("Incorrect name", METHOD_1, moduleATest1.getName()); 551 assertEquals("Incorrect result", TestStatus.PASS, moduleATest1.getResultStatus()); 552 assertNull("Unexpected bugreport", moduleATest1.getBugReport()); 553 assertNull("Unexpected log", moduleATest1.getLog()); 554 assertNull("Unexpected screenshot", moduleATest1.getScreenshot()); 555 assertNull("Unexpected message", moduleATest1.getMessage()); 556 assertNull("Unexpected stack trace", moduleATest1.getStackTrace()); 557 assertNull("Unexpected report", moduleATest1.getReportLog()); 558 559 IModuleResult moduleB = modules.get(1); 560 assertEquals("Expected 2 passes", 2, moduleB.countResults(TestStatus.PASS)); 561 assertEquals("Expected 1 failure", 1, moduleB.countResults(TestStatus.FAIL)); 562 assertEquals("Incorrect ABI", ABI, moduleB.getAbi()); 563 assertEquals("Incorrect name", NAME_B, moduleB.getName()); 564 assertEquals("Incorrect ID", ID_B, moduleB.getId()); 565 assertEquals("Incorrect runtime", Integer.parseInt(RUNTIME_B), moduleB.getRuntime()); 566 List<ICaseResult> moduleBCases = moduleB.getResults(); 567 assertEquals("Expected 1 test case", 1, moduleBCases.size()); 568 ICaseResult moduleBCase = moduleBCases.get(0); 569 assertEquals("Incorrect name", CLASS_B, moduleBCase.getName()); 570 List<ITestResult> moduleBResults = moduleBCase.getResults(); 571 assertEquals("Expected 3 results", 3, moduleBResults.size()); 572 ITestResult moduleBTest3 = moduleBResults.get(0); 573 assertEquals("Incorrect name", METHOD_3, moduleBTest3.getName()); 574 assertEquals("Incorrect result", TestStatus.FAIL, moduleBTest3.getResultStatus()); 575 assertEquals("Incorrect bugreport", BUG_REPORT, moduleBTest3.getBugReport()); 576 assertEquals("Incorrect log", LOGCAT, moduleBTest3.getLog()); 577 assertEquals("Incorrect screenshot", SCREENSHOT, moduleBTest3.getScreenshot()); 578 assertEquals("Incorrect message", MESSAGE, moduleBTest3.getMessage()); 579 assertEquals("Incorrect stack trace", STACK_TRACE, moduleBTest3.getStackTrace()); 580 assertNull("Unexpected report", moduleBTest3.getReportLog()); 581 List<TestResultHistory> resultHistories = moduleBTest3.getTestResultHistories(); 582 // Check if unit tests do parsing result, because tests for CTS Verifier do not parse it. 583 if (checkResultHistories) { 584 // For xTS except CTS Verifier. 585 assertNull("Unexpected test result history list", resultHistories); 586 } else { 587 // For CTS Verifier. 588 assertNotNull("Expected test result history list", resultHistories); 589 assertEquals("Expected 1 test result history", 1, resultHistories.size()); 590 for (TestResultHistory resultHistory : resultHistories) { 591 assertNotNull("Expected test result history", resultHistory); 592 assertEquals("Incorrect test name", METHOD_3, resultHistory.getTestName()); 593 for (TestResultHistory.ExecutionRecord execRecord : 594 resultHistory.getExecutionRecords()) { 595 assertEquals( 596 "Incorrect test start time", TEST_START_MS, execRecord.getStartTime()); 597 assertEquals("Incorrect test end time", TEST_END_MS, execRecord.getEndTime()); 598 assertEquals( 599 "Incorrect test is automated", 600 TEST_IS_AUTOMATED, 601 execRecord.getIsAutomated()); 602 } 603 } 604 } 605 ITestResult moduleBTest4 = moduleBResults.get(1); 606 assertEquals("Incorrect name", METHOD_4, moduleBTest4.getName()); 607 assertEquals("Incorrect result", TestStatus.PASS, moduleBTest4.getResultStatus()); 608 assertNull("Unexpected bugreport", moduleBTest4.getBugReport()); 609 assertNull("Unexpected log", moduleBTest4.getLog()); 610 assertNull("Unexpected screenshot", moduleBTest4.getScreenshot()); 611 assertNull("Unexpected message", moduleBTest4.getMessage()); 612 assertNull("Unexpected stack trace", moduleBTest4.getStackTrace()); 613 if (!newTestFormat) { 614 ReportLog report = moduleBTest4.getReportLog(); 615 assertNotNull("Expected report", report); 616 ReportLog.Metric summary = report.getSummary(); 617 assertNotNull("Expected report summary", summary); 618 assertEquals("Incorrect source", SUMMARY_SOURCE, summary.getSource()); 619 assertEquals("Incorrect message", SUMMARY_MESSAGE, summary.getMessage()); 620 assertEquals("Incorrect type", ResultType.HIGHER_BETTER, summary.getType()); 621 assertEquals("Incorrect unit", ResultUnit.SCORE, summary.getUnit()); 622 assertTrue( 623 "Incorrect values", 624 Arrays.equals(new double[] {SUMMARY_VALUE}, summary.getValues())); 625 } 626 ITestResult moduleBTest5 = moduleBResults.get(2); 627 assertEquals("Incorrect name", METHOD_5, moduleBTest5.getName()); 628 assertEquals("Incorrect result", TestStatus.PASS, moduleBTest5.getResultStatus()); 629 assertTrue("Expected skipped", moduleBTest5.isSkipped()); 630 assertNull("Unexpected bugreport", moduleBTest5.getBugReport()); 631 assertNull("Unexpected log", moduleBTest5.getLog()); 632 assertNull("Unexpected screenshot", moduleBTest5.getScreenshot()); 633 assertNull("Unexpected message", moduleBTest5.getMessage()); 634 assertNull("Unexpected stack trace", moduleBTest5.getStackTrace()); 635 assertNull("Unexpected report", moduleBTest5.getReportLog()); 636 } 637 638 /** Return all XML nodes that match the given xPathExpression. */ getXmlNodes(String xml, String xPathExpression)639 private NodeList getXmlNodes(String xml, String xPathExpression) 640 throws XPathExpressionException { 641 642 InputSource inputSource = new InputSource(new StringReader(xml)); 643 XPath xpath = XPathFactory.newInstance().newXPath(); 644 return (NodeList) xpath.evaluate(xPathExpression, inputSource, XPathConstants.NODESET); 645 } 646 647 /** Assert that the XML contains a node matching the given xPathExpression. */ assertXmlContainsNode(String xml, String xPathExpression)648 private NodeList assertXmlContainsNode(String xml, String xPathExpression) 649 throws XPathExpressionException { 650 NodeList nodes = getXmlNodes(xml, xPathExpression); 651 assertNotNull( 652 String.format("XML '%s' returned null for xpath '%s'.", xml, xPathExpression), 653 nodes); 654 assertTrue( 655 String.format( 656 "XML '%s' should have returned at least 1 node for xpath '%s', " 657 + "but returned %s nodes instead.", 658 xml, xPathExpression, nodes.getLength()), 659 nodes.getLength() >= 1); 660 return nodes; 661 } 662 663 /** 664 * Assert that the XML contains a node matching the given xPathExpression and that the node has 665 * a given value. 666 */ assertXmlContainsAttribute( String xml, String xPathExpression, String attributeName, String attributeValue)667 private void assertXmlContainsAttribute( 668 String xml, String xPathExpression, String attributeName, String attributeValue) 669 throws XPathExpressionException { 670 NodeList nodes = assertXmlContainsNode(xml, xPathExpression); 671 boolean found = false; 672 673 for (int i = 0; i < nodes.getLength(); i++) { 674 Element element = (Element) nodes.item(i); 675 String value = element.getAttribute(attributeName); 676 if (attributeValue.equals(value)) { 677 found = true; 678 break; 679 } 680 } 681 682 assertTrue( 683 String.format( 684 "xPath '%s' should contain attribute '%s' but does not. XML: '%s'", 685 xPathExpression, attributeName, xml), 686 found); 687 } 688 } 689
