1 /* 2 * Copyright (C) 2015 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License 15 */ 16 package com.android.compatibility.common.util; 17 18 import com.android.tradefed.util.FileUtil; 19 import com.android.tradefed.util.AbiUtils; 20 21 import org.w3c.dom.Element; 22 import org.w3c.dom.NodeList; 23 import org.xml.sax.InputSource; 24 25 import junit.framework.TestCase; 26 27 import java.io.File; 28 import java.io.FileWriter; 29 import java.io.IOException; 30 import java.io.StringReader; 31 import java.net.InetAddress; 32 import java.net.UnknownHostException; 33 import java.util.ArrayList; 34 import java.util.Arrays; 35 import java.util.Collection; 36 import java.util.HashMap; 37 import java.util.HashSet; 38 import java.util.List; 39 import java.util.Map; 40 import java.util.Set; 41 42 import javax.xml.xpath.XPath; 43 import javax.xml.xpath.XPathConstants; 44 import javax.xml.xpath.XPathExpressionException; 45 import javax.xml.xpath.XPathFactory; 46 47 /** 48 * Unit tests for {@link ResultHandler} 49 */ 50 public class ResultHandlerTest extends TestCase { 51 52 private static final String SUITE_NAME = "CTS"; 53 private static final String SUITE_VERSION = "5.0"; 54 private static final String SUITE_PLAN = "cts"; 55 private static final String SUITE_BUILD = "12345"; 56 private static final String REPORT_VERSION = "5.0"; 57 private static final String OS_NAME = System.getProperty("os.name"); 58 private static final String OS_VERSION = System.getProperty("os.version"); 59 private static final String OS_ARCH = System.getProperty("os.arch"); 60 private static final String JAVA_VENDOR = System.getProperty("java.vendor"); 61 private static final String JAVA_VERSION = System.getProperty("java.version"); 62 private static final String NAME_A = "ModuleA"; 63 private static final String NAME_B = "ModuleB"; 64 private static final String DONE_A = "false"; 65 private static final String DONE_B = "true"; 66 private static final String RUNTIME_A = "100"; 67 private static final String RUNTIME_B = "200"; 68 private static final String ABI = "mips64"; 69 private static final String ID_A = AbiUtils.createId(ABI, NAME_A); 70 private static final String ID_B = AbiUtils.createId(ABI, NAME_B); 71 72 private static final String BUILD_FINGERPRINT = "build_fingerprint"; 73 private static final String BUILD_FINGERPRINT_UNALTERED = "build_fingerprint_unaltered"; 74 private static final String BUILD_ID = "build_id"; 75 private static final String BUILD_PRODUCT = "build_product"; 76 private static final String RUN_HISTORY = "run_history"; 77 private static final String EXAMPLE_BUILD_ID = "XYZ"; 78 private static final String EXAMPLE_BUILD_PRODUCT = "wolverine"; 79 private static final String EXAMPLE_BUILD_FINGERPRINT = "example_build_fingerprint"; 80 private static final String EXAMPLE_BUILD_FINGERPRINT_UNALTERED = "example_build_fingerprint_unaltered"; 81 private static final String EXAMPLE_RUN_HISTORY = 82 "[{\"startTime\":10000000000000,\"endTime\":10000000000001}," 83 + "{\"startTime\":10000000000002,\"endTime\":10000000000003}]"; 84 85 private static final String DEVICE_A = "device123"; 86 private static final String DEVICE_B = "device456"; 87 private static final String DEVICES = "device456,device123"; 88 private static final String CLASS_A = "android.test.Foor"; 89 private static final String CLASS_B = "android.test.Bar"; 90 private static final String METHOD_1 = "testBlah1"; 91 private static final String METHOD_2 = "testBlah2"; 92 private static final String METHOD_3 = "testBlah3"; 93 private static final String METHOD_4 = "testBlah4"; 94 private static final String METHOD_5 = "testBlah5"; 95 private static final String METHOD_6_BRACKETS = "testBlah6[suffix]"; 96 private static final String SUB_METHOD_3 = "subTestBlah3"; 97 private static final String SUB_METHOD_6_BRACKETS = "subTestBlah6[suffix]"; 98 private static final String SUMMARY_SOURCE = String.format("%s#%s:20", CLASS_B, METHOD_4); 99 private static final String SUMMARY_MESSAGE = "Headline"; 100 private static final double SUMMARY_VALUE = 9001; 101 private static final String MESSAGE = "Something small is not alright"; 102 private static final String STACK_TRACE = "Something small is not alright\n " + 103 "at four.big.insects.Marley.sing(Marley.java:10)"; 104 private static final String BUG_REPORT = "https://cnsviewer.corp.google.com/cns/bugreport.txt"; 105 private static final String LOGCAT = "https://cnsviewer.corp.google.com/cns/logcat.gz"; 106 private static final String SCREENSHOT = "https://cnsviewer.corp.google.com/screenshot.png"; 107 private static final long START_MS = 1431586801000L; 108 private static final long END_MS = 1431673199000L; 109 private static final String START_DISPLAY = "Fri Aug 20 15:13:03 PDT 2010"; 110 private static final String END_DISPLAY = "Fri Aug 20 15:13:04 PDT 2010"; 111 private static final long TEST_START_MS = 1000000000011L; 112 private static final long TEST_END_MS = 1000000000012L; 113 private static final boolean TEST_IS_AUTOMATED = false; 114 115 private static final String REFERENCE_URL = "http://android.com"; 116 private static final String LOG_URL = "file:///path/to/logs"; 117 private static final String COMMAND_LINE_ARGS = "cts -m CtsMyModuleTestCases"; 118 private static final String XML_BASE = 119 "<?xml version='1.0' encoding='UTF-8' standalone='no' ?>" + 120 "<?xml-stylesheet type=\"text/xsl\" href=\"compatibility_result.xsl\"?>\n" + 121 "<Result start=\"%d\" end=\"%d\" start_display=\"%s\"" + 122 "end_display=\"%s\" suite_name=\"%s\" suite_version=\"%s\" " + 123 "suite_plan=\"%s\" suite_build_number=\"%s\" report_version=\"%s\" " + 124 "devices=\"%s\" host_name=\"%s\"" + 125 "os_name=\"%s\" os_version=\"%s\" os_arch=\"%s\" java_vendor=\"%s\"" + 126 "java_version=\"%s\" reference_url=\"%s\" log_url=\"%s\"" + 127 "command_line_args=\"%s\">\n" + 128 "%s%s%s" + 129 "</Result>"; 130 private static final String XML_BUILD_INFO = 131 " <Build " + 132 BUILD_FINGERPRINT + "=\"%s\" " + 133 BUILD_ID + "=\"%s\" " + 134 BUILD_PRODUCT + "=\"%s\" " + 135 " />\n"; 136 private static final String XML_BUILD_INFO_WITH_UNALTERED_BUILD_FINGERPRINT = 137 " <Build " + 138 BUILD_FINGERPRINT + "=\"%s\" " + 139 BUILD_FINGERPRINT_UNALTERED + "=\"%s\" " + 140 BUILD_ID + "=\"%s\" " + 141 BUILD_PRODUCT + "=\"%s\" " + 142 " />\n"; 143 private static final String XML_SUMMARY = 144 " <Summary pass=\"%d\" failed=\"%d\" " + 145 "modules_done=\"1\" modules_total=\"1\" />\n"; 146 private static final String XML_MODULE = 147 " <Module name=\"%s\" abi=\"%s\" device=\"%s\" runtime=\"%s\" done=\"%s\">\n" + 148 "%s" + 149 " </Module>\n"; 150 private static final String XML_CASE = 151 " <TestCase name=\"%s\">\n" + 152 "%s" + 153 " </TestCase>\n"; 154 private static final String XML_TEST_PASS = 155 " <Test result=\"pass\" name=\"%s\"/>\n"; 156 private static final String XML_TEST_SKIP = 157 " <Test result=\"pass\" name=\"%s\" skipped=\"true\"/>\n"; 158 private static final String XML_TEST_FAIL = 159 " <Test result=\"fail\" name=\"%s\">\n" + 160 " <Failure message=\"%s\">\n" + 161 " <StackTrace>%s</StackTrace>\n" + 162 " </Failure>\n" + 163 " <BugReport>%s</BugReport>\n" + 164 " <Logcat>%s</Logcat>\n" + 165 " <Screenshot>%s</Screenshot>\n" + 166 " </Test>\n"; 167 private static final String XML_TEST_RESULT = 168 " <Test result=\"pass\" name=\"%s\">\n" + 169 " <Summary>\n" + 170 " <Metric source=\"%s\" message=\"%s\" score_type=\"%s\" score_unit=\"%s\">\n" + 171 " <Value>%s</Value>\n" + 172 " </Metric>\n" + 173 " </Summary>\n" + 174 " </Test>\n"; 175 private static final String NEW_XML_TEST_RESULT = 176 " <Test result=\"pass\" name=\"%s\">\n" 177 + " <Metric key=\"%s\">%s</Metric>\n" 178 + " </Test>\n"; 179 180 private File resultsDir = null; 181 private File resultDir = null; 182 183 @Override setUp()184 public void setUp() throws Exception { 185 resultsDir = FileUtil.createTempDir("results"); 186 resultDir = FileUtil.createTempDir("12345", resultsDir); 187 } 188 189 @Override tearDown()190 public void tearDown() throws Exception { 191 FileUtil.recursiveDelete(resultsDir); 192 } 193 testSerialization()194 public void testSerialization() throws Exception { 195 IInvocationResult result = new InvocationResult(); 196 result.setStartTime(START_MS); 197 result.setTestPlan(SUITE_PLAN); 198 result.addDeviceSerial(DEVICE_A); 199 result.addDeviceSerial(DEVICE_B); 200 result.addInvocationInfo(BUILD_FINGERPRINT, EXAMPLE_BUILD_FINGERPRINT); 201 result.addInvocationInfo(BUILD_ID, EXAMPLE_BUILD_ID); 202 result.addInvocationInfo(BUILD_PRODUCT, EXAMPLE_BUILD_PRODUCT); 203 result.addInvocationInfo(RUN_HISTORY, EXAMPLE_RUN_HISTORY); 204 Collection<InvocationResult.RunHistory> runHistories = 205 ((InvocationResult) result).getRunHistories(); 206 InvocationResult.RunHistory runHistory1 = new InvocationResult.RunHistory(); 207 runHistory1.startTime = 10000000000000L; 208 runHistory1.endTime = 10000000000001L; 209 runHistories.add(runHistory1); 210 InvocationResult.RunHistory runHistory2 = new InvocationResult.RunHistory(); 211 runHistory2.startTime = 10000000000002L; 212 runHistory2.endTime = 10000000000003L; 213 runHistories.add(runHistory2); 214 215 // Module A: test1 passes, test2 not executed 216 IModuleResult moduleA = result.getOrCreateModule(ID_A); 217 moduleA.setDone(false); 218 moduleA.addRuntime(Integer.parseInt(RUNTIME_A)); 219 ICaseResult moduleACase = moduleA.getOrCreateResult(CLASS_A); 220 ITestResult moduleATest1 = moduleACase.getOrCreateResult(METHOD_1); 221 moduleATest1.setResultStatus(TestStatus.PASS); 222 ITestResult moduleATest2 = moduleACase.getOrCreateResult(METHOD_2); 223 moduleATest2.setResultStatus(null); // not executed test 224 // Module B: test3 fails, test4 passes with report log, test5 passes with skip 225 IModuleResult moduleB = result.getOrCreateModule(ID_B); 226 moduleB.setDone(true); 227 moduleB.addRuntime(Integer.parseInt(RUNTIME_B)); 228 ICaseResult moduleBCase = moduleB.getOrCreateResult(CLASS_B); 229 ITestResult moduleBTest3 = moduleBCase.getOrCreateResult(METHOD_3); 230 moduleBTest3.setResultStatus(TestStatus.FAIL); 231 moduleBTest3.setMessage(MESSAGE); 232 moduleBTest3.setStackTrace(STACK_TRACE); 233 moduleBTest3.setBugReport(BUG_REPORT); 234 moduleBTest3.setLog(LOGCAT); 235 moduleBTest3.setScreenshot(SCREENSHOT); 236 ITestResult moduleBTest4 = moduleBCase.getOrCreateResult(METHOD_4); 237 moduleBTest4.setResultStatus(TestStatus.PASS); 238 ReportLog report = new ReportLog(); 239 ReportLog.Metric summary = new ReportLog.Metric(SUMMARY_SOURCE, SUMMARY_MESSAGE, 240 SUMMARY_VALUE, ResultType.HIGHER_BETTER, ResultUnit.SCORE); 241 report.setSummary(summary); 242 moduleBTest4.setReportLog(report); 243 ITestResult moduleBTest5 = moduleBCase.getOrCreateResult(METHOD_5); 244 moduleBTest5.skipped(); 245 246 Map<String, String> testAttributes = new HashMap<String, String>(); 247 testAttributes.put("foo1", "bar1"); 248 testAttributes.put("foo2", "bar2"); 249 // Serialize to file 250 File res = 251 ResultHandler.writeResults( 252 SUITE_NAME, 253 SUITE_VERSION, 254 SUITE_PLAN, 255 SUITE_BUILD, 256 result, 257 resultDir, 258 START_MS, 259 END_MS, 260 REFERENCE_URL, 261 LOG_URL, 262 COMMAND_LINE_ARGS, 263 testAttributes); 264 String content = FileUtil.readStringFromFile(res); 265 assertXmlContainsAttribute(content, "Result", "foo1", "bar1"); 266 assertXmlContainsAttribute(content, "Result", "foo2", "bar2"); 267 assertXmlContainsAttribute(content, "Result/Build", "run_history", EXAMPLE_RUN_HISTORY); 268 assertXmlContainsNode(content, "Result/RunHistory"); 269 assertXmlContainsAttribute(content, "Result/RunHistory/Run", "start", "10000000000000"); 270 assertXmlContainsAttribute(content, "Result/RunHistory/Run", "end", "10000000000001"); 271 assertXmlContainsAttribute(content, "Result/RunHistory/Run", "start", "10000000000002"); 272 assertXmlContainsAttribute(content, "Result/RunHistory/Run", "end", "10000000000003"); 273 274 // Parse the results and assert correctness 275 result = ResultHandler.getResultFromDir(resultDir); 276 checkResult(result, false); 277 checkRunHistory(result); 278 } 279 280 /* 281 * Test serialization for CTS Verifier since test results with test result history is only in 282 * CTS Verifier and was not parsed by suite harness. 283 */ testSerialization_whenTestResultWithTestResultHistoryWithoutParsing()284 public void testSerialization_whenTestResultWithTestResultHistoryWithoutParsing() 285 throws Exception { 286 IInvocationResult result = new InvocationResult(); 287 result.setStartTime(START_MS); 288 result.setTestPlan(SUITE_PLAN); 289 result.addDeviceSerial(DEVICE_A); 290 result.addDeviceSerial(DEVICE_B); 291 result.addInvocationInfo(BUILD_FINGERPRINT, EXAMPLE_BUILD_FINGERPRINT); 292 result.addInvocationInfo(BUILD_ID, EXAMPLE_BUILD_ID); 293 result.addInvocationInfo(BUILD_PRODUCT, EXAMPLE_BUILD_PRODUCT); 294 295 // Module A: test1 passes, test2 not executed 296 IModuleResult moduleA = result.getOrCreateModule(ID_A); 297 moduleA.setDone(false); 298 moduleA.addRuntime(Integer.parseInt(RUNTIME_A)); 299 ICaseResult moduleACase = moduleA.getOrCreateResult(CLASS_A); 300 ITestResult moduleATest1 = moduleACase.getOrCreateResult(METHOD_1); 301 moduleATest1.setResultStatus(TestStatus.PASS); 302 // Module B: test3 fails with test result history, test4 passes with report log, 303 // test5 passes with skip 304 IModuleResult moduleB = result.getOrCreateModule(ID_B); 305 moduleB.setDone(true); 306 moduleB.addRuntime(Integer.parseInt(RUNTIME_B)); 307 ICaseResult moduleBCase = moduleB.getOrCreateResult(CLASS_B); 308 Set<TestResultHistory.ExecutionRecord> executionRecords = 309 new HashSet<TestResultHistory.ExecutionRecord>(); 310 executionRecords.add( 311 new TestResultHistory.ExecutionRecord( 312 TEST_START_MS, TEST_END_MS, TEST_IS_AUTOMATED)); 313 314 ITestResult moduleBTest3 = moduleBCase.getOrCreateResult(METHOD_3); 315 moduleBTest3.setResultStatus(TestStatus.FAIL); 316 moduleBTest3.setMessage(MESSAGE); 317 moduleBTest3.setStackTrace(STACK_TRACE); 318 moduleBTest3.setBugReport(BUG_REPORT); 319 moduleBTest3.setLog(LOGCAT); 320 moduleBTest3.setScreenshot(SCREENSHOT); 321 List<TestResultHistory> resultHistories = new ArrayList<TestResultHistory>(); 322 TestResultHistory resultHistory = 323 new TestResultHistory(METHOD_3 + ":" + SUB_METHOD_3, executionRecords); 324 resultHistories.add(resultHistory); 325 moduleBTest3.setTestResultHistories(resultHistories); 326 ITestResult moduleBTest4 = moduleBCase.getOrCreateResult(METHOD_4); 327 moduleBTest4.setResultStatus(TestStatus.PASS); 328 ReportLog report = new ReportLog(); 329 ReportLog.Metric summary = 330 new ReportLog.Metric( 331 SUMMARY_SOURCE, 332 SUMMARY_MESSAGE, 333 SUMMARY_VALUE, 334 ResultType.HIGHER_BETTER, 335 ResultUnit.SCORE); 336 report.setSummary(summary); 337 moduleBTest4.setReportLog(report); 338 ITestResult moduleBTest5 = moduleBCase.getOrCreateResult(METHOD_5); 339 moduleBTest5.skipped(); 340 // For test name with bracket as suffix in CTS Verifier. 341 ITestResult moduleBTest6 = moduleBCase.getOrCreateResult(METHOD_6_BRACKETS); 342 moduleBTest6.setResultStatus(TestStatus.FAIL); 343 List<TestResultHistory> resultHistories2 = new ArrayList<TestResultHistory>(); 344 TestResultHistory resultHistory2 = 345 new TestResultHistory( 346 METHOD_6_BRACKETS + ":" + SUB_METHOD_6_BRACKETS, executionRecords); 347 resultHistories2.add(resultHistory2); 348 moduleBTest6.setTestResultHistories(resultHistories2); 349 350 // Serialize to file 351 File res = 352 ResultHandler.writeResults( 353 SUITE_NAME, 354 SUITE_VERSION, 355 SUITE_PLAN, 356 SUITE_BUILD, 357 result, 358 resultDir, 359 START_MS, 360 END_MS, 361 REFERENCE_URL, 362 LOG_URL, 363 COMMAND_LINE_ARGS, 364 null); 365 String content = FileUtil.readStringFromFile(res); 366 assertXmlContainsNode(content, "Result/Module/TestCase/Test/RunHistory"); 367 assertXmlContainsAttribute( 368 content, 369 "Result/Module/TestCase/Test/RunHistory/Run", 370 "start", 371 Long.toString(TEST_START_MS)); 372 assertXmlContainsAttribute( 373 content, 374 "Result/Module/TestCase/Test/RunHistory/Run", 375 "end", 376 Long.toString(TEST_END_MS)); 377 assertXmlContainsAttribute( 378 content, 379 "Result/Module/TestCase/Test/RunHistory/Run", 380 "isAutomated", 381 Boolean.toString(TEST_IS_AUTOMATED)); 382 checkResult(result, EXAMPLE_BUILD_FINGERPRINT, false, false); 383 } 384 testParsing()385 public void testParsing() throws Exception { 386 File resultDir = writeResultDir(resultsDir, false); 387 // Parse the results and assert correctness 388 checkResult(ResultHandler.getResultFromDir(resultDir), false); 389 } 390 testParsing_newTestFormat()391 public void testParsing_newTestFormat() throws Exception { 392 File resultDir = writeResultDir(resultsDir, true); 393 // Parse the results and assert correctness 394 checkResult(ResultHandler.getResultFromDir(resultDir), true); 395 } 396 testParsing_usesUnalteredBuildFingerprintWhenPresent()397 public void testParsing_usesUnalteredBuildFingerprintWhenPresent() throws Exception { 398 String buildInfo = String.format(XML_BUILD_INFO_WITH_UNALTERED_BUILD_FINGERPRINT, 399 EXAMPLE_BUILD_FINGERPRINT, EXAMPLE_BUILD_FINGERPRINT_UNALTERED, 400 EXAMPLE_BUILD_ID, EXAMPLE_BUILD_PRODUCT); 401 File resultDir = writeResultDir(resultsDir, buildInfo, false); 402 checkResult( 403 ResultHandler.getResultFromDir(resultDir), 404 EXAMPLE_BUILD_FINGERPRINT_UNALTERED, 405 false, 406 true); 407 } 408 testParsing_whenUnalteredBuildFingerprintIsEmpty_usesRegularBuildFingerprint()409 public void testParsing_whenUnalteredBuildFingerprintIsEmpty_usesRegularBuildFingerprint() throws Exception { 410 String buildInfo = String.format(XML_BUILD_INFO_WITH_UNALTERED_BUILD_FINGERPRINT, 411 EXAMPLE_BUILD_FINGERPRINT, "", EXAMPLE_BUILD_ID, EXAMPLE_BUILD_PRODUCT); 412 File resultDir = writeResultDir(resultsDir, buildInfo, false); 413 checkResult( 414 ResultHandler.getResultFromDir(resultDir), EXAMPLE_BUILD_FINGERPRINT, false, true); 415 } 416 testGetLightResults()417 public void testGetLightResults() throws Exception { 418 File resultDir = writeResultDir(resultsDir, false); 419 List<IInvocationResult> lightResults = ResultHandler.getLightResults(resultsDir); 420 assertEquals("Expected one result", 1, lightResults.size()); 421 IInvocationResult lightResult = lightResults.get(0); 422 checkLightResult(lightResult); 423 } 424 writeResultDir(File resultsDir, boolean newTestFormat)425 static File writeResultDir(File resultsDir, boolean newTestFormat) throws IOException { 426 String buildInfo = String.format(XML_BUILD_INFO, EXAMPLE_BUILD_FINGERPRINT, 427 EXAMPLE_BUILD_ID, EXAMPLE_BUILD_PRODUCT); 428 return writeResultDir(resultsDir, buildInfo, newTestFormat); 429 } 430 431 /* 432 * Helper to write a result to the results dir, for testing. 433 * @return the written resultDir 434 */ writeResultDir(File resultsDir, String buildInfo, boolean newTestFormat)435 static File writeResultDir(File resultsDir, String buildInfo, boolean newTestFormat) 436 throws IOException { 437 File resultDir = null; 438 FileWriter writer = null; 439 try { 440 resultDir = FileUtil.createTempDir("12345", resultsDir); 441 // Create the result file 442 File resultFile = new File(resultDir, ResultHandler.TEST_RESULT_FILE_NAME); 443 writer = new FileWriter(resultFile); 444 String summary = String.format(XML_SUMMARY, 2, 1); 445 String moduleATest = String.format(XML_TEST_PASS, METHOD_1); 446 String moduleACases = String.format(XML_CASE, CLASS_A, moduleATest); 447 String moduleA = String.format(XML_MODULE, NAME_A, ABI, DEVICE_A, RUNTIME_A, DONE_A, 448 moduleACases); 449 String moduleBTest3 = String.format(XML_TEST_FAIL, METHOD_3, MESSAGE, STACK_TRACE, 450 BUG_REPORT, LOGCAT, SCREENSHOT); 451 String moduleBTest4 = ""; 452 if (newTestFormat) { 453 moduleBTest4 = 454 String.format( 455 NEW_XML_TEST_RESULT, 456 METHOD_4, 457 SUMMARY_MESSAGE, 458 Double.toString(SUMMARY_VALUE)); 459 } else { 460 moduleBTest4 = 461 String.format( 462 XML_TEST_RESULT, 463 METHOD_4, 464 SUMMARY_SOURCE, 465 SUMMARY_MESSAGE, 466 ResultType.HIGHER_BETTER.toReportString(), 467 ResultUnit.SCORE.toReportString(), 468 Double.toString(SUMMARY_VALUE)); 469 } 470 471 String moduleBTest5 = String.format(XML_TEST_SKIP, METHOD_5); 472 String moduleBTests = String.join("", moduleBTest3, moduleBTest4, moduleBTest5); 473 String moduleBCases = String.format(XML_CASE, CLASS_B, moduleBTests); 474 String moduleB = String.format(XML_MODULE, NAME_B, ABI, DEVICE_B, RUNTIME_B, DONE_B, 475 moduleBCases); 476 String modules = String.join("", moduleA, moduleB); 477 String hostName = ""; 478 try { 479 hostName = InetAddress.getLocalHost().getHostName(); 480 } catch (UnknownHostException ignored) {} 481 String output = String.format(XML_BASE, START_MS, END_MS, START_DISPLAY, END_DISPLAY, 482 SUITE_NAME, SUITE_VERSION, SUITE_PLAN, SUITE_BUILD, REPORT_VERSION, DEVICES, 483 hostName, OS_NAME, OS_VERSION, OS_ARCH, JAVA_VENDOR, 484 JAVA_VERSION, REFERENCE_URL, LOG_URL, COMMAND_LINE_ARGS, 485 buildInfo, summary, modules); 486 writer.write(output); 487 writer.flush(); 488 } finally { 489 if (writer != null) { 490 writer.close(); 491 } 492 } 493 return resultDir; 494 } 495 checkLightResult(IInvocationResult lightResult)496 static void checkLightResult(IInvocationResult lightResult) throws Exception { 497 assertEquals("Expected 3 passes", 3, lightResult.countResults(TestStatus.PASS)); 498 assertEquals("Expected 1 failure", 1, lightResult.countResults(TestStatus.FAIL)); 499 500 Map<String, String> buildInfo = lightResult.getInvocationInfo(); 501 assertEquals("Incorrect Build ID", EXAMPLE_BUILD_ID, buildInfo.get(BUILD_ID)); 502 assertEquals("Incorrect Build Product", 503 EXAMPLE_BUILD_PRODUCT, buildInfo.get(BUILD_PRODUCT)); 504 505 Set<String> serials = lightResult.getDeviceSerials(); 506 assertTrue("Missing device", serials.contains(DEVICE_A)); 507 assertTrue("Missing device", serials.contains(DEVICE_B)); 508 assertEquals("Expected 2 devices", 2, serials.size()); 509 assertTrue("Incorrect devices", serials.contains(DEVICE_A) && serials.contains(DEVICE_B)); 510 assertEquals("Incorrect start time", START_MS, lightResult.getStartTime()); 511 assertEquals("Incorrect test plan", SUITE_PLAN, lightResult.getTestPlan()); 512 List<IModuleResult> modules = lightResult.getModules(); 513 assertEquals("Expected 1 completed module", 1, lightResult.getModuleCompleteCount()); 514 assertEquals("Expected 2 total modules", 2, modules.size()); 515 } 516 checkResult(IInvocationResult result, boolean newTestFormat)517 static void checkResult(IInvocationResult result, boolean newTestFormat) throws Exception { 518 checkResult(result, EXAMPLE_BUILD_FINGERPRINT, newTestFormat, true); 519 } 520 checkRunHistory(IInvocationResult result)521 static void checkRunHistory(IInvocationResult result) { 522 Map<String, String> buildInfo = result.getInvocationInfo(); 523 assertEquals("Incorrect run history", EXAMPLE_RUN_HISTORY, buildInfo.get(RUN_HISTORY)); 524 } 525 checkResult( IInvocationResult result, String expectedBuildFingerprint, boolean newTestFormat, boolean checkResultHistories)526 static void checkResult( 527 IInvocationResult result, String expectedBuildFingerprint, boolean newTestFormat, 528 boolean checkResultHistories) throws Exception { 529 assertEquals("Expected 3 passes", 3, result.countResults(TestStatus.PASS)); 530 if (checkResultHistories) { 531 assertEquals("Expected 1 failure", 1, result.countResults(TestStatus.FAIL)); 532 } else { 533 // CTS Verifier needs to check the condition of the test name with bracket, 534 // so adds one more test with failure result. 535 assertEquals("Expected 2 failure", 2, result.countResults(TestStatus.FAIL)); 536 } 537 538 Map<String, String> buildInfo = result.getInvocationInfo(); 539 assertEquals("Incorrect Build Fingerprint", expectedBuildFingerprint, result.getBuildFingerprint()); 540 assertEquals("Incorrect Build ID", EXAMPLE_BUILD_ID, buildInfo.get(BUILD_ID)); 541 assertEquals("Incorrect Build Product", 542 EXAMPLE_BUILD_PRODUCT, buildInfo.get(BUILD_PRODUCT)); 543 544 Set<String> serials = result.getDeviceSerials(); 545 assertTrue("Missing device", serials.contains(DEVICE_A)); 546 assertTrue("Missing device", serials.contains(DEVICE_B)); 547 assertEquals("Expected 2 devices", 2, serials.size()); 548 assertTrue("Incorrect devices", serials.contains(DEVICE_A) && serials.contains(DEVICE_B)); 549 assertEquals("Incorrect start time", START_MS, result.getStartTime()); 550 assertEquals("Incorrect test plan", SUITE_PLAN, result.getTestPlan()); 551 552 List<IModuleResult> modules = result.getModules(); 553 assertEquals("Expected 2 modules", 2, modules.size()); 554 555 IModuleResult moduleA = modules.get(0); 556 assertEquals("Expected 1 pass", 1, moduleA.countResults(TestStatus.PASS)); 557 assertEquals("Expected 0 failures", 0, moduleA.countResults(TestStatus.FAIL)); 558 assertEquals("Incorrect ABI", ABI, moduleA.getAbi()); 559 assertEquals("Incorrect name", NAME_A, moduleA.getName()); 560 assertEquals("Incorrect ID", ID_A, moduleA.getId()); 561 assertEquals("Incorrect runtime", Integer.parseInt(RUNTIME_A), moduleA.getRuntime()); 562 List<ICaseResult> moduleACases = moduleA.getResults(); 563 assertEquals("Expected 1 test case", 1, moduleACases.size()); 564 ICaseResult moduleACase = moduleACases.get(0); 565 assertEquals("Incorrect name", CLASS_A, moduleACase.getName()); 566 List<ITestResult> moduleAResults = moduleACase.getResults(); 567 assertEquals("Expected 1 result", 1, moduleAResults.size()); 568 ITestResult moduleATest1 = moduleAResults.get(0); 569 assertEquals("Incorrect name", METHOD_1, moduleATest1.getName()); 570 assertEquals("Incorrect result", TestStatus.PASS, moduleATest1.getResultStatus()); 571 assertNull("Unexpected bugreport", moduleATest1.getBugReport()); 572 assertNull("Unexpected log", moduleATest1.getLog()); 573 assertNull("Unexpected screenshot", moduleATest1.getScreenshot()); 574 assertNull("Unexpected message", moduleATest1.getMessage()); 575 assertNull("Unexpected stack trace", moduleATest1.getStackTrace()); 576 assertNull("Unexpected report", moduleATest1.getReportLog()); 577 578 IModuleResult moduleB = modules.get(1); 579 assertEquals("Expected 2 passes", 2, moduleB.countResults(TestStatus.PASS)); 580 if (checkResultHistories) { 581 assertEquals("Expected 1 failure", 1, moduleB.countResults(TestStatus.FAIL)); 582 } else { 583 // CTS Verifier needs to check the condition of the test name with bracket, 584 // so adds one more test with failure result. 585 assertEquals("Expected 2 failure", 2, moduleB.countResults(TestStatus.FAIL)); 586 } 587 assertEquals("Incorrect ABI", ABI, moduleB.getAbi()); 588 assertEquals("Incorrect name", NAME_B, moduleB.getName()); 589 assertEquals("Incorrect ID", ID_B, moduleB.getId()); 590 assertEquals("Incorrect runtime", Integer.parseInt(RUNTIME_B), moduleB.getRuntime()); 591 List<ICaseResult> moduleBCases = moduleB.getResults(); 592 assertEquals("Expected 1 test case", 1, moduleBCases.size()); 593 ICaseResult moduleBCase = moduleBCases.get(0); 594 assertEquals("Incorrect name", CLASS_B, moduleBCase.getName()); 595 List<ITestResult> moduleBResults = moduleBCase.getResults(); 596 if (checkResultHistories) { 597 assertEquals("Expected 3 results", 3, moduleBResults.size()); 598 } else { 599 // CTS Verifier needs to check the condition of the test name with bracket, 600 // so adds one more test with failure result. 601 assertEquals("Expected 4 results", 4, moduleBResults.size()); 602 } 603 ITestResult moduleBTest3 = moduleBResults.get(0); 604 assertEquals("Incorrect name", METHOD_3, moduleBTest3.getName()); 605 assertEquals("Incorrect result", TestStatus.FAIL, moduleBTest3.getResultStatus()); 606 assertEquals("Incorrect bugreport", BUG_REPORT, moduleBTest3.getBugReport()); 607 assertEquals("Incorrect log", LOGCAT, moduleBTest3.getLog()); 608 assertEquals("Incorrect screenshot", SCREENSHOT, moduleBTest3.getScreenshot()); 609 assertEquals("Incorrect message", MESSAGE, moduleBTest3.getMessage()); 610 assertEquals("Incorrect stack trace", STACK_TRACE, moduleBTest3.getStackTrace()); 611 assertNull("Unexpected report", moduleBTest3.getReportLog()); 612 List<TestResultHistory> resultHistories = moduleBTest3.getTestResultHistories(); 613 // Check if unit tests do parsing result, because tests for CTS Verifier do not parse it. 614 if (checkResultHistories) { 615 // For xTS except CTS Verifier. 616 assertNull("Unexpected test result history list", resultHistories); 617 } else { 618 // For CTS Verifier. 619 assertNotNull("Expected test result history list", resultHistories); 620 assertEquals("Expected 1 test result history", 1, resultHistories.size()); 621 for (TestResultHistory resultHistory : resultHistories) { 622 assertNotNull("Expected test result history", resultHistory); 623 assertEquals( 624 "Incorrect test name", 625 SUB_METHOD_3, 626 resultHistory.getSubTestName(METHOD_3, resultHistory.getTestName())); 627 for (TestResultHistory.ExecutionRecord execRecord : 628 resultHistory.getExecutionRecords()) { 629 assertEquals( 630 "Incorrect test start time", TEST_START_MS, execRecord.getStartTime()); 631 assertEquals("Incorrect test end time", TEST_END_MS, execRecord.getEndTime()); 632 assertEquals( 633 "Incorrect test is automated", 634 TEST_IS_AUTOMATED, 635 execRecord.getIsAutomated()); 636 } 637 } 638 } 639 ITestResult moduleBTest4 = moduleBResults.get(1); 640 assertEquals("Incorrect name", METHOD_4, moduleBTest4.getName()); 641 assertEquals("Incorrect result", TestStatus.PASS, moduleBTest4.getResultStatus()); 642 assertNull("Unexpected bugreport", moduleBTest4.getBugReport()); 643 assertNull("Unexpected log", moduleBTest4.getLog()); 644 assertNull("Unexpected screenshot", moduleBTest4.getScreenshot()); 645 assertNull("Unexpected message", moduleBTest4.getMessage()); 646 assertNull("Unexpected stack trace", moduleBTest4.getStackTrace()); 647 if (!newTestFormat) { 648 ReportLog report = moduleBTest4.getReportLog(); 649 assertNotNull("Expected report", report); 650 ReportLog.Metric summary = report.getSummary(); 651 assertNotNull("Expected report summary", summary); 652 assertEquals("Incorrect source", SUMMARY_SOURCE, summary.getSource()); 653 assertEquals("Incorrect message", SUMMARY_MESSAGE, summary.getMessage()); 654 assertEquals("Incorrect type", ResultType.HIGHER_BETTER, summary.getType()); 655 assertEquals("Incorrect unit", ResultUnit.SCORE, summary.getUnit()); 656 assertTrue( 657 "Incorrect values", 658 Arrays.equals(new double[] {SUMMARY_VALUE}, summary.getValues())); 659 } 660 ITestResult moduleBTest5 = moduleBResults.get(2); 661 assertEquals("Incorrect name", METHOD_5, moduleBTest5.getName()); 662 assertEquals("Incorrect result", TestStatus.PASS, moduleBTest5.getResultStatus()); 663 assertTrue("Expected skipped", moduleBTest5.isSkipped()); 664 assertNull("Unexpected bugreport", moduleBTest5.getBugReport()); 665 assertNull("Unexpected log", moduleBTest5.getLog()); 666 assertNull("Unexpected screenshot", moduleBTest5.getScreenshot()); 667 assertNull("Unexpected message", moduleBTest5.getMessage()); 668 assertNull("Unexpected stack trace", moduleBTest5.getStackTrace()); 669 assertNull("Unexpected report", moduleBTest5.getReportLog()); 670 if (!checkResultHistories) { 671 // CTS Verifier needs to check the condition of the test name with bracket, 672 // so adds one more test with failure result. 673 ITestResult moduleBTest6 = moduleBResults.get(3); 674 assertEquals("Incorrect name", METHOD_6_BRACKETS, moduleBTest6.getName()); 675 assertEquals("Incorrect result", TestStatus.FAIL, moduleBTest6.getResultStatus()); 676 List<TestResultHistory> resultHistories2 = moduleBTest6.getTestResultHistories(); 677 assertNotNull("Expected test result history list", resultHistories2); 678 assertEquals("Expected 1 test result history", 1, resultHistories2.size()); 679 for (TestResultHistory resultHistory : resultHistories2) { 680 assertNotNull("Expected test result history", resultHistory); 681 assertEquals( 682 "Incorrect test name", 683 SUB_METHOD_6_BRACKETS, 684 resultHistory.getSubTestName( 685 METHOD_6_BRACKETS, resultHistory.getTestName())); 686 for (TestResultHistory.ExecutionRecord execRecord : 687 resultHistory.getExecutionRecords()) { 688 assertEquals( 689 "Incorrect test start time", TEST_START_MS, execRecord.getStartTime()); 690 assertEquals("Incorrect test end time", TEST_END_MS, execRecord.getEndTime()); 691 assertEquals( 692 "Incorrect test is automated", 693 TEST_IS_AUTOMATED, 694 execRecord.getIsAutomated()); 695 } 696 } 697 } 698 } 699 700 /** Return all XML nodes that match the given xPathExpression. */ getXmlNodes(String xml, String xPathExpression)701 private NodeList getXmlNodes(String xml, String xPathExpression) 702 throws XPathExpressionException { 703 704 InputSource inputSource = new InputSource(new StringReader(xml)); 705 XPath xpath = XPathFactory.newInstance().newXPath(); 706 return (NodeList) xpath.evaluate(xPathExpression, inputSource, XPathConstants.NODESET); 707 } 708 709 /** Assert that the XML contains a node matching the given xPathExpression. */ assertXmlContainsNode(String xml, String xPathExpression)710 private NodeList assertXmlContainsNode(String xml, String xPathExpression) 711 throws XPathExpressionException { 712 NodeList nodes = getXmlNodes(xml, xPathExpression); 713 assertNotNull( 714 String.format("XML '%s' returned null for xpath '%s'.", xml, xPathExpression), 715 nodes); 716 assertTrue( 717 String.format( 718 "XML '%s' should have returned at least 1 node for xpath '%s', " 719 + "but returned %s nodes instead.", 720 xml, xPathExpression, nodes.getLength()), 721 nodes.getLength() >= 1); 722 return nodes; 723 } 724 725 /** 726 * Assert that the XML contains a node matching the given xPathExpression and that the node has 727 * a given value. 728 */ assertXmlContainsAttribute( String xml, String xPathExpression, String attributeName, String attributeValue)729 private void assertXmlContainsAttribute( 730 String xml, String xPathExpression, String attributeName, String attributeValue) 731 throws XPathExpressionException { 732 NodeList nodes = assertXmlContainsNode(xml, xPathExpression); 733 boolean found = false; 734 735 for (int i = 0; i < nodes.getLength(); i++) { 736 Element element = (Element) nodes.item(i); 737 String value = element.getAttribute(attributeName); 738 if (attributeValue.equals(value)) { 739 found = true; 740 break; 741 } 742 } 743 744 assertTrue( 745 String.format( 746 "xPath '%s' should contain attribute '%s' but does not. XML: '%s'", 747 xPathExpression, attributeName, xml), 748 found); 749 } 750 } 751
