1 /* 2 * Copyright (c) 2017 Google Inc. All Rights Reserved. 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); you 5 * may not use this file except in compliance with the License. You may 6 * obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 * implied. See the License for the specific language governing 14 * permissions and limitations under the License. 15 */ 16 17 package com.android.vts.util; 18 19 import com.android.vts.entity.CodeCoverageEntity; 20 import com.android.vts.entity.DeviceInfoEntity; 21 import com.android.vts.entity.ProfilingPointRunEntity; 22 import com.android.vts.entity.TestCaseRunEntity; 23 import com.android.vts.entity.TestCaseRunEntity.TestCase; 24 import com.android.vts.entity.TestEntity; 25 import com.android.vts.entity.TestRunEntity; 26 import com.android.vts.proto.VtsReportMessage.TestCaseResult; 27 import com.android.vts.util.UrlUtil.LinkDisplay; 28 import com.google.appengine.api.datastore.DatastoreService; 29 import com.google.appengine.api.datastore.DatastoreServiceFactory; 30 import com.google.appengine.api.datastore.Entity; 31 import com.google.appengine.api.datastore.Key; 32 import com.google.appengine.api.datastore.KeyFactory; 33 import com.google.appengine.api.datastore.Query; 34 import java.util.ArrayList; 35 import java.util.Arrays; 36 import java.util.HashMap; 37 import java.util.HashSet; 38 import java.util.List; 39 import java.util.Map; 40 import java.util.Set; 41 import java.util.logging.Level; 42 import java.util.logging.Logger; 43 import org.apache.commons.lang.StringUtils; 44 45 /** Helper object for describing test results data. */ 46 public class TestResults { 47 private final Logger logger = Logger.getLogger(getClass().getName()); 48 49 private List<TestRunEntity> testRuns; // list of all test runs 50 private Map<Key, List<TestCaseRunEntity>> 51 testCaseRunMap; // map from test run key to the test run information 52 private Map<Key, List<DeviceInfoEntity>> deviceInfoMap; // map from test run key to device info 53 private Map<String, Integer> testCaseNameMap; // map from test case name to its order 54 private Set<String> profilingPointNameSet; // set of profiling point names 55 56 public String testName; 57 public String[] headerRow; // row to display above the test results table 58 public String[][] timeGrid; // grid of data storing timestamps to render as dates 59 public String[][] durationGrid; // grid of data storing timestamps to render as time intervals 60 public String[][] summaryGrid; // grid of data displaying a summary of the test run 61 public String[][] resultsGrid; // grid of data displaying test case results 62 public String[] profilingPointNames; // list of profiling point names in the test run 63 public Map<String, List<String[]>> logInfoMap; // map from test run index to url/display pairs 64 public int[] totResultCounts; // array of test result counts for the tip-of-tree runs 65 public String totBuildId = ""; // build ID of tip-of-tree run 66 public long startTime = Long.MAX_VALUE; // oldest timestamp displayed in the results table 67 public long endTime = Long.MIN_VALUE; // newest timestamp displayed in the results table 68 69 // Row labels for the test time-formatted information. 70 private static final String[] TIME_INFO_NAMES = {"Test Start", "Test End"}; 71 72 // Row labels for the test duration information. 73 private static final String[] DURATION_INFO_NAMES = {"<b>Test Duration</b>"}; 74 75 // Row labels for the test summary grid. 76 private static final String[] SUMMARY_NAMES = { 77 "Total", "Passing #", "Non-Passing #", "Passing %", "Covered Lines", "Coverage %", "Links" 78 }; 79 80 // Row labels for the device summary information in the table header. 81 private static final String[] HEADER_NAMES = { 82 "<b>Stats Type \\ Device Build ID</b>", 83 "Branch", 84 "Build Target", 85 "Device", 86 "ABI Target", 87 "VTS Build ID", 88 "Hostname" 89 }; 90 91 /** 92 * Create a test results object. 93 * 94 * @param testName The name of the test. 95 */ TestResults(String testName)96 public TestResults(String testName) { 97 this.testName = testName; 98 this.testRuns = new ArrayList<>(); 99 this.deviceInfoMap = new HashMap<>(); 100 this.testCaseRunMap = new HashMap<>(); 101 this.testCaseNameMap = new HashMap<>(); 102 this.logInfoMap = new HashMap<>(); 103 this.profilingPointNameSet = new HashSet<>(); 104 } 105 106 /** 107 * Add a test run to the test results. 108 * 109 * @param testRun The Entity containing the test run information. 110 * @param testCaseRuns The collection of test case executions within the test run. 111 */ addTestRun(Entity testRun, Iterable<Entity> testCaseRuns)112 public void addTestRun(Entity testRun, Iterable<Entity> testCaseRuns) { 113 TestRunEntity testRunEntity = TestRunEntity.fromEntity(testRun); 114 if (testRunEntity == null) return; 115 if (testRunEntity.getStartTimestamp() < startTime) { 116 startTime = testRunEntity.getStartTimestamp(); 117 } 118 if (testRunEntity.getStartTimestamp() > endTime) { 119 endTime = testRunEntity.getStartTimestamp(); 120 } 121 testRuns.add(testRunEntity); 122 testCaseRunMap.put(testRun.getKey(), new ArrayList<TestCaseRunEntity>()); 123 124 // Process the test cases in the test run 125 for (Entity e : testCaseRuns) { 126 TestCaseRunEntity testCaseRunEntity = TestCaseRunEntity.fromEntity(e); 127 if (testCaseRunEntity == null) continue; 128 testCaseRunMap.get(testRun.getKey()).add(testCaseRunEntity); 129 for (TestCase testCase : testCaseRunEntity.testCases) { 130 if (!testCaseNameMap.containsKey(testCase.name)) { 131 testCaseNameMap.put(testCase.name, testCaseNameMap.size()); 132 } 133 } 134 } 135 } 136 137 /** Creates a test case breakdown of the most recent test run. */ generateToTBreakdown()138 private void generateToTBreakdown() { 139 totResultCounts = new int[TestCaseResult.values().length]; 140 if (testRuns.size() == 0) return; 141 142 TestRunEntity mostRecentRun = testRuns.get(0); 143 List<TestCaseRunEntity> testCaseResults = testCaseRunMap.get(mostRecentRun.getKey()); 144 List<DeviceInfoEntity> deviceInfos = deviceInfoMap.get(mostRecentRun.getKey()); 145 if (deviceInfos.size() > 0) { 146 DeviceInfoEntity totDevice = deviceInfos.get(0); 147 totBuildId = totDevice.getBuildId(); 148 } 149 // Count array for each test result 150 for (TestCaseRunEntity testCaseRunEntity : testCaseResults) { 151 for (TestCase testCase : testCaseRunEntity.testCases) { 152 totResultCounts[testCase.result]++; 153 } 154 } 155 } 156 157 /** 158 * Get the number of test runs observed. 159 * 160 * @return The number of test runs observed. 161 */ getSize()162 public int getSize() { 163 return testRuns.size(); 164 } 165 166 /** Fetch and process profiling point names for the set of test runs. */ processProfilingPoints()167 private void processProfilingPoints() { 168 DatastoreService datastore = DatastoreServiceFactory.getDatastoreService(); 169 Key testKey = KeyFactory.createKey(TestEntity.KIND, this.testName); 170 Query.Filter profilingFilter = 171 FilterUtil.getProfilingTimeFilter( 172 testKey, TestRunEntity.KIND, this.startTime, this.endTime); 173 Query profilingPointQuery = 174 new Query(ProfilingPointRunEntity.KIND) 175 .setAncestor(testKey) 176 .setFilter(profilingFilter) 177 .setKeysOnly(); 178 Iterable<Entity> profilingPoints = datastore.prepare(profilingPointQuery).asIterable(); 179 // Process the profiling point observations in the test run 180 for (Entity e : profilingPoints) { 181 if (e.getKey().getName() != null) { 182 profilingPointNameSet.add(e.getKey().getName()); 183 } 184 } 185 } 186 187 /** Fetch and process device information for the set of test runs. */ processDeviceInfos()188 private void processDeviceInfos() { 189 DatastoreService datastore = DatastoreServiceFactory.getDatastoreService(); 190 Key testKey = KeyFactory.createKey(TestEntity.KIND, this.testName); 191 Query.Filter deviceFilter = 192 FilterUtil.getDeviceTimeFilter( 193 testKey, TestRunEntity.KIND, this.startTime, this.endTime); 194 Query deviceQuery = 195 new Query(DeviceInfoEntity.KIND) 196 .setAncestor(testKey) 197 .setFilter(deviceFilter) 198 .setKeysOnly(); 199 List<Key> deviceGets = new ArrayList<>(); 200 for (Entity device : 201 datastore.prepare(deviceQuery).asIterable(DatastoreHelper.getLargeBatchOptions())) { 202 if (testCaseRunMap.containsKey(device.getParent())) { 203 deviceGets.add(device.getKey()); 204 } 205 } 206 Map<Key, Entity> devices = datastore.get(deviceGets); 207 for (Key key : devices.keySet()) { 208 Entity device = devices.get(key); 209 if (!testCaseRunMap.containsKey(device.getParent())) return; 210 DeviceInfoEntity deviceEntity = DeviceInfoEntity.fromEntity(device); 211 if (deviceEntity == null) return; 212 if (!deviceInfoMap.containsKey(device.getParent())) { 213 deviceInfoMap.put(device.getParent(), new ArrayList<DeviceInfoEntity>()); 214 } 215 deviceInfoMap.get(device.getParent()).add(deviceEntity); 216 } 217 } 218 219 /** Post-process the test runs to generate reports of the results. */ processReport()220 public void processReport() { 221 if (getSize() > 0) { 222 processDeviceInfos(); 223 processProfilingPoints(); 224 } 225 testRuns.sort((t1, t2) -> new Long(t2.getStartTimestamp()).compareTo(t1.getStartTimestamp())); 226 generateToTBreakdown(); 227 228 headerRow = new String[testRuns.size() + 1]; 229 headerRow[0] = StringUtils.join(HEADER_NAMES, "<br>"); 230 231 summaryGrid = new String[SUMMARY_NAMES.length][testRuns.size() + 1]; 232 for (int i = 0; i < SUMMARY_NAMES.length; i++) { 233 summaryGrid[i][0] = "<b>" + SUMMARY_NAMES[i] + "</b>"; 234 } 235 236 timeGrid = new String[TIME_INFO_NAMES.length][testRuns.size() + 1]; 237 for (int i = 0; i < TIME_INFO_NAMES.length; i++) { 238 timeGrid[i][0] = "<b>" + TIME_INFO_NAMES[i] + "</b>"; 239 } 240 241 durationGrid = new String[DURATION_INFO_NAMES.length][testRuns.size() + 1]; 242 for (int i = 0; i < DURATION_INFO_NAMES.length; i++) { 243 durationGrid[i][0] = "<b>" + DURATION_INFO_NAMES[i] + "</b>"; 244 } 245 246 resultsGrid = new String[testCaseNameMap.size()][testRuns.size() + 1]; 247 // first column for results grid 248 for (String testCaseName : testCaseNameMap.keySet()) { 249 resultsGrid[testCaseNameMap.get(testCaseName)][0] = testCaseName; 250 } 251 252 // Iterate through the test runs 253 for (int col = 0; col < testRuns.size(); col++) { 254 TestRunEntity testRun = testRuns.get(col); 255 CodeCoverageEntity codeCoverageEntity = testRun.getCodeCoverageEntity(); 256 257 // Process the device information 258 List<DeviceInfoEntity> devices = deviceInfoMap.get(testRun.getKey()); 259 List<String> buildIdList = new ArrayList<>(); 260 List<String> buildAliasList = new ArrayList<>(); 261 List<String> buildFlavorList = new ArrayList<>(); 262 List<String> productVariantList = new ArrayList<>(); 263 List<String> abiInfoList = new ArrayList<>(); 264 for (DeviceInfoEntity deviceInfoEntity : devices) { 265 buildAliasList.add(deviceInfoEntity.getBranch()); 266 buildFlavorList.add(deviceInfoEntity.getBuildFlavor()); 267 productVariantList.add(deviceInfoEntity.getProduct()); 268 buildIdList.add(deviceInfoEntity.getBuildId()); 269 String abi = ""; 270 String abiName = deviceInfoEntity.getAbiName(); 271 String abiBitness = deviceInfoEntity.getAbiBitness(); 272 if (abiName.length() > 0) { 273 abi += abiName; 274 if (abiBitness.length() > 0) { 275 abi += " (" + abiBitness + " bit)"; 276 } 277 } 278 abiInfoList.add(abi); 279 } 280 281 String buildAlias = StringUtils.join(buildAliasList, ","); 282 String buildFlavor = StringUtils.join(buildFlavorList, ","); 283 String productVariant = StringUtils.join(productVariantList, ","); 284 String buildIds = StringUtils.join(buildIdList, ","); 285 String abiInfo = StringUtils.join(abiInfoList, ","); 286 String vtsBuildId = testRun.getTestBuildId(); 287 288 int totalCount = 0; 289 int passCount = (int) testRun.getPassCount(); 290 int nonpassCount = (int) testRun.getFailCount(); 291 TestCaseResult aggregateStatus = TestCaseResult.UNKNOWN_RESULT; 292 293 long totalLineCount = 0; 294 long coveredLineCount = 0; 295 if (testRun.getHasCodeCoverage()) { 296 totalLineCount = codeCoverageEntity.getTotalLineCount(); 297 coveredLineCount = codeCoverageEntity.getCoveredLineCount(); 298 } 299 300 // Process test case results 301 for (TestCaseRunEntity testCaseEntity : testCaseRunMap.get(testRun.getKey())) { 302 // Update the aggregated test run status 303 totalCount += testCaseEntity.testCases.size(); 304 for (TestCase testCase : testCaseEntity.testCases) { 305 int result = testCase.result; 306 String name = testCase.name; 307 if (result == TestCaseResult.TEST_CASE_RESULT_PASS.getNumber()) { 308 if (aggregateStatus == TestCaseResult.UNKNOWN_RESULT) { 309 aggregateStatus = TestCaseResult.TEST_CASE_RESULT_PASS; 310 } 311 } else if (result != TestCaseResult.TEST_CASE_RESULT_SKIP.getNumber()) { 312 aggregateStatus = TestCaseResult.TEST_CASE_RESULT_FAIL; 313 } 314 315 String systraceUrl = null; 316 317 if (testCaseEntity.getSystraceUrl() != null) { 318 String url = testCaseEntity.getSystraceUrl(); 319 LinkDisplay validatedLink = UrlUtil.processUrl(url); 320 if (validatedLink != null) { 321 systraceUrl = validatedLink.url; 322 } else { 323 logger.log(Level.WARNING, "Invalid systrace URL : " + url); 324 } 325 } 326 327 int index = testCaseNameMap.get(name); 328 String classNames = "test-case-status "; 329 String glyph = ""; 330 TestCaseResult testCaseResult = TestCaseResult.valueOf(result); 331 if (testCaseResult != null) classNames += testCaseResult.toString(); 332 else classNames += TestCaseResult.UNKNOWN_RESULT.toString(); 333 334 if (systraceUrl != null) { 335 classNames += " width-1"; 336 glyph += 337 "<a href=\"" 338 + systraceUrl 339 + "\" " 340 + "class=\"waves-effect waves-light btn red right inline-btn\">" 341 + "<i class=\"material-icons inline-icon\">info_outline</i></a>"; 342 } 343 resultsGrid[index][col + 1] = 344 "<div class=\"" + classNames + "\"> </div>" + glyph; 345 } 346 } 347 String passInfo; 348 try { 349 double passPct = 350 Math.round((100 * passCount / (passCount + nonpassCount)) * 100f) / 100f; 351 passInfo = Double.toString(passPct) + "%"; 352 } catch (ArithmeticException e) { 353 passInfo = " - "; 354 } 355 356 // Process coverage metadata 357 String coverageInfo; 358 String coveragePctInfo; 359 try { 360 double coveragePct = 361 Math.round((100 * coveredLineCount / totalLineCount) * 100f) / 100f; 362 coveragePctInfo = 363 Double.toString(coveragePct) 364 + "%" 365 + "<a href=\"/show_coverage?testName=" 366 + testName 367 + "&startTime=" 368 + testRun.getStartTimestamp() 369 + "\" class=\"waves-effect waves-light btn red right inline-btn\">" 370 + "<i class=\"material-icons inline-icon\">menu</i></a>"; 371 coverageInfo = coveredLineCount + "/" + totalLineCount; 372 } catch (ArithmeticException e) { 373 coveragePctInfo = " - "; 374 coverageInfo = " - "; 375 } 376 377 // Process log information 378 String linkSummary = " - "; 379 List<String[]> linkEntries = new ArrayList<>(); 380 logInfoMap.put(Integer.toString(col), linkEntries); 381 382 if (testRun.getLogLinks() != null) { 383 for (String rawUrl : testRun.getLogLinks()) { 384 LinkDisplay validatedLink = UrlUtil.processUrl(rawUrl); 385 if (validatedLink == null) { 386 logger.log(Level.WARNING, "Invalid logging URL : " + rawUrl); 387 continue; 388 } 389 String[] logInfo = 390 new String[] { 391 validatedLink.name, 392 validatedLink.url // TODO: process the name from the URL 393 }; 394 linkEntries.add(logInfo); 395 } 396 } 397 if (linkEntries.size() > 0) { 398 linkSummary = Integer.toString(linkEntries.size()); 399 linkSummary += 400 "<i class=\"waves-effect waves-light btn red right inline-btn" 401 + " info-btn material-icons inline-icon\"" 402 + " data-col=\"" 403 + Integer.toString(col) 404 + "\"" 405 + ">launch</i>"; 406 } 407 408 String icon = "<div class='status-icon " + aggregateStatus.toString() + "'> </div>"; 409 String hostname = testRun.getHostName(); 410 411 // Populate the header row 412 headerRow[col + 1] = 413 "<span class='valign-wrapper'><b>" 414 + buildIds 415 + "</b>" 416 + icon 417 + "</span>" 418 + buildAlias 419 + "<br>" 420 + buildFlavor 421 + "<br>" 422 + productVariant 423 + "<br>" 424 + abiInfo 425 + "<br>" 426 + vtsBuildId 427 + "<br>" 428 + hostname; 429 430 // Populate the test summary grid 431 summaryGrid[0][col + 1] = Integer.toString(totalCount); 432 summaryGrid[1][col + 1] = Integer.toString(passCount); 433 summaryGrid[2][col + 1] = Integer.toString(nonpassCount); 434 summaryGrid[3][col + 1] = passInfo; 435 summaryGrid[4][col + 1] = coverageInfo; 436 summaryGrid[5][col + 1] = coveragePctInfo; 437 summaryGrid[6][col + 1] = linkSummary; 438 439 // Populate the test time info grid 440 timeGrid[0][col + 1] = Long.toString(testRun.getStartTimestamp()); 441 timeGrid[1][col + 1] = Long.toString(testRun.getEndTimestamp()); 442 443 // Populate the test duration info grid 444 durationGrid[0][col + 1] = Long.toString(testRun.getEndTimestamp() - testRun.getStartTimestamp()); 445 } 446 447 profilingPointNames = 448 profilingPointNameSet.toArray(new String[profilingPointNameSet.size()]); 449 Arrays.sort(profilingPointNames); 450 } 451 } 452