1 /* 2 * Copyright (C) 2015 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.compatibility.common.tradefed.command; 17 18 import com.android.compatibility.common.tradefed.build.CompatibilityBuildHelper; 19 import com.android.compatibility.common.tradefed.build.CompatibilityBuildProvider; 20 import com.android.compatibility.common.tradefed.result.SubPlanHelper; 21 import com.android.compatibility.common.tradefed.result.suite.CertificationResultXml; 22 import com.android.compatibility.common.tradefed.testtype.ModuleRepo; 23 import com.android.compatibility.common.tradefed.testtype.suite.CompatibilityTestSuite; 24 import com.android.compatibility.common.util.ResultHandler; 25 import com.android.tradefed.build.BuildRetrievalError; 26 import com.android.tradefed.build.IBuildInfo; 27 import com.android.tradefed.command.Console; 28 import com.android.tradefed.config.ArgsOptionParser; 29 import com.android.tradefed.config.ConfigurationException; 30 import com.android.tradefed.config.ConfigurationFactory; 31 import com.android.tradefed.config.IConfiguration; 32 import com.android.tradefed.config.IConfigurationFactory; 33 import com.android.tradefed.device.DeviceNotAvailableException; 34 import com.android.tradefed.device.ITestDevice; 35 import com.android.tradefed.log.LogUtil.CLog; 36 import com.android.tradefed.result.suite.SuiteResultHolder; 37 import com.android.tradefed.testtype.Abi; 38 import com.android.tradefed.testtype.IAbi; 39 import com.android.tradefed.testtype.IRemoteTest; 40 import com.android.tradefed.testtype.IRuntimeHintProvider; 41 import com.android.tradefed.testtype.suite.TestSuiteInfo; 42 import com.android.tradefed.testtype.suite.params.ModuleParameters; 43 import com.android.tradefed.util.AbiUtils; 44 import com.android.tradefed.util.FileUtil; 45 import com.android.tradefed.util.MultiMap; 46 import com.android.tradefed.util.Pair; 47 import com.android.tradefed.util.RegexTrie; 48 import com.android.tradefed.util.TableFormatter; 49 import com.android.tradefed.util.TimeUtil; 50 import com.android.tradefed.util.VersionParser; 51 52 import com.google.common.base.Joiner; 53 54 import java.io.File; 55 import java.io.FileNotFoundException; 56 import java.io.IOException; 57 import java.io.PrintWriter; 58 import java.util.ArrayList; 59 import java.util.Arrays; 60 import java.util.Collections; 61 import java.util.Comparator; 62 import java.util.HashSet; 63 import java.util.Iterator; 64 import java.util.LinkedHashMap; 65 import java.util.LinkedHashSet; 66 import java.util.List; 67 import java.util.Map; 68 import java.util.Set; 69 70 /** 71 * An extension of Tradefed's console which adds features specific to compatibility testing. 72 */ 73 public class CompatibilityConsole extends Console { 74 75 /** 76 * Hard coded list of modules to be excluded from manual module sharding 77 * @see #splitModules(int) 78 */ 79 private final static Set<String> MODULE_SPLIT_EXCLUSIONS = new HashSet<>(); 80 static { 81 MODULE_SPLIT_EXCLUSIONS.add("CtsDeqpTestCases"); 82 } 83 private final static String ADD_PATTERN = "a(?:dd)?"; 84 private static final String LATEST_RESULT_DIR = "latest"; 85 private CompatibilityBuildHelper mBuildHelper; 86 private IBuildInfo mBuildInfo; 87 88 /** 89 * {@inheritDoc} 90 */ 91 @Override run()92 public void run() { 93 String buildNumber = TestSuiteInfo.getInstance().getBuildNumber(); 94 String versionFile = VersionParser.fetchVersion(); 95 if (versionFile != null) { 96 buildNumber = versionFile; 97 } 98 printLine( 99 String.format( 100 "Android %s %s (%s)", 101 TestSuiteInfo.getInstance().getFullName(), 102 TestSuiteInfo.getInstance().getVersion(), 103 buildNumber)); 104 printLine("Use \"help\" or \"help all\" to get more information on running commands."); 105 super.run(); 106 } 107 108 /** 109 * Adds the 'list plans', 'list modules' and 'list results' commands 110 */ 111 @Override setCustomCommands(RegexTrie<Runnable> trie, List<String> genericHelp, Map<String, String> commandHelp)112 protected void setCustomCommands(RegexTrie<Runnable> trie, List<String> genericHelp, 113 Map<String, String> commandHelp) { 114 115 genericHelp.add("Enter 'help add' for help with 'add subplan' commands"); 116 genericHelp.add("\t----- " + TestSuiteInfo.getInstance().getFullName() + " usage ----- "); 117 genericHelp.add("Usage: run <plan> [--module <module name>] [ options ]"); 118 genericHelp.add("Example: run cts --module CtsGestureTestCases --bugreport-on-failure"); 119 genericHelp.add(""); 120 121 trie.put(new Runnable() { 122 @Override 123 public void run() { 124 listPlans(); 125 } 126 }, LIST_PATTERN, "p(?:lans)?"); 127 trie.put( 128 new Runnable() { 129 @Override 130 public void run() { 131 listModules(null); 132 } 133 }, 134 LIST_PATTERN, 135 "m(?:odules)?"); 136 trie.put( 137 new ArgRunnable<CaptureList>() { 138 @Override 139 public void run(CaptureList args) { 140 String parameter = args.get(2).get(0); 141 listModules(parameter); 142 } 143 }, 144 LIST_PATTERN, 145 "m(?:odules)?", 146 "(.*)"); 147 trie.put(new Runnable() { 148 @Override 149 public void run() { 150 listResults(); 151 } 152 }, LIST_PATTERN, "r(?:esults)?"); 153 trie.put(new Runnable() { 154 @Override 155 public void run() { 156 listSubPlans(); 157 } 158 }, LIST_PATTERN, "s(?:ubplans)?"); 159 trie.put(new ArgRunnable<CaptureList>() { 160 @Override 161 public void run(CaptureList args) { 162 // Skip 2 tokens to get past split and modules pattern 163 String arg = args.get(2).get(0); 164 int shards = Integer.parseInt(arg); 165 if (shards <= 1) { 166 printLine("number of shards should be more than 1"); 167 return; 168 } 169 splitModules(shards); 170 } 171 }, "split", "m(?:odules)?", "(\\d+)"); 172 trie.put(new ArgRunnable<CaptureList>() { 173 @Override 174 public void run(CaptureList args) { 175 // Skip 2 tokens to get past "add" and "subplan" 176 String[] flatArgs = new String[args.size() - 2]; 177 for (int i = 2; i < args.size(); i++) { 178 flatArgs[i - 2] = args.get(i).get(0); 179 } 180 addSubPlan(flatArgs); 181 } 182 }, ADD_PATTERN, "s(?:ubplan)?", null); 183 trie.put(new ArgRunnable<CaptureList>() { 184 @Override 185 public void run(CaptureList args) { 186 printLine("'add subplan' requires parameters, use 'help add' to get more details"); 187 } 188 }, ADD_PATTERN, "s(?:ubplan)?"); 189 trie.put(new Runnable() { 190 @Override 191 public void run() { 192 printLine(String.format("Android %s %s (%s)", 193 TestSuiteInfo.getInstance().getFullName(), 194 TestSuiteInfo.getInstance().getVersion(), 195 TestSuiteInfo.getInstance().getBuildNumber())); 196 } 197 }, "version"); // override tradefed 'version' command to print test suite name and version 198 199 // find existing help for 'LIST_PATTERN' commands, and append these commands help 200 String listHelp = commandHelp.get(LIST_PATTERN); 201 if (listHelp == null) { 202 // no help? Unexpected, but soldier on 203 listHelp = new String(); 204 } 205 String combinedHelp = 206 listHelp 207 + LINE_SEPARATOR 208 + "\t----- " 209 + TestSuiteInfo.getInstance().getFullName() 210 + " specific options ----- " 211 + LINE_SEPARATOR 212 + "\tp[lans] List all plans available" 213 + LINE_SEPARATOR 214 + "\tm[odules] List all modules available" 215 + LINE_SEPARATOR 216 + String.format( 217 "\tm[odules] [module parameter] List all modules matching the " 218 + "parameter. (available params: %s)", 219 Arrays.asList(ModuleParameters.values())) 220 + LINE_SEPARATOR 221 + "\tr[esults] List all results" 222 + LINE_SEPARATOR; 223 commandHelp.put(LIST_PATTERN, combinedHelp); 224 225 // Update existing RUN_PATTERN with CTS specific extra run possibilities. 226 String runHelp = commandHelp.get(RUN_PATTERN); 227 if (runHelp == null) { 228 runHelp = new String(); 229 } 230 String combinedRunHelp = runHelp + 231 LINE_SEPARATOR + 232 "\t----- " + TestSuiteInfo.getInstance().getFullName() 233 + " specific options ----- " + LINE_SEPARATOR + 234 "\t<plan> --module/-m <module> Run a test module" + LINE_SEPARATOR + 235 "\t<plan> --module/-m <module> --test/-t <test_name> Run a specific test from" + 236 " the module. Test name can be <package>.<class>, <package>.<class>#<method> or " 237 + "<native_binary_name>" + LINE_SEPARATOR + 238 "\t\tAvailable Options:" + LINE_SEPARATOR + 239 "\t\t\t--serial/-s <device_id>: The device to run the test on" + LINE_SEPARATOR + 240 "\t\t\t--abi/-a <abi> : The ABI to run the test against" + LINE_SEPARATOR + 241 "\t\t\t--logcat-on-failure : Capture logcat when a test fails" 242 + LINE_SEPARATOR + 243 "\t\t\t--bugreport-on-failure : Capture a bugreport when a test fails" 244 + LINE_SEPARATOR + 245 "\t\t\t--screenshot-on-failure: Capture a screenshot when a test fails" 246 + LINE_SEPARATOR + 247 "\t\t\t--shard-count <shards>: Shards a run into the given number of independent " + 248 "chunks, to run on multiple devices in parallel." + LINE_SEPARATOR + 249 "\t ----- In order to retry a previous run -----" + LINE_SEPARATOR + 250 "\tretry --retry <session id to retry> [--retry-type <FAILED | NOT_EXECUTED>]" 251 + LINE_SEPARATOR + 252 "\t\tWithout --retry-type, retry will run both FAIL and NOT_EXECUTED tests" 253 + LINE_SEPARATOR; 254 commandHelp.put(RUN_PATTERN, combinedRunHelp); 255 256 commandHelp.put(ADD_PATTERN, String.format( 257 "%s help:" + LINE_SEPARATOR + 258 "\tadd s[ubplan]: create a subplan from a previous session" + LINE_SEPARATOR + 259 "\t\tAvailable Options:" + LINE_SEPARATOR + 260 "\t\t\t--session <session_id>: The session used to create a subplan" 261 + LINE_SEPARATOR + 262 "\t\t\t--name/-n <subplan_name>: The name of the new subplan" + LINE_SEPARATOR + 263 "\t\t\t--result-type <status>: Which results to include in the subplan. " 264 + "One of passed, failed, not_executed. Repeatable" + LINE_SEPARATOR, 265 ADD_PATTERN)); 266 } 267 268 /** 269 * {@inheritDoc} 270 */ 271 @Override getConsolePrompt()272 protected String getConsolePrompt() { 273 return String.format("%s-tf > ", TestSuiteInfo.getInstance().getName().toLowerCase()); 274 } 275 276 /** 277 * List all the modules available in the suite, if a specific parameter is requested, only 278 * display that one. 279 * 280 * @param moduleParameter The parameter requested to be displayed. Null if all should be shown. 281 */ listModules(String moduleParameter)282 private void listModules(String moduleParameter) { 283 CompatibilityTestSuite test = new CompatibilityTestSuite() { 284 @Override 285 public Set<IAbi> getAbis(ITestDevice device) throws DeviceNotAvailableException { 286 Set<String> abiStrings = getAbisForBuildTargetArch(); 287 Set<IAbi> abis = new LinkedHashSet<>(); 288 for (String abi : abiStrings) { 289 if (AbiUtils.isAbiSupportedByCompatibility(abi)) { 290 abis.add(new Abi(abi, AbiUtils.getBitness(abi))); 291 } 292 } 293 return abis; 294 } 295 }; 296 if (getBuild() != null) { 297 test.setEnableParameterizedModules(true); 298 test.setEnableOptionalParameterizedModules(true); 299 if (moduleParameter != null) { 300 test.setModuleParameter(ModuleParameters.valueOf(moduleParameter.toUpperCase())); 301 } 302 test.setBuild(getBuild()); 303 LinkedHashMap<String, IConfiguration> configs = test.loadTests(); 304 printLine(String.format("%s", Joiner.on("\n").join(configs.keySet()))); 305 } else { 306 printLine("Error fetching information about modules."); 307 } 308 } 309 listPlans()310 private void listPlans() { 311 printLine("Available plans include:"); 312 ConfigurationFactory.getInstance().printHelp(System.out); 313 } 314 splitModules(int shards)315 private void splitModules(int shards) { 316 File[] files = null; 317 try { 318 files = getBuildHelper().getTestsDir().listFiles(new ModuleRepo.ConfigFilter()); 319 } catch (FileNotFoundException e) { 320 printLine(e.getMessage()); 321 e.printStackTrace(); 322 } 323 // parse through all config files to get runtime hints 324 if (files != null && files.length > 0) { 325 IConfigurationFactory configFactory = ConfigurationFactory.getInstance(); 326 List<Pair<String, Long>> moduleRuntime = new ArrayList<>(); 327 // parse through all config files to calculate module execution time 328 for (File file : files) { 329 IConfiguration config = null; 330 String moduleName = file.getName().split("\\.")[0]; 331 if (MODULE_SPLIT_EXCLUSIONS.contains(moduleName)) { 332 continue; 333 } 334 try { 335 config = configFactory.createConfigurationFromArgs(new String[]{ 336 file.getAbsolutePath(), 337 }); 338 } catch (ConfigurationException ce) { 339 printLine("Error loading config file: " + file.getAbsolutePath()); 340 CLog.e(ce); 341 continue; 342 } 343 long runtime = 0; 344 for (IRemoteTest test : config.getTests()) { 345 if (test instanceof IRuntimeHintProvider) { 346 runtime += ((IRuntimeHintProvider) test).getRuntimeHint(); 347 } else { 348 CLog.w("Using default 1m runtime estimation for test type %s", 349 test.getClass().getSimpleName()); 350 runtime += 60 * 1000; 351 } 352 } 353 moduleRuntime.add(new Pair<String, Long>(moduleName, runtime)); 354 } 355 // sort list modules in descending order of runtime hint 356 Collections.sort(moduleRuntime, new Comparator<Pair<String, Long>>() { 357 @Override 358 public int compare(Pair<String, Long> o1, Pair<String, Long> o2) { 359 return o2.second.compareTo(o1.second); 360 } 361 }); 362 // partition list of modules based on the runtime hint 363 List<List<Pair<String, Long>>> splittedModules = new ArrayList<>(); 364 for (int i = 0; i < shards; i++) { 365 splittedModules.add(new ArrayList<>()); 366 } 367 int shardIndex = 0; 368 int increment = 1; 369 long[] shardTimes = new long[shards]; 370 // go through the sorted list, distribute modules into shards in zig-zag pattern to get 371 // an even execution time among shards 372 for (Pair<String, Long> module : moduleRuntime) { 373 splittedModules.get(shardIndex).add(module); 374 // also collect total runtime per shard 375 shardTimes[shardIndex] += module.second; 376 shardIndex += increment; 377 // zig-zagging: first distribute modules from shard 0 to N, then N down to 0, repeat 378 if (shardIndex == shards) { 379 increment = -1; 380 shardIndex = shards - 1; 381 } 382 if (shardIndex == -1) { 383 increment = 1; 384 shardIndex = 0; 385 } 386 } 387 shardIndex = 0; 388 // print the final shared lists 389 for (List<Pair<String, Long>> shardedModules : splittedModules) { 390 StringBuilder lineBuffer = new StringBuilder(); 391 lineBuffer.append(String.format("shard #%d (%s):", 392 shardIndex, TimeUtil.formatElapsedTime(shardTimes[shardIndex]))); 393 Iterator<Pair<String, Long>> itr = shardedModules.iterator(); 394 lineBuffer.append(itr.next().first); 395 while (itr.hasNext()) { 396 lineBuffer.append(','); 397 lineBuffer.append(itr.next().first); 398 } 399 shardIndex++; 400 printLine(lineBuffer.toString()); 401 } 402 } else { 403 printLine("No modules found"); 404 } 405 } 406 listResults()407 private void listResults() { 408 TableFormatter tableFormatter = new TableFormatter(); 409 List<List<String>> table = new ArrayList<>(); 410 411 List<File> resultDirs = null; 412 Map<SuiteResultHolder, File> holders = new LinkedHashMap<>(); 413 try { 414 resultDirs = getResults(getBuildHelper().getResultsDir()); 415 } catch (FileNotFoundException e) { 416 throw new RuntimeException("Error while parsing results directory", e); 417 } 418 CertificationResultXml xmlParser = new CertificationResultXml(); 419 for (File resultDir : resultDirs) { 420 if (LATEST_RESULT_DIR.equals(resultDir.getName())) { 421 continue; 422 } 423 try { 424 holders.put(xmlParser.parseResults(resultDir, true), resultDir); 425 } catch (IOException e) { 426 e.printStackTrace(); 427 } 428 } 429 430 if (holders.isEmpty()) { 431 printLine(String.format("No results found")); 432 return; 433 } 434 int i = 0; 435 for (SuiteResultHolder holder : holders.keySet()) { 436 String moduleProgress = String.format("%d of %d", 437 holder.completeModules, holder.totalModules); 438 439 table.add( 440 Arrays.asList( 441 Integer.toString(i), 442 Long.toString(holder.passedTests), 443 Long.toString(holder.failedTests), 444 moduleProgress, 445 holders.get(holder).getName(), 446 holder.context 447 .getAttributes() 448 .get(CertificationResultXml.SUITE_PLAN_ATTR) 449 .get(0), 450 Joiner.on(", ").join(holder.context.getShardsSerials().values()), 451 printAttributes(holder.context.getAttributes(), "build_id"), 452 printAttributes(holder.context.getAttributes(), "build_product"))); 453 i++; 454 } 455 456 // add the table header to the beginning of the list 457 table.add(0, Arrays.asList("Session", "Pass", "Fail", "Modules Complete", 458 "Result Directory", "Test Plan", "Device serial(s)", "Build ID", "Product")); 459 tableFormatter.displayTable(table, new PrintWriter(System.out, true)); 460 } 461 printAttributes(MultiMap<String, String> map, String key)462 private String printAttributes(MultiMap<String, String> map, String key) { 463 if (map.get(key) == null) { 464 return "unknown"; 465 } 466 return map.get(key).get(0); 467 } 468 469 /** 470 * Returns the list of all results directories. 471 */ getResults(File resultsDir)472 private List<File> getResults(File resultsDir) { 473 return ResultHandler.getResultDirectories(resultsDir); 474 } 475 listSubPlans()476 private void listSubPlans() { 477 File[] files = null; 478 try { 479 files = getBuildHelper().getSubPlansDir().listFiles(); 480 } catch (FileNotFoundException e) { 481 printLine(e.getMessage()); 482 e.printStackTrace(); 483 } 484 if (files != null && files.length > 0) { 485 List<String> subPlans = new ArrayList<>(); 486 for (File subPlanFile : files) { 487 subPlans.add(FileUtil.getBaseName(subPlanFile.getName())); 488 } 489 Collections.sort(subPlans); 490 for (String subPlan : subPlans) { 491 printLine(subPlan); 492 } 493 } else { 494 printLine("No subplans found"); 495 } 496 } 497 addSubPlan(String[] flatArgs)498 private void addSubPlan(String[] flatArgs) { 499 SubPlanHelper creator = new SubPlanHelper(); 500 try { 501 ArgsOptionParser optionParser = new ArgsOptionParser(creator); 502 optionParser.parse(Arrays.asList(flatArgs)); 503 creator.createAndSerializeSubPlan(getBuildHelper()); 504 } catch (ConfigurationException e) { 505 printLine("Error: " + e.getMessage()); 506 printLine(ArgsOptionParser.getOptionHelp(false, creator)); 507 } 508 509 } 510 getBuildHelper()511 private CompatibilityBuildHelper getBuildHelper() { 512 if (mBuildHelper == null) { 513 IBuildInfo build = getBuild(); 514 if (build == null) { 515 return null; 516 } 517 mBuildHelper = new CompatibilityBuildHelper(build); 518 } 519 return mBuildHelper; 520 } 521 getBuild()522 private IBuildInfo getBuild() { 523 if (mBuildInfo == null) { 524 try { 525 CompatibilityBuildProvider buildProvider = new CompatibilityBuildProvider(); 526 mBuildInfo = buildProvider.getBuild(); 527 } catch (BuildRetrievalError e) { 528 e.printStackTrace(); 529 } 530 } 531 return mBuildInfo; 532 } 533 main(String[] args)534 public static void main(String[] args) throws InterruptedException, ConfigurationException { 535 Console console = new CompatibilityConsole(); 536 Console.startConsole(console, args); 537 } 538 } 539