1 /* 2 * Copyright (C) 2015 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.compatibility.common.tradefed.testtype; 17 18 import com.android.compatibility.common.util.AbiUtils; 19 import com.android.compatibility.common.util.TestFilter; 20 import com.android.ddmlib.Log.LogLevel; 21 import com.android.tradefed.build.IBuildInfo; 22 import com.android.tradefed.config.ConfigurationException; 23 import com.android.tradefed.config.ConfigurationFactory; 24 import com.android.tradefed.config.IConfiguration; 25 import com.android.tradefed.config.IConfigurationFactory; 26 import com.android.tradefed.log.LogUtil.CLog; 27 import com.android.tradefed.testtype.IAbi; 28 import com.android.tradefed.testtype.IBuildReceiver; 29 import com.android.tradefed.testtype.IRemoteTest; 30 import com.android.tradefed.testtype.IShardableTest; 31 import com.android.tradefed.testtype.ITestFileFilterReceiver; 32 import com.android.tradefed.testtype.ITestFilterReceiver; 33 import com.android.tradefed.util.FileUtil; 34 import com.android.tradefed.util.TimeUtil; 35 36 import java.io.File; 37 import java.io.FilenameFilter; 38 import java.io.IOException; 39 import java.io.PrintWriter; 40 import java.util.ArrayList; 41 import java.util.Collection; 42 import java.util.Collections; 43 import java.util.Comparator; 44 import java.util.HashMap; 45 import java.util.HashSet; 46 import java.util.List; 47 import java.util.Map; 48 import java.util.Map.Entry; 49 import java.util.Set; 50 import java.util.concurrent.TimeUnit; 51 52 /** 53 * Retrieves Compatibility test module definitions from the repository. 54 */ 55 public class ModuleRepo implements IModuleRepo { 56 57 private static final String CONFIG_EXT = ".config"; 58 private static final Map<String, Integer> ENDING_MODULES = new HashMap<>(); 59 static { 60 ENDING_MODULES.put("CtsMonkeyTestCases", 1); 61 } 62 private static final long SMALL_TEST = TimeUnit.MINUTES.toMillis(2); // Small tests < 2mins 63 private static final long MEDIUM_TEST = TimeUnit.MINUTES.toMillis(10); // Medium tests < 10mins 64 65 private int mShards; 66 private int mModulesPerShard; 67 private int mSmallModulesPerShard; 68 private int mMediumModulesPerShard; 69 private int mLargeModulesPerShard; 70 private int mModuleCount = 0; 71 private Set<String> mSerials = new HashSet<>(); 72 private Map<String, Set<String>> mDeviceTokens = new HashMap<>(); 73 private Map<String, Map<String, String>> mTestArgs = new HashMap<>(); 74 private Map<String, Map<String, String>> mModuleArgs = new HashMap<>(); 75 private boolean mIncludeAll; 76 private Map<String, List<TestFilter>> mIncludeFilters = new HashMap<>(); 77 private Map<String, List<TestFilter>> mExcludeFilters = new HashMap<>(); 78 private IConfigurationFactory mConfigFactory = ConfigurationFactory.getInstance(); 79 80 private volatile boolean mInitialized = false; 81 82 // Holds all the small tests waiting to be run. 83 private List<IModuleDef> mSmallModules = new ArrayList<>(); 84 // Holds all the medium tests waiting to be run. 85 private List<IModuleDef> mMediumModules = new ArrayList<>(); 86 // Holds all the large tests waiting to be run. 87 private List<IModuleDef> mLargeModules = new ArrayList<>(); 88 // Holds all the tests with tokens waiting to be run. Meaning the DUT must have a specific token. 89 private List<IModuleDef> mTokenModules = new ArrayList<>(); 90 91 /** 92 * {@inheritDoc} 93 */ 94 @Override getNumberOfShards()95 public int getNumberOfShards() { 96 return mShards; 97 } 98 99 /** 100 * {@inheritDoc} 101 */ 102 @Override getModulesPerShard()103 public int getModulesPerShard() { 104 return mModulesPerShard; 105 } 106 107 /** 108 * {@inheritDoc} 109 */ 110 @Override getDeviceTokens()111 public Map<String, Set<String>> getDeviceTokens() { 112 return mDeviceTokens; 113 } 114 115 /** 116 * A {@link FilenameFilter} to find all modules in a directory who match the given pattern. 117 */ 118 public static class NameFilter implements FilenameFilter { 119 120 private String mPattern; 121 NameFilter(String pattern)122 public NameFilter(String pattern) { 123 mPattern = pattern; 124 } 125 126 /** 127 * {@inheritDoc} 128 */ 129 @Override accept(File dir, String name)130 public boolean accept(File dir, String name) { 131 return name.contains(mPattern) && name.endsWith(CONFIG_EXT); 132 } 133 } 134 135 /** 136 * {@inheritDoc} 137 */ 138 @Override getSerials()139 public Set<String> getSerials() { 140 return mSerials; 141 } 142 143 /** 144 * {@inheritDoc} 145 */ 146 @Override getSmallModules()147 public List<IModuleDef> getSmallModules() { 148 return mSmallModules; 149 } 150 151 /** 152 * {@inheritDoc} 153 */ 154 @Override getMediumModules()155 public List<IModuleDef> getMediumModules() { 156 return mMediumModules; 157 } 158 159 /** 160 * {@inheritDoc} 161 */ 162 @Override getLargeModules()163 public List<IModuleDef> getLargeModules() { 164 return mLargeModules; 165 } 166 167 /** 168 * {@inheritDoc} 169 */ 170 @Override getTokenModules()171 public List<IModuleDef> getTokenModules() { 172 return mTokenModules; 173 } 174 175 /** 176 * {@inheritDoc} 177 */ 178 @Override getModuleIds()179 public String[] getModuleIds() { 180 Set<String> moduleIdSet = new HashSet<>(); 181 for (IModuleDef moduleDef : mSmallModules) { 182 moduleIdSet.add(moduleDef.getId()); 183 } 184 for (IModuleDef moduleDef : mMediumModules) { 185 moduleIdSet.add(moduleDef.getId()); 186 } 187 for (IModuleDef moduleDef : mLargeModules) { 188 moduleIdSet.add(moduleDef.getId()); 189 } 190 for (IModuleDef moduleDef : mTokenModules) { 191 moduleIdSet.add(moduleDef.getId()); 192 } 193 return moduleIdSet.toArray(new String[moduleIdSet.size()]); 194 } 195 196 /** 197 * {@inheritDoc} 198 */ 199 @Override isInitialized()200 public boolean isInitialized() { 201 return mInitialized; 202 } 203 204 /** 205 * {@inheritDoc} 206 */ 207 @Override initialize(int shards, File testsDir, Set<IAbi> abis, List<String> deviceTokens, List<String> testArgs, List<String> moduleArgs, List<String> includeFilters, List<String> excludeFilters, IBuildInfo buildInfo)208 public void initialize(int shards, File testsDir, Set<IAbi> abis, List<String> deviceTokens, 209 List<String> testArgs, List<String> moduleArgs, List<String> includeFilters, 210 List<String> excludeFilters, IBuildInfo buildInfo) { 211 CLog.d("Initializing ModuleRepo\nShards:%d\nTests Dir:%s\nABIs:%s\nDevice Tokens:%s\n" + 212 "Test Args:%s\nModule Args:%s\nIncludes:%s\nExcludes:%s", 213 shards, testsDir.getAbsolutePath(), abis, deviceTokens, testArgs, moduleArgs, 214 includeFilters, excludeFilters); 215 mInitialized = true; 216 mShards = shards; 217 for (String line : deviceTokens) { 218 String[] parts = line.split(":"); 219 if (parts.length == 2) { 220 String key = parts[0]; 221 String value = parts[1]; 222 Set<String> list = mDeviceTokens.get(key); 223 if (list == null) { 224 list = new HashSet<>(); 225 mDeviceTokens.put(key, list); 226 } 227 list.add(value); 228 } else { 229 throw new IllegalArgumentException( 230 String.format("Could not parse device token: %s", line)); 231 } 232 } 233 putArgs(testArgs, mTestArgs); 234 putArgs(moduleArgs, mModuleArgs); 235 mIncludeAll = includeFilters.isEmpty(); 236 // Include all the inclusions 237 addFilters(includeFilters, mIncludeFilters, abis); 238 // Exclude all the exclusions 239 addFilters(excludeFilters, mExcludeFilters, abis); 240 241 File[] configFiles = testsDir.listFiles(new ConfigFilter()); 242 if (configFiles.length == 0) { 243 throw new IllegalArgumentException( 244 String.format("No config files found in %s", testsDir.getAbsolutePath())); 245 } 246 for (File configFile : configFiles) { 247 final String name = configFile.getName().replace(CONFIG_EXT, ""); 248 final String[] pathArg = new String[] { configFile.getAbsolutePath() }; 249 try { 250 // Invokes parser to process the test module config file 251 // Need to generate a different config for each ABI as we cannot guarantee the 252 // configs are idempotent. This however means we parse the same file multiple times 253 for (IAbi abi : abis) { 254 IConfiguration config = mConfigFactory.createConfigurationFromArgs(pathArg); 255 String id = AbiUtils.createId(abi.getName(), name); 256 if (!shouldRunModule(id)) { 257 // If the module should not run tests based on the state of filters, 258 // skip this name/abi combination. 259 continue; 260 } 261 { 262 Map<String, String> args = new HashMap<>(); 263 if (mModuleArgs.containsKey(name)) { 264 args.putAll(mModuleArgs.get(name)); 265 } 266 if (mModuleArgs.containsKey(id)) { 267 args.putAll(mModuleArgs.get(id)); 268 } 269 if (args != null && args.size() > 0) { 270 for (Entry<String, String> entry : args.entrySet()) { 271 config.injectOptionValue(entry.getKey(), entry.getValue()); 272 } 273 } 274 } 275 List<IRemoteTest> tests = config.getTests(); 276 for (IRemoteTest test : tests) { 277 String className = test.getClass().getName(); 278 Map<String, String> args = new HashMap<>(); 279 if (mTestArgs.containsKey(className)) { 280 args.putAll(mTestArgs.get(className)); 281 } 282 if (args != null && args.size() > 0) { 283 for (Entry<String, String> entry : args.entrySet()) { 284 config.injectOptionValue(entry.getKey(), entry.getValue()); 285 } 286 } 287 addFiltersToTest(test, abi, name); 288 } 289 List<IRemoteTest> shardedTests = tests; 290 if (mShards > 1) { 291 shardedTests = splitShardableTests(tests, buildInfo); 292 } 293 for (IRemoteTest test : shardedTests) { 294 if (test instanceof IBuildReceiver) { 295 ((IBuildReceiver)test).setBuild(buildInfo); 296 } 297 addModuleDef(name, abi, test, pathArg); 298 } 299 } 300 } catch (ConfigurationException e) { 301 throw new RuntimeException(String.format("error parsing config file: %s", 302 configFile.getName()), e); 303 } 304 } 305 mModulesPerShard = mModuleCount / shards; 306 if (mModuleCount % shards != 0) { 307 mModulesPerShard++; // Round up 308 } 309 mSmallModulesPerShard = mSmallModules.size() / shards; 310 mMediumModulesPerShard = mMediumModules.size() / shards; 311 mLargeModulesPerShard = mLargeModules.size() / shards; 312 } 313 splitShardableTests(List<IRemoteTest> tests, IBuildInfo buildInfo)314 private static List<IRemoteTest> splitShardableTests(List<IRemoteTest> tests, 315 IBuildInfo buildInfo) { 316 ArrayList<IRemoteTest> shardedList = new ArrayList<>(tests.size()); 317 for (IRemoteTest test : tests) { 318 if (test instanceof IShardableTest) { 319 if (test instanceof IBuildReceiver) { 320 ((IBuildReceiver)test).setBuild(buildInfo); 321 } 322 shardedList.addAll(((IShardableTest)test).split()); 323 } else { 324 shardedList.add(test); 325 } 326 } 327 return shardedList; 328 } 329 addFilters(List<String> stringFilters, Map<String, List<TestFilter>> filters, Set<IAbi> abis)330 private static void addFilters(List<String> stringFilters, 331 Map<String, List<TestFilter>> filters, Set<IAbi> abis) { 332 for (String filterString : stringFilters) { 333 TestFilter filter = TestFilter.createFrom(filterString); 334 String abi = filter.getAbi(); 335 if (abi == null) { 336 for (IAbi a : abis) { 337 addFilter(a.getName(), filter, filters); 338 } 339 } else { 340 addFilter(abi, filter, filters); 341 } 342 } 343 } 344 addFilter(String abi, TestFilter filter, Map<String, List<TestFilter>> filters)345 private static void addFilter(String abi, TestFilter filter, 346 Map<String, List<TestFilter>> filters) { 347 getFilter(filters, AbiUtils.createId(abi, filter.getName())).add(filter); 348 } 349 getFilter(Map<String, List<TestFilter>> filters, String id)350 private static List<TestFilter> getFilter(Map<String, List<TestFilter>> filters, String id) { 351 List<TestFilter> fs = filters.get(id); 352 if (fs == null) { 353 fs = new ArrayList<>(); 354 filters.put(id, fs); 355 } 356 return fs; 357 } 358 addModuleDef(String name, IAbi abi, IRemoteTest test, String[] configPaths)359 private void addModuleDef(String name, IAbi abi, IRemoteTest test, 360 String[] configPaths) throws ConfigurationException { 361 // Invokes parser to process the test module config file 362 IConfiguration config = mConfigFactory.createConfigurationFromArgs(configPaths); 363 addModuleDef(new ModuleDef(name, abi, test, config.getTargetPreparers())); 364 } 365 addModuleDef(IModuleDef moduleDef)366 private void addModuleDef(IModuleDef moduleDef) { 367 Set<String> tokens = moduleDef.getTokens(); 368 if (tokens != null && !tokens.isEmpty()) { 369 mTokenModules.add(moduleDef); 370 } else if (moduleDef.getRuntimeHint() < SMALL_TEST) { 371 mSmallModules.add(moduleDef); 372 } else if (moduleDef.getRuntimeHint() < MEDIUM_TEST) { 373 mMediumModules.add(moduleDef); 374 } else { 375 mLargeModules.add(moduleDef); 376 } 377 mModuleCount++; 378 } 379 addFiltersToTest(IRemoteTest test, IAbi abi, String name)380 private void addFiltersToTest(IRemoteTest test, IAbi abi, String name) { 381 String moduleId = AbiUtils.createId(abi.getName(), name); 382 if (!(test instanceof ITestFilterReceiver)) { 383 throw new IllegalArgumentException(String.format( 384 "Test in module %s must implement ITestFilterReceiver.", moduleId)); 385 } 386 List<TestFilter> mdIncludes = getFilter(mIncludeFilters, moduleId); 387 List<TestFilter> mdExcludes = getFilter(mExcludeFilters, moduleId); 388 if (!mdIncludes.isEmpty()) { 389 addTestIncludes((ITestFilterReceiver) test, mdIncludes, name); 390 } 391 if (!mdExcludes.isEmpty()) { 392 addTestExcludes((ITestFilterReceiver) test, mdExcludes, name); 393 } 394 } 395 shouldRunModule(String moduleId)396 private boolean shouldRunModule(String moduleId) { 397 List<TestFilter> mdIncludes = getFilter(mIncludeFilters, moduleId); 398 List<TestFilter> mdExcludes = getFilter(mExcludeFilters, moduleId); 399 // if including all modules or includes exist for this module, and there are not excludes 400 // for the entire module, this module should be run. 401 return (mIncludeAll || !mdIncludes.isEmpty()) && !containsModuleExclude(mdExcludes); 402 } 403 addTestIncludes(ITestFilterReceiver test, List<TestFilter> includes, String name)404 private void addTestIncludes(ITestFilterReceiver test, List<TestFilter> includes, 405 String name) { 406 if (test instanceof ITestFileFilterReceiver) { 407 File includeFile = createFilterFile(name, ".include", includes); 408 ((ITestFileFilterReceiver)test).setIncludeTestFile(includeFile); 409 } else { 410 // add test includes one at a time 411 for (TestFilter include : includes) { 412 String filterTestName = include.getTest(); 413 if (filterTestName != null) { 414 test.addIncludeFilter(filterTestName); 415 } 416 } 417 } 418 } 419 addTestExcludes(ITestFilterReceiver test, List<TestFilter> excludes, String name)420 private void addTestExcludes(ITestFilterReceiver test, List<TestFilter> excludes, 421 String name) { 422 if (test instanceof ITestFileFilterReceiver) { 423 File excludeFile = createFilterFile(name, ".exclude", excludes); 424 ((ITestFileFilterReceiver)test).setExcludeTestFile(excludeFile); 425 } else { 426 // add test excludes one at a time 427 for (TestFilter exclude : excludes) { 428 test.addExcludeFilter(exclude.getTest()); 429 } 430 } 431 } 432 createFilterFile(String prefix, String suffix, List<TestFilter> filters)433 private File createFilterFile(String prefix, String suffix, List<TestFilter> filters) { 434 File filterFile = null; 435 PrintWriter out = null; 436 try { 437 filterFile = FileUtil.createTempFile(prefix, suffix); 438 out = new PrintWriter(filterFile); 439 for (TestFilter filter : filters) { 440 String filterTest = filter.getTest(); 441 if (filterTest != null) { 442 out.println(filterTest); 443 } 444 } 445 out.flush(); 446 } catch (IOException e) { 447 throw new RuntimeException("Failed to create filter file"); 448 } finally { 449 if (out != null) { 450 out.close(); 451 } 452 } 453 filterFile.deleteOnExit(); 454 return filterFile; 455 } 456 457 /* 458 * Returns true iff one or more test filters in excludes apply to the entire module. 459 */ containsModuleExclude(Collection<TestFilter> excludes)460 private boolean containsModuleExclude(Collection<TestFilter> excludes) { 461 for (TestFilter exclude : excludes) { 462 if (exclude.getTest() == null) { 463 return true; 464 } 465 } 466 return false; 467 } 468 469 /** 470 * A {@link FilenameFilter} to find all the config files in a directory. 471 */ 472 public static class ConfigFilter implements FilenameFilter { 473 474 /** 475 * {@inheritDoc} 476 */ 477 @Override accept(File dir, String name)478 public boolean accept(File dir, String name) { 479 CLog.d("%s/%s", dir.getAbsolutePath(), name); 480 return name.endsWith(CONFIG_EXT); 481 } 482 } 483 484 /** 485 * {@inheritDoc} 486 */ 487 @Override getModules(String serial)488 public synchronized List<IModuleDef> getModules(String serial) { 489 List<IModuleDef> modules = new ArrayList<>(mModulesPerShard); 490 Set<String> tokens = mDeviceTokens.get(serial); 491 getModulesWithTokens(tokens, modules); 492 getModules(modules); 493 mSerials.add(serial); 494 if (mSerials.size() == mShards) { 495 for (IModuleDef def : mTokenModules) { 496 CLog.logAndDisplay(LogLevel.WARN, 497 String.format("No devices found with %s, running %s on %s", 498 def.getTokens(), def.getId(), serial)); 499 modules.add(def); 500 } 501 // Add left over modules 502 modules.addAll(mLargeModules); 503 modules.addAll(mMediumModules); 504 modules.addAll(mSmallModules); 505 } 506 long estimatedTime = 0; 507 for (IModuleDef def : modules) { 508 estimatedTime += def.getRuntimeHint(); 509 } 510 Collections.sort(modules, new ExecutionOrderComparator()); 511 CLog.logAndDisplay(LogLevel.INFO, String.format( 512 "%s running %s modules, expected to complete in %s", 513 serial, modules.size(), TimeUtil.formatElapsedTime(estimatedTime))); 514 return modules; 515 } 516 517 /** 518 * Iterates through the remaining tests that require tokens and if the device has all the 519 * required tokens it will queue that module to run on that device, else the module gets put 520 * back into the list. 521 */ getModulesWithTokens(Set<String> tokens, List<IModuleDef> modules)522 private void getModulesWithTokens(Set<String> tokens, List<IModuleDef> modules) { 523 if (tokens != null) { 524 List<IModuleDef> copy = mTokenModules; 525 mTokenModules = new ArrayList<>(); 526 for (IModuleDef module : copy) { 527 // If a device has all the tokens required by the module then it can run it. 528 if (tokens.containsAll(module.getTokens())) { 529 modules.add(module); 530 } else { 531 mTokenModules.add(module); 532 } 533 } 534 } 535 } 536 537 /** 538 * Adds count modules that do not require tokens, to run on a device. 539 */ getModules(List<IModuleDef> modules)540 private void getModules(List<IModuleDef> modules) { 541 // Take the normal share of modules unless the device already has token modules. 542 takeModule(mSmallModules, modules, mSmallModulesPerShard - modules.size()); 543 takeModule(mMediumModules, modules, mMediumModulesPerShard); 544 takeModule(mLargeModules, modules, mLargeModulesPerShard); 545 // If one bucket runs out, take from any of the others. 546 boolean success = true; 547 while (success && modules.size() < mModulesPerShard) { 548 // Take modules from the buckets until it has enough, or there are no more modules. 549 success = takeModule(mSmallModules, modules, 1) 550 || takeModule(mMediumModules, modules, 1) 551 || takeModule(mLargeModules, modules, 1); 552 } 553 } 554 555 /** 556 * Takes count modules from the first list and move it to the second. 557 */ takeModule( List<IModuleDef> source, List<IModuleDef> destination, int count)558 private static boolean takeModule( 559 List<IModuleDef> source, List<IModuleDef> destination, int count) { 560 if (source.isEmpty()) { 561 return false; 562 } 563 if (count > source.size()) { 564 count = source.size(); 565 } 566 for (int i = 0; i < count; i++) { 567 destination.add(source.remove(source.size() - 1));// Take from the end of the arraylist. 568 } 569 return true; 570 } 571 572 /** 573 * @return the {@link List} of modules whose name contains the given pattern. 574 */ getModuleNamesMatching(File directory, String pattern)575 public static List<String> getModuleNamesMatching(File directory, String pattern) { 576 String[] names = directory.list(new NameFilter(pattern)); 577 List<String> modules = new ArrayList<String>(names.length); 578 for (String name : names) { 579 int index = name.indexOf(CONFIG_EXT); 580 if (index > 0) { 581 String module = name.substring(0, index); 582 if (module.equals(pattern)) { 583 // Pattern represents a single module, just return a single-item list 584 modules = new ArrayList<>(1); 585 modules.add(module); 586 return modules; 587 } 588 modules.add(module); 589 } 590 } 591 return modules; 592 } 593 putArgs(List<String> args, Map<String, Map<String, String>> argsMap)594 private static void putArgs(List<String> args, Map<String, Map<String, String>> argsMap) { 595 for (String arg : args) { 596 String[] parts = arg.split(":"); 597 String target = parts[0]; 598 String key = parts[1]; 599 String value = parts[2]; 600 Map<String, String> map = argsMap.get(target); 601 if (map == null) { 602 map = new HashMap<>(); 603 argsMap.put(target, map); 604 } 605 map.put(key, value); 606 } 607 } 608 609 private static class ExecutionOrderComparator implements Comparator<IModuleDef> { 610 611 @Override compare(IModuleDef def1, IModuleDef def2)612 public int compare(IModuleDef def1, IModuleDef def2) { 613 int value1 = 0; 614 int value2 = 0; 615 if (ENDING_MODULES.containsKey(def1.getName())) { 616 value1 = ENDING_MODULES.get(def1.getName()); 617 } 618 if (ENDING_MODULES.containsKey(def2.getName())) { 619 value2 = ENDING_MODULES.get(def2.getName()); 620 } 621 if (value1 == 0 && value2 == 0) { 622 return (int) Math.signum(def2.getRuntimeHint() - def1.getRuntimeHint()); 623 } 624 return (int) Math.signum(value1 - value2); 625 } 626 } 627 } 628