1 /*
2  * Copyright (C) 2015 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 package com.android.compatibility.common.tradefed.testtype;
17 
18 import com.android.compatibility.common.util.AbiUtils;
19 import com.android.compatibility.common.util.TestFilter;
20 import com.android.ddmlib.Log.LogLevel;
21 import com.android.tradefed.build.IBuildInfo;
22 import com.android.tradefed.config.ConfigurationException;
23 import com.android.tradefed.config.ConfigurationFactory;
24 import com.android.tradefed.config.IConfiguration;
25 import com.android.tradefed.config.IConfigurationFactory;
26 import com.android.tradefed.log.LogUtil.CLog;
27 import com.android.tradefed.testtype.IAbi;
28 import com.android.tradefed.testtype.IBuildReceiver;
29 import com.android.tradefed.testtype.IRemoteTest;
30 import com.android.tradefed.testtype.IShardableTest;
31 import com.android.tradefed.testtype.ITestFileFilterReceiver;
32 import com.android.tradefed.testtype.ITestFilterReceiver;
33 import com.android.tradefed.util.FileUtil;
34 import com.android.tradefed.util.TimeUtil;
35 
36 import java.io.File;
37 import java.io.FilenameFilter;
38 import java.io.IOException;
39 import java.io.PrintWriter;
40 import java.util.ArrayList;
41 import java.util.Collection;
42 import java.util.Collections;
43 import java.util.Comparator;
44 import java.util.HashMap;
45 import java.util.HashSet;
46 import java.util.List;
47 import java.util.Map;
48 import java.util.Map.Entry;
49 import java.util.Set;
50 import java.util.concurrent.TimeUnit;
51 
52 /**
53  * Retrieves Compatibility test module definitions from the repository.
54  */
55 public class ModuleRepo implements IModuleRepo {
56 
57     private static final String CONFIG_EXT = ".config";
58     private static final Map<String, Integer> ENDING_MODULES = new HashMap<>();
59     static {
60         ENDING_MODULES.put("CtsMonkeyTestCases", 1);
61     }
62     private static final long SMALL_TEST = TimeUnit.MINUTES.toMillis(2); // Small tests < 2mins
63     private static final long MEDIUM_TEST = TimeUnit.MINUTES.toMillis(10); // Medium tests < 10mins
64 
65     private int mShards;
66     private int mModulesPerShard;
67     private int mSmallModulesPerShard;
68     private int mMediumModulesPerShard;
69     private int mLargeModulesPerShard;
70     private int mModuleCount = 0;
71     private Set<String> mSerials = new HashSet<>();
72     private Map<String, Set<String>> mDeviceTokens = new HashMap<>();
73     private Map<String, Map<String, String>> mTestArgs = new HashMap<>();
74     private Map<String, Map<String, String>> mModuleArgs = new HashMap<>();
75     private boolean mIncludeAll;
76     private Map<String, List<TestFilter>> mIncludeFilters = new HashMap<>();
77     private Map<String, List<TestFilter>> mExcludeFilters = new HashMap<>();
78     private IConfigurationFactory mConfigFactory = ConfigurationFactory.getInstance();
79 
80     private volatile boolean mInitialized = false;
81 
82     // Holds all the small tests waiting to be run.
83     private List<IModuleDef> mSmallModules = new ArrayList<>();
84     // Holds all the medium tests waiting to be run.
85     private List<IModuleDef> mMediumModules = new ArrayList<>();
86     // Holds all the large tests waiting to be run.
87     private List<IModuleDef> mLargeModules = new ArrayList<>();
88     // Holds all the tests with tokens waiting to be run. Meaning the DUT must have a specific token.
89     private List<IModuleDef> mTokenModules = new ArrayList<>();
90 
91     /**
92      * {@inheritDoc}
93      */
94     @Override
getNumberOfShards()95     public int getNumberOfShards() {
96         return mShards;
97     }
98 
99     /**
100      * {@inheritDoc}
101      */
102     @Override
getModulesPerShard()103     public int getModulesPerShard() {
104         return mModulesPerShard;
105     }
106 
107     /**
108      * {@inheritDoc}
109      */
110     @Override
getDeviceTokens()111     public Map<String, Set<String>> getDeviceTokens() {
112         return mDeviceTokens;
113     }
114 
115     /**
116      * A {@link FilenameFilter} to find all modules in a directory who match the given pattern.
117      */
118     public static class NameFilter implements FilenameFilter {
119 
120         private String mPattern;
121 
NameFilter(String pattern)122         public NameFilter(String pattern) {
123             mPattern = pattern;
124         }
125 
126         /**
127          * {@inheritDoc}
128          */
129         @Override
accept(File dir, String name)130         public boolean accept(File dir, String name) {
131             return name.contains(mPattern) && name.endsWith(CONFIG_EXT);
132         }
133     }
134 
135     /**
136      * {@inheritDoc}
137      */
138     @Override
getSerials()139     public Set<String> getSerials() {
140         return mSerials;
141     }
142 
143     /**
144      * {@inheritDoc}
145      */
146     @Override
getSmallModules()147     public List<IModuleDef> getSmallModules() {
148         return mSmallModules;
149     }
150 
151     /**
152      * {@inheritDoc}
153      */
154     @Override
getMediumModules()155     public List<IModuleDef> getMediumModules() {
156         return mMediumModules;
157     }
158 
159     /**
160      * {@inheritDoc}
161      */
162     @Override
getLargeModules()163     public List<IModuleDef> getLargeModules() {
164         return mLargeModules;
165     }
166 
167     /**
168      * {@inheritDoc}
169      */
170     @Override
getTokenModules()171     public List<IModuleDef> getTokenModules() {
172         return mTokenModules;
173     }
174 
175     /**
176      * {@inheritDoc}
177      */
178     @Override
getModuleIds()179     public String[] getModuleIds() {
180         Set<String> moduleIdSet = new HashSet<>();
181         for (IModuleDef moduleDef : mSmallModules) {
182             moduleIdSet.add(moduleDef.getId());
183         }
184         for (IModuleDef moduleDef : mMediumModules) {
185             moduleIdSet.add(moduleDef.getId());
186         }
187         for (IModuleDef moduleDef : mLargeModules) {
188             moduleIdSet.add(moduleDef.getId());
189         }
190         for (IModuleDef moduleDef : mTokenModules) {
191             moduleIdSet.add(moduleDef.getId());
192         }
193         return moduleIdSet.toArray(new String[moduleIdSet.size()]);
194     }
195 
196     /**
197      * {@inheritDoc}
198      */
199     @Override
isInitialized()200     public boolean isInitialized() {
201         return mInitialized;
202     }
203 
204     /**
205      * {@inheritDoc}
206      */
207     @Override
initialize(int shards, File testsDir, Set<IAbi> abis, List<String> deviceTokens, List<String> testArgs, List<String> moduleArgs, List<String> includeFilters, List<String> excludeFilters, IBuildInfo buildInfo)208     public void initialize(int shards, File testsDir, Set<IAbi> abis, List<String> deviceTokens,
209             List<String> testArgs, List<String> moduleArgs, List<String> includeFilters,
210             List<String> excludeFilters, IBuildInfo buildInfo) {
211         CLog.d("Initializing ModuleRepo\nShards:%d\nTests Dir:%s\nABIs:%s\nDevice Tokens:%s\n" +
212                 "Test Args:%s\nModule Args:%s\nIncludes:%s\nExcludes:%s",
213                 shards, testsDir.getAbsolutePath(), abis, deviceTokens, testArgs, moduleArgs,
214                 includeFilters, excludeFilters);
215         mInitialized = true;
216         mShards = shards;
217         for (String line : deviceTokens) {
218             String[] parts = line.split(":");
219             if (parts.length == 2) {
220                 String key = parts[0];
221                 String value = parts[1];
222                 Set<String> list = mDeviceTokens.get(key);
223                 if (list == null) {
224                     list = new HashSet<>();
225                     mDeviceTokens.put(key, list);
226                 }
227                 list.add(value);
228             } else {
229                 throw new IllegalArgumentException(
230                         String.format("Could not parse device token: %s", line));
231             }
232         }
233         putArgs(testArgs, mTestArgs);
234         putArgs(moduleArgs, mModuleArgs);
235         mIncludeAll = includeFilters.isEmpty();
236         // Include all the inclusions
237         addFilters(includeFilters, mIncludeFilters, abis);
238         // Exclude all the exclusions
239         addFilters(excludeFilters, mExcludeFilters, abis);
240 
241         File[] configFiles = testsDir.listFiles(new ConfigFilter());
242         if (configFiles.length == 0) {
243             throw new IllegalArgumentException(
244                     String.format("No config files found in %s", testsDir.getAbsolutePath()));
245         }
246         for (File configFile : configFiles) {
247             final String name = configFile.getName().replace(CONFIG_EXT, "");
248             final String[] pathArg = new String[] { configFile.getAbsolutePath() };
249             try {
250                 // Invokes parser to process the test module config file
251                 // Need to generate a different config for each ABI as we cannot guarantee the
252                 // configs are idempotent. This however means we parse the same file multiple times
253                 for (IAbi abi : abis) {
254                     IConfiguration config = mConfigFactory.createConfigurationFromArgs(pathArg);
255                     String id = AbiUtils.createId(abi.getName(), name);
256                     if (!shouldRunModule(id)) {
257                         // If the module should not run tests based on the state of filters,
258                         // skip this name/abi combination.
259                         continue;
260                     }
261                     {
262                         Map<String, String> args = new HashMap<>();
263                         if (mModuleArgs.containsKey(name)) {
264                             args.putAll(mModuleArgs.get(name));
265                         }
266                         if (mModuleArgs.containsKey(id)) {
267                             args.putAll(mModuleArgs.get(id));
268                         }
269                         if (args != null && args.size() > 0) {
270                             for (Entry<String, String> entry : args.entrySet()) {
271                                 config.injectOptionValue(entry.getKey(), entry.getValue());
272                             }
273                         }
274                     }
275                     List<IRemoteTest> tests = config.getTests();
276                     for (IRemoteTest test : tests) {
277                         String className = test.getClass().getName();
278                         Map<String, String> args = new HashMap<>();
279                         if (mTestArgs.containsKey(className)) {
280                             args.putAll(mTestArgs.get(className));
281                         }
282                         if (args != null && args.size() > 0) {
283                             for (Entry<String, String> entry : args.entrySet()) {
284                                 config.injectOptionValue(entry.getKey(), entry.getValue());
285                             }
286                         }
287                         addFiltersToTest(test, abi, name);
288                     }
289                     List<IRemoteTest> shardedTests = tests;
290                     if (mShards > 1) {
291                          shardedTests = splitShardableTests(tests, buildInfo);
292                     }
293                     for (IRemoteTest test : shardedTests) {
294                         if (test instanceof IBuildReceiver) {
295                             ((IBuildReceiver)test).setBuild(buildInfo);
296                         }
297                         addModuleDef(name, abi, test, pathArg);
298                     }
299                 }
300             } catch (ConfigurationException e) {
301                 throw new RuntimeException(String.format("error parsing config file: %s",
302                         configFile.getName()), e);
303             }
304         }
305         mModulesPerShard = mModuleCount / shards;
306         if (mModuleCount % shards != 0) {
307             mModulesPerShard++; // Round up
308         }
309         mSmallModulesPerShard = mSmallModules.size() / shards;
310         mMediumModulesPerShard = mMediumModules.size() / shards;
311         mLargeModulesPerShard = mLargeModules.size() / shards;
312     }
313 
splitShardableTests(List<IRemoteTest> tests, IBuildInfo buildInfo)314     private static List<IRemoteTest> splitShardableTests(List<IRemoteTest> tests,
315             IBuildInfo buildInfo) {
316         ArrayList<IRemoteTest> shardedList = new ArrayList<>(tests.size());
317         for (IRemoteTest test : tests) {
318             if (test instanceof IShardableTest) {
319                 if (test instanceof IBuildReceiver) {
320                     ((IBuildReceiver)test).setBuild(buildInfo);
321                 }
322                 shardedList.addAll(((IShardableTest)test).split());
323             } else {
324                 shardedList.add(test);
325             }
326         }
327         return shardedList;
328     }
329 
addFilters(List<String> stringFilters, Map<String, List<TestFilter>> filters, Set<IAbi> abis)330     private static void addFilters(List<String> stringFilters,
331             Map<String, List<TestFilter>> filters, Set<IAbi> abis) {
332         for (String filterString : stringFilters) {
333             TestFilter filter = TestFilter.createFrom(filterString);
334             String abi = filter.getAbi();
335             if (abi == null) {
336                 for (IAbi a : abis) {
337                     addFilter(a.getName(), filter, filters);
338                 }
339             } else {
340                 addFilter(abi, filter, filters);
341             }
342         }
343     }
344 
addFilter(String abi, TestFilter filter, Map<String, List<TestFilter>> filters)345     private static void addFilter(String abi, TestFilter filter,
346             Map<String, List<TestFilter>> filters) {
347         getFilter(filters, AbiUtils.createId(abi, filter.getName())).add(filter);
348     }
349 
getFilter(Map<String, List<TestFilter>> filters, String id)350     private static List<TestFilter> getFilter(Map<String, List<TestFilter>> filters, String id) {
351         List<TestFilter> fs = filters.get(id);
352         if (fs == null) {
353             fs = new ArrayList<>();
354             filters.put(id, fs);
355         }
356         return fs;
357     }
358 
addModuleDef(String name, IAbi abi, IRemoteTest test, String[] configPaths)359     private void addModuleDef(String name, IAbi abi, IRemoteTest test,
360             String[] configPaths) throws ConfigurationException {
361         // Invokes parser to process the test module config file
362         IConfiguration config = mConfigFactory.createConfigurationFromArgs(configPaths);
363         addModuleDef(new ModuleDef(name, abi, test, config.getTargetPreparers()));
364     }
365 
addModuleDef(IModuleDef moduleDef)366     private void addModuleDef(IModuleDef moduleDef) {
367         Set<String> tokens = moduleDef.getTokens();
368         if (tokens != null && !tokens.isEmpty()) {
369             mTokenModules.add(moduleDef);
370         } else if (moduleDef.getRuntimeHint() < SMALL_TEST) {
371             mSmallModules.add(moduleDef);
372         } else if (moduleDef.getRuntimeHint() < MEDIUM_TEST) {
373             mMediumModules.add(moduleDef);
374         } else {
375             mLargeModules.add(moduleDef);
376         }
377         mModuleCount++;
378     }
379 
addFiltersToTest(IRemoteTest test, IAbi abi, String name)380     private void addFiltersToTest(IRemoteTest test, IAbi abi, String name) {
381         String moduleId = AbiUtils.createId(abi.getName(), name);
382         if (!(test instanceof ITestFilterReceiver)) {
383             throw new IllegalArgumentException(String.format(
384                     "Test in module %s must implement ITestFilterReceiver.", moduleId));
385         }
386         List<TestFilter> mdIncludes = getFilter(mIncludeFilters, moduleId);
387         List<TestFilter> mdExcludes = getFilter(mExcludeFilters, moduleId);
388         if (!mdIncludes.isEmpty()) {
389             addTestIncludes((ITestFilterReceiver) test, mdIncludes, name);
390         }
391         if (!mdExcludes.isEmpty()) {
392             addTestExcludes((ITestFilterReceiver) test, mdExcludes, name);
393         }
394     }
395 
shouldRunModule(String moduleId)396     private boolean shouldRunModule(String moduleId) {
397         List<TestFilter> mdIncludes = getFilter(mIncludeFilters, moduleId);
398         List<TestFilter> mdExcludes = getFilter(mExcludeFilters, moduleId);
399         // if including all modules or includes exist for this module, and there are not excludes
400         // for the entire module, this module should be run.
401         return (mIncludeAll || !mdIncludes.isEmpty()) && !containsModuleExclude(mdExcludes);
402     }
403 
addTestIncludes(ITestFilterReceiver test, List<TestFilter> includes, String name)404     private void addTestIncludes(ITestFilterReceiver test, List<TestFilter> includes,
405             String name) {
406         if (test instanceof ITestFileFilterReceiver) {
407             File includeFile = createFilterFile(name, ".include", includes);
408             ((ITestFileFilterReceiver)test).setIncludeTestFile(includeFile);
409         } else {
410             // add test includes one at a time
411             for (TestFilter include : includes) {
412                 String filterTestName = include.getTest();
413                 if (filterTestName != null) {
414                     test.addIncludeFilter(filterTestName);
415                 }
416             }
417         }
418     }
419 
addTestExcludes(ITestFilterReceiver test, List<TestFilter> excludes, String name)420     private void addTestExcludes(ITestFilterReceiver test, List<TestFilter> excludes,
421             String name) {
422         if (test instanceof ITestFileFilterReceiver) {
423             File excludeFile = createFilterFile(name, ".exclude", excludes);
424             ((ITestFileFilterReceiver)test).setExcludeTestFile(excludeFile);
425         } else {
426             // add test excludes one at a time
427             for (TestFilter exclude : excludes) {
428                 test.addExcludeFilter(exclude.getTest());
429             }
430         }
431     }
432 
createFilterFile(String prefix, String suffix, List<TestFilter> filters)433     private File createFilterFile(String prefix, String suffix, List<TestFilter> filters) {
434         File filterFile = null;
435         PrintWriter out = null;
436         try {
437             filterFile = FileUtil.createTempFile(prefix, suffix);
438             out = new PrintWriter(filterFile);
439             for (TestFilter filter : filters) {
440                 String filterTest = filter.getTest();
441                 if (filterTest != null) {
442                     out.println(filterTest);
443                 }
444             }
445             out.flush();
446         } catch (IOException e) {
447             throw new RuntimeException("Failed to create filter file");
448         } finally {
449             if (out != null) {
450                 out.close();
451             }
452         }
453         filterFile.deleteOnExit();
454         return filterFile;
455     }
456 
457     /*
458      * Returns true iff one or more test filters in excludes apply to the entire module.
459      */
containsModuleExclude(Collection<TestFilter> excludes)460     private boolean containsModuleExclude(Collection<TestFilter> excludes) {
461         for (TestFilter exclude : excludes) {
462             if (exclude.getTest() == null) {
463                 return true;
464             }
465         }
466         return false;
467     }
468 
469     /**
470      * A {@link FilenameFilter} to find all the config files in a directory.
471      */
472     public static class ConfigFilter implements FilenameFilter {
473 
474         /**
475          * {@inheritDoc}
476          */
477         @Override
accept(File dir, String name)478         public boolean accept(File dir, String name) {
479             CLog.d("%s/%s", dir.getAbsolutePath(), name);
480             return name.endsWith(CONFIG_EXT);
481         }
482     }
483 
484     /**
485      * {@inheritDoc}
486      */
487     @Override
getModules(String serial)488     public synchronized List<IModuleDef> getModules(String serial) {
489         List<IModuleDef> modules = new ArrayList<>(mModulesPerShard);
490         Set<String> tokens = mDeviceTokens.get(serial);
491         getModulesWithTokens(tokens, modules);
492         getModules(modules);
493         mSerials.add(serial);
494         if (mSerials.size() == mShards) {
495             for (IModuleDef def : mTokenModules) {
496                 CLog.logAndDisplay(LogLevel.WARN,
497                         String.format("No devices found with %s, running %s on %s",
498                                 def.getTokens(), def.getId(), serial));
499                 modules.add(def);
500             }
501             // Add left over modules
502             modules.addAll(mLargeModules);
503             modules.addAll(mMediumModules);
504             modules.addAll(mSmallModules);
505         }
506         long estimatedTime = 0;
507         for (IModuleDef def : modules) {
508             estimatedTime += def.getRuntimeHint();
509         }
510         Collections.sort(modules, new ExecutionOrderComparator());
511         CLog.logAndDisplay(LogLevel.INFO, String.format(
512                 "%s running %s modules, expected to complete in %s",
513                 serial, modules.size(), TimeUtil.formatElapsedTime(estimatedTime)));
514         return modules;
515     }
516 
517     /**
518      * Iterates through the remaining tests that require tokens and if the device has all the
519      * required tokens it will queue that module to run on that device, else the module gets put
520      * back into the list.
521      */
getModulesWithTokens(Set<String> tokens, List<IModuleDef> modules)522     private void getModulesWithTokens(Set<String> tokens, List<IModuleDef> modules) {
523         if (tokens != null) {
524             List<IModuleDef> copy = mTokenModules;
525             mTokenModules = new ArrayList<>();
526             for (IModuleDef module : copy) {
527                 // If a device has all the tokens required by the module then it can run it.
528                 if (tokens.containsAll(module.getTokens())) {
529                     modules.add(module);
530                 } else {
531                     mTokenModules.add(module);
532                 }
533             }
534         }
535     }
536 
537     /**
538      * Adds count modules that do not require tokens, to run on a device.
539      */
getModules(List<IModuleDef> modules)540     private void getModules(List<IModuleDef> modules) {
541         // Take the normal share of modules unless the device already has token modules.
542         takeModule(mSmallModules, modules, mSmallModulesPerShard - modules.size());
543         takeModule(mMediumModules, modules, mMediumModulesPerShard);
544         takeModule(mLargeModules, modules, mLargeModulesPerShard);
545         // If one bucket runs out, take from any of the others.
546         boolean success = true;
547         while (success && modules.size() < mModulesPerShard) {
548             // Take modules from the buckets until it has enough, or there are no more modules.
549             success = takeModule(mSmallModules, modules, 1)
550                     || takeModule(mMediumModules, modules, 1)
551                     || takeModule(mLargeModules, modules, 1);
552         }
553     }
554 
555     /**
556      * Takes count modules from the first list and move it to the second.
557      */
takeModule( List<IModuleDef> source, List<IModuleDef> destination, int count)558     private static boolean takeModule(
559             List<IModuleDef> source, List<IModuleDef> destination, int count) {
560         if (source.isEmpty()) {
561             return false;
562         }
563         if (count > source.size()) {
564             count = source.size();
565         }
566         for (int i = 0; i < count; i++) {
567             destination.add(source.remove(source.size() - 1));// Take from the end of the arraylist.
568         }
569         return true;
570     }
571 
572     /**
573      * @return the {@link List} of modules whose name contains the given pattern.
574      */
getModuleNamesMatching(File directory, String pattern)575     public static List<String> getModuleNamesMatching(File directory, String pattern) {
576         String[] names = directory.list(new NameFilter(pattern));
577         List<String> modules = new ArrayList<String>(names.length);
578         for (String name : names) {
579             int index = name.indexOf(CONFIG_EXT);
580             if (index > 0) {
581                 String module = name.substring(0, index);
582                 if (module.equals(pattern)) {
583                     // Pattern represents a single module, just return a single-item list
584                     modules = new ArrayList<>(1);
585                     modules.add(module);
586                     return modules;
587                 }
588                 modules.add(module);
589             }
590         }
591         return modules;
592     }
593 
putArgs(List<String> args, Map<String, Map<String, String>> argsMap)594     private static void putArgs(List<String> args, Map<String, Map<String, String>> argsMap) {
595         for (String arg : args) {
596             String[] parts = arg.split(":");
597             String target = parts[0];
598             String key = parts[1];
599             String value = parts[2];
600             Map<String, String> map = argsMap.get(target);
601             if (map == null) {
602                 map = new HashMap<>();
603                 argsMap.put(target, map);
604             }
605             map.put(key, value);
606         }
607     }
608 
609     private static class ExecutionOrderComparator implements Comparator<IModuleDef> {
610 
611         @Override
compare(IModuleDef def1, IModuleDef def2)612         public int compare(IModuleDef def1, IModuleDef def2) {
613             int value1 = 0;
614             int value2 = 0;
615             if (ENDING_MODULES.containsKey(def1.getName())) {
616                 value1 = ENDING_MODULES.get(def1.getName());
617             }
618             if (ENDING_MODULES.containsKey(def2.getName())) {
619                 value2 = ENDING_MODULES.get(def2.getName());
620             }
621             if (value1 == 0 && value2 == 0) {
622                 return (int) Math.signum(def2.getRuntimeHint() - def1.getRuntimeHint());
623             }
624             return (int) Math.signum(value1 - value2);
625         }
626     }
627 }
628