1 /*
2  * Copyright (C) 2018 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 package com.android.tradefed.testtype.suite;
17 
18 import com.android.tradefed.build.IBuildInfo;
19 import com.android.tradefed.build.IDeviceBuildInfo;
20 import com.android.tradefed.config.IConfiguration;
21 import com.android.tradefed.config.Option;
22 import com.android.tradefed.config.Option.Importance;
23 import com.android.tradefed.config.OptionClass;
24 import com.android.tradefed.device.DeviceNotAvailableException;
25 import com.android.tradefed.log.LogUtil.CLog;
26 import com.android.tradefed.testtype.IAbi;
27 import com.android.tradefed.testtype.IRemoteTest;
28 import com.android.tradefed.util.ArrayUtil;
29 
30 import java.io.File;
31 import java.io.FileNotFoundException;
32 import java.util.ArrayList;
33 import java.util.HashMap;
34 import java.util.HashSet;
35 import java.util.LinkedHashMap;
36 import java.util.List;
37 import java.util.Map;
38 import java.util.Set;
39 
40 /** A Test for running Compatibility Test Suite with new suite system. */
41 @OptionClass(alias = "base-suite")
42 public class BaseTestSuite extends ITestSuite {
43 
44     public static final String INCLUDE_FILTER_OPTION = "include-filter";
45     public static final String EXCLUDE_FILTER_OPTION = "exclude-filter";
46     public static final String MODULE_OPTION = "module";
47     public static final String TEST_ARG_OPTION = "test-arg";
48     public static final String TEST_OPTION = "test";
49     public static final char TEST_OPTION_SHORT_NAME = 't';
50     private static final String MODULE_ARG_OPTION = "module-arg";
51 
52     @Option(
53         name = INCLUDE_FILTER_OPTION,
54         description = "the include module filters to apply.",
55         importance = Importance.ALWAYS
56     )
57     private Set<String> mIncludeFilters = new HashSet<>();
58 
59     @Option(
60         name = EXCLUDE_FILTER_OPTION,
61         description = "the exclude module filters to apply.",
62         importance = Importance.ALWAYS
63     )
64     private Set<String> mExcludeFilters = new HashSet<>();
65 
66     @Option(
67         name = MODULE_OPTION,
68         shortName = 'm',
69         description = "the test module to run. Only works for configuration in the tests dir.",
70         importance = Importance.IF_UNSET
71     )
72     private String mModuleName = null;
73 
74     @Option(
75         name = TEST_OPTION,
76         shortName = TEST_OPTION_SHORT_NAME,
77         description = "the test to run.",
78         importance = Importance.IF_UNSET
79     )
80     private String mTestName = null;
81 
82     @Option(
83         name = MODULE_ARG_OPTION,
84         description =
85                 "the arguments to pass to a module. The expected format is"
86                         + "\"<module-name>:<arg-name>:[<arg-key>:=]<arg-value>\"",
87         importance = Importance.ALWAYS
88     )
89     private List<String> mModuleArgs = new ArrayList<>();
90 
91     @Option(
92         name = TEST_ARG_OPTION,
93         description =
94                 "the arguments to pass to a test. The expected format is"
95                         + "\"<test-class>:<arg-name>:[<arg-key>:=]<arg-value>\"",
96         importance = Importance.ALWAYS
97     )
98     private List<String> mTestArgs = new ArrayList<>();
99 
100     @Option(
101         name = "run-suite-tag",
102         description =
103                 "The tag that must be run. If specified, only configurations containing the "
104                         + "matching suite tag will be able to run."
105     )
106     private String mSuiteTag = null;
107 
108     @Option(
109         name = "suite-config-prefix",
110         description = "Search only configs with given prefix for suite tags."
111     )
112     private String mSuitePrefix = null;
113 
114     @Option(
115         name = "skip-loading-config-jar",
116         description = "Whether or not to skip loading configurations from the JAR on the classpath."
117     )
118     private boolean mSkipJarLoading = false;
119 
120     @Option(
121         name = "config-patterns",
122         description =
123                 "The pattern(s) of the configurations that should be loaded from a directory."
124                         + " If none is explicitly specified, .*.xml and .*.config will be used."
125                         + " Can be repeated."
126     )
127     private List<String> mConfigPatterns = new ArrayList<>();
128 
129     private SuiteModuleLoader mModuleRepo;
130     private Map<String, List<SuiteTestFilter>> mIncludeFiltersParsed = new HashMap<>();
131     private Map<String, List<SuiteTestFilter>> mExcludeFiltersParsed = new HashMap<>();
132 
133     /** {@inheritDoc} */
134     @Override
loadTests()135     public LinkedHashMap<String, IConfiguration> loadTests() {
136         try {
137             File testsDir = getTestsDir();
138             setupFilters(testsDir);
139             Set<IAbi> abis = getAbis(getDevice());
140 
141             // Create and populate the filters here
142             SuiteModuleLoader.addFilters(mIncludeFilters, mIncludeFiltersParsed, abis);
143             SuiteModuleLoader.addFilters(mExcludeFilters, mExcludeFiltersParsed, abis);
144 
145             CLog.d(
146                     "Initializing ModuleRepo\nABIs:%s\n"
147                             + "Test Args:%s\nModule Args:%s\nIncludes:%s\nExcludes:%s",
148                     abis, mTestArgs, mModuleArgs, mIncludeFiltersParsed, mExcludeFiltersParsed);
149             mModuleRepo =
150                     createModuleLoader(
151                             mIncludeFiltersParsed, mExcludeFiltersParsed, mTestArgs, mModuleArgs);
152             // Actual loading of the configurations.
153             return loadingStrategy(abis, testsDir, mSuitePrefix, mSuiteTag);
154         } catch (DeviceNotAvailableException | FileNotFoundException e) {
155             throw new RuntimeException(e);
156         }
157     }
158 
159     /**
160      * Default loading strategy will load from the resources and the tests directory. Can be
161      * extended or replaced.
162      *
163      * @param abis The set of abis to run against.
164      * @param testsDir The tests directory.
165      * @param suitePrefix A prefix to filter the resource directory.
166      * @param suiteTag The suite tag a module should have to be included. Can be null.
167      * @return A list of loaded configuration for the suite.
168      */
loadingStrategy( Set<IAbi> abis, File testsDir, String suitePrefix, String suiteTag)169     public LinkedHashMap<String, IConfiguration> loadingStrategy(
170             Set<IAbi> abis, File testsDir, String suitePrefix, String suiteTag) {
171         LinkedHashMap<String, IConfiguration> loadedConfigs = new LinkedHashMap<>();
172         // Load configs that are part of the resources
173         if (!mSkipJarLoading) {
174             loadedConfigs.putAll(
175                     getModuleLoader().loadConfigsFromJars(abis, suitePrefix, suiteTag));
176         }
177 
178         // Load the configs that are part of the tests dir
179         if (mConfigPatterns.isEmpty()) {
180             // If no special pattern was configured, use the default configuration patterns we know
181             mConfigPatterns.add(".*\\.config");
182             mConfigPatterns.add(".*\\.xml");
183         }
184         loadedConfigs.putAll(
185                 getModuleLoader()
186                         .loadConfigsFromDirectory(
187                                 testsDir, abis, suitePrefix, suiteTag, mConfigPatterns));
188         return loadedConfigs;
189     }
190 
getTestsDir()191     public File getTestsDir() throws FileNotFoundException {
192         IBuildInfo build = getBuildInfo();
193         if (build instanceof IDeviceBuildInfo) {
194             return ((IDeviceBuildInfo) build).getTestsDir();
195         }
196         // TODO: handle multi build?
197         throw new FileNotFoundException("Could not found a tests dir folder.");
198     }
199 
200     /** {@inheritDoc} */
201     @Override
setBuild(IBuildInfo buildInfo)202     public void setBuild(IBuildInfo buildInfo) {
203         super.setBuild(buildInfo);
204     }
205 
206     /** Sets include-filters for the compatibility test */
setIncludeFilter(Set<String> includeFilters)207     public void setIncludeFilter(Set<String> includeFilters) {
208         mIncludeFilters.addAll(includeFilters);
209     }
210 
211     /** Gets a copy of include-filters for the compatibility test */
getIncludeFilter()212     protected Set<String> getIncludeFilter() {
213         return new HashSet<String>(mIncludeFilters);
214     }
215 
216     /** Sets exclude-filters for the compatibility test */
setExcludeFilter(Set<String> excludeFilters)217     public void setExcludeFilter(Set<String> excludeFilters) {
218         mExcludeFilters.addAll(excludeFilters);
219     }
220 
221     /** Returns the current {@link SuiteModuleLoader}. */
getModuleLoader()222     public SuiteModuleLoader getModuleLoader() {
223         return mModuleRepo;
224     }
225 
226     /** Adds module args */
addModuleArgs(Set<String> moduleArgs)227     public void addModuleArgs(Set<String> moduleArgs) {
228         mModuleArgs.addAll(moduleArgs);
229     }
230 
231     /**
232      * Create the {@link SuiteModuleLoader} responsible to load the {@link IConfiguration} and
233      * assign them some of the options.
234      *
235      * @param includeFiltersFormatted The formatted and parsed include filters.
236      * @param excludeFiltersFormatted The formatted and parsed exclude filters.
237      * @param testArgs the list of test ({@link IRemoteTest}) arguments.
238      * @param moduleArgs the list of module arguments.
239      * @return the created {@link SuiteModuleLoader}.
240      */
createModuleLoader( Map<String, List<SuiteTestFilter>> includeFiltersFormatted, Map<String, List<SuiteTestFilter>> excludeFiltersFormatted, List<String> testArgs, List<String> moduleArgs)241     public SuiteModuleLoader createModuleLoader(
242             Map<String, List<SuiteTestFilter>> includeFiltersFormatted,
243             Map<String, List<SuiteTestFilter>> excludeFiltersFormatted,
244             List<String> testArgs,
245             List<String> moduleArgs) {
246         return new SuiteModuleLoader(
247                 includeFiltersFormatted, excludeFiltersFormatted, testArgs, moduleArgs);
248     }
249 
250     /**
251      * Sets the include/exclude filters up based on if a module name was given.
252      *
253      * @throws FileNotFoundException if any file is not found.
254      */
setupFilters(File testsDir)255     protected void setupFilters(File testsDir) throws FileNotFoundException {
256         if (mModuleName != null) {
257             // If this option (-m / --module) is set only the matching unique module should run.
258             Set<File> modules =
259                     SuiteModuleLoader.getModuleNamesMatching(
260                             testsDir, mSuitePrefix, String.format("%s.*.config", mModuleName));
261             // If multiple modules match, do exact match.
262             if (modules.size() > 1) {
263                 Set<File> newModules = new HashSet<>();
264                 String exactModuleName = String.format("%s.config", mModuleName);
265                 for (File module : modules) {
266                     if (module.getName().equals(exactModuleName)) {
267                         newModules.add(module);
268                         modules = newModules;
269                         break;
270                     }
271                 }
272             }
273             if (modules.size() == 0) {
274                 throw new IllegalArgumentException(
275                         String.format("No modules found matching %s", mModuleName));
276             } else if (modules.size() > 1) {
277                 throw new IllegalArgumentException(
278                         String.format(
279                                 "Multiple modules found matching %s:\n%s\nWhich one did you "
280                                         + "mean?\n",
281                                 mModuleName, ArrayUtil.join("\n", modules)));
282             } else {
283                 File mod = modules.iterator().next();
284                 String moduleName = mod.getName().replace(".config", "");
285                 checkFilters(mIncludeFilters, moduleName);
286                 checkFilters(mExcludeFilters, moduleName);
287                 mIncludeFilters.add(
288                         new SuiteTestFilter(getRequestedAbi(), moduleName, mTestName).toString());
289             }
290         } else if (mTestName != null) {
291             throw new IllegalArgumentException(
292                     "Test name given without module name. Add --module <module-name>");
293         }
294     }
295 
296     /* Helper method designed to remove filters in a list not applicable to the given module */
checkFilters(Set<String> filters, String moduleName)297     private static void checkFilters(Set<String> filters, String moduleName) {
298         Set<String> cleanedFilters = new HashSet<String>();
299         for (String filter : filters) {
300             if (moduleName.equals(SuiteTestFilter.createFrom(filter).getName())) {
301                 cleanedFilters.add(filter); // Module name matches, filter passes
302             }
303         }
304         filters.clear();
305         filters.addAll(cleanedFilters);
306     }
307 }
308