1 /*
2  * Copyright (C) 2017 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 package com.android.compatibility.common.tradefed.loading;
17 
18 import static org.junit.Assert.assertTrue;
19 import static org.junit.Assert.fail;
20 
21 import com.android.compatibility.common.tradefed.build.CompatibilityBuildHelper;
22 import com.android.tradefed.build.FolderBuildInfo;
23 import com.android.tradefed.config.ConfigurationDescriptor;
24 import com.android.tradefed.config.ConfigurationException;
25 import com.android.tradefed.config.ConfigurationFactory;
26 import com.android.tradefed.config.IConfiguration;
27 import com.android.tradefed.invoker.ExecutionFiles.FilesKey;
28 import com.android.tradefed.invoker.InvocationContext;
29 import com.android.tradefed.invoker.TestInformation;
30 import com.android.tradefed.testtype.suite.ITestSuite;
31 import com.android.tradefed.testtype.suite.TestSuiteInfo;
32 import com.android.tradefed.testtype.suite.params.ModuleParameters;
33 import com.android.tradefed.util.FileUtil;
34 
35 import com.google.common.base.Strings;
36 
37 import org.junit.Assert;
38 import org.junit.Test;
39 import org.junit.runner.RunWith;
40 import org.junit.runners.JUnit4;
41 
42 import java.io.File;
43 import java.util.ArrayList;
44 import java.util.Arrays;
45 import java.util.HashMap;
46 import java.util.HashSet;
47 import java.util.List;
48 import java.util.Map;
49 import java.util.Set;
50 
51 /**
52  * Test CTS specific config requirements.
53  */
54 @RunWith(JUnit4.class)
55 public class CtsConfigLoadingTest {
56 
57     private static final String METADATA_COMPONENT = "component";
58     private static final Set<String> KNOWN_COMPONENTS =
59             new HashSet<>(
60                     Arrays.asList(
61                             // modifications to the list below must be reviewed
62                             "abuse",
63                             "adservices",
64                             "art",
65                             "auth",
66                             "auto",
67                             "autofill",
68                             "backup",
69                             "bionic",
70                             "bluetooth",
71                             "camera",
72                             "contentcapture",
73                             "deviceinfo",
74                             "deqp",
75                             "devtools",
76                             "framework",
77                             "graphics",
78                             "hdmi",
79                             "inputmethod",
80                             "libcore",
81                             "libnativehelper",
82                             "location",
83                             "media",
84                             "metrics",
85                             "misc",
86                             "mocking",
87                             "networking",
88                             "neuralnetworks",
89                             "nfc",
90                             "packagemanager",
91                             "permissions",
92                             "print",
93                             "renderscript",
94                             "security",
95                             "statsd",
96                             "systems",
97                             "sysui",
98                             "telecom",
99                             "threadnetwork",
100                             "tv",
101                             "uitoolkit",
102                             "uwb",
103                             "vr",
104                             "webview",
105                             "wifi"));
106     private static final Set<String> KNOWN_MISC_MODULES =
107             new HashSet<>(
108                     Arrays.asList(
109                             // Modifications to the list below must be approved by someone in
110                             // test/suite_harness/OWNERS.
111                             "CtsSliceTestCases.config",
112                             "CtsSampleDeviceTestCases.config",
113                             "CtsUsbTests.config",
114                             "CtsGpuToolsHostTestCases.config",
115                             "CtsEdiHostTestCases.config",
116                             "CtsClassLoaderFactoryPathClassLoaderTestCases.config",
117                             "CtsSampleHostTestCases.config",
118                             "CtsHardwareTestCases.config",
119                             "CtsAndroidAppTestCases.config",
120                             "CtsClassLoaderFactoryInMemoryDexClassLoaderTestCases.config",
121                             "CtsAppComponentFactoryTestCases.config",
122                             "CtsSeccompHostTestCases.config"));
123 
124 
125     /**
126      * Families of module parameterization that MUST be specified explicitly in the module
127      * AndroidTest.xml.
128      */
129     private static final Set<String> MANDATORY_PARAMETERS_FAMILY = new HashSet<>();
130 
131     static {
132         MANDATORY_PARAMETERS_FAMILY.add(ModuleParameters.INSTANT_APP_FAMILY);
133         MANDATORY_PARAMETERS_FAMILY.add(ModuleParameters.MULTI_ABI_FAMILY);
134         MANDATORY_PARAMETERS_FAMILY.add(ModuleParameters.SECONDARY_USER_FAMILY);
135     }
136 
137     /**
138      * AllowList to start enforcing metadata on modules. No additional entry will be allowed! This
139      * is meant to burn down the remaining modules definition.
140      */
141     private static final Set<String> ALLOWLIST_MODULE_PARAMETERS = new HashSet<>();
142 
143     static {
144     }
145 
146     /**
147      * Test that configuration shipped in Tradefed can be parsed.
148      * -> Exclude deprecated ApkInstaller.
149      * -> Check if host-side tests are non empty.
150      */
151     @Test
testConfigurationLoad()152     public void testConfigurationLoad() throws Exception {
153         String rootVar = String.format("%s_ROOT", getSuiteName().toUpperCase());
154         String suiteRoot = System.getProperty(rootVar);
155         if (Strings.isNullOrEmpty(suiteRoot)) {
156             fail(String.format("Should run within a suite context: %s doesn't exist", rootVar));
157         }
158         File testcases =
159                 new File(
160                         suiteRoot,
161                         String.format("/android-%s/testcases/", getSuiteName().toLowerCase()));
162         if (!testcases.exists()) {
163             fail(String.format("%s does not exists", testcases));
164             return;
165         }
166         Set<File> listConfigs = FileUtil.findFilesObject(testcases, ".*\\.config");
167         assertTrue(listConfigs.size() > 0);
168         // Create a FolderBuildInfo to similate the CompatibilityBuildProvider
169         FolderBuildInfo stubFolder = new FolderBuildInfo("-1", "-1");
170         stubFolder.setRootDir(new File(suiteRoot));
171         stubFolder.addBuildAttribute(CompatibilityBuildHelper.SUITE_NAME, getSuiteName().toUpperCase());
172         stubFolder.addBuildAttribute("ROOT_DIR", suiteRoot);
173         TestInformation stubTestInfo = TestInformation.newBuilder()
174                 .setInvocationContext(new InvocationContext()).build();
175         stubTestInfo.executionFiles().put(FilesKey.TESTS_DIRECTORY, new File(suiteRoot));
176 
177         List<String> missingMandatoryParameters = new ArrayList<>();
178         // We expect to be able to load every single config in testcases/
179         for (File config : listConfigs) {
180             IConfiguration c = ConfigurationFactory.getInstance()
181                     .createConfigurationFromArgs(new String[] {config.getAbsolutePath()});
182 
183             ConfigurationDescriptor cd = c.getConfigurationDescription();
184             Assert.assertNotNull(config + ": configuration descriptor is null", cd);
185             List<String> component = cd.getMetaData(METADATA_COMPONENT);
186             Assert.assertNotNull(String.format("Missing module metadata field \"component\", "
187                     + "please add the following line to your AndroidTest.xml:\n"
188                     + "<option name=\"config-descriptor:metadata\" key=\"component\" "
189                     + "value=\"...\" />\nwhere \"value\" must be one of: %s\n"
190                     + "config: %s", KNOWN_COMPONENTS, config),
191                     component);
192             Assert.assertEquals(String.format("Module config contains more than one \"component\" "
193                     + "metadata field: %s\nconfig: %s", component, config),
194                     1, component.size());
195             String cmp = component.get(0);
196             Assert.assertTrue(String.format("Module config contains unknown \"component\" metadata "
197                     + "field \"%s\", supported ones are: %s\nconfig: %s",
198                     cmp, KNOWN_COMPONENTS, config), KNOWN_COMPONENTS.contains(cmp));
199 
200             if ("misc".equals(cmp)) {
201                 String configFileName = config.getName();
202                 Assert.assertTrue(
203                         String.format(
204                                 "Adding new module %s to \"misc\" component is restricted, "
205                                         + "please pick a component that your module fits in",
206                                 configFileName),
207                         KNOWN_MISC_MODULES.contains(configFileName));
208             }
209 
210             // Check that specified parameters are expected
211             boolean res =
212                     checkModuleParameters(
213                             config.getName(), cd.getMetaData(ITestSuite.PARAMETER_KEY));
214             if (!res) {
215                 missingMandatoryParameters.add(config.getName());
216             }
217 
218             String suiteName = getSuiteName().toLowerCase();
219             // Ensure each CTS module is tagged with <option name="test-suite-tag" value="cts" />
220             Assert.assertTrue(
221                     String.format(
222                             "Module config %s does not contains "
223                                     + "'<option name=\"test-suite-tag\" value=\"%s\" />'",
224                             config.getName(), suiteName),
225                     cd.getSuiteTags().contains(suiteName));
226 
227             // Ensure options have been set
228             c.validateOptions();
229         }
230 
231         // Exempt the allow list
232         missingMandatoryParameters.removeAll(ALLOWLIST_MODULE_PARAMETERS);
233         // Ensure the mandatory fields are filled
234         if (!missingMandatoryParameters.isEmpty()) {
235             String msg =
236                     String.format(
237                             "The following %s modules are missing some of the mandatory "
238                                     + "parameters [instant_app, not_instant_app, "
239                                     + "multi_abi, not_multi_abi, "
240                                     + "secondary_user, not_secondary_user]: '%s'",
241                             missingMandatoryParameters.size(), missingMandatoryParameters);
242             throw new ConfigurationException(msg);
243         }
244     }
245 
246     /** Test that all parameter metadata can be resolved. */
checkModuleParameters(String configName, List<String> parameters)247     private boolean checkModuleParameters(String configName, List<String> parameters)
248             throws ConfigurationException {
249         if (parameters == null) {
250             return false;
251         }
252         Map<String, Boolean> families = createFamilyCheckMap();
253         for (String param : parameters) {
254             try {
255                 ModuleParameters p = ModuleParameters.valueOf(param.toUpperCase());
256                 if (families.containsKey(p.getFamily())) {
257                     families.put(p.getFamily(), true);
258                 }
259             } catch (IllegalArgumentException e) {
260                 throw new ConfigurationException(
261                         String.format("Config: %s includes an unknown parameter '%s'.",
262                                 configName, param));
263             }
264         }
265         if (families.containsValue(false)) {
266             return false;
267         }
268         return true;
269     }
270 
createFamilyCheckMap()271     private Map<String, Boolean> createFamilyCheckMap() {
272         Map<String, Boolean> families = new HashMap<>();
273         for (String family : MANDATORY_PARAMETERS_FAMILY) {
274             families.put(family, false);
275         }
276         return families;
277     }
278 
getSuiteName()279     private String getSuiteName() {
280         return TestSuiteInfo.getInstance().getName();
281     }
282 }
283