1 /* 2 * Copyright (C) 2017 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.compatibility.common.tradefed.presubmit; 17 18 import static org.junit.Assert.assertTrue; 19 import static org.junit.Assert.fail; 20 21 import com.android.compatibility.common.tradefed.build.CompatibilityBuildHelper; 22 import com.android.compatibility.common.tradefed.targetprep.ApkInstaller; 23 import com.android.compatibility.common.tradefed.targetprep.PreconditionPreparer; 24 import com.android.compatibility.common.tradefed.testtype.JarHostTest; 25 import com.android.tradefed.build.FolderBuildInfo; 26 import com.android.tradefed.config.ConfigurationDescriptor; 27 import com.android.tradefed.config.ConfigurationException; 28 import com.android.tradefed.config.ConfigurationFactory; 29 import com.android.tradefed.config.IConfiguration; 30 import com.android.tradefed.invoker.ExecutionFiles.FilesKey; 31 import com.android.tradefed.invoker.TestInformation; 32 import com.android.tradefed.invoker.shard.token.TokenProperty; 33 import com.android.tradefed.targetprep.ITargetPreparer; 34 import com.android.tradefed.testtype.AndroidJUnitTest; 35 import com.android.tradefed.testtype.HostTest; 36 import com.android.tradefed.testtype.IRemoteTest; 37 import com.android.tradefed.testtype.ITestFilterReceiver; 38 import com.android.tradefed.testtype.suite.ITestSuite; 39 import com.android.tradefed.testtype.suite.params.ModuleParameters; 40 41 import org.junit.Assert; 42 import org.junit.Test; 43 import org.junit.runner.RunWith; 44 import org.junit.runners.JUnit4; 45 46 import java.io.File; 47 import java.io.FilenameFilter; 48 import java.util.ArrayList; 49 import java.util.Arrays; 50 import java.util.HashMap; 51 import java.util.HashSet; 52 import java.util.List; 53 import java.util.Map; 54 import java.util.Set; 55 56 /** 57 * Test that configuration in CTS can load and have expected properties. 58 */ 59 @RunWith(JUnit4.class) 60 public class CtsConfigLoadingTest { 61 62 private static final String METADATA_COMPONENT = "component"; 63 private static final Set<String> KNOWN_COMPONENTS = 64 new HashSet<>( 65 Arrays.asList( 66 // modifications to the list below must be reviewed 67 "abuse", 68 "art", 69 "auth", 70 "auto", 71 "autofill", 72 "backup", 73 "bionic", 74 "bluetooth", 75 "camera", 76 "contentcapture", 77 "deviceinfo", 78 "deqp", 79 "devtools", 80 "framework", 81 "graphics", 82 "hdmi", 83 "inputmethod", 84 "libcore", 85 "libnativehelper", 86 "location", 87 "media", 88 "metrics", 89 "misc", 90 "mocking", 91 "networking", 92 "neuralnetworks", 93 "print", 94 "renderscript", 95 "security", 96 "statsd", 97 "systems", 98 "sysui", 99 "telecom", 100 "tv", 101 "uitoolkit", 102 "vr", 103 "webview", 104 "wifi")); 105 private static final Set<String> KNOWN_MISC_MODULES = 106 new HashSet<>( 107 Arrays.asList( 108 // Modifications to the list below must be approved by someone in 109 // test/suite_harness/OWNERS. 110 "CtsSliceTestCases.config", 111 "CtsSampleDeviceTestCases.config", 112 "CtsUsbTests.config", 113 "CtsGpuToolsHostTestCases.config", 114 "CtsEdiHostTestCases.config", 115 "CtsClassLoaderFactoryPathClassLoaderTestCases.config", 116 "CtsSampleHostTestCases.config", 117 "CtsHardwareTestCases.config", 118 "CtsMonkeyTestCases.config", 119 "CtsAndroidAppTestCases.config", 120 "CtsClassLoaderFactoryInMemoryDexClassLoaderTestCases.config", 121 "CtsAppComponentFactoryTestCases.config", 122 "CtsSeccompHostTestCases.config")); 123 124 /** 125 * List of the officially supported runners in CTS, they meet all the interfaces criteria as 126 * well as support sharding very well. Any new addition should go through a review. 127 */ 128 private static final Set<String> SUPPORTED_CTS_TEST_TYPE = new HashSet<>(Arrays.asList( 129 // Cts runners 130 "com.android.compatibility.common.tradefed.testtype.JarHostTest", 131 "com.android.compatibility.testtype.DalvikTest", 132 "com.android.compatibility.testtype.LibcoreTest", 133 "com.drawelements.deqp.runner.DeqpTestRunner", 134 // Tradefed runners 135 "com.android.tradefed.testtype.AndroidJUnitTest", 136 "com.android.tradefed.testtype.HostTest", 137 "com.android.tradefed.testtype.GTest" 138 )); 139 140 /** 141 * In Most cases we impose the usage of the AndroidJUnitRunner because it supports all the 142 * features required (filtering, sharding, etc.). We do not typically expect people to need a 143 * different runner. 144 */ 145 private static final Set<String> ALLOWED_INSTRUMENTATION_RUNNER_NAME = new HashSet<>(); 146 static { 147 ALLOWED_INSTRUMENTATION_RUNNER_NAME.add("android.support.test.runner.AndroidJUnitRunner"); 148 ALLOWED_INSTRUMENTATION_RUNNER_NAME.add("androidx.test.runner.AndroidJUnitRunner"); 149 } 150 private static final Set<String> RUNNER_EXCEPTION = new HashSet<>(); 151 static { 152 // Used for a bunch of system-api cts tests 153 RUNNER_EXCEPTION.add("repackaged.android.test.InstrumentationTestRunner"); 154 // Used by a UiRendering scenario where an activity is persisted between tests 155 RUNNER_EXCEPTION.add("android.uirendering.cts.runner.UiRenderingRunner"); 156 } 157 158 /** 159 * Families of module parameterization that MUST be specified explicitly in the module 160 * AndroidTest.xml. 161 */ 162 private static final Set<String> MANDATORY_PARAMETERS_FAMILY = new HashSet<>(); 163 164 static { 165 MANDATORY_PARAMETERS_FAMILY.add(ModuleParameters.INSTANT_APP_FAMILY); 166 MANDATORY_PARAMETERS_FAMILY.add(ModuleParameters.MULTI_ABI_FAMILY); 167 MANDATORY_PARAMETERS_FAMILY.add(ModuleParameters.SECONDARY_USER_FAMILY); 168 } 169 170 /** 171 * AllowList to start enforcing metadata on modules. No additional entry will be allowed! This 172 * is meant to burn down the remaining modules definition. 173 */ 174 private static final Set<String> ALLOWLIST_MODULE_PARAMETERS = new HashSet<>(); 175 176 static { 177 } 178 179 /** 180 * Test that configuration shipped in Tradefed can be parsed. 181 * -> Exclude deprecated ApkInstaller. 182 * -> Check if host-side tests are non empty. 183 */ 184 @Test testConfigurationLoad()185 public void testConfigurationLoad() throws Exception { 186 String ctsRoot = System.getProperty("CTS_ROOT"); 187 File testcases = new File(ctsRoot, "/android-cts/testcases/"); 188 if (!testcases.exists()) { 189 fail(String.format("%s does not exists", testcases)); 190 return; 191 } 192 File[] listConfig = testcases.listFiles(new FilenameFilter() { 193 @Override 194 public boolean accept(File dir, String name) { 195 if (name.endsWith(".config")) { 196 return true; 197 } 198 return false; 199 } 200 }); 201 assertTrue(listConfig.length > 0); 202 // Create a FolderBuildInfo to similate the CompatibilityBuildProvider 203 FolderBuildInfo stubFolder = new FolderBuildInfo("-1", "-1"); 204 stubFolder.setRootDir(new File(ctsRoot)); 205 stubFolder.addBuildAttribute(CompatibilityBuildHelper.SUITE_NAME, "CTS"); 206 stubFolder.addBuildAttribute("ROOT_DIR", ctsRoot); 207 TestInformation stubTestInfo = TestInformation.newBuilder().build(); 208 stubTestInfo.executionFiles().put(FilesKey.TESTS_DIRECTORY, new File(ctsRoot)); 209 210 List<String> missingMandatoryParameters = new ArrayList<>(); 211 // We expect to be able to load every single config in testcases/ 212 for (File config : listConfig) { 213 IConfiguration c = ConfigurationFactory.getInstance() 214 .createConfigurationFromArgs(new String[] {config.getAbsolutePath()}); 215 // Ensure the deprecated ApkInstaller is not used anymore. 216 for (ITargetPreparer prep : c.getTargetPreparers()) { 217 if (prep.getClass().isAssignableFrom(ApkInstaller.class)) { 218 throw new ConfigurationException( 219 String.format("%s: Use com.android.tradefed.targetprep.suite." 220 + "SuiteApkInstaller instead of com.android.compatibility." 221 + "common.tradefed.targetprep.ApkInstaller, options will be " 222 + "the same.", config)); 223 } 224 if (prep.getClass().isAssignableFrom(PreconditionPreparer.class)) { 225 throw new ConfigurationException( 226 String.format( 227 "%s: includes a PreconditionPreparer (%s) which is not allowed" 228 + " in modules.", 229 config.getName(), prep.getClass())); 230 } 231 } 232 // We can ensure that Host side tests are not empty. 233 for (IRemoteTest test : c.getTests()) { 234 // Check that all the tests runners are well supported. 235 if (!SUPPORTED_CTS_TEST_TYPE.contains(test.getClass().getCanonicalName())) { 236 throw new ConfigurationException( 237 String.format( 238 "testtype %s is not officially supported by CTS. " 239 + "The supported ones are: %s", 240 test.getClass().getCanonicalName(), SUPPORTED_CTS_TEST_TYPE)); 241 } 242 if (test instanceof HostTest) { 243 HostTest hostTest = (HostTest) test; 244 // We inject a made up folder so that it can find the tests. 245 hostTest.setBuild(stubFolder); 246 hostTest.setTestInformation(stubTestInfo); 247 int testCount = hostTest.countTestCases(); 248 if (testCount == 0) { 249 throw new ConfigurationException( 250 String.format("%s: %s reports 0 test cases.", 251 config.getName(), test)); 252 } 253 } 254 // Tests are expected to implement that interface. 255 if (!(test instanceof ITestFilterReceiver)) { 256 throw new IllegalArgumentException(String.format( 257 "Test in module %s must implement ITestFilterReceiver.", 258 config.getName())); 259 } 260 // Ensure that the device runner is the AJUR one if explicitly specified. 261 if (test instanceof AndroidJUnitTest) { 262 AndroidJUnitTest instru = (AndroidJUnitTest) test; 263 if (instru.getRunnerName() != null && 264 !ALLOWED_INSTRUMENTATION_RUNNER_NAME.contains(instru.getRunnerName())) { 265 // Some runner are exempt 266 if (!RUNNER_EXCEPTION.contains(instru.getRunnerName())) { 267 throw new ConfigurationException( 268 String.format("%s: uses '%s' instead of on of '%s' that are " 269 + "expected", config.getName(), instru.getRunnerName(), 270 ALLOWED_INSTRUMENTATION_RUNNER_NAME)); 271 } 272 } 273 } 274 } 275 ConfigurationDescriptor cd = c.getConfigurationDescription(); 276 Assert.assertNotNull(config + ": configuration descriptor is null", cd); 277 List<String> component = cd.getMetaData(METADATA_COMPONENT); 278 Assert.assertNotNull(String.format("Missing module metadata field \"component\", " 279 + "please add the following line to your AndroidTest.xml:\n" 280 + "<option name=\"config-descriptor:metadata\" key=\"component\" " 281 + "value=\"...\" />\nwhere \"value\" must be one of: %s\n" 282 + "config: %s", KNOWN_COMPONENTS, config), 283 component); 284 Assert.assertEquals(String.format("Module config contains more than one \"component\" " 285 + "metadata field: %s\nconfig: %s", component, config), 286 1, component.size()); 287 String cmp = component.get(0); 288 Assert.assertTrue(String.format("Module config contains unknown \"component\" metadata " 289 + "field \"%s\", supported ones are: %s\nconfig: %s", 290 cmp, KNOWN_COMPONENTS, config), KNOWN_COMPONENTS.contains(cmp)); 291 292 if ("misc".equals(cmp)) { 293 String configFileName = config.getName(); 294 Assert.assertTrue( 295 String.format( 296 "Adding new module %s to \"misc\" component is restricted, " 297 + "please pick a component that your module fits in", 298 configFileName), 299 KNOWN_MISC_MODULES.contains(configFileName)); 300 } 301 302 // Check that specified parameters are expected 303 boolean res = 304 checkModuleParameters( 305 config.getName(), cd.getMetaData(ITestSuite.PARAMETER_KEY)); 306 if (!res) { 307 missingMandatoryParameters.add(config.getName()); 308 } 309 // Check that specified tokens are expected 310 checkTokens(config.getName(), cd.getMetaData(ITestSuite.TOKEN_KEY)); 311 312 // Ensure each CTS module is tagged with <option name="test-suite-tag" value="cts" /> 313 Assert.assertTrue(String.format( 314 "Module config %s does not contains " 315 + "'<option name=\"test-suite-tag\" value=\"cts\" />'", config.getName()), 316 cd.getSuiteTags().contains("cts")); 317 318 // Check not-shardable: JarHostTest cannot create empty shards so it should never need 319 // to be not-shardable. 320 if (cd.isNotShardable()) { 321 for (IRemoteTest test : c.getTests()) { 322 if (test.getClass().isAssignableFrom(JarHostTest.class)) { 323 throw new ConfigurationException( 324 String.format("config: %s. JarHostTest does not need the " 325 + "not-shardable option.", config.getName())); 326 } 327 } 328 } 329 // Ensure options have been set 330 c.validateOptions(); 331 } 332 333 // Exempt the allow list 334 missingMandatoryParameters.removeAll(ALLOWLIST_MODULE_PARAMETERS); 335 // Ensure the mandatory fields are filled 336 if (!missingMandatoryParameters.isEmpty()) { 337 String msg = 338 String.format( 339 "The following %s modules are missing some of the mandatory " 340 + "parameters [instant_app, not_instant_app, " 341 + "multi_abi, not_multi_abi, " 342 + "secondary_user, not_secondary_user]: '%s'", 343 missingMandatoryParameters.size(), missingMandatoryParameters); 344 throw new ConfigurationException(msg); 345 } 346 } 347 348 /** Test that all parameter metadata can be resolved. */ checkModuleParameters(String configName, List<String> parameters)349 private boolean checkModuleParameters(String configName, List<String> parameters) 350 throws ConfigurationException { 351 if (parameters == null) { 352 return false; 353 } 354 Map<String, Boolean> families = createFamilyCheckMap(); 355 for (String param : parameters) { 356 try { 357 ModuleParameters p = ModuleParameters.valueOf(param.toUpperCase()); 358 if (families.containsKey(p.getFamily())) { 359 families.put(p.getFamily(), true); 360 } 361 } catch (IllegalArgumentException e) { 362 throw new ConfigurationException( 363 String.format("Config: %s includes an unknown parameter '%s'.", 364 configName, param)); 365 } 366 } 367 if (families.containsValue(false)) { 368 return false; 369 } 370 return true; 371 } 372 373 /** Test that all tokens can be resolved. */ checkTokens(String configName, List<String> tokens)374 private void checkTokens(String configName, List<String> tokens) throws ConfigurationException { 375 if (tokens == null) { 376 return; 377 } 378 for (String token : tokens) { 379 try { 380 TokenProperty.valueOf(token.toUpperCase()); 381 } catch (IllegalArgumentException e) { 382 throw new ConfigurationException( 383 String.format( 384 "Config: %s includes an unknown token '%s'.", configName, token)); 385 } 386 } 387 } 388 createFamilyCheckMap()389 private Map<String, Boolean> createFamilyCheckMap() { 390 Map<String, Boolean> families = new HashMap<>(); 391 for (String family : MANDATORY_PARAMETERS_FAMILY) { 392 families.put(family, false); 393 } 394 return families; 395 } 396 } 397