1 /* 2 * Copyright (C) 2022 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.compatibility.common.tradefed.loading; 17 18 import static org.junit.Assert.assertTrue; 19 import static org.junit.Assert.fail; 20 21 import com.android.compatibility.common.tradefed.build.CompatibilityBuildHelper; 22 import com.android.compatibility.common.tradefed.targetprep.ApkInstaller; 23 import com.android.compatibility.common.tradefed.targetprep.PreconditionPreparer; 24 import com.android.compatibility.common.tradefed.testtype.JarHostTest; 25 import com.android.tradefed.build.FolderBuildInfo; 26 import com.android.tradefed.config.ConfigurationDescriptor; 27 import com.android.tradefed.config.ConfigurationException; 28 import com.android.tradefed.config.ConfigurationFactory; 29 import com.android.tradefed.config.IConfiguration; 30 import com.android.tradefed.config.IDeviceConfiguration; 31 import com.android.tradefed.invoker.ExecutionFiles.FilesKey; 32 import com.android.tradefed.invoker.InvocationContext; 33 import com.android.tradefed.invoker.TestInformation; 34 import com.android.tradefed.invoker.shard.token.TokenProperty; 35 import com.android.tradefed.targetprep.DeviceSetup; 36 import com.android.tradefed.targetprep.ITargetPreparer; 37 import com.android.tradefed.targetprep.PythonVirtualenvPreparer; 38 import com.android.tradefed.testtype.AndroidJUnitTest; 39 import com.android.tradefed.testtype.GTest; 40 import com.android.tradefed.testtype.HostTest; 41 import com.android.tradefed.testtype.IRemoteTest; 42 import com.android.tradefed.testtype.ITestFilterReceiver; 43 import com.android.tradefed.testtype.suite.ITestSuite; 44 import com.android.tradefed.testtype.suite.TestSuiteInfo; 45 import com.android.tradefed.testtype.suite.ValidateSuiteConfigHelper; 46 import com.android.tradefed.util.FileUtil; 47 import com.android.tradefed.util.ModuleTestTypeUtil; 48 49 import com.google.common.base.Strings; 50 51 import org.junit.Assert; 52 import org.junit.Test; 53 import org.junit.runner.RunWith; 54 import org.junit.runners.JUnit4; 55 56 import java.io.File; 57 import java.io.IOException; 58 import java.util.Arrays; 59 import java.util.HashSet; 60 import java.util.List; 61 import java.util.Set; 62 import java.util.regex.Pattern; 63 64 /** 65 * Test that configuration in *TS can load and have expected properties. 66 */ 67 @RunWith(JUnit4.class) 68 public class CommonConfigLoadingTest { 69 70 private static final Pattern TODO_BUG_PATTERN = Pattern.compile(".*TODO\\(b/[0-9]+\\).*", Pattern.DOTALL); 71 72 /** 73 * List of the officially supported runners in CTS, they meet all the interfaces criteria as 74 * well as support sharding very well. Any new addition should go through a review. 75 */ 76 private static final Set<String> SUPPORTED_SUITE_TEST_TYPE = new HashSet<>(Arrays.asList( 77 // Suite runners 78 "com.android.compatibility.common.tradefed.testtype.JarHostTest", 79 "com.android.compatibility.testtype.DalvikTest", 80 "com.android.compatibility.testtype.LibcoreTest", 81 "com.drawelements.deqp.runner.DeqpTestRunner", 82 // Tradefed runners 83 "com.android.tradefed.testtype.AndroidJUnitTest", 84 "com.android.tradefed.testtype.ArtRunTest", 85 "com.android.tradefed.testtype.HostTest", 86 "com.android.tradefed.testtype.GTest", 87 "com.android.tradefed.testtype.mobly.MoblyBinaryHostTest", 88 "com.android.tradefed.testtype.pandora.PtsBotTest", 89 // VTS specific runners 90 "com.android.tradefed.testtype.binary.KernelTargetTest", 91 "com.android.tradefed.testtype.python.PythonBinaryHostTest", 92 "com.android.tradefed.testtype.binary.ExecutableTargetTest", 93 "com.android.tradefed.testtype.binary.ExecutableHostTest", 94 "com.android.tradefed.testtype.rust.RustBinaryTest" 95 )); 96 97 /** 98 * In Most cases we impose the usage of the AndroidJUnitRunner because it supports all the 99 * features required (filtering, sharding, etc.). We do not typically expect people to need a 100 * different runner. 101 */ 102 private static final Set<String> ALLOWED_INSTRUMENTATION_RUNNER_NAME = new HashSet<>(); 103 static { 104 ALLOWED_INSTRUMENTATION_RUNNER_NAME.add("android.support.test.runner.AndroidJUnitRunner"); 105 ALLOWED_INSTRUMENTATION_RUNNER_NAME.add("androidx.test.runner.AndroidJUnitRunner"); 106 } 107 private static final Set<String> RUNNER_EXCEPTION = new HashSet<>(); 108 static { 109 // Used for a bunch of system-api cts tests 110 RUNNER_EXCEPTION.add("repackaged.android.test.InstrumentationTestRunner"); 111 // Used by a UiRendering scenario where an activity is persisted between tests 112 RUNNER_EXCEPTION.add("android.uirendering.cts.runner.UiRenderingRunner"); 113 // Used by a Widget scenario where an activity is persisted between tests 114 RUNNER_EXCEPTION.add("android.widget.cts.runner.WidgetRunner"); 115 // Used by a text scenario where an activity is persisted between tests 116 RUNNER_EXCEPTION.add("android.text.cts.runner.CtsTextRunner"); 117 // Used to avoid crashing runner on -eng build due to Log.wtf() - b/216648699 118 RUNNER_EXCEPTION.add("com.android.server.uwb.CustomTestRunner"); 119 RUNNER_EXCEPTION.add("com.android.server.wifi.CustomTestRunner"); 120 // HealthConnect APK use Hilt for dependency injection. For test setup it needs 121 // to replace the main Application class with Test Application so Hilt can swap 122 // dependencies for testing. 123 RUNNER_EXCEPTION.add("com.android.healthconnect.controller.tests.HiltTestRunner"); 124 } 125 126 /** 127 * Test that configuration shipped in Tradefed can be parsed. 128 * -> Exclude deprecated ApkInstaller. 129 * -> Check if host-side tests are non empty. 130 */ 131 @Test testConfigurationLoad()132 public void testConfigurationLoad() throws Exception { 133 String rootVar = String.format("%s_ROOT", getSuiteName().toUpperCase()); 134 String suiteRoot = System.getProperty(rootVar); 135 if (Strings.isNullOrEmpty(suiteRoot)) { 136 fail(String.format("Should run within a suite context: %s doesn't exist", rootVar)); 137 } 138 File testcases = new File(suiteRoot, String.format("/android-%s/testcases/", getSuiteName().toLowerCase())); 139 if (!testcases.exists()) { 140 fail(String.format("%s does not exist", testcases)); 141 return; 142 } 143 Set<File> listConfigs = FileUtil.findFilesObject(testcases, ".*\\.config"); 144 assertTrue(listConfigs.size() > 0); 145 // Create a FolderBuildInfo to similate the CompatibilityBuildProvider 146 FolderBuildInfo stubFolder = new FolderBuildInfo("-1", "-1"); 147 stubFolder.setRootDir(new File(suiteRoot)); 148 stubFolder.addBuildAttribute(CompatibilityBuildHelper.SUITE_NAME, getSuiteName().toUpperCase()); 149 stubFolder.addBuildAttribute("ROOT_DIR", suiteRoot); 150 TestInformation stubTestInfo = TestInformation.newBuilder() 151 .setInvocationContext(new InvocationContext()).build(); 152 stubTestInfo.executionFiles().put(FilesKey.TESTS_DIRECTORY, new File(suiteRoot)); 153 154 // We expect to be able to load every single config in testcases/ 155 for (File config : listConfigs) { 156 IConfiguration c = ConfigurationFactory.getInstance() 157 .createConfigurationFromArgs(new String[] {config.getAbsolutePath()}); 158 if (c.getDeviceConfig().size() > 2) { 159 throw new ConfigurationException(String.format("%s declares more than 2 devices.", config)); 160 } 161 int deviceCount = 0; 162 for (IDeviceConfiguration dConfig : c.getDeviceConfig()) { 163 // Ensure the deprecated ApkInstaller is not used anymore. 164 for (ITargetPreparer prep : dConfig.getTargetPreparers()) { 165 if (prep.getClass().isAssignableFrom(ApkInstaller.class)) { 166 throw new ConfigurationException( 167 String.format("%s: Use com.android.tradefed.targetprep.suite." 168 + "SuiteApkInstaller instead of com.android.compatibility." 169 + "common.tradefed.targetprep.ApkInstaller, options will be " 170 + "the same.", config)); 171 } 172 if (prep.getClass().isAssignableFrom(PreconditionPreparer.class)) { 173 throw new ConfigurationException( 174 String.format( 175 "%s: includes a PreconditionPreparer (%s) which is not " 176 + "allowed in modules.", 177 config.getName(), prep.getClass())); 178 } 179 if (prep.getClass().isAssignableFrom(DeviceSetup.class)) { 180 DeviceSetup deviceSetup = (DeviceSetup) prep; 181 if (!deviceSetup.isForceSkipSystemProps()) { 182 throw new ConfigurationException( 183 String.format("%s: %s needs to be configured with " 184 + "<option name=\"force-skip-system-props\" " 185 + "value=\"true\" /> in *TS.", 186 config.getName(), prep.getClass())); 187 } 188 } 189 if (prep.getClass().isAssignableFrom(PythonVirtualenvPreparer.class)) { 190 // Ensure each modules has a tracking bug to be imported. 191 checkPythonModules(config, deviceCount); 192 } 193 } 194 deviceCount++; 195 } 196 // We can ensure that Host side tests are not empty. 197 for (IRemoteTest test : c.getTests()) { 198 // Check that all the tests runners are well supported. 199 if (!SUPPORTED_SUITE_TEST_TYPE.contains(test.getClass().getCanonicalName())) { 200 throw new ConfigurationException( 201 String.format( 202 "testtype %s is not officially supported by *TS. " 203 + "The supported ones are: %s", 204 test.getClass().getCanonicalName(), SUPPORTED_SUITE_TEST_TYPE)); 205 } 206 if (test instanceof HostTest) { 207 HostTest hostTest = (HostTest) test; 208 // We inject a made up folder so that it can find the tests. 209 hostTest.setBuild(stubFolder); 210 hostTest.setTestInformation(stubTestInfo); 211 int testCount = hostTest.countTestCases(); 212 if (testCount == 0) { 213 throw new ConfigurationException( 214 String.format("%s: %s reports 0 test cases.", 215 config.getName(), test)); 216 } 217 } 218 if (test instanceof GTest) { 219 if (((GTest) test).isRebootBeforeTestEnabled()) { 220 throw new ConfigurationException(String.format( 221 "%s: instead of reboot-before-test use a RebootTargetPreparer " 222 + "which is more optimized during sharding.", config.getName())); 223 } 224 } 225 // Tests are expected to implement that interface. 226 if (!(test instanceof ITestFilterReceiver)) { 227 throw new IllegalArgumentException(String.format( 228 "Test in module %s must implement ITestFilterReceiver.", 229 config.getName())); 230 } 231 // Ensure that the device runner is the AJUR one if explicitly specified. 232 if (test instanceof AndroidJUnitTest) { 233 AndroidJUnitTest instru = (AndroidJUnitTest) test; 234 if (instru.getRunnerName() != null && 235 !ALLOWED_INSTRUMENTATION_RUNNER_NAME.contains(instru.getRunnerName())) { 236 // Some runner are exempt 237 if (!RUNNER_EXCEPTION.contains(instru.getRunnerName())) { 238 throw new ConfigurationException( 239 String.format("%s: uses '%s' instead of on of '%s' that are " 240 + "expected", config.getName(), instru.getRunnerName(), 241 ALLOWED_INSTRUMENTATION_RUNNER_NAME)); 242 } 243 } 244 } 245 } 246 247 ConfigurationDescriptor cd = c.getConfigurationDescription(); 248 Assert.assertNotNull(config + ": configuration descriptor is null", cd); 249 250 // Check that specified tokens are expected 251 checkTokens(config.getName(), cd.getMetaData(ITestSuite.TOKEN_KEY)); 252 253 // Check not-shardable: JarHostTest cannot create empty shards so it should never need 254 // to be not-shardable. 255 if (cd.isNotShardable()) { 256 for (IRemoteTest test : c.getTests()) { 257 if (test.getClass().isAssignableFrom(JarHostTest.class)) { 258 throw new ConfigurationException( 259 String.format("config: %s. JarHostTest does not need the " 260 + "not-shardable option.", config.getName())); 261 } 262 } 263 } 264 // Ensure options have been set 265 c.validateOptions(); 266 267 // Check that no performance test module is included 268 if (ModuleTestTypeUtil.isPerformanceModule(c)) { 269 throw new ConfigurationException( 270 String.format("config: %s. Performance test modules are not allowed in xTS", 271 config.getName())); 272 } 273 274 // Vailidate the module config doesn't contain inclusion tags 275 ValidateSuiteConfigHelper.validateConfigFile(config); 276 } 277 } 278 279 /** Test that all tokens can be resolved. */ checkTokens(String configName, List<String> tokens)280 private void checkTokens(String configName, List<String> tokens) throws ConfigurationException { 281 if (tokens == null) { 282 return; 283 } 284 for (String token : tokens) { 285 try { 286 TokenProperty.valueOf(token.toUpperCase()); 287 } catch (IllegalArgumentException e) { 288 throw new ConfigurationException( 289 String.format( 290 "Config: %s includes an unknown token '%s'.", configName, token)); 291 } 292 } 293 } 294 295 /** 296 * For each usage of python virtualenv preparer, make sure we have tracking bugs to import as 297 * source the python libs. 298 */ checkPythonModules(File config, int deviceCount)299 private void checkPythonModules(File config, int deviceCount) 300 throws IOException, ConfigurationException { 301 if (deviceCount != 0) { 302 throw new ConfigurationException( 303 String.format("%s: PythonVirtualenvPreparer should only be declared for " 304 + "the first <device> tag in the config", config.getName())); 305 } 306 if (!TODO_BUG_PATTERN.matcher(FileUtil.readStringFromFile(config)).matches()) { 307 throw new ConfigurationException( 308 String.format("%s: Contains some virtualenv python lib usage but no " 309 + "tracking bug to import them as source.", config.getName())); 310 } 311 } 312 getSuiteName()313 private String getSuiteName() { 314 return TestSuiteInfo.getInstance().getName(); 315 } 316 } 317