1 /* 2 * Copyright (C) 2019 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.compat.cts; 18 19 import static com.google.common.truth.Truth.assertThat; 20 import static com.google.common.truth.Truth.assertWithMessage; 21 22 import com.android.compatibility.common.tradefed.build.CompatibilityBuildHelper; 23 import com.android.ddmlib.testrunner.RemoteAndroidTestRunner; 24 import com.android.ddmlib.testrunner.TestResult.TestStatus; 25 import com.android.internal.os.StatsdConfigProto; 26 import com.android.os.AtomsProto; 27 import com.android.os.AtomsProto.Atom; 28 import com.android.os.StatsLog.ConfigMetricsReport; 29 import com.android.os.StatsLog.ConfigMetricsReportList; 30 import com.android.tradefed.build.IBuildInfo; 31 import com.android.tradefed.device.CollectingByteOutputReceiver; 32 import com.android.tradefed.device.CollectingOutputReceiver; 33 import com.android.tradefed.device.DeviceNotAvailableException; 34 import com.android.tradefed.device.ITestDevice; 35 import com.android.tradefed.log.LogUtil.CLog; 36 import com.android.tradefed.result.CollectingTestListener; 37 import com.android.tradefed.result.TestDescription; 38 import com.android.tradefed.result.TestResult; 39 import com.android.tradefed.result.TestRunResult; 40 import com.android.tradefed.testtype.DeviceTestCase; 41 import com.android.tradefed.testtype.IBuildReceiver; 42 43 import com.google.common.io.Files; 44 import com.google.protobuf.InvalidProtocolBufferException; 45 46 import java.io.File; 47 import java.io.FileNotFoundException; 48 import java.io.IOException; 49 import java.util.Arrays; 50 import java.util.List; 51 import java.util.Map; 52 import java.util.Set; 53 import java.util.stream.Collectors; 54 55 import javax.annotation.Nonnull; 56 57 // Shamelessly plagiarised from incident's ProtoDumpTestCase and statsd's BaseTestCase family 58 public class CompatChangeGatingTestCase extends DeviceTestCase implements IBuildReceiver { 59 protected IBuildInfo mCtsBuild; 60 61 private static final String UPDATE_CONFIG_CMD = "cat %s | cmd stats config update %d"; 62 private static final String DUMP_REPORT_CMD = 63 "cmd stats dump-report %d --include_current_bucket --proto"; 64 private static final String REMOVE_CONFIG_CMD = "cmd stats config remove %d"; 65 66 private static final String TEST_RUNNER = "androidx.test.runner.AndroidJUnitRunner"; 67 68 @Override setUp()69 protected void setUp() throws Exception { 70 super.setUp(); 71 assertThat(mCtsBuild).isNotNull(); 72 } 73 74 @Override setBuild(IBuildInfo buildInfo)75 public void setBuild(IBuildInfo buildInfo) { 76 mCtsBuild = buildInfo; 77 } 78 79 /** 80 * Install a device side test package. 81 * 82 * @param appFileName Apk file name, such as "CtsNetStatsApp.apk". 83 * @param grantPermissions whether to give runtime permissions. 84 */ installPackage(String appFileName, boolean grantPermissions)85 protected void installPackage(String appFileName, boolean grantPermissions) 86 throws FileNotFoundException, DeviceNotAvailableException { 87 CLog.d("Installing app " + appFileName); 88 CompatibilityBuildHelper buildHelper = new CompatibilityBuildHelper(mCtsBuild); 89 final String result = getDevice().installPackage(buildHelper.getTestFile(appFileName), true, 90 grantPermissions); 91 assertWithMessage("Failed to install %s: %s", appFileName, result).that(result).isNull(); 92 } 93 94 /** 95 * Uninstall a device side test package. 96 * 97 * @param appFileName Apk file name, such as "CtsNetStatsApp.apk". 98 * @param shouldSucceed Whether to assert on failure. 99 */ uninstallPackage(String packageName, boolean shouldSucceed)100 protected void uninstallPackage(String packageName, boolean shouldSucceed) 101 throws DeviceNotAvailableException { 102 final String result = getDevice().uninstallPackage(packageName); 103 if (shouldSucceed) { 104 assertWithMessage("uninstallPackage(%s) failed: %s", packageName, result) 105 .that(result).isNull(); 106 } 107 } 108 109 /** 110 * Run a device side compat test. 111 * 112 * @param pkgName Test package name, such as 113 * "com.android.server.cts.netstats". 114 * @param testClassName Test class name; either a fully qualified name, or "." 115 * + a class name. 116 * @param testMethodName Test method name. 117 * @param enabledChanges Set of compat changes to enable. 118 * @param disabledChanges Set of compat changes to disable. 119 */ runDeviceCompatTest(@onnull String pkgName, @Nonnull String testClassName, @Nonnull String testMethodName, Set<Long> enabledChanges, Set<Long> disabledChanges)120 protected void runDeviceCompatTest(@Nonnull String pkgName, @Nonnull String testClassName, 121 @Nonnull String testMethodName, 122 Set<Long> enabledChanges, Set<Long> disabledChanges) 123 throws DeviceNotAvailableException { 124 runDeviceCompatTestReported(pkgName, testClassName, testMethodName, enabledChanges, 125 disabledChanges, enabledChanges, disabledChanges); 126 } 127 128 /** 129 * Run a device side compat test where not all changes are reported through statsd. 130 * 131 * @param pkgName Test package name, such as 132 * "com.android.server.cts.netstats". 133 * @param testClassName Test class name; either a fully qualified name, or "." 134 * + a class name. 135 * @param testMethodName Test method name. 136 * @param enabledChanges Set of compat changes to enable. 137 * @param disabledChanges Set of compat changes to disable. 138 * @param reportedEnabledChanges Expected enabled changes in statsd report. 139 * @param reportedDisabledChanges Expected disabled changes in statsd report. 140 */ runDeviceCompatTestReported(@onnull String pkgName, @Nonnull String testClassName, @Nonnull String testMethodName, Set<Long> enabledChanges, Set<Long> disabledChanges, Set<Long> reportedEnabledChanges, Set<Long> reportedDisabledChanges)141 protected void runDeviceCompatTestReported(@Nonnull String pkgName, @Nonnull String testClassName, 142 @Nonnull String testMethodName, 143 Set<Long> enabledChanges, Set<Long> disabledChanges, 144 Set<Long> reportedEnabledChanges, Set<Long> reportedDisabledChanges) 145 throws DeviceNotAvailableException { 146 147 // Set compat overrides 148 setCompatConfig(enabledChanges, disabledChanges, pkgName); 149 // Send statsd config 150 final long configId = getClass().getCanonicalName().hashCode(); 151 createAndUploadStatsdConfig(configId, pkgName); 152 153 try { 154 // Run device-side test 155 if (testClassName.startsWith(".")) { 156 testClassName = pkgName + testClassName; 157 } 158 RemoteAndroidTestRunner testRunner = new RemoteAndroidTestRunner(pkgName, TEST_RUNNER, 159 getDevice().getIDevice()); 160 testRunner.setMethodName(testClassName, testMethodName); 161 CollectingTestListener listener = new CollectingTestListener(); 162 assertThat(getDevice().runInstrumentationTests(testRunner, listener)).isTrue(); 163 164 // Check that device side test occurred as expected 165 final TestRunResult result = listener.getCurrentRunResults(); 166 assertWithMessage("Failed to successfully run device tests for %s: %s", 167 result.getName(), result.getRunFailureMessage()) 168 .that(result.isRunFailure()).isFalse(); 169 assertWithMessage("Should run only exactly one test method!") 170 .that(result.getNumTests()).isEqualTo(1); 171 if (result.hasFailedTests()) { 172 // build a meaningful error message 173 StringBuilder errorBuilder = new StringBuilder("On-device test failed:\n"); 174 for (Map.Entry<TestDescription, TestResult> resultEntry : 175 result.getTestResults().entrySet()) { 176 if (!resultEntry.getValue().getStatus().equals(TestStatus.PASSED)) { 177 errorBuilder.append(resultEntry.getKey().toString()); 178 errorBuilder.append(":\n"); 179 errorBuilder.append(resultEntry.getValue().getStackTrace()); 180 } 181 } 182 throw new AssertionError(errorBuilder.toString()); 183 } 184 185 } finally { 186 // Cleanup compat overrides 187 resetCompatConfig(pkgName, enabledChanges, disabledChanges); 188 // Validate statsd report 189 validatePostRunStatsdReport(configId, pkgName, reportedEnabledChanges, 190 reportedDisabledChanges); 191 } 192 193 } 194 195 /** 196 * Gets the statsd report. Note that this also deletes that report from statsd. 197 */ getReportList(long configId)198 private List<ConfigMetricsReport> getReportList(long configId) throws DeviceNotAvailableException { 199 try { 200 final CollectingByteOutputReceiver receiver = new CollectingByteOutputReceiver(); 201 getDevice().executeShellCommand(String.format(DUMP_REPORT_CMD, configId), receiver); 202 return ConfigMetricsReportList.parser() 203 .parseFrom(receiver.getOutput()) 204 .getReportsList(); 205 } catch (InvalidProtocolBufferException e) { 206 throw new IllegalStateException("Failed to fetch and parse the statsd output report.", 207 e); 208 } 209 } 210 211 /** 212 * Creates and uploads a statsd config that matches the AppCompatibilityChangeReported atom 213 * logged by a given package name. 214 * 215 * @param configId A unique config id. 216 * @param pkgName The package name of the app that is expected to report the atom. It will be 217 * the only allowed log source. 218 */ createAndUploadStatsdConfig(long configId, String pkgName)219 protected void createAndUploadStatsdConfig(long configId, String pkgName) 220 throws DeviceNotAvailableException { 221 final String atomName = "Atom" + System.nanoTime(); 222 final String eventName = "Event" + System.nanoTime(); 223 final ITestDevice device = getDevice(); 224 225 StatsdConfigProto.StatsdConfig.Builder configBuilder = 226 StatsdConfigProto.StatsdConfig.newBuilder() 227 .setId(configId) 228 .addAllowedLogSource(pkgName) 229 .addWhitelistedAtomIds(Atom.APP_COMPATIBILITY_CHANGE_REPORTED_FIELD_NUMBER); 230 StatsdConfigProto.SimpleAtomMatcher.Builder simpleAtomMatcherBuilder = 231 StatsdConfigProto.SimpleAtomMatcher 232 .newBuilder().setAtomId( 233 Atom.APP_COMPATIBILITY_CHANGE_REPORTED_FIELD_NUMBER); 234 configBuilder.addAtomMatcher( 235 StatsdConfigProto.AtomMatcher.newBuilder() 236 .setId(atomName.hashCode()) 237 .setSimpleAtomMatcher(simpleAtomMatcherBuilder)); 238 configBuilder.addEventMetric( 239 StatsdConfigProto.EventMetric.newBuilder() 240 .setId(eventName.hashCode()) 241 .setWhat(atomName.hashCode())); 242 StatsdConfigProto.StatsdConfig config = configBuilder.build(); 243 try { 244 File configFile = File.createTempFile("statsdconfig", ".config"); 245 configFile.deleteOnExit(); 246 Files.write(config.toByteArray(), configFile); 247 String remotePath = "/data/local/tmp/" + configFile.getName(); 248 device.pushFile(configFile, remotePath); 249 device.executeShellCommand(String.format(UPDATE_CONFIG_CMD, remotePath, configId)); 250 device.executeShellCommand("rm " + remotePath); 251 } catch (IOException e) { 252 throw new RuntimeException("IO error when writing to temp file.", e); 253 } 254 // Purge data 255 getReportList(configId); 256 } 257 258 /** 259 * Gets the uid of the test app. 260 */ getUid(@onnull String packageName)261 protected int getUid(@Nonnull String packageName) throws DeviceNotAvailableException { 262 int currentUser = getDevice().getCurrentUser(); 263 String uidLine = getDevice() 264 .executeShellCommand( 265 "cmd package list packages -U --user " + currentUser + " " 266 + packageName); 267 String[] uidLineParts = uidLine.split(":"); 268 // 3rd entry is package uid 269 assertThat(uidLineParts.length).isGreaterThan(2); 270 int uid = Integer.parseInt(uidLineParts[2].trim()); 271 assertThat(uid).isGreaterThan(10000); 272 return uid; 273 } 274 275 /** 276 * Set the compat config using adb. 277 * 278 * @param enabledChanges Changes to be enabled. 279 * @param disabledChanges Changes to be disabled. 280 * @param packageName Package name for the app whose config is being changed. 281 */ setCompatConfig(Set<Long> enabledChanges, Set<Long> disabledChanges, @Nonnull String packageName)282 protected void setCompatConfig(Set<Long> enabledChanges, Set<Long> disabledChanges, 283 @Nonnull String packageName) throws DeviceNotAvailableException { 284 for (Long enabledChange : enabledChanges) { 285 runCommand("am compat enable " + enabledChange + " " + packageName); 286 } 287 for (Long disabledChange : disabledChanges) { 288 runCommand("am compat disable " + disabledChange + " " + packageName); 289 } 290 } 291 292 /** 293 * Reset changes to default for a package. 294 */ resetCompatChanges(Set<Long> changes, @Nonnull String packageName)295 protected void resetCompatChanges(Set<Long> changes, @Nonnull String packageName) 296 throws DeviceNotAvailableException { 297 for (Long change : changes) { 298 runCommand("am compat reset " + change + " " + packageName); 299 } 300 } 301 302 /** 303 * Remove statsd config for a given id. 304 */ removeStatsdConfig(long configId)305 private void removeStatsdConfig(long configId) throws DeviceNotAvailableException { 306 getDevice().executeShellCommand( 307 String.join(" ", REMOVE_CONFIG_CMD, String.valueOf(configId))); 308 } 309 310 /** 311 * Get the compat changes that were logged. 312 */ getReportedChanges(long configId, String pkgName)313 private Map<Long, Boolean> getReportedChanges(long configId, String pkgName) 314 throws DeviceNotAvailableException { 315 final int packageUid = getUid(pkgName); 316 return getReportList(configId).stream() 317 .flatMap(report -> report.getMetricsList().stream()) 318 .flatMap(metric -> metric.getEventMetrics().getDataList().stream()) 319 .filter(eventMetricData -> eventMetricData.hasAtom()) 320 .map(eventMetricData -> eventMetricData.getAtom()) 321 .map(atom -> atom.getAppCompatibilityChangeReported()) 322 .filter(atom -> atom != null && atom.getUid() == packageUid) // Should be redundant 323 .collect(Collectors.toMap( 324 atom -> atom.getChangeId(), // Key 325 atom -> atom.getState() == // Value 326 AtomsProto.AppCompatibilityChangeReported.State.ENABLED, 327 (a, b) -> { 328 if (a != b) { 329 throw new IllegalStateException("inconsistent compatibility states"); 330 } 331 return a; 332 })); 333 } 334 335 /** 336 * Cleanup the altered change ids under test. 337 * 338 * @param pkgName Package name of the app under test. 339 * @param enabledChanges Set of changes that were enabled during the test and need to be 340 * reset to the default value. 341 * @param disabledChanges Set of changes that were disabled during the test and need to 342 * be reset to the default value. 343 */ 344 protected void resetCompatConfig( String pkgName, Set<Long> enabledChanges, 345 Set<Long> disabledChanges) throws DeviceNotAvailableException { 346 // Clear overrides. 347 resetCompatChanges(enabledChanges, pkgName); 348 resetCompatChanges(disabledChanges, pkgName); 349 } 350 351 /** 352 * Validate that all overridden changes were logged while running the test. 353 * 354 * @param configId The unique config id used to track change id queries. 355 * @param pkgName Package name of the app under test. 356 * @param loggedEnabledChanges Changes expected to be logged as enabled during the test. 357 * @param loggedDisabledChanges Changes expected to be logged as disabled during the test. 358 */ 359 protected void validatePostRunStatsdReport(long configId, String pkgName, 360 Set<Long> loggedEnabledChanges, Set<Long> loggedDisabledChanges) 361 throws DeviceNotAvailableException { 362 // Clear statsd report data and remove config 363 Map<Long, Boolean> reportedChanges = getReportedChanges(configId, pkgName); 364 removeStatsdConfig(configId); 365 366 for (Long enabledChange : loggedEnabledChanges) { 367 assertThat(reportedChanges) 368 .containsEntry(enabledChange, true); 369 } 370 for (Long disabledChange : loggedDisabledChanges) { 371 assertThat(reportedChanges) 372 .containsEntry(disabledChange, false); 373 } 374 } 375 376 /** 377 * Execute the given command, and returns the output. 378 */ 379 protected String runCommand(String command) throws DeviceNotAvailableException { 380 final CollectingOutputReceiver receiver = new CollectingOutputReceiver(); 381 getDevice().executeShellCommand(command, receiver); 382 return receiver.getOutput(); 383 } 384 385 /** 386 * Get the on device compat config. 387 */ 388 protected List<Change> getOnDeviceCompatConfig() throws Exception { 389 String config = runCommand("dumpsys platform_compat"); 390 return Arrays.stream(config.split("\n")) 391 .map(Change::fromString) 392 .collect(Collectors.toList()); 393 } 394 395 protected Change getOnDeviceChangeIdConfig(long changeId) throws Exception { 396 List<Change> changes = getOnDeviceCompatConfig(); 397 for (Change change : changes) { 398 if (change.changeId == changeId) { 399 return change; 400 } 401 } 402 return null; 403 } 404 } 405