1 /* 2 * Copyright (C) 2016 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.compatibility.testtype; 18 19 import com.android.compatibility.common.tradefed.build.CompatibilityBuildHelper; 20 import com.android.ddmlib.IShellOutputReceiver; 21 import com.android.ddmlib.Log; 22 import com.android.ddmlib.Log.LogLevel; 23 import com.android.ddmlib.MultiLineReceiver; 24 import com.android.ddmlib.testrunner.TestIdentifier; 25 import com.android.tradefed.build.IBuildInfo; 26 import com.android.tradefed.config.Option; 27 import com.android.tradefed.config.OptionCopier; 28 import com.android.tradefed.device.DeviceNotAvailableException; 29 import com.android.tradefed.device.ITestDevice; 30 import com.android.tradefed.result.ITestInvocationListener; 31 import com.android.tradefed.testtype.IAbi; 32 import com.android.tradefed.testtype.IAbiReceiver; 33 import com.android.tradefed.testtype.IBuildReceiver; 34 import com.android.tradefed.testtype.IDeviceTest; 35 import com.android.tradefed.testtype.IRemoteTest; 36 import com.android.tradefed.testtype.IRuntimeHintProvider; 37 import com.android.tradefed.testtype.IShardableTest; 38 import com.android.tradefed.testtype.ITestCollector; 39 import com.android.tradefed.testtype.ITestFileFilterReceiver; 40 import com.android.tradefed.testtype.ITestFilterReceiver; 41 import com.android.tradefed.util.AbiUtils; 42 import com.android.tradefed.util.ArrayUtil; 43 import com.android.tradefed.util.FileUtil; 44 import com.android.tradefed.util.TimeVal; 45 import com.google.common.base.Splitter; 46 47 import vogar.ExpectationStore; 48 import vogar.ModeId; 49 50 import java.io.BufferedReader; 51 import java.io.File; 52 import java.io.FilenameFilter; 53 import java.io.FileReader; 54 import java.io.IOException; 55 import java.io.PrintWriter; 56 import java.util.ArrayList; 57 import java.util.Arrays; 58 import java.util.Collection; 59 import java.util.Collections; 60 import java.util.HashSet; 61 import java.util.List; 62 import java.util.Set; 63 import java.util.concurrent.TimeUnit; 64 65 /** 66 * A wrapper to run tests against Dalvik. 67 */ 68 public class DalvikTest implements IAbiReceiver, IBuildReceiver, IDeviceTest, IRemoteTest, 69 IRuntimeHintProvider, IShardableTest, ITestCollector, ITestFileFilterReceiver, 70 ITestFilterReceiver { 71 72 private static final String TAG = DalvikTest.class.getSimpleName(); 73 74 /** 75 * TEST_PACKAGES is a Set containing the names of packages on the classpath known to contain 76 * tests to be run under DalvikTest. The TEST_PACKAGES set is used to shard DalvikTest into 77 * multiple DalvikTests, each responsible for running one of these packages' tests. 78 */ 79 private static final Set<String> TEST_PACKAGES = new HashSet<>(); 80 private static final String JDWP_PACKAGE_BASE = "org.apache.harmony.jpda.tests.jdwp.%s"; 81 static { 82 // Though uppercase, these are package names, not class names String.format(JDWP_PACKAGE_BASE, "ArrayReference")83 TEST_PACKAGES.add(String.format(JDWP_PACKAGE_BASE, "ArrayReference")); String.format(JDWP_PACKAGE_BASE, "ArrayType")84 TEST_PACKAGES.add(String.format(JDWP_PACKAGE_BASE, "ArrayType")); String.format(JDWP_PACKAGE_BASE, "ClassLoaderReference")85 TEST_PACKAGES.add(String.format(JDWP_PACKAGE_BASE, "ClassLoaderReference")); String.format(JDWP_PACKAGE_BASE, "ClassObjectReference")86 TEST_PACKAGES.add(String.format(JDWP_PACKAGE_BASE, "ClassObjectReference")); String.format(JDWP_PACKAGE_BASE, "ClassType")87 TEST_PACKAGES.add(String.format(JDWP_PACKAGE_BASE, "ClassType")); String.format(JDWP_PACKAGE_BASE, "DebuggerOnDemand")88 TEST_PACKAGES.add(String.format(JDWP_PACKAGE_BASE, "DebuggerOnDemand")); String.format(JDWP_PACKAGE_BASE, "Deoptimization")89 TEST_PACKAGES.add(String.format(JDWP_PACKAGE_BASE, "Deoptimization")); String.format(JDWP_PACKAGE_BASE, "EventModifiers")90 TEST_PACKAGES.add(String.format(JDWP_PACKAGE_BASE, "EventModifiers")); String.format(JDWP_PACKAGE_BASE, "Events")91 TEST_PACKAGES.add(String.format(JDWP_PACKAGE_BASE, "Events")); String.format(JDWP_PACKAGE_BASE, "InterfaceType")92 TEST_PACKAGES.add(String.format(JDWP_PACKAGE_BASE, "InterfaceType")); String.format(JDWP_PACKAGE_BASE, "Method")93 TEST_PACKAGES.add(String.format(JDWP_PACKAGE_BASE, "Method")); String.format(JDWP_PACKAGE_BASE, "MultiSession")94 TEST_PACKAGES.add(String.format(JDWP_PACKAGE_BASE, "MultiSession")); String.format(JDWP_PACKAGE_BASE, "ObjectReference")95 TEST_PACKAGES.add(String.format(JDWP_PACKAGE_BASE, "ObjectReference")); String.format(JDWP_PACKAGE_BASE, "ReferenceType")96 TEST_PACKAGES.add(String.format(JDWP_PACKAGE_BASE, "ReferenceType")); String.format(JDWP_PACKAGE_BASE, "StackFrame")97 TEST_PACKAGES.add(String.format(JDWP_PACKAGE_BASE, "StackFrame")); String.format(JDWP_PACKAGE_BASE, "StringReference")98 TEST_PACKAGES.add(String.format(JDWP_PACKAGE_BASE, "StringReference")); String.format(JDWP_PACKAGE_BASE, "ThreadGroupReference")99 TEST_PACKAGES.add(String.format(JDWP_PACKAGE_BASE, "ThreadGroupReference")); String.format(JDWP_PACKAGE_BASE, "ThreadReference")100 TEST_PACKAGES.add(String.format(JDWP_PACKAGE_BASE, "ThreadReference")); String.format(JDWP_PACKAGE_BASE, "VirtualMachine")101 TEST_PACKAGES.add(String.format(JDWP_PACKAGE_BASE, "VirtualMachine")); 102 } 103 104 private static final String EXPECTATIONS_EXT = ".expectations"; 105 // Command to run the VM, args are bitness, classpath, dalvik-args, abi, runner-args, 106 // include and exclude filters, and exclude filters file. 107 private static final String COMMAND = "dalvikvm%s -classpath %s %s " 108 + "com.android.compatibility.dalvik.DalvikTestRunner --abi=%s %s %s %s %s %s %s"; 109 private static final String INCLUDE_FILE = "/data/local/tmp/dalvik/includes"; 110 private static final String EXCLUDE_FILE = "/data/local/tmp/dalvik/excludes"; 111 private static String START_RUN = "start-run"; 112 private static String END_RUN = "end-run"; 113 private static String START_TEST = "start-test"; 114 private static String END_TEST = "end-test"; 115 private static String FAILURE = "failure"; 116 117 @Option(name = "run-name", description = "The name to use when reporting results") 118 private String mRunName; 119 120 @Option(name = "classpath", description = "Holds the paths to search when loading tests") 121 private List<String> mClasspath = new ArrayList<>(); 122 123 @Option(name = "dalvik-arg", description = "Holds arguments to pass to Dalvik") 124 private List<String> mDalvikArgs = new ArrayList<>(); 125 126 @Option(name = "runner-arg", 127 description = "Holds arguments to pass to the device-side test runner") 128 private List<String> mRunnerArgs = new ArrayList<>(); 129 130 @Option(name = "include-filter", 131 description = "The include filters of the test name to run.") 132 private List<String> mIncludeFilters = new ArrayList<>(); 133 134 @Option(name = "exclude-filter", 135 description = "The exclude filters of the test name to run.") 136 private List<String> mExcludeFilters = new ArrayList<>(); 137 138 @Option(name = "test-file-include-filter", 139 description="A file containing a list of line separated test classes and optionally" 140 + " methods to include") 141 private File mIncludeTestFile = null; 142 143 @Option(name = "test-file-exclude-filter", 144 description="A file containing a list of line separated test classes and optionally" 145 + " methods to exclude") 146 private File mExcludeTestFile = null; 147 148 @Option(name = "runtime-hint", 149 isTimeVal = true, 150 description="The hint about the test's runtime.") 151 private long mRuntimeHint = 60000;// 1 minute 152 153 @Option(name = "known-failures", 154 description = "Comma-separated list of files specifying known-failures to be skipped") 155 private String mKnownFailures; 156 157 @Option(name = "collect-tests-only", 158 description = "Only invoke the instrumentation to collect list of applicable test " 159 + "cases. All test run callbacks will be triggered, but test execution will " 160 + "not be actually carried out.") 161 private boolean mCollectTestsOnly = false; 162 163 @Option(name = "per-test-timeout", 164 description = "The maximum amount of time during which the DalvikTestRunner may " 165 + "yield no output. Because the runner outputs results for each test, this " 166 + "is essentially a per-test timeout") 167 private long mPerTestTimeout = 10; // 10 minutes 168 169 private IAbi mAbi; 170 private CompatibilityBuildHelper mBuildHelper; 171 private ITestDevice mDevice; 172 173 /** 174 * {@inheritDoc} 175 */ 176 @Override setAbi(IAbi abi)177 public void setAbi(IAbi abi) { 178 mAbi = abi; 179 } 180 181 /** 182 * {@inheritDoc} 183 */ 184 @Override getAbi()185 public IAbi getAbi() { 186 return mAbi; 187 } 188 189 /** 190 * {@inheritDoc} 191 */ 192 @Override setBuild(IBuildInfo build)193 public void setBuild(IBuildInfo build) { 194 mBuildHelper = new CompatibilityBuildHelper(build); 195 } 196 197 /** 198 * {@inheritDoc} 199 */ 200 @Override setDevice(ITestDevice device)201 public void setDevice(ITestDevice device) { 202 mDevice = device; 203 } 204 205 /** 206 * {@inheritDoc} 207 */ 208 @Override getDevice()209 public ITestDevice getDevice() { 210 return mDevice; 211 } 212 213 /** 214 * {@inheritDoc} 215 */ 216 @Override addIncludeFilter(String filter)217 public void addIncludeFilter(String filter) { 218 mIncludeFilters.add(filter); 219 } 220 221 /** 222 * {@inheritDoc} 223 */ 224 @Override addAllIncludeFilters(Set<String> filters)225 public void addAllIncludeFilters(Set<String> filters) { 226 mIncludeFilters.addAll(filters); 227 } 228 229 /** 230 * {@inheritDoc} 231 */ 232 @Override addExcludeFilter(String filter)233 public void addExcludeFilter(String filter) { 234 mExcludeFilters.add(filter); 235 } 236 237 /** 238 * {@inheritDoc} 239 */ 240 @Override addAllExcludeFilters(Set<String> filters)241 public void addAllExcludeFilters(Set<String> filters) { 242 mExcludeFilters.addAll(filters); 243 } 244 245 /** 246 * {@inheritDoc} 247 */ 248 @Override setIncludeTestFile(File testFile)249 public void setIncludeTestFile(File testFile) { 250 mIncludeTestFile = testFile; 251 } 252 253 /** 254 * {@inheritDoc} 255 */ 256 @Override setExcludeTestFile(File testFile)257 public void setExcludeTestFile(File testFile) { 258 mExcludeTestFile = testFile; 259 } 260 261 /** 262 * {@inheritDoc} 263 */ 264 @Override getRuntimeHint()265 public long getRuntimeHint() { 266 return mRuntimeHint; 267 } 268 269 /** 270 * {@inheritDoc} 271 */ 272 @Override setCollectTestsOnly(boolean shouldCollectTest)273 public void setCollectTestsOnly(boolean shouldCollectTest) { 274 mCollectTestsOnly = shouldCollectTest; 275 } 276 277 /** 278 * {@inheritDoc} 279 */ 280 @Override run(final ITestInvocationListener listener)281 public void run(final ITestInvocationListener listener) throws DeviceNotAvailableException { 282 String abiName = mAbi.getName(); 283 String bitness = AbiUtils.getBitness(abiName); 284 285 File tmpExcludeFile = null; 286 try { 287 // push one file of exclude filters to the device 288 tmpExcludeFile = getExcludeFile(); 289 if (!mDevice.pushFile(tmpExcludeFile, EXCLUDE_FILE)) { 290 Log.logAndDisplay(LogLevel.ERROR, TAG, "Couldn't push file: " + tmpExcludeFile); 291 } 292 } catch (IOException e) { 293 throw new RuntimeException("Failed to parse expectations", e); 294 } finally { 295 FileUtil.deleteFile(tmpExcludeFile); 296 } 297 298 // push one file of include filters to the device, if file exists 299 if (mIncludeTestFile != null) { 300 String path = mIncludeTestFile.getAbsolutePath(); 301 if (!mIncludeTestFile.isFile() || !mIncludeTestFile.canRead()) { 302 throw new RuntimeException(String.format("Failed to read include file %s", path)); 303 } 304 if (!mDevice.pushFile(mIncludeTestFile, INCLUDE_FILE)) { 305 Log.logAndDisplay(LogLevel.ERROR, TAG, "Couldn't push file: " + path); 306 } 307 } 308 309 310 // Create command 311 mDalvikArgs.add("-Duser.name=shell"); 312 mDalvikArgs.add("-Duser.language=en"); 313 mDalvikArgs.add("-Duser.region=US"); 314 mDalvikArgs.add("-Xcheck:jni"); 315 mDalvikArgs.add("-Xjnigreflimit:2000"); 316 317 String dalvikArgs = ArrayUtil.join(" ", mDalvikArgs); 318 dalvikArgs = dalvikArgs.replace("|#ABI#|", bitness); 319 320 String runnerArgs = ArrayUtil.join(" ", mRunnerArgs); 321 // Filters 322 StringBuilder includeFilters = new StringBuilder(); 323 if (!mIncludeFilters.isEmpty()) { 324 includeFilters.append("--include-filter="); 325 includeFilters.append(ArrayUtil.join(",", mIncludeFilters)); 326 } 327 StringBuilder excludeFilters = new StringBuilder(); 328 if (!mExcludeFilters.isEmpty()) { 329 excludeFilters.append("--exclude-filter="); 330 excludeFilters.append(ArrayUtil.join(",", mExcludeFilters)); 331 } 332 // Filter files 333 String includeFile = String.format("--include-filter-file=%s", INCLUDE_FILE); 334 String excludeFile = String.format("--exclude-filter-file=%s", EXCLUDE_FILE); 335 // Communicate with DalvikTestRunner if tests should only be collected 336 String collectTestsOnlyString = (mCollectTestsOnly) ? "--collect-tests-only" : ""; 337 final String command = String.format(COMMAND, bitness, 338 ArrayUtil.join(File.pathSeparator, mClasspath), 339 dalvikArgs, abiName, runnerArgs, 340 includeFilters, excludeFilters, includeFile, excludeFile, collectTestsOnlyString); 341 IShellOutputReceiver receiver = new MultiLineReceiver() { 342 private TestIdentifier test; 343 344 @Override 345 public boolean isCancelled() { 346 return false; 347 } 348 349 @Override 350 public void processNewLines(String[] lines) { 351 for (String line : lines) { 352 String[] parts = line.split(":", 2); 353 String tag = parts[0]; 354 if (tag.equals(START_RUN)) { 355 listener.testRunStarted(mRunName, Integer.parseInt(parts[1])); 356 Log.logAndDisplay(LogLevel.INFO, TAG, command); 357 Log.logAndDisplay(LogLevel.INFO, TAG, line); 358 } else if (tag.equals(END_RUN)) { 359 listener.testRunEnded(Integer.parseInt(parts[1]), 360 Collections.<String, String>emptyMap()); 361 Log.logAndDisplay(LogLevel.INFO, TAG, line); 362 } else if (tag.equals(START_TEST)) { 363 test = getTestIdentifier(parts[1]); 364 listener.testStarted(test); 365 } else if (tag.equals(FAILURE)) { 366 listener.testFailed(test, parts[1]); 367 } else if (tag.equals(END_TEST)) { 368 listener.testEnded(getTestIdentifier(parts[1]), 369 Collections.<String, String>emptyMap()); 370 } else { 371 Log.logAndDisplay(LogLevel.INFO, TAG, line); 372 } 373 } 374 } 375 376 private TestIdentifier getTestIdentifier(String name) { 377 String[] parts = name.split("#"); 378 String className = parts[0]; 379 String testName = ""; 380 if (parts.length > 1) { 381 testName = parts[1]; 382 } 383 return new TestIdentifier(className, testName); 384 } 385 386 }; 387 mDevice.executeShellCommand(command, receiver, mPerTestTimeout, TimeUnit.MINUTES, 1); 388 } 389 390 /* 391 * Due to known failures, there are typically too many excludes to pass via command line. 392 * Collect excludes from .expectation files in the testcases directory, from files in the 393 * module's resources directory, and from mExcludeTestFile, if set. 394 */ getExcludeFile()395 private File getExcludeFile() throws IOException { 396 File excludeFile = null; 397 PrintWriter out = null; 398 399 try { 400 excludeFile = File.createTempFile("excludes", "txt"); 401 out = new PrintWriter(excludeFile); 402 // create expectation store from set of expectation files found in testcases dir 403 Set<File> expectationFiles = new HashSet<>(); 404 for (File f : mBuildHelper.getTestsDir().listFiles( 405 new ExpectationFileFilter(mRunName))) { 406 expectationFiles.add(f); 407 } 408 ExpectationStore testsDirStore = 409 ExpectationStore.parse(expectationFiles, ModeId.DEVICE); 410 // create expectation store from expectation files found in module resources dir 411 ExpectationStore resourceStore = null; 412 if (mKnownFailures != null) { 413 Splitter splitter = Splitter.on(',').trimResults(); 414 Set<String> knownFailuresFiles = 415 new HashSet<>(splitter.splitToList(mKnownFailures)); 416 resourceStore = ExpectationStore.parseResources( 417 getClass(), knownFailuresFiles, ModeId.DEVICE); 418 } 419 // Add expectations from testcases dir 420 for (String exclude : testsDirStore.getAllFailures().keySet()) { 421 out.println(exclude); 422 } 423 for (String exclude : testsDirStore.getAllOutComes().keySet()) { 424 out.println(exclude); 425 } 426 // Add expectations from resources dir 427 if (resourceStore != null) { 428 for (String exclude : resourceStore.getAllFailures().keySet()) { 429 out.println(exclude); 430 } 431 for (String exclude : resourceStore.getAllOutComes().keySet()) { 432 out.println(exclude); 433 } 434 } 435 // Add excludes from test-file-exclude-filter option 436 for (String exclude : getFiltersFromFile(mExcludeTestFile)) { 437 out.println(exclude); 438 } 439 out.flush(); 440 } finally { 441 if (out != null) { 442 out.close(); 443 } 444 } 445 return excludeFile; 446 } 447 448 449 /* 450 * Helper method that reads filters from a file into a set. 451 * Returns an empty set given a null file 452 */ getFiltersFromFile(File f)453 private static Set<String> getFiltersFromFile(File f) throws IOException { 454 Set<String> filters = new HashSet<String>(); 455 if (f != null) { 456 BufferedReader reader = new BufferedReader(new FileReader(f)); 457 String filter = null; 458 while ((filter = reader.readLine()) != null) { 459 filters.add(filter); 460 } 461 reader.close(); 462 } 463 return filters; 464 } 465 466 /** 467 * {@inheritDoc} 468 */ 469 @Override split()470 public Collection<IRemoteTest> split() { 471 List<IRemoteTest> shards = new ArrayList<>(); 472 // A DalvikTest to run any tests not contained in packages from TEST_PACKAGES, may be empty 473 DalvikTest catchAll = new DalvikTest(); 474 OptionCopier.copyOptionsNoThrow(this, catchAll); 475 catchAll.mAbi = mAbi; 476 shards.add(catchAll); 477 // estimate catchAll's runtime to be that of a single package in TEST_PACKAGES 478 long runtimeHint = mRuntimeHint / TEST_PACKAGES.size(); 479 catchAll.mRuntimeHint = runtimeHint; 480 for (String packageName: TEST_PACKAGES) { 481 catchAll.addExcludeFilter(packageName); 482 // create one shard for package 'packageName' 483 DalvikTest test = new DalvikTest(); 484 OptionCopier.copyOptionsNoThrow(this, test); 485 test.addIncludeFilter(packageName); 486 test.mRuntimeHint = runtimeHint / TEST_PACKAGES.size(); 487 test.mAbi = mAbi; 488 shards.add(test); 489 } 490 // return a shard for each package in TEST_PACKAGE, plus a shard for any other tests 491 return shards; 492 } 493 494 /** 495 * A {@link FilenameFilter} to find all the expectation files in a directory. 496 */ 497 public static class ExpectationFileFilter implements FilenameFilter { 498 499 private String mName; 500 ExpectationFileFilter(String name)501 public ExpectationFileFilter(String name) { 502 mName = name; 503 } 504 /** 505 * {@inheritDoc} 506 */ 507 @Override accept(File dir, String name)508 public boolean accept(File dir, String name) { 509 return name.startsWith(mName) && name.endsWith(EXPECTATIONS_EXT); 510 } 511 } 512 } 513