1#!/usr/bin/env python3 2# 3# [VPYTHON:BEGIN] 4# python_version: "3.8" 5# [VPYTHON:END] 6# 7# Copyright (C) 2021 The Android Open Source Project 8# 9# Licensed under the Apache License, Version 2.0 (the "License"); 10# you may not use this file except in compliance with the License. 11# You may obtain a copy of the License at 12# 13# http://www.apache.org/licenses/LICENSE-2.0 14# 15# Unless required by applicable law or agreed to in writing, software 16# distributed under the License is distributed on an "AS IS" BASIS, 17# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18# See the License for the specific language governing permissions and 19# limitations under the License. 20 21import sys, os, argparse, subprocess, shlex, re, concurrent.futures, multiprocessing 22 23def parse_args(): 24 parser = argparse.ArgumentParser(description="Run libcore tests using the vogar testing tool.") 25 parser.add_argument('--mode', choices=['device', 'host', 'jvm'], required=True, 26 help='Specify where tests should be run.') 27 parser.add_argument('--variant', choices=['X32', 'X64'], 28 help='Which dalvikvm variant to execute with.') 29 parser.add_argument('-j', '--jobs', type=int, 30 help='Number of tests to run simultaneously.') 31 parser.add_argument('--timeout', type=int, 32 help='How long to run the test before aborting (seconds).') 33 parser.add_argument('--debug', action='store_true', 34 help='Use debug version of ART (device|host only).') 35 parser.add_argument('--dry-run', action='store_true', 36 help='Print vogar command-line, but do not run.') 37 parser.add_argument('--no-getrandom', action='store_false', dest='getrandom', 38 help='Ignore failures from getrandom() (for kernel < 3.17).') 39 parser.add_argument('--no-jit', action='store_false', dest='jit', 40 help='Disable JIT (device|host only).') 41 parser.add_argument('--gcstress', action='store_true', 42 help='Enable GC stress configuration (device|host only).') 43 parser.add_argument('tests', nargs="*", 44 help='Name(s) of the test(s) to run') 45 return parser.parse_args() 46 47ART_TEST_ANDROID_ROOT = os.environ.get("ART_TEST_ANDROID_ROOT", "/system") 48ART_TEST_CHROOT = os.environ.get("ART_TEST_CHROOT") 49ANDROID_PRODUCT_OUT = os.environ.get("ANDROID_PRODUCT_OUT") 50 51LIBCORE_TEST_NAMES = [ 52 # Naive critical path optimization: Run the longest tests first. 53 "org.apache.harmony.tests.java.util", # 90min under gcstress 54 "libcore.java.lang", # 90min under gcstress 55 "jsr166", # 60min under gcstress 56 "libcore.java.util", # 60min under gcstress 57 "libcore.java.math", # 50min under gcstress 58 "org.apache.harmony.crypto", # 30min under gcstress 59 "org.apache.harmony.tests.java.io", # 30min under gcstress 60 "org.apache.harmony.tests.java.text", # 30min under gcstress 61 # Split highmemorytest to individual classes since it is too big. 62 "libcore.highmemorytest.java.text.DateFormatTest", 63 "libcore.highmemorytest.java.text.DecimalFormatTest", 64 "libcore.highmemorytest.java.text.SimpleDateFormatTest", 65 "libcore.highmemorytest.java.time.format.DateTimeFormatterTest", 66 "libcore.highmemorytest.java.util.CalendarTest", 67 "libcore.highmemorytest.java.util.CurrencyTest", 68 "libcore.highmemorytest.libcore.icu.LocaleDataTest", 69 # All other tests in alphabetical order. 70 "libcore.android.system", 71 "libcore.build", 72 "libcore.dalvik.system", 73 "libcore.java.awt", 74 "libcore.java.text", 75 "libcore.javax.crypto", 76 "libcore.javax.net", 77 "libcore.javax.security", 78 "libcore.javax.sql", 79 "libcore.javax.xml", 80 "libcore.libcore.icu", 81 "libcore.libcore.internal", 82 "libcore.libcore.io", 83 "libcore.libcore.net", 84 "libcore.libcore.reflect", 85 "libcore.libcore.util", 86 "libcore.sun.invoke", 87 "libcore.sun.misc", 88 "libcore.sun.net", 89 "libcore.sun.security", 90 "libcore.sun.util", 91 "libcore.xml", 92 "org.apache.harmony.annotation", 93 "org.apache.harmony.luni", 94 "org.apache.harmony.nio", 95 "org.apache.harmony.regex", 96 "org.apache.harmony.testframework", 97 "org.apache.harmony.tests.java.lang", 98 "org.apache.harmony.tests.java.math", 99 "org.apache.harmony.tests.javax.security", 100 "tests.java.lang.String", 101] 102# "org.apache.harmony.security", # We don't have rights to revert changes in case of failures. 103 104# Note: This must start with the CORE_IMG_JARS in Android.common_path.mk 105# because that's what we use for compiling the boot.art image. 106# It may contain additional modules from TEST_CORE_JARS. 107BOOT_CLASSPATH = [ 108 "/apex/com.android.art/javalib/core-oj.jar", 109 "/apex/com.android.art/javalib/core-libart.jar", 110 "/apex/com.android.art/javalib/okhttp.jar", 111 "/apex/com.android.art/javalib/bouncycastle.jar", 112 "/apex/com.android.art/javalib/apache-xml.jar", 113 "/apex/com.android.i18n/javalib/core-icu4j.jar", 114 "/apex/com.android.conscrypt/javalib/conscrypt.jar", 115] 116 117CLASSPATH = ["core-tests", "jsr166-tests", "mockito-target"] 118 119def get_jar_filename(classpath): 120 base_path = (ANDROID_PRODUCT_OUT + "/../..") if ANDROID_PRODUCT_OUT else "out/target" 121 base_path = os.path.normpath(base_path) # Normalize ".." components for readability. 122 return f"{base_path}/common/obj/JAVA_LIBRARIES/{classpath}_intermediates/classes.jar" 123 124def get_timeout_secs(): 125 default_timeout_secs = 600 126 if args.mode == "device" and args.gcstress: 127 default_timeout_secs = 1200 128 if args.debug: 129 default_timeout_secs = 1800 130 return args.timeout or default_timeout_secs 131 132def get_expected_failures(): 133 failures = ["art/tools/libcore_failures.txt"] 134 if args.mode != "jvm": 135 if args.gcstress: 136 failures.append("art/tools/libcore_gcstress_failures.txt") 137 if args.gcstress and args.debug: 138 failures.append("art/tools/libcore_gcstress_debug_failures.txt") 139 if args.debug and not args.gcstress and args.getrandom: 140 failures.append("art/tools/libcore_debug_failures.txt") 141 if not args.getrandom: 142 failures.append("art/tools/libcore_fugu_failures.txt") 143 return failures 144 145def get_test_names(): 146 if args.tests: 147 return args.tests 148 test_names = list(LIBCORE_TEST_NAMES) 149 # See b/78228743 and b/178351808. 150 if args.gcstress or args.debug or args.mode == "jvm": 151 test_names = list(t for t in test_names if not t.startswith("libcore.highmemorytest")) 152 return test_names 153 154def get_vogar_command(test_name): 155 cmd = ["vogar"] 156 if args.mode == "device": 157 cmd.append("--mode=device --vm-arg -Ximage:/apex/com.android.art/javalib/boot.art") 158 cmd.append("--vm-arg -Xbootclasspath:" + ":".join(BOOT_CLASSPATH)) 159 if args.mode == "host": 160 # We explicitly give a wrong path for the image, to ensure vogar 161 # will create a boot image with the default compiler. Note that 162 # giving an existing image on host does not work because of 163 # classpath/resources differences when compiling the boot image. 164 cmd.append("--mode=host --vm-arg -Ximage:/non/existent/vogar.art") 165 if args.mode == "jvm": 166 cmd.append("--mode=jvm") 167 if args.variant: 168 cmd.append("--variant=" + args.variant) 169 if args.gcstress: 170 cmd.append("--vm-arg -Xgc:gcstress") 171 if args.debug: 172 cmd.append("--vm-arg -XXlib:libartd.so --vm-arg -XX:SlowDebug=true") 173 174 if args.mode == "device": 175 if ART_TEST_CHROOT: 176 cmd.append(f"--chroot {ART_TEST_CHROOT} --device-dir=/tmp/vogar/test-{test_name}") 177 else: 178 cmd.append("--device-dir=/data/local/tmp/vogar/test-{test_name}") 179 cmd.append(f"--vm-command={ART_TEST_ANDROID_ROOT}/bin/art") 180 else: 181 cmd.append(f"--device-dir=/tmp/vogar/test-{test_name}") 182 183 if args.mode != "jvm": 184 cmd.append("--timeout {}".format(get_timeout_secs())) 185 186 # Suppress explicit gc logs that are triggered an absurd number of times by these tests. 187 cmd.append("--vm-arg -XX:AlwaysLogExplicitGcs:false") 188 cmd.append("--toolchain d8 --language CUR") 189 if args.jit: 190 cmd.append("--vm-arg -Xcompiler-option --vm-arg --compiler-filter=quicken") 191 cmd.append("--vm-arg -Xusejit:{}".format(str(args.jit).lower())) 192 193 if args.gcstress: 194 # Bump pause threshold as long pauses cause explicit gc logging to occur irrespective 195 # of -XX:AlwayLogExplicitGcs:false. 196 cmd.append("--vm-arg -XX:LongPauseLogThreshold=15") # 15 ms (default: 5ms)) 197 198 # Suppress color codes if not attached to a terminal 199 if not sys.stdout.isatty(): 200 cmd.append("--no-color") 201 202 cmd.extend("--expectations " + f for f in get_expected_failures()) 203 cmd.extend("--classpath " + get_jar_filename(cp) for cp in CLASSPATH) 204 cmd.append(test_name) 205 return cmd 206 207def get_target_cpu_count(): 208 adb_command = 'adb shell cat /sys/devices/system/cpu/present' 209 with subprocess.Popen(adb_command.split(), 210 stderr=subprocess.STDOUT, 211 stdout=subprocess.PIPE, 212 universal_newlines=True) as proc: 213 assert(proc.wait() == 0) # Check the exit code. 214 match = re.match(r'\d*-(\d*)', proc.stdout.read()) 215 assert(match) 216 return int(match.group(1)) + 1 # Add one to convert from "last-index" to "count" 217 218def main(): 219 global args 220 args = parse_args() 221 222 if not os.path.exists('build/envsetup.sh'): 223 raise AssertionError("Script needs to be run at the root of the android tree") 224 for jar in map(get_jar_filename, CLASSPATH): 225 if not os.path.exists(jar): 226 raise AssertionError(f"Missing {jar}. Run buildbot-build.sh first.") 227 228 if not args.jobs: 229 if args.mode == "device": 230 args.jobs = get_target_cpu_count() 231 else: 232 args.jobs = multiprocessing.cpu_count() 233 if args.gcstress: 234 # TODO: Investigate and fix the underlying issues. 235 args.jobs = args.jobs // 2 236 237 def run_test(test_name): 238 cmd = " ".join(get_vogar_command(test_name)) 239 if args.dry_run: 240 return test_name, cmd, "Dry-run: skipping execution", 0 241 with subprocess.Popen(shlex.split(cmd), 242 stderr=subprocess.STDOUT, 243 stdout=subprocess.PIPE, 244 universal_newlines=True) as proc: 245 return test_name, cmd, proc.communicate()[0], proc.wait() 246 247 failed_regex = re.compile(r"^.* FAIL \((?:EXEC_FAILED|ERROR)\)$", re.MULTILINE) 248 failed_tests, max_exit_code = [], 0 249 with concurrent.futures.ThreadPoolExecutor(max_workers=args.jobs) as pool: 250 futures = [pool.submit(run_test, test_name) for test_name in get_test_names()] 251 print(f"Running {len(futures)} tasks on {args.jobs} core(s)...\n") 252 for i, future in enumerate(concurrent.futures.as_completed(futures)): 253 test_name, cmd, stdout, exit_code = future.result() 254 if exit_code != 0 or args.dry_run: 255 print(cmd) 256 print(stdout.strip()) 257 else: 258 print(stdout.strip().split("\n")[-1]) # Vogar final summary line. 259 failed_match = failed_regex.findall(stdout) 260 failed_tests.extend(failed_match) 261 max_exit_code = max(max_exit_code, exit_code) 262 result = "PASSED" if exit_code == 0 else f"FAILED ({len(failed_match)} test(s) failed)" 263 print(f"[{i+1}/{len(futures)}] Test set {test_name} {result}\n") 264 print(f"Overall, {len(failed_tests)} test(s) failed:") 265 print("\n".join(failed_tests)) 266 sys.exit(max_exit_code) 267 268if __name__ == '__main__': 269 main() 270