1#!/usr/bin/env python3
2#
3# [VPYTHON:BEGIN]
4# python_version: "3.8"
5# [VPYTHON:END]
6#
7# Copyright (C) 2021 The Android Open Source Project
8#
9# Licensed under the Apache License, Version 2.0 (the "License");
10# you may not use this file except in compliance with the License.
11# You may obtain a copy of the License at
12#
13#      http://www.apache.org/licenses/LICENSE-2.0
14#
15# Unless required by applicable law or agreed to in writing, software
16# distributed under the License is distributed on an "AS IS" BASIS,
17# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18# See the License for the specific language governing permissions and
19# limitations under the License.
20
21import sys, os, argparse, subprocess, shlex, re, concurrent.futures, multiprocessing
22
23def parse_args():
24  parser = argparse.ArgumentParser(description="Run libcore tests using the vogar testing tool.")
25  parser.add_argument('--mode', choices=['device', 'host', 'jvm'], required=True,
26                      help='Specify where tests should be run.')
27  parser.add_argument('--variant', choices=['X32', 'X64'],
28                      help='Which dalvikvm variant to execute with.')
29  parser.add_argument('-j', '--jobs', type=int,
30                      help='Number of tests to run simultaneously.')
31  parser.add_argument('--timeout', type=int,
32                      help='How long to run the test before aborting (seconds).')
33  parser.add_argument('--debug', action='store_true',
34                      help='Use debug version of ART (device|host only).')
35  parser.add_argument('--dry-run', action='store_true',
36                      help='Print vogar command-line, but do not run.')
37  parser.add_argument('--no-getrandom', action='store_false', dest='getrandom',
38                      help='Ignore failures from getrandom() (for kernel < 3.17).')
39  parser.add_argument('--no-jit', action='store_false', dest='jit',
40                      help='Disable JIT (device|host only).')
41  parser.add_argument('--gcstress', action='store_true',
42                      help='Enable GC stress configuration (device|host only).')
43  parser.add_argument('tests', nargs="*",
44                      help='Name(s) of the test(s) to run')
45  return parser.parse_args()
46
47ART_TEST_ANDROID_ROOT = os.environ.get("ART_TEST_ANDROID_ROOT", "/system")
48ART_TEST_CHROOT = os.environ.get("ART_TEST_CHROOT")
49ANDROID_PRODUCT_OUT = os.environ.get("ANDROID_PRODUCT_OUT")
50
51LIBCORE_TEST_NAMES = [
52  # Naive critical path optimization: Run the longest tests first.
53  "org.apache.harmony.tests.java.util",  # 90min under gcstress
54  "libcore.java.lang",                   # 90min under gcstress
55  "jsr166",                              # 60min under gcstress
56  "libcore.java.util",                   # 60min under gcstress
57  "libcore.java.math",                   # 50min under gcstress
58  "org.apache.harmony.crypto",           # 30min under gcstress
59  "org.apache.harmony.tests.java.io",    # 30min under gcstress
60  "org.apache.harmony.tests.java.text",  # 30min under gcstress
61  # Split highmemorytest to individual classes since it is too big.
62  "libcore.highmemorytest.java.text.DateFormatTest",
63  "libcore.highmemorytest.java.text.DecimalFormatTest",
64  "libcore.highmemorytest.java.text.SimpleDateFormatTest",
65  "libcore.highmemorytest.java.time.format.DateTimeFormatterTest",
66  "libcore.highmemorytest.java.util.CalendarTest",
67  "libcore.highmemorytest.java.util.CurrencyTest",
68  "libcore.highmemorytest.libcore.icu.LocaleDataTest",
69  # All other tests in alphabetical order.
70  "libcore.android.system",
71  "libcore.build",
72  "libcore.dalvik.system",
73  "libcore.java.awt",
74  "libcore.java.text",
75  "libcore.javax.crypto",
76  "libcore.javax.net",
77  "libcore.javax.security",
78  "libcore.javax.sql",
79  "libcore.javax.xml",
80  "libcore.libcore.icu",
81  "libcore.libcore.internal",
82  "libcore.libcore.io",
83  "libcore.libcore.net",
84  "libcore.libcore.reflect",
85  "libcore.libcore.util",
86  "libcore.sun.invoke",
87  "libcore.sun.misc",
88  "libcore.sun.net",
89  "libcore.sun.security",
90  "libcore.sun.util",
91  "libcore.xml",
92  "org.apache.harmony.annotation",
93  "org.apache.harmony.luni",
94  "org.apache.harmony.nio",
95  "org.apache.harmony.regex",
96  "org.apache.harmony.testframework",
97  "org.apache.harmony.tests.java.lang",
98  "org.apache.harmony.tests.java.math",
99  "org.apache.harmony.tests.javax.security",
100  "tests.java.lang.String",
101]
102# "org.apache.harmony.security",  # We don't have rights to revert changes in case of failures.
103
104# Note: This must start with the CORE_IMG_JARS in Android.common_path.mk
105# because that's what we use for compiling the boot.art image.
106# It may contain additional modules from TEST_CORE_JARS.
107BOOT_CLASSPATH = [
108  "/apex/com.android.art/javalib/core-oj.jar",
109  "/apex/com.android.art/javalib/core-libart.jar",
110  "/apex/com.android.art/javalib/okhttp.jar",
111  "/apex/com.android.art/javalib/bouncycastle.jar",
112  "/apex/com.android.art/javalib/apache-xml.jar",
113  "/apex/com.android.i18n/javalib/core-icu4j.jar",
114  "/apex/com.android.conscrypt/javalib/conscrypt.jar",
115]
116
117CLASSPATH = ["core-tests", "jsr166-tests", "mockito-target"]
118
119def get_jar_filename(classpath):
120  base_path = (ANDROID_PRODUCT_OUT + "/../..") if ANDROID_PRODUCT_OUT else "out/target"
121  base_path = os.path.normpath(base_path)  # Normalize ".." components for readability.
122  return f"{base_path}/common/obj/JAVA_LIBRARIES/{classpath}_intermediates/classes.jar"
123
124def get_timeout_secs():
125  default_timeout_secs = 600
126  if args.mode == "device" and args.gcstress:
127    default_timeout_secs = 1200
128    if args.debug:
129      default_timeout_secs = 1800
130  return args.timeout or default_timeout_secs
131
132def get_expected_failures():
133  failures = ["art/tools/libcore_failures.txt"]
134  if args.mode != "jvm":
135    if args.gcstress:
136      failures.append("art/tools/libcore_gcstress_failures.txt")
137    if args.gcstress and args.debug:
138      failures.append("art/tools/libcore_gcstress_debug_failures.txt")
139    if args.debug and not args.gcstress and args.getrandom:
140      failures.append("art/tools/libcore_debug_failures.txt")
141    if not args.getrandom:
142      failures.append("art/tools/libcore_fugu_failures.txt")
143  return failures
144
145def get_test_names():
146  if args.tests:
147    return args.tests
148  test_names = list(LIBCORE_TEST_NAMES)
149  # See b/78228743 and b/178351808.
150  if args.gcstress or args.debug or args.mode == "jvm":
151    test_names = list(t for t in test_names if not t.startswith("libcore.highmemorytest"))
152  return test_names
153
154def get_vogar_command(test_name):
155  cmd = ["vogar"]
156  if args.mode == "device":
157    cmd.append("--mode=device --vm-arg -Ximage:/apex/com.android.art/javalib/boot.art")
158    cmd.append("--vm-arg -Xbootclasspath:" + ":".join(BOOT_CLASSPATH))
159  if args.mode == "host":
160    # We explicitly give a wrong path for the image, to ensure vogar
161    # will create a boot image with the default compiler. Note that
162    # giving an existing image on host does not work because of
163    # classpath/resources differences when compiling the boot image.
164    cmd.append("--mode=host --vm-arg -Ximage:/non/existent/vogar.art")
165  if args.mode == "jvm":
166    cmd.append("--mode=jvm")
167  if args.variant:
168    cmd.append("--variant=" + args.variant)
169  if args.gcstress:
170    cmd.append("--vm-arg -Xgc:gcstress")
171  if args.debug:
172    cmd.append("--vm-arg -XXlib:libartd.so --vm-arg -XX:SlowDebug=true")
173
174  if args.mode == "device":
175    if ART_TEST_CHROOT:
176      cmd.append(f"--chroot {ART_TEST_CHROOT} --device-dir=/tmp/vogar/test-{test_name}")
177    else:
178      cmd.append("--device-dir=/data/local/tmp/vogar/test-{test_name}")
179    cmd.append(f"--vm-command={ART_TEST_ANDROID_ROOT}/bin/art")
180  else:
181    cmd.append(f"--device-dir=/tmp/vogar/test-{test_name}")
182
183  if args.mode != "jvm":
184    cmd.append("--timeout {}".format(get_timeout_secs()))
185
186    # Suppress explicit gc logs that are triggered an absurd number of times by these tests.
187    cmd.append("--vm-arg -XX:AlwaysLogExplicitGcs:false")
188    cmd.append("--toolchain d8 --language CUR")
189    if args.jit:
190      cmd.append("--vm-arg -Xcompiler-option --vm-arg --compiler-filter=quicken")
191    cmd.append("--vm-arg -Xusejit:{}".format(str(args.jit).lower()))
192
193    if args.gcstress:
194      # Bump pause threshold as long pauses cause explicit gc logging to occur irrespective
195      # of -XX:AlwayLogExplicitGcs:false.
196      cmd.append("--vm-arg -XX:LongPauseLogThreshold=15") # 15 ms (default: 5ms))
197
198  # Suppress color codes if not attached to a terminal
199  if not sys.stdout.isatty():
200    cmd.append("--no-color")
201
202  cmd.extend("--expectations " + f for f in get_expected_failures())
203  cmd.extend("--classpath " + get_jar_filename(cp) for cp in CLASSPATH)
204  cmd.append(test_name)
205  return cmd
206
207def get_target_cpu_count():
208  adb_command = 'adb shell cat /sys/devices/system/cpu/present'
209  with subprocess.Popen(adb_command.split(),
210                        stderr=subprocess.STDOUT,
211                        stdout=subprocess.PIPE,
212                        universal_newlines=True) as proc:
213    assert(proc.wait() == 0)  # Check the exit code.
214    match = re.match(r'\d*-(\d*)', proc.stdout.read())
215    assert(match)
216    return int(match.group(1)) + 1  # Add one to convert from "last-index" to "count"
217
218def main():
219  global args
220  args = parse_args()
221
222  if not os.path.exists('build/envsetup.sh'):
223    raise AssertionError("Script needs to be run at the root of the android tree")
224  for jar in map(get_jar_filename, CLASSPATH):
225    if not os.path.exists(jar):
226      raise AssertionError(f"Missing {jar}. Run buildbot-build.sh first.")
227
228  if not args.jobs:
229    if args.mode == "device":
230      args.jobs = get_target_cpu_count()
231    else:
232      args.jobs = multiprocessing.cpu_count()
233      if args.gcstress:
234        # TODO: Investigate and fix the underlying issues.
235        args.jobs = args.jobs // 2
236
237  def run_test(test_name):
238    cmd = " ".join(get_vogar_command(test_name))
239    if args.dry_run:
240      return test_name, cmd, "Dry-run: skipping execution", 0
241    with subprocess.Popen(shlex.split(cmd),
242                          stderr=subprocess.STDOUT,
243                          stdout=subprocess.PIPE,
244                          universal_newlines=True) as proc:
245      return test_name, cmd, proc.communicate()[0], proc.wait()
246
247  failed_regex = re.compile(r"^.* FAIL \((?:EXEC_FAILED|ERROR)\)$", re.MULTILINE)
248  failed_tests, max_exit_code = [], 0
249  with concurrent.futures.ThreadPoolExecutor(max_workers=args.jobs) as pool:
250    futures = [pool.submit(run_test, test_name) for test_name in get_test_names()]
251    print(f"Running {len(futures)} tasks on {args.jobs} core(s)...\n")
252    for i, future in enumerate(concurrent.futures.as_completed(futures)):
253      test_name, cmd, stdout, exit_code = future.result()
254      if exit_code != 0 or args.dry_run:
255        print(cmd)
256        print(stdout.strip())
257      else:
258        print(stdout.strip().split("\n")[-1])  # Vogar final summary line.
259      failed_match = failed_regex.findall(stdout)
260      failed_tests.extend(failed_match)
261      max_exit_code = max(max_exit_code, exit_code)
262      result = "PASSED" if exit_code == 0 else f"FAILED ({len(failed_match)} test(s) failed)"
263      print(f"[{i+1}/{len(futures)}] Test set {test_name} {result}\n")
264  print(f"Overall, {len(failed_tests)} test(s) failed:")
265  print("\n".join(failed_tests))
266  sys.exit(max_exit_code)
267
268if __name__ == '__main__':
269  main()
270