1#! /usr/bin/env python3 2# 3# Copyright 2020 The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17# Regenerate some ART test related files. 18 19# This script handles only a subset of ART run-tests at the moment; additional 20# cases will be added later. 21 22import argparse 23import collections 24import json 25import logging 26import os 27import re 28import sys 29import textwrap 30import xml.dom.minidom 31 32logging.basicConfig(format='%(levelname)s: %(message)s') 33 34ME = os.path.basename(sys.argv[0]) 35 36# Common advisory placed at the top of all generated files. 37ADVISORY = f"Generated by `{ME}`. Do not edit manually." 38 39# Default indentation unit. 40INDENT = " " 41 42# Indentation unit for XML files. 43XML_INDENT = " " 44 45def reindent(str, indent = ""): 46 """Reindent literal string while removing common leading spaces.""" 47 return textwrap.indent(textwrap.dedent(str), indent) 48 49def copyright_header_text(year): 50 """Return the copyright header text used in XML files.""" 51 return reindent(f"""\ 52 Copyright (C) {year} The Android Open Source Project 53 54 Licensed under the Apache License, Version 2.0 (the "License"); 55 you may not use this file except in compliance with the License. 56 You may obtain a copy of the License at 57 58 http://www.apache.org/licenses/LICENSE-2.0 59 60 Unless required by applicable law or agreed to in writing, software 61 distributed under the License is distributed on an "AS IS" BASIS, 62 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 63 See the License for the specific language governing permissions and 64 limitations under the License. 65 """, " ") 66 67def split_list(l, n): 68 """Return a list of `n` sublists of (contiguous) elements of list `l`.""" 69 assert n > 0 70 (d, m) = divmod(len(l), n) 71 # If the length of `l` is divisible by `n`, use that that divisor (`d`) as size of each sublist; 72 # otherwise, the next integer value (`d + 1`). 73 s = d if m == 0 else d + 1 74 result = [l[i:i + s] for i in range(0, len(l), s)] 75 assert len(result) == n 76 return result 77 78# The prefix used in the Soong module name of all ART run-tests. 79ART_RUN_TEST_MODULE_NAME_PREFIX = "art-run-test-" 80 81# Number of shards used to declare ART run-tests in the sharded ART MTS test plan. 82NUM_MTS_ART_RUN_TEST_SHARDS = 1 83 84# Known failing ART run-tests. 85# TODO(rpl): Investigate and address the causes of failures. 86known_failing_tests = [ 87 "004-SignalTest", 88 "004-UnsafeTest", 89 "030-bad-finalizer", 90 "034-call-null", 91 "038-inner-null", 92 "044-proxy", 93 "051-thread", 94 "054-uncaught", 95 "086-null-super", 96 "087-gc-after-link", 97 "096-array-copy-concurrent-gc", 98 "1004-checker-volatile-ref-load", 99 "115-native-bridge", 100 "116-nodex2oat", 101 "1336-short-finalizer-timeout", 102 "1337-gc-coverage", 103 "1339-dead-reference-safe", 104 "134-nodex2oat-nofallback", 105 "136-daemon-jni-shutdown", 106 "139-register-natives", 107 "148-multithread-gc-annotations", 108 "149-suspend-all-stress", 109 "150-loadlibrary", 110 "154-gc-loop", 111 "158-app-image-class-table", 112 "169-threadgroup-jni", 113 "172-app-image-twice", 114 "177-visibly-initialized-deadlock", 115 "178-app-image-native-method", 116 "179-nonvirtual-jni", 117 "450-checker-types", 118 "1900-track-alloc", 119 "1901-get-bytecodes", 120 "1902-suspend", 121 "1903-suspend-self", 122 "1904-double-suspend", 123 "1905-suspend-native", 124 "1906-suspend-list-me-first", 125 "1907-suspend-list-self-twice", 126 "1908-suspend-native-resume-self", 127 "1909-per-agent-tls", 128 "1910-transform-with-default", 129 "1911-get-local-var-table", 130 "1912-get-set-local-primitive", 131 "1913-get-set-local-objects", 132 "1914-get-local-instance", 133 "1915-get-set-local-current-thread", 134 "1916-get-set-current-frame", 135 "1917-get-stack-frame", 136 "1919-vminit-thread-start-timing", 137 "1920-suspend-native-monitor", 138 "1921-suspend-native-recursive-monitor", 139 "1922-owned-monitors-info", 140 "1923-frame-pop", 141 "1924-frame-pop-toggle", 142 "1925-self-frame-pop", 143 "1926-missed-frame-pop", 144 "1927-exception-event", 145 "1928-exception-event-exception", 146 "1930-monitor-info", 147 "1931-monitor-events", 148 "1932-monitor-events-misc", 149 "1933-monitor-current-contended", 150 "1934-jvmti-signal-thread", 151 "1935-get-set-current-frame-jit", 152 "1936-thread-end-events", 153 "1937-transform-soft-fail", 154 "1938-transform-abstract-single-impl", 155 "1939-proxy-frames", 156 "1941-dispose-stress", 157 "1942-suspend-raw-monitor-exit", 158 "1943-suspend-raw-monitor-wait", 159 "1945-proxy-method-arguments", 160 "1947-breakpoint-redefine-deopt", 161 "1949-short-dex-file", 162 "1951-monitor-enter-no-suspend", 163 "1953-pop-frame", 164 "1954-pop-frame-jit", 165 "1955-pop-frame-jit-called", 166 "1956-pop-frame-jit-calling", 167 "1957-error-ext", 168 "1958-transform-try-jit", 169 "1959-redefine-object-instrument", 170 "1960-obsolete-jit-multithread-native", 171 "1961-obsolete-jit-multithread", 172 "1962-multi-thread-events", 173 "1963-add-to-dex-classloader-in-memory", 174 "1967-get-set-local-bad-slot", 175 "1968-force-early-return", 176 "1969-force-early-return-void", 177 "1970-force-early-return-long", 178 "1971-multi-force-early-return", 179 "1972-jni-id-swap-indices", 180 "1973-jni-id-swap-pointer", 181 "1974-resize-array", 182 "1975-hello-structural-transformation", 183 "1976-hello-structural-static-methods", 184 "1977-hello-structural-obsolescence", 185 "1978-regular-obsolete-then-structural-obsolescence", 186 "1979-threaded-structural-transformation", 187 "1980-obsolete-object-cleared", 188 "1982-no-virtuals-structural-redefinition", 189 "1984-structural-redefine-field-trace", 190 "1985-structural-redefine-stack-scope", 191 "1986-structural-redefine-multi-thread-stack-scope", 192 "1987-structural-redefine-recursive-stack-scope", 193 "1988-multi-structural-redefine", 194 "1989-transform-bad-monitor", 195 "1990-structural-bad-verify", 196 "1991-hello-structural-retransform", 197 "1992-retransform-no-such-field", 198 "1993-fallback-non-structural", 199 "1994-final-virtual-structural", 200 "1995-final-virtual-structural-multithread", 201 "1996-final-override-virtual-structural", 202 "1997-structural-shadow-method", 203 "1998-structural-shadow-field", 204 "1999-virtual-structural", 205 "2003-double-virtual-structural", 206 "2004-double-virtual-structural-abstract", 207 "2005-pause-all-redefine-multithreaded", 208 "2008-redefine-then-old-reflect-field", 209 "2011-stack-walk-concurrent-instrument", 210 "203-multi-checkpoint", 211 "2031-zygote-compiled-frame-deopt", 212 "2033-shutdown-mechanics", 213 "2036-jni-filechannel", 214 "2037-thread-name-inherit", 215 "305-other-fault-handler", 216 "449-checker-bce", 217 "454-get-vreg", 218 "461-get-reference-vreg", 219 "466-get-live-vreg", 220 "497-inlining-and-class-loader", 221 "530-regression-lse", 222 "555-UnsafeGetLong-regression", 223 "566-polymorphic-inlining", 224 "595-profile-saving", 225 "597-deopt-busy-loop", 226 "597-deopt-invoke-stub", 227 "597-deopt-new-string", 228 "602-deoptimizeable", 229 "604-hot-static-interface", 230 "616-cha-abstract", 231 "616-cha-interface", 232 "616-cha-miranda", 233 "616-cha-native", 234 "616-cha-regression-proxy-method", 235 "616-cha", 236 "623-checker-loop-regressions", 237 "626-set-resolved-string", 238 "629-vdex-speed", 239 "638-checker-inline-cache-intrinsic", 240 "642-fp-callees", 241 "647-jni-get-field-id", 242 "652-deopt-intrinsic", 243 "655-jit-clinit", 244 "656-loop-deopt", 245 "660-clinit", 246 "661-oat-writer-layout", 247 "664-aget-verifier", 248 "667-jit-jni-stub", 249 "674-hotness-compiled", 250 "679-locks", 251 "680-checker-deopt-dex-pc-0", 252 "685-deoptimizeable", 253 "687-deopt", 254 "689-zygote-jit-deopt", 255 "693-vdex-inmem-loader-evict", 256 "707-checker-invalid-profile", 257 "708-jit-cache-churn", 258 "717-integer-value-of", 259 "720-thread-priority", 260 # 728-imt-conflict-zygote: Custom `run` script + dependency on `libarttest`. 261 "728-imt-conflict-zygote", 262 # 730-cha-deopt: Fails with: 263 # 264 # Test command execution failed with status FAILED: CommandResult: exit code=1, out=, err=Exception in thread "main" java.lang.ArrayIndexOutOfBoundsException: length=0; index=0 265 # at Main.main(Main.java:24) 266 # 267 "730-cha-deopt", 268 # 813-fp-args: Dependency on `libarttest`. 269 "813-fp-args", 270 # 821-many-args: Fails with: 271 # 272 # Test command execution failed with status FAILED: CommandResult: exit code=1, out=, err=Exception in thread "main" java.lang.ArrayIndexOutOfBoundsException: length=0; index=0 273 # at Main.main(Main.java:20) 274 # 275 "821-many-args", 276 "900-hello-plugin", 277 "901-hello-ti-agent", 278 "902-hello-transformation", 279 "903-hello-tagging", 280 "904-object-allocation", 281 "905-object-free", 282 "906-iterate-heap", 283 "907-get-loaded-classes", 284 "908-gc-start-finish", 285 "910-methods", 286 "911-get-stack-trace", 287 "913-heaps", 288 "914-hello-obsolescence", 289 "915-obsolete-2", 290 "916-obsolete-jit", 291 "917-fields-transformation", 292 "918-fields", 293 "919-obsolete-fields", 294 "920-objects", 295 "921-hello-failure", 296 "922-properties", 297 "923-monitors", 298 "924-threads", 299 "925-threadgroups", 300 "926-multi-obsolescence", 301 "927-timers", 302 "928-jni-table", 303 "930-hello-retransform", 304 "931-agent-thread", 305 "932-transform-saves", 306 "933-misc-events", 307 "937-hello-retransform-package", 308 "939-hello-transformation-bcp", 309 "940-recursive-obsolete", 310 "941-recursive-obsolete-jit", 311 "942-private-recursive", 312 "943-private-recursive-jit", 313 "944-transform-classloaders", 314 "945-obsolete-native", 315 "946-obsolete-throw", 316 "947-reflect-method", 317 "949-in-memory-transform", 318 "950-redefine-intrinsic", 319 "951-threaded-obsolete", 320 "982-ok-no-retransform", 321 "983-source-transform-verify", 322 "984-obsolete-invoke", 323 "985-re-obsolete", 324 "986-native-method-bind", 325 "987-agent-bind", 326 "988-method-trace", 327 "989-method-trace-throw", 328 "990-field-trace", 329 "991-field-trace-2", 330 "992-source-data", 331 "993-breakpoints", 332 "994-breakpoint-line", 333 "995-breakpoints-throw", 334 "996-breakpoint-obsolete", 335 "997-single-step", 336] 337 338# More known failing tests, related to Checker. 339# TODO(rpl): Investigate and address the causes of failures. 340known_failing_tests.extend([ 341 # Fails (on flame-userdebug) with: 342 # 343 # java.lang.RuntimeException: Error running Checker 344 # error: Statement could not be matched starting from line 564575 345 # RemTest.java:289: lsr x{{\d+}}, x{{\d+}}, #32 346 # ISA_FEATURES = {'a53': True, 'crc': True, 'lse': False, 'fp16': False, 'dotprod': False, 'sve': False} 347 # 348 "411-checker-hdiv-hrem-const", 349 # Fails (on aosp_cf_x86_phone-userdebug) with: 350 # 351 # Error while running Checker: java.lang.RuntimeException: Error running Checker command: 352 # error: Statement could not be matched starting from line 317325 353 # Main.java:296: InvokeStaticOrDirect 354 # ISA_FEATURES = {'ssse3': True, 'sse4': False} 355 # NewInstance = l6 356 # 357 "476-checker-ctor-fence-redun-elim", 358 # Fails (on aosp_cf_x86_phone-userdebug) with: 359 # 360 # Error while running Checker: java.lang.RuntimeException: Error running Checker command: 361 # error: Statement could not be matched starting from line 264874 362 # Main.java:77: InvokeStaticOrDirect [{{([ij]\d+,)?}}<<ClinitCheck>>] 363 # ClinitCheck = l4 364 # ISA_FEATURES = {'ssse3': True, 'sse4': False} 365 # LoadClass = l3 366 # 367 "478-checker-clinit-check-pruning", 368 # Fails (on aosp_cf_x86_phone-userdebug) with: 369 # 370 # Error while running Checker: java.lang.RuntimeException: Error running Checker command: 371 # error: Statement could not be matched starting from line 124181 372 # Main.java:178: <<Arg:z\d+>> StaticFieldGet liveness:<<ArgLiv:\d+>> ranges:{[<<ArgLiv>>,<<ArgUse:\d+>>)} uses:[<<ArgUse>>] 373 # ISA_FEATURES = {'ssse3': True, 'sse4': False} 374 # 375 "482-checker-loop-back-edge-use", 376 # Fails (on aosp_cf_x86_phone-userdebug) with: 377 # 378 # Error while running Checker: java.lang.RuntimeException: Error running Checker command: 379 # error: Statement could not be matched starting from line 7333 380 # Main.java:66: <<t1:i\d+>> Add [<<Arg>>,<<Const1>>] {{.*->e(bp|si|di)}} 381 # Arg = i0 382 # Const1 = i3 383 # ISA_FEATURES = {'ssse3': True, 'sse4': False} 384 # 385 "526-checker-caller-callee-regs", 386 # Fails (on aosp_cf_x86_phone-userdebug) with: 387 # 388 # Error while running Checker: java.lang.RuntimeException: Error running Checker command: 389 # error: NOT statement matched line 379347 390 # Main.java:538: NewInstance 391 # ISA_FEATURES = {'ssse3': True, 'sse4': False} 392 # 393 "530-checker-lse", 394 # Fails (on cf_x86_phone-userdebug_coverage) with: 395 # 396 # error: NOT statement matched line 117078 397 # Main.java:108: NewInstance 398 # ISA_FEATURES = {'ssse3': True, 'sse4': False} 399 # 400 "530-checker-lse2", 401 # Fails (on aosp_cf_x86_phone-userdebug) with: 402 # 403 # Error while running Checker: java.lang.RuntimeException: Error running Checker command: 404 # error: NOT statement matched line 238857 405 # Main.java:650: Shl 406 # ISA_FEATURES = {'ssse3': True, 'sse4': False} 407 # 408 "551-checker-shifter-operand", 409 # Fails (on aosp_cf_x86_phone-userdebug) with: 410 # 411 # Error while running Checker: java.lang.RuntimeException: Error running Checker command: 412 # error: Statement could not be matched starting from line 158575 413 # Main.java:97: X86ComputeBaseMethodAddress 414 # ISA_FEATURES = {'ssse3': True, 'sse4': False} 415 # 416 "552-checker-sharpening", 417 # Fails (on flame-userdebug) with: 418 # 419 # java.lang.RuntimeException: Error running Checker 420 # error: Statement could not be matched starting from line 11964 421 # Main.java:59: <<ConstM42:i\d+>> IntConstant -42 422 # ISA_FEATURES = {'a53': True, 'crc': True, 'lse': False, 'fp16': False, 'dotprod': False, 'sve': False} 423 # 424 # 425 "562-checker-no-intermediate", 426 # Fails (on cf_x86_phone-userdebug_coverage) with: 427 # 428 # error: Statement could not be matched starting from line 5260 429 # Main.java:24: InstanceFieldSet 430 # ISA_FEATURES = {'ssse3': True, 'sse4': False} 431 # 432 "583-checker-zero", 433 # Fails (on aosp_cf_x86_phone-userdebug) with: 434 # 435 # Error while running Checker: java.lang.RuntimeException: Error running Checker command: 436 # error: Statement could not be matched starting from line 149082 437 # Main.java:312: <<LoadClass:l\d+>> LoadClass class_name:Main 438 # ISA_FEATURES = {'ssse3': True, 'sse4': False} 439 # Int42 = i8 440 # Int43 = i10 441 # 442 "639-checker-code-sinking", 443 # Fails (on aosp_cf_x86_phone-userdebug) with: 444 # 445 # error: Statement could not be matched starting from line 12527 446 # Main.java:24: InvokeVirtual method_name:java.lang.StringBuilder.toString intrinsic:StringBuilderToString 447 # ISA_FEATURES = {'ssse3': True, 'sse4': False} 448 # 449 "729-checker-polymorphic-intrinsic", 450]) 451 452known_failing_tests = frozenset(known_failing_tests) 453 454# Percentage of ART run-tests (among the ones expected to succeed) to include in 455# the `presubmit` test group in `TEST_MAPPING` file -- the rest will be included 456# in `postsubmit` test group. 457# This value has to be a number between 0 and 100. 458presubmit_tests_percentage = 100 459 460# Percentage of ART run-tests (among the ones expected to succeed) to include in 461# the `mainline-presubmit` test group in `TEST_MAPPING` file. 462# This value has to be a number between 0 and 100. 463mainline_presubmit_tests_percentage = 100 464 465# ART gtests that do not need root access to the device. 466art_gtest_user_module_names = [ 467 "art_standalone_cmdline_tests", 468 "art_standalone_compiler_tests", 469 "art_standalone_dex2oat_tests", 470 "art_standalone_dexdump_tests", 471 "art_standalone_dexlist_tests", 472 "art_standalone_libartbase_tests", 473 "art_standalone_libartpalette_tests", 474 "art_standalone_libdexfile_support_tests", 475 "art_standalone_libdexfile_tests", 476 "art_standalone_libprofile_tests", 477 "art_standalone_oatdump_tests", 478 "art_standalone_odrefresh_tests", 479 "art_standalone_runtime_compiler_tests", 480 "art_standalone_runtime_tests", 481 "art_standalone_sigchain_tests", 482] 483 484# ART gtests that need root access to the device. 485art_gtest_eng_only_module_names = [ 486 "art_standalone_dexoptanalyzer_tests", 487 "art_standalone_profman_tests", 488] 489 490# All supported ART gtests. 491art_gtest_module_names = sorted(art_gtest_user_module_names + art_gtest_eng_only_module_names) 492 493 494# Is `run_test` a Checker test (i.e. a test containing Checker 495# assertions)? 496def is_checker_test(run_test): 497 return re.match("^[0-9]+-checker-", run_test) 498 499# Is `run_test` expected to succeed? 500def is_expected_succeeding(run_test): 501 return run_test not in known_failing_tests 502 503 504class Generator: 505 def __init__(self, top_dir): 506 """Generator of ART test files for an Android source tree anchored at `top_dir`.""" 507 # Path to the Android top source tree. 508 self.top_dir = top_dir 509 # Path to the ART directory 510 self.art_dir = os.path.join(top_dir, "art") 511 # Path to the ART tests directory. 512 self.art_test_dir = os.path.join(self.art_dir, "test") 513 # Path to the MTS configuration directory. 514 self.mts_config_dir = os.path.join( 515 top_dir, "test", "mts", "tools", "mts-tradefed", "res", "config") 516 517 def enumerate_run_tests(self): 518 return sorted([run_test 519 for run_test in os.listdir(self.art_test_dir) 520 if re.match("^[0-9]{3,}-", run_test)]) 521 522 # Is building `run_test` supported? 523 # TODO(b/147814778): Add build support for more tests. 524 def is_buildable(self, run_test): 525 run_test_path = os.path.join(self.art_test_dir, run_test) 526 527 # Ignore tests with non-default build rules. 528 if os.path.isfile(os.path.join(run_test_path, "build")): 529 return False 530 # Ignore tests with no `src` directory. 531 if not os.path.isdir(os.path.join(run_test_path, "src")): 532 return False 533 # Ignore tests with sources outside the `src` directory. 534 for subdir in ["jasmin", 535 "jasmin-multidex", 536 "smali", 537 "smali-ex", 538 "smali-multidex", 539 "src-art", 540 "src-dex2oat-unresolved", 541 "src-ex", 542 "src-ex2", 543 "src-multidex", 544 "src2"]: 545 if os.path.isdir(os.path.join(run_test_path, subdir)): 546 return False 547 # Ignore test with a copy of `sun.misc.Unsafe`. 548 if os.path.isfile(os.path.join(run_test_path, "src", "sun", "misc", "Unsafe.java")): 549 return False 550 # Ignore tests with Hidden API specs. 551 if os.path.isfile(os.path.join(run_test_path, "hiddenapi-flags.csv")): 552 return False 553 # All other tests are considered buildable. 554 return True 555 556 def regen_bp_files(self, run_tests, buildable_tests): 557 for run_test in run_tests: 558 # Remove any previously generated file. 559 bp_file = os.path.join(self.art_test_dir, run_test, "Android.bp") 560 if os.path.exists(bp_file): 561 logging.debug(f"Removing `{bp_file}`.") 562 os.remove(bp_file) 563 564 for run_test in buildable_tests: 565 self.regen_bp_file(run_test) 566 567 def regen_bp_file(self, run_test): 568 """Regenerate Blueprint file for an ART run-test.""" 569 570 bp_file = os.path.join(self.art_test_dir, run_test, "Android.bp") 571 572 run_test_module_name = ART_RUN_TEST_MODULE_NAME_PREFIX + run_test 573 574 if is_expected_succeeding(run_test): 575 test_config_template = "art-run-test-target-template" 576 else: 577 test_config_template = "art-run-test-target-no-test-suite-tag-template" 578 579 if is_checker_test(run_test): 580 include_src = """\ 581 582 // Include the Java source files in the test's artifacts, to make Checker assertions 583 // available to the TradeFed test runner. 584 include_srcs: true,""" 585 else: 586 include_src = "" 587 with open(bp_file, "w") as f: 588 logging.debug(f"Writing `{bp_file}`.") 589 f.write(textwrap.dedent(f"""\ 590 // {ADVISORY} 591 592 // Build rules for ART run-test `{run_test}`. 593 594 package {{ 595 // See: http://go/android-license-faq 596 // A large-scale-change added 'default_applicable_licenses' to import 597 // all of the 'license_kinds' from "art_license" 598 // to get the below license kinds: 599 // SPDX-license-identifier-Apache-2.0 600 default_applicable_licenses: ["art_license"], 601 }} 602 603 // Test's Dex code. 604 java_test {{ 605 name: "{run_test_module_name}", 606 defaults: ["art-run-test-defaults"], 607 test_config_template: ":{test_config_template}", 608 srcs: ["src/**/*.java"], 609 data: [ 610 ":{run_test_module_name}-expected-stdout", 611 ":{run_test_module_name}-expected-stderr", 612 ],{include_src} 613 }} 614 615 // Test's expected standard output. 616 genrule {{ 617 name: "{run_test_module_name}-expected-stdout", 618 out: ["{run_test_module_name}-expected-stdout.txt"], 619 srcs: ["expected-stdout.txt"], 620 cmd: "cp -f $(in) $(out)", 621 }} 622 623 // Test's expected standard error. 624 genrule {{ 625 name: "{run_test_module_name}-expected-stderr", 626 out: ["{run_test_module_name}-expected-stderr.txt"], 627 srcs: ["expected-stderr.txt"], 628 cmd: "cp -f $(in) $(out)", 629 }} 630 """)) 631 632 def regen_test_mapping_file(self, art_run_tests, num_presubmit_run_tests, 633 num_mainline_presubmit_run_tests): 634 """Regenerate ART's `TEST_MAPPING`.""" 635 636 run_test_module_names = [ART_RUN_TEST_MODULE_NAME_PREFIX + t for t in art_run_tests] 637 638 # Mainline presubmits. 639 mainline_presubmit_run_tests = run_test_module_names[0:num_mainline_presubmit_run_tests] 640 mainline_presubmit_tests = mainline_presubmit_run_tests + art_gtest_module_names 641 mainline_presubmit_tests_with_apex = [t + "[com.google.android.art.apex]" 642 for t 643 in mainline_presubmit_tests] 644 mainline_presubmit_tests_dict = [{"name": t} for t in mainline_presubmit_tests_with_apex] 645 646 # Presubmits. 647 other_presubmit_tests = [ 648 "CtsJdwpTestCases", 649 "BootImageProfileTest", 650 ] 651 presubmit_run_tests = run_test_module_names[0:num_presubmit_run_tests] 652 presubmit_tests = other_presubmit_tests + presubmit_run_tests + art_gtest_module_names 653 presubmit_tests_dict = [{"name": t} for t in presubmit_tests] 654 655 # Postsubmits. 656 postsubmit_run_tests = run_test_module_names[num_presubmit_run_tests:] 657 postsubmit_tests_dict = [{"name": t} for t in postsubmit_run_tests] 658 659 # Use an `OrderedDict` container to preserve the order in which items are inserted. 660 # Do not produce an entry for a test group if it is empty. 661 test_mapping_dict = collections.OrderedDict([ 662 (test_group_name, test_group_dict) 663 for (test_group_name, test_group_dict) 664 in [ 665 ("mainline-presubmit", mainline_presubmit_tests_dict), 666 ("presubmit", presubmit_tests_dict), 667 ("postsubmit", postsubmit_tests_dict), 668 ] 669 if test_group_dict 670 ]) 671 test_mapping_contents = json.dumps(test_mapping_dict, indent = INDENT) 672 673 test_mapping_file = os.path.join(self.art_dir, "TEST_MAPPING") 674 with open(test_mapping_file, "w") as f: 675 logging.debug(f"Writing `{test_mapping_file}`.") 676 f.write(f"// {ADVISORY}\n") 677 f.write(test_mapping_contents) 678 f.write("\n") 679 680 def create_mts_test_shard(self, description, tests, shard_num, copyright_year, comments = []): 681 """Factory method instantiating an `MtsTestShard`.""" 682 return self.MtsTestShard(self.mts_config_dir, 683 description, tests, shard_num, copyright_year, comments) 684 685 class MtsTestShard: 686 """Class encapsulating data and generation logic for an ART MTS test shard.""" 687 688 def __init__(self, mts_config_dir, description, tests, shard_num, copyright_year, comments): 689 self.mts_config_dir = mts_config_dir 690 self.description = description 691 self.tests = tests 692 self.shard_num = shard_num 693 self.copyright_year = copyright_year 694 self.comments = comments 695 696 def shard_id(self): 697 return f"{self.shard_num:02}" 698 699 def test_plan_name(self): 700 return "mts-art-shard-" + self.shard_id() 701 702 def test_list_name(self): 703 return "mts-art-tests-list-user-shard-" + self.shard_id() 704 705 def regen_test_plan_file(self): 706 """Regenerate ART MTS test plan file shard (`mts-art-shard-<shard_num>.xml`).""" 707 root = xml.dom.minidom.Document() 708 709 advisory_header = root.createComment(f" {ADVISORY} ") 710 root.appendChild(advisory_header) 711 copyright_header = root.createComment(copyright_header_text(self.copyright_year)) 712 root.appendChild(copyright_header) 713 714 configuration = root.createElement("configuration") 715 root.appendChild(configuration) 716 configuration.setAttribute( 717 "description", 718 f"Run mts-art-shard-{self.shard_id()} from a preexisting MTS installation.") 719 720 # Included XML files. 721 included_xml_files = ["mts", self.test_list_name()] 722 for xml_file in included_xml_files: 723 include = root.createElement("include") 724 include.setAttribute("name", xml_file) 725 configuration.appendChild(include) 726 727 # Test plan name. 728 option = root.createElement("option") 729 option.setAttribute("name", "plan") 730 option.setAttribute("value", self.test_plan_name()) 731 configuration.appendChild(option) 732 733 xml_str = root.toprettyxml(indent = XML_INDENT, encoding = "utf-8") 734 735 test_plan_file = os.path.join(self.mts_config_dir, self.test_plan_name() + ".xml") 736 with open(test_plan_file, "wb") as f: 737 logging.debug(f"Writing `{test_plan_file}`.") 738 f.write(xml_str) 739 740 def regen_test_list_file(self): 741 """Regenerate ART MTS test list file (`mts-art-tests-list-user-shard-<shard_num>.xml`).""" 742 root = xml.dom.minidom.Document() 743 744 advisory_header = root.createComment(f" {ADVISORY} ") 745 root.appendChild(advisory_header) 746 copyright_header = root.createComment(copyright_header_text(self.copyright_year)) 747 root.appendChild(copyright_header) 748 749 configuration = root.createElement("configuration") 750 root.appendChild(configuration) 751 configuration.setAttribute( 752 "description", 753 f"List of ART MTS tests that do not need root access (shard {self.shard_id()})" 754 ) 755 756 # Test declarations. 757 # ------------------ 758 759 def append_test_declaration(test): 760 option = root.createElement("option") 761 option.setAttribute("name", "compatibility:include-filter") 762 option.setAttribute("value", test) 763 configuration.appendChild(option) 764 765 test_declarations_comments = [self.description + "."] 766 test_declarations_comments.extend(self.comments) 767 for c in test_declarations_comments: 768 xml_comment = root.createComment(f" {c} ") 769 configuration.appendChild(xml_comment) 770 for t in self.tests: 771 append_test_declaration(t) 772 773 # `MainlineTestModuleController` configurations. 774 # ---------------------------------------------- 775 776 def append_module_controller_configuration(test): 777 option = root.createElement("option") 778 option.setAttribute("name", "compatibility:module-arg") 779 option.setAttribute("value", f"{test}:enable:true") 780 configuration.appendChild(option) 781 782 module_controller_configuration_comments = [ 783 f"Enable MainlineTestModuleController for {self.description}."] 784 module_controller_configuration_comments.extend(self.comments) 785 for c in module_controller_configuration_comments: 786 xml_comment = root.createComment(f" {c} ") 787 configuration.appendChild(xml_comment) 788 for t in self.tests: 789 append_module_controller_configuration(t) 790 791 xml_str = root.toprettyxml(indent = XML_INDENT, encoding = "utf-8") 792 793 test_list_file = os.path.join(self.mts_config_dir, self.test_list_name() + ".xml") 794 with open(test_list_file, "wb") as f: 795 logging.debug(f"Writing `{test_list_file}`.") 796 f.write(xml_str) 797 798 def regen_mts_art_tests_list_user_file(self, num_mts_art_run_test_shards): 799 """Regenerate ART MTS test list file (`mts-art-tests-list-user.xml`).""" 800 root = xml.dom.minidom.Document() 801 802 advisory_header = root.createComment(f" {ADVISORY} ") 803 root.appendChild(advisory_header) 804 copyright_header = root.createComment(copyright_header_text(2020)) 805 root.appendChild(copyright_header) 806 807 configuration = root.createElement("configuration") 808 root.appendChild(configuration) 809 configuration.setAttribute("description", "List of ART MTS tests that do not need root access.") 810 811 # Included XML files. 812 for s in range(num_mts_art_run_test_shards): 813 include = root.createElement("include") 814 include.setAttribute("name", f"mts-art-tests-list-user-shard-{s:02}") 815 configuration.appendChild(include) 816 817 xml_str = root.toprettyxml(indent = XML_INDENT, encoding = "utf-8") 818 819 mts_art_tests_list_user_file = os.path.join(self.mts_config_dir, "mts-art-tests-list-user.xml") 820 with open(mts_art_tests_list_user_file, "wb") as f: 821 logging.debug(f"Writing `{mts_art_tests_list_user_file}`.") 822 f.write(xml_str) 823 824 def regen_art_mts_files(self, art_run_tests): 825 """Regenerate ART MTS definition files.""" 826 827 # Remove any previously MTS ART test plan shard (`mts-art-shard-[0-9]+.xml`) 828 # and any test list shard (`mts-art-tests-list-user-shard-[0-9]+.xml`). 829 old_test_plan_shards = sorted([ 830 test_plan_shard 831 for test_plan_shard in os.listdir(self.mts_config_dir) 832 if re.match("^mts-art-(tests-list-user-)?shard-[0-9]+.xml$", test_plan_shard)]) 833 for shard in old_test_plan_shards: 834 shard_path = os.path.join(self.mts_config_dir, shard) 835 if os.path.exists(shard_path): 836 logging.debug(f"Removing `{shard_path}`.") 837 os.remove(shard_path) 838 839 mts_test_shards = [] 840 841 # ART test (gtest & run-test) shard(s). 842 # TODO: Also handle the case of gtests requiring root access to the device 843 # (`art_gtest_eng_only_module_names`). 844 art_run_test_module_names = [ART_RUN_TEST_MODULE_NAME_PREFIX + t for t in art_run_tests] 845 art_run_test_shards = split_list(art_run_test_module_names, NUM_MTS_ART_RUN_TEST_SHARDS) 846 for i in range(len(art_run_test_shards)): 847 art_tests_shard_i_tests = art_run_test_shards[i] 848 # Append ART gtests to the last ART run-test shard for now. 849 # If needed, consider moving them to their own shard to increase 850 # the parallelization of code coverage runs. 851 if i + 1 == len(art_run_test_shards): 852 art_tests_shard_i_tests.extend(art_gtest_user_module_names) 853 art_tests_shard_i = self.create_mts_test_shard( 854 "ART run-tests", art_tests_shard_i_tests, i, 2020, 855 ["TODO(rpl): Find a way to express this list in a more concise fashion."]) 856 mts_test_shards.append(art_tests_shard_i) 857 858 # CTS Libcore non-OJ tests (`CtsLibcoreTestCases`) shard. 859 cts_libcore_tests_shard_num = len(mts_test_shards) 860 cts_libcore_tests_shard = self.create_mts_test_shard( 861 "CTS Libcore non-OJ tests", ["CtsLibcoreTestCases"], cts_libcore_tests_shard_num, 2020) 862 mts_test_shards.append(cts_libcore_tests_shard) 863 864 # Other CTS Libcore tests shard. 865 other_cts_libcore_tests_shard_num = len(mts_test_shards) 866 other_cts_libcore_tests_shard_tests = [ 867 "CtsLibcoreApiEvolutionTestCases", 868 "CtsLibcoreFileIOTestCases", 869 "CtsLibcoreJsr166TestCases", 870 "CtsLibcoreLegacy22TestCases", 871 "CtsLibcoreOjTestCases", 872 "CtsLibcoreWycheproofBCTestCases", 873 "MtsLibcoreOkHttpTestCases", 874 ] 875 other_cts_libcore_tests_shard = self.create_mts_test_shard( 876 "CTS Libcore OJ tests", other_cts_libcore_tests_shard_tests, 877 other_cts_libcore_tests_shard_num, 2021) 878 mts_test_shards.append(other_cts_libcore_tests_shard) 879 880 for s in mts_test_shards: 881 s.regen_test_plan_file() 882 s.regen_test_list_file() 883 884 self.regen_mts_art_tests_list_user_file(len(mts_test_shards)) 885 886 def regen_test_files(self, regen_art_mts): 887 """Regenerate ART test files. 888 889 Args: 890 regen_art_mts: If true, also regenerate the ART MTS definition. 891 """ 892 run_tests = self.enumerate_run_tests() 893 894 # Create a list of the tests that can currently be built, and for 895 # which a Blueprint file is to be generated. 896 buildable_tests = list(filter(self.is_buildable, run_tests)) 897 898 # Create a list of the tests that can be built and are expected to 899 # succeed. These tests are to be added to ART's `TEST_MAPPING` 900 # file and also tagged as part of TradeFed's `art-target-run-test` 901 # test suite via the `test-suite-tag` option in their 902 # configuration file. 903 expected_succeeding_tests = list(filter(is_expected_succeeding, buildable_tests)) 904 905 # Regenerate Blueprint files. 906 # --------------------------- 907 908 self.regen_bp_files(run_tests, buildable_tests) 909 910 buildable_tests_percentage = int(len(buildable_tests) * 100 / len(run_tests)) 911 912 print(f"Generated Blueprint files for {len(buildable_tests)} ART run-tests out of" 913 f" {len(run_tests)} ({buildable_tests_percentage}%).") 914 915 # Regenerate `TEST_MAPPING` file. 916 # ------------------------------- 917 918 # Note: We only include ART run-tests expected to succeed for now. 919 920 # Note: We only include a (growing) fraction of the supported ART 921 # run-tests (see `presubmit_tests_percentage`) into the 922 # `presubmit` test group (the other ART run-tests are added to the 923 # `postsubmit` test group), as we initially had issues with 924 # Android presubmits when the whole set of supported ART run-tests 925 # was included in one go (b/169310621). This progressive rollout 926 # allows us to better monitor future potential presubmit failures. 927 # 928 # Likewise for tests in the `mainline-presubmit` group. 929 num_presubmit_run_tests = int(len(expected_succeeding_tests) * presubmit_tests_percentage / 100) 930 num_mainline_presubmit_run_tests = int( 931 len(expected_succeeding_tests) * mainline_presubmit_tests_percentage / 100) 932 self.regen_test_mapping_file( 933 expected_succeeding_tests, num_presubmit_run_tests, num_mainline_presubmit_run_tests) 934 935 expected_succeeding_tests_percentage = int( 936 len(expected_succeeding_tests) * 100 / len(run_tests)) 937 938 num_postsubmit_tests = len(expected_succeeding_tests) - num_presubmit_run_tests 939 postsubmit_tests_percentage = 100 - presubmit_tests_percentage 940 941 print(f"Generated TEST_MAPPING entries for {len(expected_succeeding_tests)} ART run-tests out" 942 f" of {len(run_tests)} ({expected_succeeding_tests_percentage}%):") 943 for (num_tests, test_kind, tests_percentage, test_group_name) in [ 944 (num_mainline_presubmit_run_tests, "ART run-tests", mainline_presubmit_tests_percentage, 945 "mainline-presubmit"), 946 (len(art_gtest_module_names), "ART gtests", 100, "mainline-presubmit"), 947 (num_presubmit_run_tests, "ART run-tests", presubmit_tests_percentage, "presubmit"), 948 (len(art_gtest_module_names), "ART gtests", 100, "presubmit"), 949 (num_postsubmit_tests, "ART run-tests", postsubmit_tests_percentage, "postsubmit"), 950 ]: 951 print( 952 f" {num_tests:3d} {test_kind} ({tests_percentage}%) in `{test_group_name}` test group.") 953 954 # Regenerate ART MTS definition (optional). 955 # ----------------------------------------- 956 957 if regen_art_mts: 958 self.regen_art_mts_files(expected_succeeding_tests) 959 print(f"Generated ART MTS entries for {len(expected_succeeding_tests)} ART run-tests out" 960 f" of {len(run_tests)} ({expected_succeeding_tests_percentage}%).") 961 962def main(): 963 if "ANDROID_BUILD_TOP" not in os.environ: 964 logging.error("ANDROID_BUILD_TOP environment variable is empty; did you forget to run `lunch`?") 965 sys.exit(1) 966 967 parser = argparse.ArgumentParser( 968 formatter_class=argparse.RawDescriptionHelpFormatter, 969 description=textwrap.dedent("Regenerate some ART test related files."), 970 epilog=textwrap.dedent("""\ 971 Regenerate ART run-tests Blueprint files, ART's `TEST_MAPPING` file, and 972 optionally the ART MTS (Mainline Test Suite) definition. 973 """)) 974 parser.add_argument("-m", "--regen-art-mts", help="regenerate the ART MTS definition as well", 975 action="store_true") 976 parser.add_argument("-v", "--verbose", help="enable verbose output", action="store_true") 977 args = parser.parse_args() 978 979 if args.verbose: 980 logging.getLogger().setLevel(logging.DEBUG) 981 982 generator = Generator(os.path.join(os.environ["ANDROID_BUILD_TOP"])) 983 generator.regen_test_files(args.regen_art_mts) 984 985 986if __name__ == "__main__": 987 main() 988