1#!/usr/bin/env python3 2# 3# Copyright 2017, The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17"""ART Run-Test TestRunner 18 19The testrunner runs the ART run-tests by simply invoking the script. 20It fetches the list of eligible tests from art/test directory, and list of 21disabled tests from art/test/knownfailures.json. It runs the tests by 22invoking art/test/run-test script and checks the exit value to decide if the 23test passed or failed. 24 25Before invoking the script, first build all the tests dependencies. 26There are two major build targets for building target and host tests 27dependencies: 281) test-art-host-run-test 292) test-art-target-run-test 30 31There are various options to invoke the script which are: 32-t: Either the test name as in art/test or the test name including the variant 33 information. Eg, "-t 001-HelloWorld", 34 "-t test-art-host-run-test-debug-prebuild-optimizing-relocate-ntrace-cms-checkjni-picimage-npictest-ndebuggable-001-HelloWorld32" 35-j: Number of thread workers to be used. Eg - "-j64" 36--dry-run: Instead of running the test name, just print its name. 37--verbose 38-b / --build-dependencies: to build the dependencies before running the test 39 40To specify any specific variants for the test, use --<<variant-name>>. 41For eg, for compiler type as optimizing, use --optimizing. 42 43 44In the end, the script will print the failed and skipped tests if any. 45 46""" 47import argparse 48import collections 49import fnmatch 50import itertools 51import json 52import multiprocessing 53import os 54import re 55import subprocess 56import sys 57import tempfile 58import threading 59import time 60 61import env 62from target_config import target_config 63from device_config import device_config 64 65# timeout for individual tests. 66# TODO: make it adjustable per tests and for buildbots 67timeout = 3000 # 50 minutes 68 69# DISABLED_TEST_CONTAINER holds information about the disabled tests. It is a map 70# that has key as the test name (like 001-HelloWorld), and value as set of 71# variants that the test is disabled for. 72DISABLED_TEST_CONTAINER = {} 73 74# The Dict contains the list of all possible variants for a given type. For example, 75# for key TARGET, the value would be target and host. The list is used to parse 76# the test name given as the argument to run. 77VARIANT_TYPE_DICT = {} 78 79# The set contains all the variants of each time. 80TOTAL_VARIANTS_SET = set() 81 82# The colors are used in the output. When a test passes, COLOR_PASS is used, 83# and so on. 84COLOR_ERROR = '\033[91m' 85COLOR_PASS = '\033[92m' 86COLOR_SKIP = '\033[93m' 87COLOR_NORMAL = '\033[0m' 88 89# The mutex object is used by the threads for exclusive access of test_count 90# to make any changes in its value. 91test_count_mutex = threading.Lock() 92 93# The set contains the list of all the possible run tests that are in art/test 94# directory. 95RUN_TEST_SET = set() 96 97# The semaphore object is used by the testrunner to limit the number of 98# threads to the user requested concurrency value. 99semaphore = threading.Semaphore(1) 100 101# The mutex object is used to provide exclusive access to a thread to print 102# its output. 103print_mutex = threading.Lock() 104failed_tests = [] 105skipped_tests = [] 106 107# Flags 108n_thread = -1 109test_count = 0 110total_test_count = 0 111verbose = False 112dry_run = False 113ignore_skips = False 114build = False 115gdb = False 116gdb_arg = '' 117stop_testrunner = False 118dex2oat_jobs = -1 # -1 corresponds to default threads for dex2oat 119run_all_configs = False 120 121# Dict containing extra arguments 122extra_arguments = { "host" : [], "target" : [] } 123 124# Dict to store user requested test variants. 125# key: variant_type. 126# value: set of variants user wants to run of type <key>. 127_user_input_variants = collections.defaultdict(set) 128 129def gather_test_info(): 130 """The method gathers test information about the test to be run which includes 131 generating the list of total tests from the art/test directory and the list 132 of disabled test. It also maps various variants to types. 133 """ 134 global TOTAL_VARIANTS_SET 135 global DISABLED_TEST_CONTAINER 136 # TODO: Avoid duplication of the variant names in different lists. 137 VARIANT_TYPE_DICT['pictest'] = {'pictest', 'npictest'} 138 VARIANT_TYPE_DICT['run'] = {'ndebug', 'debug'} 139 VARIANT_TYPE_DICT['target'] = {'target', 'host', 'jvm'} 140 VARIANT_TYPE_DICT['trace'] = {'trace', 'ntrace', 'stream'} 141 VARIANT_TYPE_DICT['image'] = {'picimage', 'no-image', 'multipicimage'} 142 VARIANT_TYPE_DICT['debuggable'] = {'ndebuggable', 'debuggable'} 143 VARIANT_TYPE_DICT['gc'] = {'gcstress', 'gcverify', 'cms'} 144 VARIANT_TYPE_DICT['prebuild'] = {'no-prebuild', 'no-dex2oat', 'prebuild'} 145 VARIANT_TYPE_DICT['cdex_level'] = {'cdex-none', 'cdex-fast'} 146 VARIANT_TYPE_DICT['relocate'] = {'relocate-npatchoat', 'relocate', 'no-relocate'} 147 VARIANT_TYPE_DICT['jni'] = {'jni', 'forcecopy', 'checkjni'} 148 VARIANT_TYPE_DICT['address_sizes'] = {'64', '32'} 149 VARIANT_TYPE_DICT['jvmti'] = {'no-jvmti', 'jvmti-stress', 'redefine-stress', 'trace-stress', 150 'field-stress', 'step-stress'} 151 VARIANT_TYPE_DICT['compiler'] = {'interp-ac', 'interpreter', 'jit', 'optimizing', 152 'regalloc_gc', 'speed-profile'} 153 154 for v_type in VARIANT_TYPE_DICT: 155 TOTAL_VARIANTS_SET = TOTAL_VARIANTS_SET.union(VARIANT_TYPE_DICT.get(v_type)) 156 157 test_dir = env.ANDROID_BUILD_TOP + '/art/test' 158 for f in os.listdir(test_dir): 159 if fnmatch.fnmatch(f, '[0-9]*'): 160 RUN_TEST_SET.add(f) 161 DISABLED_TEST_CONTAINER = get_disabled_test_info() 162 163 164def setup_test_env(): 165 """The method sets default value for the various variants of the tests if they 166 are already not set. 167 """ 168 if env.ART_TEST_BISECTION: 169 env.ART_TEST_RUN_TEST_NO_PREBUILD = True 170 env.ART_TEST_RUN_TEST_PREBUILD = False 171 # Bisection search writes to standard output. 172 env.ART_TEST_QUIET = False 173 174 global _user_input_variants 175 global run_all_configs 176 if run_all_configs: 177 target_types = _user_input_variants['target'] 178 _user_input_variants = VARIANT_TYPE_DICT 179 _user_input_variants['target'] = target_types 180 181 if not _user_input_variants['target']: 182 _user_input_variants['target'].add('host') 183 _user_input_variants['target'].add('target') 184 185 if not _user_input_variants['prebuild']: # Default 186 _user_input_variants['prebuild'].add('prebuild') 187 188 if not _user_input_variants['cdex_level']: # Default 189 _user_input_variants['cdex_level'].add('cdex-fast') 190 191 # By default only run without jvmti 192 if not _user_input_variants['jvmti']: 193 _user_input_variants['jvmti'].add('no-jvmti') 194 195 # By default we run all 'compiler' variants. 196 if not _user_input_variants['compiler'] and _user_input_variants['target'] != 'jvm': 197 _user_input_variants['compiler'].add('optimizing') 198 _user_input_variants['compiler'].add('jit') 199 _user_input_variants['compiler'].add('interpreter') 200 _user_input_variants['compiler'].add('interp-ac') 201 _user_input_variants['compiler'].add('speed-profile') 202 203 if not _user_input_variants['relocate']: # Default 204 _user_input_variants['relocate'].add('no-relocate') 205 206 if not _user_input_variants['trace']: # Default 207 _user_input_variants['trace'].add('ntrace') 208 209 if not _user_input_variants['gc']: # Default 210 _user_input_variants['gc'].add('cms') 211 212 if not _user_input_variants['jni']: # Default 213 _user_input_variants['jni'].add('checkjni') 214 215 if not _user_input_variants['image']: # Default 216 _user_input_variants['image'].add('picimage') 217 218 if not _user_input_variants['pictest']: # Default 219 _user_input_variants['pictest'].add('npictest') 220 221 if not _user_input_variants['debuggable']: # Default 222 _user_input_variants['debuggable'].add('ndebuggable') 223 224 if not _user_input_variants['run']: # Default 225 _user_input_variants['run'].add('debug') 226 227 _user_input_variants['address_sizes_target'] = collections.defaultdict(set) 228 if not _user_input_variants['address_sizes']: 229 _user_input_variants['address_sizes_target']['target'].add( 230 env.ART_PHONY_TEST_TARGET_SUFFIX) 231 _user_input_variants['address_sizes_target']['host'].add( 232 env.ART_PHONY_TEST_HOST_SUFFIX) 233 if env.ART_TEST_RUN_TEST_2ND_ARCH: 234 _user_input_variants['address_sizes_target']['host'].add( 235 env.ART_2ND_PHONY_TEST_HOST_SUFFIX) 236 _user_input_variants['address_sizes_target']['target'].add( 237 env.ART_2ND_PHONY_TEST_TARGET_SUFFIX) 238 else: 239 _user_input_variants['address_sizes_target']['host'] = _user_input_variants['address_sizes'] 240 _user_input_variants['address_sizes_target']['target'] = _user_input_variants['address_sizes'] 241 242 global n_thread 243 if n_thread is -1: 244 if 'target' in _user_input_variants['target']: 245 n_thread = get_default_threads('target') 246 else: 247 n_thread = get_default_threads('host') 248 print_text("Concurrency: " + str(n_thread) + "\n") 249 250 global extra_arguments 251 for target in _user_input_variants['target']: 252 extra_arguments[target] = find_extra_device_arguments(target) 253 254 global semaphore 255 semaphore = threading.Semaphore(n_thread) 256 257 if not sys.stdout.isatty(): 258 global COLOR_ERROR 259 global COLOR_PASS 260 global COLOR_SKIP 261 global COLOR_NORMAL 262 COLOR_ERROR = '' 263 COLOR_PASS = '' 264 COLOR_SKIP = '' 265 COLOR_NORMAL = '' 266 267def find_extra_device_arguments(target): 268 """ 269 Gets any extra arguments from the device_config. 270 """ 271 device_name = target 272 if target == 'target': 273 device_name = get_device_name() 274 return device_config.get(device_name, { 'run-test-args' : [] })['run-test-args'] 275 276def get_device_name(): 277 """ 278 Gets the value of ro.product.name from remote device. 279 """ 280 proc = subprocess.Popen(['adb', 'shell', 'getprop', 'ro.product.name'], 281 stderr=subprocess.STDOUT, 282 stdout = subprocess.PIPE, 283 universal_newlines=True) 284 # only wait 2 seconds. 285 output = proc.communicate(timeout = 2)[0] 286 success = not proc.wait() 287 if success: 288 return output.strip() 289 else: 290 print_text("Unable to determine device type!\n") 291 print_text("Continuing anyway.\n") 292 return "UNKNOWN_TARGET" 293 294def run_tests(tests): 295 """Creates thread workers to run the tests. 296 297 The method generates command and thread worker to run the tests. Depending on 298 the user input for the number of threads to be used, the method uses a 299 semaphore object to keep a count in control for the thread workers. When a new 300 worker is created, it acquires the semaphore object, and when the number of 301 workers reaches the maximum allowed concurrency, the method wait for an 302 existing thread worker to release the semaphore object. Worker releases the 303 semaphore object when they finish printing the output. 304 305 Args: 306 tests: The set of tests to be run. 307 """ 308 options_all = '' 309 310 # jvm does not run with all these combinations, 311 # or at least it doesn't make sense for most of them. 312 # TODO: support some jvm variants like jvmti ? 313 target_input_variants = _user_input_variants['target'] 314 uncombinated_target_input_variants = [] 315 if 'jvm' in target_input_variants: 316 _user_input_variants['target'].remove('jvm') 317 uncombinated_target_input_variants.append('jvm') 318 319 global total_test_count 320 total_test_count = len(tests) 321 if target_input_variants: 322 for variant_type in VARIANT_TYPE_DICT: 323 if not (variant_type == 'target' or 'address_sizes' in variant_type): 324 total_test_count *= len(_user_input_variants[variant_type]) 325 target_address_combinations = 0 326 for target in target_input_variants: 327 for address_size in _user_input_variants['address_sizes_target'][target]: 328 target_address_combinations += 1 329 target_address_combinations += len(uncombinated_target_input_variants) 330 total_test_count *= target_address_combinations 331 332 if env.ART_TEST_WITH_STRACE: 333 options_all += ' --strace' 334 335 if env.ART_TEST_RUN_TEST_ALWAYS_CLEAN: 336 options_all += ' --always-clean' 337 338 if env.ART_TEST_BISECTION: 339 options_all += ' --bisection-search' 340 341 if env.ART_TEST_ANDROID_ROOT: 342 options_all += ' --android-root ' + env.ART_TEST_ANDROID_ROOT 343 344 if gdb: 345 options_all += ' --gdb' 346 if gdb_arg: 347 options_all += ' --gdb-arg ' + gdb_arg 348 349 if dex2oat_jobs != -1: 350 options_all += ' --dex2oat-jobs ' + str(dex2oat_jobs) 351 352 def iter_config(tests, input_variants, user_input_variants): 353 config = itertools.product(tests, input_variants, user_input_variants['run'], 354 user_input_variants['prebuild'], user_input_variants['compiler'], 355 user_input_variants['relocate'], user_input_variants['trace'], 356 user_input_variants['gc'], user_input_variants['jni'], 357 user_input_variants['image'], user_input_variants['pictest'], 358 user_input_variants['debuggable'], user_input_variants['jvmti'], 359 user_input_variants['cdex_level']) 360 return config 361 362 # [--host, --target] combines with all the other user input variants. 363 config = iter_config(tests, target_input_variants, _user_input_variants) 364 # [--jvm] currently combines with nothing else. most of the extra flags we'd insert 365 # would be unrecognizable by the 'java' binary, so avoid inserting any extra flags for now. 366 uncombinated_config = iter_config(tests, uncombinated_target_input_variants, { 'run': [''], 367 'prebuild': [''], 'compiler': [''], 368 'relocate': [''], 'trace': [''], 369 'gc': [''], 'jni': [''], 370 'image': [''], 'pictest': [''], 371 'debuggable': [''], 'jvmti': [''], 372 'cdex_level': ['']}) 373 374 def start_combination(config_tuple, address_size): 375 test, target, run, prebuild, compiler, relocate, trace, gc, \ 376 jni, image, pictest, debuggable, jvmti, cdex_level = config_tuple 377 378 if stop_testrunner: 379 # When ART_TEST_KEEP_GOING is set to false, then as soon as a test 380 # fails, stop_testrunner is set to True. When this happens, the method 381 # stops creating any any thread and wait for all the exising threads 382 # to end. 383 while threading.active_count() > 2: 384 time.sleep(0.1) 385 return 386 # NB The order of components here should match the order of 387 # components in the regex parser in parse_test_name. 388 test_name = 'test-art-' 389 test_name += target + '-run-test-' 390 test_name += run + '-' 391 test_name += prebuild + '-' 392 test_name += compiler + '-' 393 test_name += relocate + '-' 394 test_name += trace + '-' 395 test_name += gc + '-' 396 test_name += jni + '-' 397 test_name += image + '-' 398 test_name += pictest + '-' 399 test_name += debuggable + '-' 400 test_name += jvmti + '-' 401 test_name += cdex_level + '-' 402 test_name += test 403 test_name += address_size 404 405 variant_set = {target, run, prebuild, compiler, relocate, trace, gc, jni, 406 image, pictest, debuggable, jvmti, cdex_level, address_size} 407 408 options_test = options_all 409 410 if target == 'host': 411 options_test += ' --host' 412 elif target == 'jvm': 413 options_test += ' --jvm' 414 415 if run == 'ndebug': 416 options_test += ' -O' 417 418 if prebuild == 'prebuild': 419 options_test += ' --prebuild' 420 elif prebuild == 'no-prebuild': 421 options_test += ' --no-prebuild' 422 elif prebuild == 'no-dex2oat': 423 options_test += ' --no-prebuild --no-dex2oat' 424 425 # Add option and remove the cdex- prefix. 426 options_test += ' --compact-dex-level ' + cdex_level.replace('cdex-','') 427 428 if compiler == 'optimizing': 429 options_test += ' --optimizing' 430 elif compiler == 'regalloc_gc': 431 options_test += ' --optimizing -Xcompiler-option --register-allocation-strategy=graph-color' 432 elif compiler == 'interpreter': 433 options_test += ' --interpreter' 434 elif compiler == 'interp-ac': 435 options_test += ' --interpreter --verify-soft-fail' 436 elif compiler == 'jit': 437 options_test += ' --jit' 438 elif compiler == 'speed-profile': 439 options_test += ' --random-profile' 440 441 if relocate == 'relocate': 442 options_test += ' --relocate' 443 elif relocate == 'no-relocate': 444 options_test += ' --no-relocate' 445 elif relocate == 'relocate-npatchoat': 446 options_test += ' --relocate --no-patchoat' 447 448 if trace == 'trace': 449 options_test += ' --trace' 450 elif trace == 'stream': 451 options_test += ' --trace --stream' 452 453 if gc == 'gcverify': 454 options_test += ' --gcverify' 455 elif gc == 'gcstress': 456 options_test += ' --gcstress' 457 458 if jni == 'forcecopy': 459 options_test += ' --runtime-option -Xjniopts:forcecopy' 460 elif jni == 'checkjni': 461 options_test += ' --runtime-option -Xcheck:jni' 462 463 if image == 'no-image': 464 options_test += ' --no-image' 465 elif image == 'multipicimage': 466 options_test += ' --multi-image' 467 468 if pictest == 'pictest': 469 options_test += ' --pic-test' 470 471 if debuggable == 'debuggable': 472 options_test += ' --debuggable' 473 474 if jvmti == 'jvmti-stress': 475 options_test += ' --jvmti-trace-stress --jvmti-redefine-stress --jvmti-field-stress' 476 elif jvmti == 'field-stress': 477 options_test += ' --jvmti-field-stress' 478 elif jvmti == 'trace-stress': 479 options_test += ' --jvmti-trace-stress' 480 elif jvmti == 'redefine-stress': 481 options_test += ' --jvmti-redefine-stress' 482 elif jvmti == 'step-stress': 483 options_test += ' --jvmti-step-stress' 484 485 if address_size == '64': 486 options_test += ' --64' 487 488 if env.DEX2OAT_HOST_INSTRUCTION_SET_FEATURES: 489 options_test += ' --instruction-set-features' + env.DEX2OAT_HOST_INSTRUCTION_SET_FEATURES 490 491 elif address_size == '32': 492 if env.HOST_2ND_ARCH_PREFIX_DEX2OAT_HOST_INSTRUCTION_SET_FEATURES: 493 options_test += ' --instruction-set-features ' + \ 494 env.HOST_2ND_ARCH_PREFIX_DEX2OAT_HOST_INSTRUCTION_SET_FEATURES 495 496 # Use the default run-test behavior unless ANDROID_COMPILE_WITH_JACK is explicitly set. 497 if env.ANDROID_COMPILE_WITH_JACK == True: 498 options_test += ' --build-with-jack' 499 elif env.ANDROID_COMPILE_WITH_JACK == False: 500 options_test += ' --build-with-javac-dx' 501 502 if env.USE_D8_BY_DEFAULT == True: 503 options_test += ' --build-with-d8' 504 505 # TODO(http://36039166): This is a temporary solution to 506 # fix build breakages. 507 options_test = (' --output-path %s') % ( 508 tempfile.mkdtemp(dir=env.ART_HOST_TEST_DIR)) + options_test 509 510 run_test_sh = env.ANDROID_BUILD_TOP + '/art/test/run-test' 511 command = ' '.join((run_test_sh, options_test, ' '.join(extra_arguments[target]), test)) 512 513 semaphore.acquire() 514 worker = threading.Thread(target=run_test, args=(command, test, variant_set, test_name)) 515 worker.daemon = True 516 worker.start() 517 518 for config_tuple in config: 519 target = config_tuple[1] 520 for address_size in _user_input_variants['address_sizes_target'][target]: 521 start_combination(config_tuple, address_size) 522 523 for config_tuple in uncombinated_config: 524 start_combination(config_tuple, "") # no address size 525 526 while threading.active_count() > 2: 527 time.sleep(0.1) 528 529 530def run_test(command, test, test_variant, test_name): 531 """Runs the test. 532 533 It invokes art/test/run-test script to run the test. The output of the script 534 is checked, and if it ends with "Succeeded!", it assumes that the tests 535 passed, otherwise, put it in the list of failed test. Before actually running 536 the test, it also checks if the test is placed in the list of disabled tests, 537 and if yes, it skips running it, and adds the test in the list of skipped 538 tests. The method uses print_text method to actually print the output. After 539 successfully running and capturing the output for the test, it releases the 540 semaphore object. 541 542 Args: 543 command: The command to be used to invoke the script 544 test: The name of the test without the variant information. 545 test_variant: The set of variant for the test. 546 test_name: The name of the test along with the variants. 547 """ 548 global stop_testrunner 549 try: 550 if is_test_disabled(test, test_variant): 551 test_skipped = True 552 else: 553 test_skipped = False 554 if gdb: 555 proc = subprocess.Popen(command.split(), stderr=subprocess.STDOUT, universal_newlines=True) 556 else: 557 proc = subprocess.Popen(command.split(), stderr=subprocess.STDOUT, stdout = subprocess.PIPE, 558 universal_newlines=True) 559 script_output = proc.communicate(timeout=timeout)[0] 560 test_passed = not proc.wait() 561 562 if not test_skipped: 563 if test_passed: 564 print_test_info(test_name, 'PASS') 565 else: 566 failed_tests.append((test_name, str(command) + "\n" + script_output)) 567 if not env.ART_TEST_KEEP_GOING: 568 stop_testrunner = True 569 print_test_info(test_name, 'FAIL', ('%s\n%s') % ( 570 command, script_output)) 571 elif not dry_run: 572 print_test_info(test_name, 'SKIP') 573 skipped_tests.append(test_name) 574 else: 575 print_test_info(test_name, '') 576 except subprocess.TimeoutExpired as e: 577 failed_tests.append((test_name, 'Timed out in %d seconds' % timeout)) 578 print_test_info(test_name, 'TIMEOUT', 'Timed out in %d seconds\n%s' % ( 579 timeout, command)) 580 except Exception as e: 581 failed_tests.append((test_name, str(e))) 582 print_test_info(test_name, 'FAIL', 583 ('%s\n%s\n\n') % (command, str(e))) 584 finally: 585 semaphore.release() 586 587 588def print_test_info(test_name, result, failed_test_info=""): 589 """Print the continous test information 590 591 If verbose is set to True, it continuously prints test status information 592 on a new line. 593 If verbose is set to False, it keeps on erasing test 594 information by overriding it with the latest test information. Also, 595 in this case it stictly makes sure that the information length doesn't 596 exceed the console width. It does so by shortening the test_name. 597 598 When a test fails, it prints the output of the run-test script and 599 command used to invoke the script. It doesn't override the failing 600 test information in either of the cases. 601 """ 602 603 global test_count 604 info = '' 605 if not verbose: 606 # Without --verbose, the testrunner erases passing test info. It 607 # does that by overriding the printed text with white spaces all across 608 # the console width. 609 console_width = int(os.popen('stty size', 'r').read().split()[1]) 610 info = '\r' + ' ' * console_width + '\r' 611 try: 612 print_mutex.acquire() 613 test_count += 1 614 percent = (test_count * 100) / total_test_count 615 progress_info = ('[ %d%% %d/%d ]') % ( 616 percent, 617 test_count, 618 total_test_count) 619 620 if result == 'FAIL' or result == 'TIMEOUT': 621 if not verbose: 622 info += ('%s %s %s\n') % ( 623 progress_info, 624 test_name, 625 COLOR_ERROR + result + COLOR_NORMAL) 626 else: 627 info += ('%s %s %s\n%s\n') % ( 628 progress_info, 629 test_name, 630 COLOR_ERROR + result + COLOR_NORMAL, 631 failed_test_info) 632 else: 633 result_text = '' 634 if result == 'PASS': 635 result_text += COLOR_PASS + 'PASS' + COLOR_NORMAL 636 elif result == 'SKIP': 637 result_text += COLOR_SKIP + 'SKIP' + COLOR_NORMAL 638 639 if verbose: 640 info += ('%s %s %s\n') % ( 641 progress_info, 642 test_name, 643 result_text) 644 else: 645 total_output_length = 2 # Two spaces 646 total_output_length += len(progress_info) 647 total_output_length += len(result) 648 allowed_test_length = console_width - total_output_length 649 test_name_len = len(test_name) 650 if allowed_test_length < test_name_len: 651 test_name = ('...%s') % ( 652 test_name[-(allowed_test_length - 3):]) 653 info += ('%s %s %s') % ( 654 progress_info, 655 test_name, 656 result_text) 657 print_text(info) 658 except Exception as e: 659 print_text(('%s\n%s\n') % (test_name, str(e))) 660 failed_tests.append(test_name) 661 finally: 662 print_mutex.release() 663 664def verify_knownfailure_entry(entry): 665 supported_field = { 666 'tests' : (list, str), 667 'test_patterns' : (list,), 668 'description' : (list, str), 669 'bug' : (str,), 670 'variant' : (str,), 671 'env_vars' : (dict,), 672 } 673 for field in entry: 674 field_type = type(entry[field]) 675 if field_type not in supported_field[field]: 676 raise ValueError('%s is not supported type for %s\n%s' % ( 677 str(field_type), 678 field, 679 str(entry))) 680 681def get_disabled_test_info(): 682 """Generate set of known failures. 683 684 It parses the art/test/knownfailures.json file to generate the list of 685 disabled tests. 686 687 Returns: 688 The method returns a dict of tests mapped to the variants list 689 for which the test should not be run. 690 """ 691 known_failures_file = env.ANDROID_BUILD_TOP + '/art/test/knownfailures.json' 692 with open(known_failures_file) as known_failures_json: 693 known_failures_info = json.loads(known_failures_json.read()) 694 695 disabled_test_info = {} 696 for failure in known_failures_info: 697 verify_knownfailure_entry(failure) 698 tests = failure.get('tests', []) 699 if isinstance(tests, str): 700 tests = [tests] 701 patterns = failure.get("test_patterns", []) 702 if (not isinstance(patterns, list)): 703 raise ValueError("test_patters is not a list in %s" % failure) 704 705 tests += [f for f in RUN_TEST_SET if any(re.match(pat, f) is not None for pat in patterns)] 706 variants = parse_variants(failure.get('variant')) 707 env_vars = failure.get('env_vars') 708 709 if check_env_vars(env_vars): 710 for test in tests: 711 if test not in RUN_TEST_SET: 712 raise ValueError('%s is not a valid run-test' % ( 713 test)) 714 if test in disabled_test_info: 715 disabled_test_info[test] = disabled_test_info[test].union(variants) 716 else: 717 disabled_test_info[test] = variants 718 return disabled_test_info 719 720 721def check_env_vars(env_vars): 722 """Checks if the env variables are set as required to run the test. 723 724 Returns: 725 True if all the env variables are set as required, otherwise False. 726 """ 727 728 if not env_vars: 729 return True 730 for key in env_vars: 731 if env.get_env(key) != env_vars.get(key): 732 return False 733 return True 734 735 736def is_test_disabled(test, variant_set): 737 """Checks if the test along with the variant_set is disabled. 738 739 Args: 740 test: The name of the test as in art/test directory. 741 variant_set: Variants to be used for the test. 742 Returns: 743 True, if the test is disabled. 744 """ 745 if dry_run: 746 return True 747 if test in env.EXTRA_DISABLED_TESTS: 748 return True 749 if ignore_skips: 750 return False 751 variants_list = DISABLED_TEST_CONTAINER.get(test, {}) 752 for variants in variants_list: 753 variants_present = True 754 for variant in variants: 755 if variant not in variant_set: 756 variants_present = False 757 break 758 if variants_present: 759 return True 760 return False 761 762 763def parse_variants(variants): 764 """Parse variants fetched from art/test/knownfailures.json. 765 """ 766 if not variants: 767 variants = '' 768 for variant in TOTAL_VARIANTS_SET: 769 variants += variant 770 variants += '|' 771 variants = variants[:-1] 772 variant_list = set() 773 or_variants = variants.split('|') 774 for or_variant in or_variants: 775 and_variants = or_variant.split('&') 776 variant = set() 777 for and_variant in and_variants: 778 and_variant = and_variant.strip() 779 if and_variant not in TOTAL_VARIANTS_SET: 780 raise ValueError('%s is not a valid variant' % ( 781 and_variant)) 782 variant.add(and_variant) 783 variant_list.add(frozenset(variant)) 784 return variant_list 785 786def print_text(output): 787 sys.stdout.write(output) 788 sys.stdout.flush() 789 790def print_analysis(): 791 if not verbose: 792 # Without --verbose, the testrunner erases passing test info. It 793 # does that by overriding the printed text with white spaces all across 794 # the console width. 795 console_width = int(os.popen('stty size', 'r').read().split()[1]) 796 eraser_text = '\r' + ' ' * console_width + '\r' 797 print_text(eraser_text) 798 799 # Prints information about the total tests run. 800 # E.g., "2/38 (5%) tests passed". 801 passed_test_count = total_test_count - len(skipped_tests) - len(failed_tests) 802 passed_test_information = ('%d/%d (%d%%) %s passed.\n') % ( 803 passed_test_count, 804 total_test_count, 805 (passed_test_count*100)/total_test_count, 806 'tests' if passed_test_count > 1 else 'test') 807 print_text(passed_test_information) 808 809 # Prints the list of skipped tests, if any. 810 if skipped_tests: 811 print_text(COLOR_SKIP + 'SKIPPED TESTS: ' + COLOR_NORMAL + '\n') 812 for test in skipped_tests: 813 print_text(test + '\n') 814 print_text('\n') 815 816 # Prints the list of failed tests, if any. 817 if failed_tests: 818 print_text(COLOR_ERROR + 'FAILED: ' + COLOR_NORMAL + '\n') 819 for test_info in failed_tests: 820 print_text(('%s\n%s\n' % (test_info[0], test_info[1]))) 821 print_text(COLOR_ERROR + '----------' + COLOR_NORMAL + '\n') 822 for failed_test in sorted([test_info[0] for test_info in failed_tests]): 823 print_text(('%s\n' % (failed_test))) 824 825 826def parse_test_name(test_name): 827 """Parses the testname provided by the user. 828 It supports two types of test_name: 829 1) Like 001-HelloWorld. In this case, it will just verify if the test actually 830 exists and if it does, it returns the testname. 831 2) Like test-art-host-run-test-debug-prebuild-interpreter-no-relocate-ntrace-cms-checkjni-picimage-npictest-ndebuggable-001-HelloWorld32 832 In this case, it will parse all the variants and check if they are placed 833 correctly. If yes, it will set the various VARIANT_TYPES to use the 834 variants required to run the test. Again, it returns the test_name 835 without the variant information like 001-HelloWorld. 836 """ 837 test_set = set() 838 for test in RUN_TEST_SET: 839 if test.startswith(test_name): 840 test_set.add(test) 841 if test_set: 842 return test_set 843 844 regex = '^test-art-' 845 regex += '(' + '|'.join(VARIANT_TYPE_DICT['target']) + ')-' 846 regex += 'run-test-' 847 regex += '(' + '|'.join(VARIANT_TYPE_DICT['run']) + ')-' 848 regex += '(' + '|'.join(VARIANT_TYPE_DICT['prebuild']) + ')-' 849 regex += '(' + '|'.join(VARIANT_TYPE_DICT['compiler']) + ')-' 850 regex += '(' + '|'.join(VARIANT_TYPE_DICT['relocate']) + ')-' 851 regex += '(' + '|'.join(VARIANT_TYPE_DICT['trace']) + ')-' 852 regex += '(' + '|'.join(VARIANT_TYPE_DICT['gc']) + ')-' 853 regex += '(' + '|'.join(VARIANT_TYPE_DICT['jni']) + ')-' 854 regex += '(' + '|'.join(VARIANT_TYPE_DICT['image']) + ')-' 855 regex += '(' + '|'.join(VARIANT_TYPE_DICT['pictest']) + ')-' 856 regex += '(' + '|'.join(VARIANT_TYPE_DICT['debuggable']) + ')-' 857 regex += '(' + '|'.join(VARIANT_TYPE_DICT['jvmti']) + ')-' 858 regex += '(' + '|'.join(VARIANT_TYPE_DICT['cdex_level']) + ')-' 859 regex += '(' + '|'.join(RUN_TEST_SET) + ')' 860 regex += '(' + '|'.join(VARIANT_TYPE_DICT['address_sizes']) + ')$' 861 match = re.match(regex, test_name) 862 if match: 863 _user_input_variants['target'].add(match.group(1)) 864 _user_input_variants['run'].add(match.group(2)) 865 _user_input_variants['prebuild'].add(match.group(3)) 866 _user_input_variants['compiler'].add(match.group(4)) 867 _user_input_variants['relocate'].add(match.group(5)) 868 _user_input_variants['trace'].add(match.group(6)) 869 _user_input_variants['gc'].add(match.group(7)) 870 _user_input_variants['jni'].add(match.group(8)) 871 _user_input_variants['image'].add(match.group(9)) 872 _user_input_variants['pictest'].add(match.group(10)) 873 _user_input_variants['debuggable'].add(match.group(11)) 874 _user_input_variants['jvmti'].add(match.group(12)) 875 _user_input_variants['cdex_level'].add(match.group(13)) 876 _user_input_variants['address_sizes'].add(match.group(15)) 877 return {match.group(14)} 878 raise ValueError(test_name + " is not a valid test") 879 880 881def setup_env_for_build_target(build_target, parser, options): 882 """Setup environment for the build target 883 884 The method setup environment for the master-art-host targets. 885 """ 886 os.environ.update(build_target['env']) 887 os.environ['SOONG_ALLOW_MISSING_DEPENDENCIES'] = 'true' 888 print_text('%s\n' % (str(os.environ))) 889 890 target_options = vars(parser.parse_args(build_target['flags'])) 891 target_options['host'] = True 892 target_options['verbose'] = True 893 target_options['build'] = True 894 target_options['n_thread'] = options['n_thread'] 895 target_options['dry_run'] = options['dry_run'] 896 897 return target_options 898 899def get_default_threads(target): 900 if target is 'target': 901 adb_command = 'adb shell cat /sys/devices/system/cpu/present' 902 cpu_info_proc = subprocess.Popen(adb_command.split(), stdout=subprocess.PIPE) 903 cpu_info = cpu_info_proc.stdout.read() 904 if type(cpu_info) is bytes: 905 cpu_info = cpu_info.decode('utf-8') 906 cpu_info_regex = '\d*-(\d*)' 907 match = re.match(cpu_info_regex, cpu_info) 908 if match: 909 return int(match.group(1)) 910 else: 911 raise ValueError('Unable to predict the concurrency for the target. ' 912 'Is device connected?') 913 else: 914 return multiprocessing.cpu_count() 915 916def parse_option(): 917 global verbose 918 global dry_run 919 global ignore_skips 920 global n_thread 921 global build 922 global gdb 923 global gdb_arg 924 global timeout 925 global dex2oat_jobs 926 global run_all_configs 927 928 parser = argparse.ArgumentParser(description="Runs all or a subset of the ART test suite.") 929 parser.add_argument('-t', '--test', action='append', dest='tests', help='name(s) of the test(s)') 930 parser.add_argument('-j', type=int, dest='n_thread') 931 parser.add_argument('--timeout', default=timeout, type=int, dest='timeout') 932 for variant in TOTAL_VARIANTS_SET: 933 flag = '--' + variant 934 parser.add_argument(flag, action='store_true', dest=variant) 935 parser.add_argument('--verbose', '-v', action='store_true', dest='verbose') 936 parser.add_argument('--dry-run', action='store_true', dest='dry_run') 937 parser.add_argument("--skip", action="append", dest="skips", default=[], 938 help="Skip the given test in all circumstances.") 939 parser.add_argument("--no-skips", dest="ignore_skips", action="store_true", default=False, 940 help="Don't skip any run-test configurations listed in knownfailures.json.") 941 parser.add_argument('--no-build-dependencies', 942 action='store_false', dest='build', 943 help="Don't build dependencies under any circumstances. This is the " + 944 "behavior if ART_TEST_RUN_TEST_ALWAYS_BUILD is not set to 'true'.") 945 parser.add_argument('-b', '--build-dependencies', 946 action='store_true', dest='build', 947 help="Build dependencies under all circumstances. By default we will " + 948 "not build dependencies unless ART_TEST_RUN_TEST_BUILD=true.") 949 parser.add_argument('--build-target', dest='build_target', help='master-art-host targets') 950 parser.set_defaults(build = env.ART_TEST_RUN_TEST_BUILD) 951 parser.add_argument('--gdb', action='store_true', dest='gdb') 952 parser.add_argument('--gdb-arg', dest='gdb_arg') 953 parser.add_argument('--dex2oat-jobs', type=int, dest='dex2oat_jobs', 954 help='Number of dex2oat jobs') 955 parser.add_argument('-a', '--all', action='store_true', dest='run_all', 956 help="Run all the possible configurations for the input test set") 957 958 options = vars(parser.parse_args()) 959 if options['build_target']: 960 options = setup_env_for_build_target(target_config[options['build_target']], 961 parser, options) 962 963 tests = None 964 env.EXTRA_DISABLED_TESTS.update(set(options['skips'])) 965 if options['tests']: 966 tests = set() 967 for test_name in options['tests']: 968 tests |= parse_test_name(test_name) 969 970 for variant_type in VARIANT_TYPE_DICT: 971 for variant in VARIANT_TYPE_DICT[variant_type]: 972 if options.get(variant): 973 _user_input_variants[variant_type].add(variant) 974 975 if options['verbose']: 976 verbose = True 977 if options['n_thread']: 978 n_thread = max(1, options['n_thread']) 979 ignore_skips = options['ignore_skips'] 980 if options['dry_run']: 981 dry_run = True 982 verbose = True 983 build = options['build'] 984 if options['gdb']: 985 n_thread = 1 986 gdb = True 987 if options['gdb_arg']: 988 gdb_arg = options['gdb_arg'] 989 timeout = options['timeout'] 990 if options['dex2oat_jobs']: 991 dex2oat_jobs = options['dex2oat_jobs'] 992 if options['run_all']: 993 run_all_configs = True 994 995 return tests 996 997def main(): 998 gather_test_info() 999 user_requested_tests = parse_option() 1000 setup_test_env() 1001 if build: 1002 build_targets = '' 1003 if 'host' in _user_input_variants['target']: 1004 build_targets += 'test-art-host-run-test-dependencies' 1005 if 'target' in _user_input_variants['target']: 1006 build_targets += 'test-art-target-run-test-dependencies' 1007 if 'jvm' in _user_input_variants['target']: 1008 build_targets += 'test-art-host-run-test-dependencies' 1009 build_command = 'make' 1010 build_command += ' DX=' 1011 build_command += ' -j' 1012 build_command += ' -C ' + env.ANDROID_BUILD_TOP 1013 build_command += ' ' + build_targets 1014 # Add 'dist' to avoid Jack issues b/36169180. 1015 build_command += ' dist' 1016 if subprocess.call(build_command.split()): 1017 sys.exit(1) 1018 if user_requested_tests: 1019 test_runner_thread = threading.Thread(target=run_tests, args=(user_requested_tests,)) 1020 else: 1021 test_runner_thread = threading.Thread(target=run_tests, args=(RUN_TEST_SET,)) 1022 test_runner_thread.daemon = True 1023 try: 1024 test_runner_thread.start() 1025 # This loops waits for all the threads to finish, unless 1026 # stop_testrunner is set to True. When ART_TEST_KEEP_GOING 1027 # is set to false, stop_testrunner is set to True as soon as 1028 # a test fails to signal the parent thread to stop 1029 # the execution of the testrunner. 1030 while threading.active_count() > 1 and not stop_testrunner: 1031 time.sleep(0.1) 1032 print_analysis() 1033 except Exception as e: 1034 print_analysis() 1035 print_text(str(e)) 1036 sys.exit(1) 1037 if failed_tests: 1038 sys.exit(1) 1039 sys.exit(0) 1040 1041if __name__ == '__main__': 1042 main() 1043