1#!/usr/bin/env python3
2#
3# Copyright 2017, The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#     http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17"""ART Run-Test TestRunner
18
19The testrunner runs the ART run-tests by simply invoking the script.
20It fetches the list of eligible tests from art/test directory, and list of
21disabled tests from art/test/knownfailures.json. It runs the tests by
22invoking art/test/run-test script and checks the exit value to decide if the
23test passed or failed.
24
25Before invoking the script, first build all the tests dependencies.
26There are two major build targets for building target and host tests
27dependencies:
281) test-art-host-run-test
292) test-art-target-run-test
30
31There are various options to invoke the script which are:
32-t: Either the test name as in art/test or the test name including the variant
33    information. Eg, "-t 001-HelloWorld",
34    "-t test-art-host-run-test-debug-prebuild-optimizing-relocate-ntrace-cms-checkjni-picimage-ndebuggable-001-HelloWorld32"
35-j: Number of thread workers to be used. Eg - "-j64"
36--dry-run: Instead of running the test name, just print its name.
37--verbose
38-b / --build-dependencies: to build the dependencies before running the test
39
40To specify any specific variants for the test, use --<<variant-name>>.
41For eg, for compiler type as optimizing, use --optimizing.
42
43
44In the end, the script will print the failed and skipped tests if any.
45
46"""
47import argparse
48import collections
49import contextlib
50import fnmatch
51import itertools
52import json
53import multiprocessing
54import os
55import re
56import shlex
57import shutil
58import subprocess
59import sys
60import tempfile
61import threading
62import time
63
64import env
65from target_config import target_config
66from device_config import device_config
67
68# timeout for individual tests.
69# TODO: make it adjustable per tests and for buildbots
70timeout = 3000 # 50 minutes
71
72# DISABLED_TEST_CONTAINER holds information about the disabled tests. It is a map
73# that has key as the test name (like 001-HelloWorld), and value as set of
74# variants that the test is disabled for.
75DISABLED_TEST_CONTAINER = {}
76
77# The Dict contains the list of all possible variants for a given type. For example,
78# for key TARGET, the value would be target and host. The list is used to parse
79# the test name given as the argument to run.
80VARIANT_TYPE_DICT = {}
81
82# The set contains all the variants of each time.
83TOTAL_VARIANTS_SET = set()
84
85# The colors are used in the output. When a test passes, COLOR_PASS is used,
86# and so on.
87COLOR_ERROR = '\033[91m'
88COLOR_PASS = '\033[92m'
89COLOR_SKIP = '\033[93m'
90COLOR_NORMAL = '\033[0m'
91
92# The mutex object is used by the threads for exclusive access of test_count
93# to make any changes in its value.
94test_count_mutex = threading.Lock()
95
96# The set contains the list of all the possible run tests that are in art/test
97# directory.
98RUN_TEST_SET = set()
99
100# The semaphore object is used by the testrunner to limit the number of
101# threads to the user requested concurrency value.
102semaphore = threading.Semaphore(1)
103
104# The mutex object is used to provide exclusive access to a thread to print
105# its output.
106print_mutex = threading.Lock()
107failed_tests = []
108skipped_tests = []
109
110# Flags
111n_thread = -1
112test_count = 0
113total_test_count = 0
114verbose = False
115dry_run = False
116ignore_skips = False
117build = False
118gdb = False
119gdb_arg = ''
120runtime_option = ''
121with_agent = []
122zipapex_loc = None
123run_test_option = []
124stop_testrunner = False
125dex2oat_jobs = -1   # -1 corresponds to default threads for dex2oat
126run_all_configs = False
127
128# Dict containing extra arguments
129extra_arguments = { "host" : [], "target" : [] }
130
131# Dict to store user requested test variants.
132# key: variant_type.
133# value: set of variants user wants to run of type <key>.
134_user_input_variants = collections.defaultdict(set)
135
136def gather_test_info():
137  """The method gathers test information about the test to be run which includes
138  generating the list of total tests from the art/test directory and the list
139  of disabled test. It also maps various variants to types.
140  """
141  global TOTAL_VARIANTS_SET
142  global DISABLED_TEST_CONTAINER
143  # TODO: Avoid duplication of the variant names in different lists.
144  VARIANT_TYPE_DICT['run'] = {'ndebug', 'debug'}
145  VARIANT_TYPE_DICT['target'] = {'target', 'host', 'jvm'}
146  VARIANT_TYPE_DICT['trace'] = {'trace', 'ntrace', 'stream'}
147  VARIANT_TYPE_DICT['image'] = {'picimage', 'no-image'}
148  VARIANT_TYPE_DICT['debuggable'] = {'ndebuggable', 'debuggable'}
149  VARIANT_TYPE_DICT['gc'] = {'gcstress', 'gcverify', 'cms'}
150  VARIANT_TYPE_DICT['prebuild'] = {'no-prebuild', 'prebuild'}
151  VARIANT_TYPE_DICT['cdex_level'] = {'cdex-none', 'cdex-fast'}
152  VARIANT_TYPE_DICT['relocate'] = {'relocate', 'no-relocate'}
153  VARIANT_TYPE_DICT['jni'] = {'jni', 'forcecopy', 'checkjni'}
154  VARIANT_TYPE_DICT['address_sizes'] = {'64', '32'}
155  VARIANT_TYPE_DICT['jvmti'] = {'no-jvmti', 'jvmti-stress', 'redefine-stress', 'trace-stress',
156                                'field-stress', 'step-stress'}
157  VARIANT_TYPE_DICT['compiler'] = {'interp-ac', 'interpreter', 'jit', 'jit-on-first-use',
158                                   'optimizing', 'regalloc_gc', 'speed-profile', 'baseline'}
159
160  for v_type in VARIANT_TYPE_DICT:
161    TOTAL_VARIANTS_SET = TOTAL_VARIANTS_SET.union(VARIANT_TYPE_DICT.get(v_type))
162
163  test_dir = env.ANDROID_BUILD_TOP + '/art/test'
164  for f in os.listdir(test_dir):
165    if fnmatch.fnmatch(f, '[0-9]*'):
166      RUN_TEST_SET.add(f)
167  DISABLED_TEST_CONTAINER = get_disabled_test_info()
168
169
170def setup_test_env():
171  """The method sets default value for the various variants of the tests if they
172  are already not set.
173  """
174  if env.ART_TEST_BISECTION:
175    env.ART_TEST_RUN_TEST_NO_PREBUILD = True
176    env.ART_TEST_RUN_TEST_PREBUILD = False
177    # Bisection search writes to standard output.
178    env.ART_TEST_QUIET = False
179
180  global _user_input_variants
181  global run_all_configs
182  # These are the default variant-options we will use if nothing in the group is specified.
183  default_variants = {
184      'target': {'host', 'target'},
185      'prebuild': {'prebuild'},
186      'cdex_level': {'cdex-fast'},
187      'jvmti': { 'no-jvmti'},
188      'compiler': {'optimizing',
189                   'jit',
190                   'interpreter',
191                   'interp-ac',
192                   'speed-profile'},
193      'relocate': {'no-relocate'},
194      'trace': {'ntrace'},
195      'gc': {'cms'},
196      'jni': {'checkjni'},
197      'image': {'picimage'},
198      'debuggable': {'ndebuggable'},
199      'run': {'debug'},
200      # address_sizes_target depends on the target so it is dealt with below.
201  }
202  # We want to pull these early since the full VARIANT_TYPE_DICT has a few additional ones we don't
203  # want to pick up if we pass --all.
204  default_variants_keys = default_variants.keys()
205  if run_all_configs:
206    default_variants = VARIANT_TYPE_DICT
207
208  for key in default_variants_keys:
209    if not _user_input_variants[key]:
210      _user_input_variants[key] = default_variants[key]
211
212  _user_input_variants['address_sizes_target'] = collections.defaultdict(set)
213  if not _user_input_variants['address_sizes']:
214    _user_input_variants['address_sizes_target']['target'].add(
215        env.ART_PHONY_TEST_TARGET_SUFFIX)
216    _user_input_variants['address_sizes_target']['host'].add(
217        env.ART_PHONY_TEST_HOST_SUFFIX)
218    if env.ART_TEST_RUN_TEST_2ND_ARCH:
219      _user_input_variants['address_sizes_target']['host'].add(
220          env.ART_2ND_PHONY_TEST_HOST_SUFFIX)
221      _user_input_variants['address_sizes_target']['target'].add(
222          env.ART_2ND_PHONY_TEST_TARGET_SUFFIX)
223  else:
224    _user_input_variants['address_sizes_target']['host'] = _user_input_variants['address_sizes']
225    _user_input_variants['address_sizes_target']['target'] = _user_input_variants['address_sizes']
226
227  global n_thread
228  if n_thread is -1:
229    if 'target' in _user_input_variants['target']:
230      n_thread = get_default_threads('target')
231    else:
232      n_thread = get_default_threads('host')
233    print_text("Concurrency: " + str(n_thread) + "\n")
234
235  global extra_arguments
236  for target in _user_input_variants['target']:
237    extra_arguments[target] = find_extra_device_arguments(target)
238
239  global semaphore
240  semaphore = threading.Semaphore(n_thread)
241
242  if not sys.stdout.isatty():
243    global COLOR_ERROR
244    global COLOR_PASS
245    global COLOR_SKIP
246    global COLOR_NORMAL
247    COLOR_ERROR = ''
248    COLOR_PASS = ''
249    COLOR_SKIP = ''
250    COLOR_NORMAL = ''
251
252def find_extra_device_arguments(target):
253  """
254  Gets any extra arguments from the device_config.
255  """
256  device_name = target
257  if target == 'target':
258    device_name = get_device_name()
259  return device_config.get(device_name, { 'run-test-args' : [] })['run-test-args']
260
261def get_device_name():
262  """
263  Gets the value of ro.product.name from remote device.
264  """
265  proc = subprocess.Popen(['adb', 'shell', 'getprop', 'ro.product.name'],
266                          stderr=subprocess.STDOUT,
267                          stdout = subprocess.PIPE,
268                          universal_newlines=True)
269  # only wait 2 seconds.
270  output = proc.communicate(timeout = 2)[0]
271  success = not proc.wait()
272  if success:
273    return output.strip()
274  else:
275    print_text("Unable to determine device type!\n")
276    print_text("Continuing anyway.\n")
277    return "UNKNOWN_TARGET"
278
279def run_tests(tests):
280  """Creates thread workers to run the tests.
281
282  The method generates command and thread worker to run the tests. Depending on
283  the user input for the number of threads to be used, the method uses a
284  semaphore object to keep a count in control for the thread workers. When a new
285  worker is created, it acquires the semaphore object, and when the number of
286  workers reaches the maximum allowed concurrency, the method wait for an
287  existing thread worker to release the semaphore object. Worker releases the
288  semaphore object when they finish printing the output.
289
290  Args:
291    tests: The set of tests to be run.
292  """
293  options_all = ''
294
295  # jvm does not run with all these combinations,
296  # or at least it doesn't make sense for most of them.
297  # TODO: support some jvm variants like jvmti ?
298  target_input_variants = _user_input_variants['target']
299  uncombinated_target_input_variants = []
300  if 'jvm' in target_input_variants:
301    _user_input_variants['target'].remove('jvm')
302    uncombinated_target_input_variants.append('jvm')
303
304  global total_test_count
305  total_test_count = len(tests)
306  if target_input_variants:
307    for variant_type in VARIANT_TYPE_DICT:
308      if not (variant_type == 'target' or 'address_sizes' in variant_type):
309        total_test_count *= len(_user_input_variants[variant_type])
310  target_address_combinations = 0
311  for target in target_input_variants:
312    for address_size in _user_input_variants['address_sizes_target'][target]:
313      target_address_combinations += 1
314  target_address_combinations += len(uncombinated_target_input_variants)
315  total_test_count *= target_address_combinations
316
317  if env.ART_TEST_WITH_STRACE:
318    options_all += ' --strace'
319
320  if env.ART_TEST_RUN_TEST_ALWAYS_CLEAN:
321    options_all += ' --always-clean'
322
323  if env.ART_TEST_BISECTION:
324    options_all += ' --bisection-search'
325
326  if gdb:
327    options_all += ' --gdb'
328    if gdb_arg:
329      options_all += ' --gdb-arg ' + gdb_arg
330
331  options_all += ' ' + ' '.join(run_test_option)
332
333  if runtime_option:
334    for opt in runtime_option:
335      options_all += ' --runtime-option ' + opt
336  if with_agent:
337    for opt in with_agent:
338      options_all += ' --with-agent ' + opt
339
340  if dex2oat_jobs != -1:
341    options_all += ' --dex2oat-jobs ' + str(dex2oat_jobs)
342
343  def iter_config(tests, input_variants, user_input_variants):
344    config = itertools.product(tests, input_variants, user_input_variants['run'],
345                                 user_input_variants['prebuild'], user_input_variants['compiler'],
346                                 user_input_variants['relocate'], user_input_variants['trace'],
347                                 user_input_variants['gc'], user_input_variants['jni'],
348                                 user_input_variants['image'],
349                                 user_input_variants['debuggable'], user_input_variants['jvmti'],
350                                 user_input_variants['cdex_level'])
351    return config
352
353  # [--host, --target] combines with all the other user input variants.
354  config = iter_config(tests, target_input_variants, _user_input_variants)
355  # [--jvm] currently combines with nothing else. most of the extra flags we'd insert
356  # would be unrecognizable by the 'java' binary, so avoid inserting any extra flags for now.
357  uncombinated_config = iter_config(tests, uncombinated_target_input_variants, { 'run': [''],
358      'prebuild': [''], 'compiler': [''],
359      'relocate': [''], 'trace': [''],
360      'gc': [''], 'jni': [''],
361      'image': [''],
362      'debuggable': [''], 'jvmti': [''],
363      'cdex_level': ['']})
364
365  def start_combination(config_tuple, global_options, address_size):
366      test, target, run, prebuild, compiler, relocate, trace, gc, \
367      jni, image, debuggable, jvmti, cdex_level = config_tuple
368
369      if stop_testrunner:
370        # When ART_TEST_KEEP_GOING is set to false, then as soon as a test
371        # fails, stop_testrunner is set to True. When this happens, the method
372        # stops creating any any thread and wait for all the exising threads
373        # to end.
374        while threading.active_count() > 2:
375          time.sleep(0.1)
376          return
377      # NB The order of components here should match the order of
378      # components in the regex parser in parse_test_name.
379      test_name = 'test-art-'
380      test_name += target + '-run-test-'
381      test_name += run + '-'
382      test_name += prebuild + '-'
383      test_name += compiler + '-'
384      test_name += relocate + '-'
385      test_name += trace + '-'
386      test_name += gc + '-'
387      test_name += jni + '-'
388      test_name += image + '-'
389      test_name += debuggable + '-'
390      test_name += jvmti + '-'
391      test_name += cdex_level + '-'
392      test_name += test
393      test_name += address_size
394
395      variant_set = {target, run, prebuild, compiler, relocate, trace, gc, jni,
396                     image, debuggable, jvmti, cdex_level, address_size}
397
398      options_test = global_options
399
400      if target == 'host':
401        options_test += ' --host'
402      elif target == 'jvm':
403        options_test += ' --jvm'
404
405      # Honor ART_TEST_CHROOT, ART_TEST_ANDROID_ROOT, ART_TEST_ANDROID_RUNTIME_ROOT,
406      # and ART_TEST_ANDROID_TZDATA_ROOT but only for target tests.
407      if target == 'target':
408        if env.ART_TEST_CHROOT:
409          options_test += ' --chroot ' + env.ART_TEST_CHROOT
410        if env.ART_TEST_ANDROID_ROOT:
411          options_test += ' --android-root ' + env.ART_TEST_ANDROID_ROOT
412        if env.ART_TEST_ANDROID_RUNTIME_ROOT:
413          options_test += ' --android-runtime-root ' + env.ART_TEST_ANDROID_RUNTIME_ROOT
414        if env.ART_TEST_ANDROID_TZDATA_ROOT:
415          options_test += ' --android-tzdata-root ' + env.ART_TEST_ANDROID_TZDATA_ROOT
416
417      if run == 'ndebug':
418        options_test += ' -O'
419
420      if prebuild == 'prebuild':
421        options_test += ' --prebuild'
422      elif prebuild == 'no-prebuild':
423        options_test += ' --no-prebuild'
424
425      if cdex_level:
426        # Add option and remove the cdex- prefix.
427        options_test += ' --compact-dex-level ' + cdex_level.replace('cdex-','')
428
429      if compiler == 'optimizing':
430        options_test += ' --optimizing'
431      elif compiler == 'regalloc_gc':
432        options_test += ' --optimizing -Xcompiler-option --register-allocation-strategy=graph-color'
433      elif compiler == 'interpreter':
434        options_test += ' --interpreter'
435      elif compiler == 'interp-ac':
436        options_test += ' --interpreter --verify-soft-fail'
437      elif compiler == 'jit':
438        options_test += ' --jit'
439      elif compiler == 'jit-on-first-use':
440        options_test += ' --jit --runtime-option -Xjitthreshold:0'
441      elif compiler == 'speed-profile':
442        options_test += ' --random-profile'
443      elif compiler == 'baseline':
444        options_test += ' --baseline'
445
446      if relocate == 'relocate':
447        options_test += ' --relocate'
448      elif relocate == 'no-relocate':
449        options_test += ' --no-relocate'
450
451      if trace == 'trace':
452        options_test += ' --trace'
453      elif trace == 'stream':
454        options_test += ' --trace --stream'
455
456      if gc == 'gcverify':
457        options_test += ' --gcverify'
458      elif gc == 'gcstress':
459        options_test += ' --gcstress'
460
461      if jni == 'forcecopy':
462        options_test += ' --runtime-option -Xjniopts:forcecopy'
463      elif jni == 'checkjni':
464        options_test += ' --runtime-option -Xcheck:jni'
465
466      if image == 'no-image':
467        options_test += ' --no-image'
468
469      if debuggable == 'debuggable':
470        options_test += ' --debuggable'
471
472      if jvmti == 'jvmti-stress':
473        options_test += ' --jvmti-trace-stress --jvmti-redefine-stress --jvmti-field-stress'
474      elif jvmti == 'field-stress':
475        options_test += ' --jvmti-field-stress'
476      elif jvmti == 'trace-stress':
477        options_test += ' --jvmti-trace-stress'
478      elif jvmti == 'redefine-stress':
479        options_test += ' --jvmti-redefine-stress'
480      elif jvmti == 'step-stress':
481        options_test += ' --jvmti-step-stress'
482
483      if address_size == '64':
484        options_test += ' --64'
485
486        if env.DEX2OAT_HOST_INSTRUCTION_SET_FEATURES:
487          options_test += ' --instruction-set-features' + env.DEX2OAT_HOST_INSTRUCTION_SET_FEATURES
488
489      elif address_size == '32':
490        if env.HOST_2ND_ARCH_PREFIX_DEX2OAT_HOST_INSTRUCTION_SET_FEATURES:
491          options_test += ' --instruction-set-features ' + \
492                          env.HOST_2ND_ARCH_PREFIX_DEX2OAT_HOST_INSTRUCTION_SET_FEATURES
493
494      # TODO(http://36039166): This is a temporary solution to
495      # fix build breakages.
496      options_test = (' --output-path %s') % (
497          tempfile.mkdtemp(dir=env.ART_HOST_TEST_DIR)) + options_test
498
499      run_test_sh = env.ANDROID_BUILD_TOP + '/art/test/run-test'
500      command = ' '.join((run_test_sh, options_test, ' '.join(extra_arguments[target]), test))
501
502      semaphore.acquire()
503      worker = threading.Thread(target=run_test, args=(command, test, variant_set, test_name))
504      worker.daemon = True
505      worker.start()
506
507  #  Use a context-manager to handle cleaning up the extracted zipapex if needed.
508  with handle_zipapex(zipapex_loc) as zipapex_opt:
509    options_all += zipapex_opt
510    for config_tuple in config:
511      target = config_tuple[1]
512      for address_size in _user_input_variants['address_sizes_target'][target]:
513        start_combination(config_tuple, options_all, address_size)
514
515    for config_tuple in uncombinated_config:
516        start_combination(config_tuple, options_all, "")  # no address size
517
518    while threading.active_count() > 2:
519      time.sleep(0.1)
520
521@contextlib.contextmanager
522def handle_zipapex(ziploc):
523  """Extracts the zipapex (if present) and handles cleanup.
524
525  If we are running out of a zipapex we want to unzip it once and have all the tests use the same
526  extracted contents. This extracts the files and handles cleanup if needed. It returns the
527  required extra arguments to pass to the run-test.
528  """
529  if ziploc is not None:
530    with tempfile.TemporaryDirectory() as tmpdir:
531      subprocess.check_call(["unzip", "-qq", ziploc, "apex_payload.zip", "-d", tmpdir])
532      subprocess.check_call(
533        ["unzip", "-qq", os.path.join(tmpdir, "apex_payload.zip"), "-d", tmpdir])
534      yield " --runtime-extracted-zipapex " + tmpdir
535  else:
536    yield ""
537
538def run_test(command, test, test_variant, test_name):
539  """Runs the test.
540
541  It invokes art/test/run-test script to run the test. The output of the script
542  is checked, and if it ends with "Succeeded!", it assumes that the tests
543  passed, otherwise, put it in the list of failed test. Before actually running
544  the test, it also checks if the test is placed in the list of disabled tests,
545  and if yes, it skips running it, and adds the test in the list of skipped
546  tests. The method uses print_text method to actually print the output. After
547  successfully running and capturing the output for the test, it releases the
548  semaphore object.
549
550  Args:
551    command: The command to be used to invoke the script
552    test: The name of the test without the variant information.
553    test_variant: The set of variant for the test.
554    test_name: The name of the test along with the variants.
555  """
556  global stop_testrunner
557  try:
558    if is_test_disabled(test, test_variant):
559      test_skipped = True
560    else:
561      test_skipped = False
562      if gdb:
563        proc = subprocess.Popen(command.split(), stderr=subprocess.STDOUT, universal_newlines=True)
564      else:
565        proc = subprocess.Popen(command.split(), stderr=subprocess.STDOUT, stdout = subprocess.PIPE,
566                                universal_newlines=True)
567      script_output = proc.communicate(timeout=timeout)[0]
568      test_passed = not proc.wait()
569
570    if not test_skipped:
571      if test_passed:
572        print_test_info(test_name, 'PASS')
573      else:
574        failed_tests.append((test_name, str(command) + "\n" + script_output))
575        if not env.ART_TEST_KEEP_GOING:
576          stop_testrunner = True
577        print_test_info(test_name, 'FAIL', ('%s\n%s') % (
578          command, script_output))
579    elif not dry_run:
580      print_test_info(test_name, 'SKIP')
581      skipped_tests.append(test_name)
582    else:
583      print_test_info(test_name, '')
584  except subprocess.TimeoutExpired as e:
585    failed_tests.append((test_name, 'Timed out in %d seconds' % timeout))
586    print_test_info(test_name, 'TIMEOUT', 'Timed out in %d seconds\n%s' % (
587        timeout, command))
588  except Exception as e:
589    failed_tests.append((test_name, str(e)))
590    print_test_info(test_name, 'FAIL',
591    ('%s\n%s\n\n') % (command, str(e)))
592  finally:
593    semaphore.release()
594
595
596def print_test_info(test_name, result, failed_test_info=""):
597  """Print the continous test information
598
599  If verbose is set to True, it continuously prints test status information
600  on a new line.
601  If verbose is set to False, it keeps on erasing test
602  information by overriding it with the latest test information. Also,
603  in this case it stictly makes sure that the information length doesn't
604  exceed the console width. It does so by shortening the test_name.
605
606  When a test fails, it prints the output of the run-test script and
607  command used to invoke the script. It doesn't override the failing
608  test information in either of the cases.
609  """
610
611  global test_count
612  info = ''
613  if not verbose:
614    # Without --verbose, the testrunner erases passing test info. It
615    # does that by overriding the printed text with white spaces all across
616    # the console width.
617    console_width = int(os.popen('stty size', 'r').read().split()[1])
618    info = '\r' + ' ' * console_width + '\r'
619  try:
620    print_mutex.acquire()
621    test_count += 1
622    percent = (test_count * 100) / total_test_count
623    progress_info = ('[ %d%% %d/%d ]') % (
624      percent,
625      test_count,
626      total_test_count)
627
628    if result == 'FAIL' or result == 'TIMEOUT':
629      if not verbose:
630        info += ('%s %s %s\n') % (
631          progress_info,
632          test_name,
633          COLOR_ERROR + result + COLOR_NORMAL)
634      else:
635        info += ('%s %s %s\n%s\n') % (
636          progress_info,
637          test_name,
638          COLOR_ERROR + result + COLOR_NORMAL,
639          failed_test_info)
640    else:
641      result_text = ''
642      if result == 'PASS':
643        result_text += COLOR_PASS + 'PASS' + COLOR_NORMAL
644      elif result == 'SKIP':
645        result_text += COLOR_SKIP + 'SKIP' + COLOR_NORMAL
646
647      if verbose:
648        info += ('%s %s %s\n') % (
649          progress_info,
650          test_name,
651          result_text)
652      else:
653        total_output_length = 2 # Two spaces
654        total_output_length += len(progress_info)
655        total_output_length += len(result)
656        allowed_test_length = console_width - total_output_length
657        test_name_len = len(test_name)
658        if allowed_test_length < test_name_len:
659          test_name = ('...%s') % (
660            test_name[-(allowed_test_length - 3):])
661        info += ('%s %s %s') % (
662          progress_info,
663          test_name,
664          result_text)
665    print_text(info)
666  except Exception as e:
667    print_text(('%s\n%s\n') % (test_name, str(e)))
668    failed_tests.append(test_name)
669  finally:
670    print_mutex.release()
671
672def verify_knownfailure_entry(entry):
673  supported_field = {
674      'tests' : (list, str),
675      'test_patterns' : (list,),
676      'description' : (list, str),
677      'bug' : (str,),
678      'variant' : (str,),
679      'env_vars' : (dict,),
680  }
681  for field in entry:
682    field_type = type(entry[field])
683    if field_type not in supported_field[field]:
684      raise ValueError('%s is not supported type for %s\n%s' % (
685          str(field_type),
686          field,
687          str(entry)))
688
689def get_disabled_test_info():
690  """Generate set of known failures.
691
692  It parses the art/test/knownfailures.json file to generate the list of
693  disabled tests.
694
695  Returns:
696    The method returns a dict of tests mapped to the variants list
697    for which the test should not be run.
698  """
699  known_failures_file = env.ANDROID_BUILD_TOP + '/art/test/knownfailures.json'
700  with open(known_failures_file) as known_failures_json:
701    known_failures_info = json.loads(known_failures_json.read())
702
703  disabled_test_info = {}
704  for failure in known_failures_info:
705    verify_knownfailure_entry(failure)
706    tests = failure.get('tests', [])
707    if isinstance(tests, str):
708      tests = [tests]
709    patterns = failure.get("test_patterns", [])
710    if (not isinstance(patterns, list)):
711      raise ValueError("test_patters is not a list in %s" % failure)
712
713    tests += [f for f in RUN_TEST_SET if any(re.match(pat, f) is not None for pat in patterns)]
714    variants = parse_variants(failure.get('variant'))
715    env_vars = failure.get('env_vars')
716
717    if check_env_vars(env_vars):
718      for test in tests:
719        if test not in RUN_TEST_SET:
720          raise ValueError('%s is not a valid run-test' % (
721              test))
722        if test in disabled_test_info:
723          disabled_test_info[test] = disabled_test_info[test].union(variants)
724        else:
725          disabled_test_info[test] = variants
726  return disabled_test_info
727
728
729def check_env_vars(env_vars):
730  """Checks if the env variables are set as required to run the test.
731
732  Returns:
733    True if all the env variables are set as required, otherwise False.
734  """
735
736  if not env_vars:
737    return True
738  for key in env_vars:
739    if env.get_env(key) != env_vars.get(key):
740      return False
741  return True
742
743
744def is_test_disabled(test, variant_set):
745  """Checks if the test along with the variant_set is disabled.
746
747  Args:
748    test: The name of the test as in art/test directory.
749    variant_set: Variants to be used for the test.
750  Returns:
751    True, if the test is disabled.
752  """
753  if dry_run:
754    return True
755  if test in env.EXTRA_DISABLED_TESTS:
756    return True
757  if ignore_skips:
758    return False
759  variants_list = DISABLED_TEST_CONTAINER.get(test, {})
760  for variants in variants_list:
761    variants_present = True
762    for variant in variants:
763      if variant not in variant_set:
764        variants_present = False
765        break
766    if variants_present:
767      return True
768  return False
769
770
771def parse_variants(variants):
772  """Parse variants fetched from art/test/knownfailures.json.
773  """
774  if not variants:
775    variants = ''
776    for variant in TOTAL_VARIANTS_SET:
777      variants += variant
778      variants += '|'
779    variants = variants[:-1]
780  variant_list = set()
781  or_variants = variants.split('|')
782  for or_variant in or_variants:
783    and_variants = or_variant.split('&')
784    variant = set()
785    for and_variant in and_variants:
786      and_variant = and_variant.strip()
787      if and_variant not in TOTAL_VARIANTS_SET:
788        raise ValueError('%s is not a valid variant' % (
789            and_variant))
790      variant.add(and_variant)
791    variant_list.add(frozenset(variant))
792  return variant_list
793
794def print_text(output):
795  sys.stdout.write(output)
796  sys.stdout.flush()
797
798def print_analysis():
799  if not verbose:
800    # Without --verbose, the testrunner erases passing test info. It
801    # does that by overriding the printed text with white spaces all across
802    # the console width.
803    console_width = int(os.popen('stty size', 'r').read().split()[1])
804    eraser_text = '\r' + ' ' * console_width + '\r'
805    print_text(eraser_text)
806
807  # Prints information about the total tests run.
808  # E.g., "2/38 (5%) tests passed".
809  passed_test_count = total_test_count - len(skipped_tests) - len(failed_tests)
810  passed_test_information = ('%d/%d (%d%%) %s passed.\n') % (
811      passed_test_count,
812      total_test_count,
813      (passed_test_count*100)/total_test_count,
814      'tests' if passed_test_count > 1 else 'test')
815  print_text(passed_test_information)
816
817  # Prints the list of skipped tests, if any.
818  if skipped_tests:
819    print_text(COLOR_SKIP + 'SKIPPED TESTS: ' + COLOR_NORMAL + '\n')
820    for test in skipped_tests:
821      print_text(test + '\n')
822    print_text('\n')
823
824  # Prints the list of failed tests, if any.
825  if failed_tests:
826    print_text(COLOR_ERROR + 'FAILED: ' + COLOR_NORMAL + '\n')
827    for test_info in failed_tests:
828      print_text(('%s\n%s\n' % (test_info[0], test_info[1])))
829    print_text(COLOR_ERROR + '----------' + COLOR_NORMAL + '\n')
830    for failed_test in sorted([test_info[0] for test_info in failed_tests]):
831      print_text(('%s\n' % (failed_test)))
832
833
834def parse_test_name(test_name):
835  """Parses the testname provided by the user.
836  It supports two types of test_name:
837  1) Like 001-HelloWorld. In this case, it will just verify if the test actually
838  exists and if it does, it returns the testname.
839  2) Like test-art-host-run-test-debug-prebuild-interpreter-no-relocate-ntrace-cms-checkjni-picimage-ndebuggable-001-HelloWorld32
840  In this case, it will parse all the variants and check if they are placed
841  correctly. If yes, it will set the various VARIANT_TYPES to use the
842  variants required to run the test. Again, it returns the test_name
843  without the variant information like 001-HelloWorld.
844  """
845  test_set = set()
846  for test in RUN_TEST_SET:
847    if test.startswith(test_name):
848      test_set.add(test)
849  if test_set:
850    return test_set
851
852  regex = '^test-art-'
853  regex += '(' + '|'.join(VARIANT_TYPE_DICT['target']) + ')-'
854  regex += 'run-test-'
855  regex += '(' + '|'.join(VARIANT_TYPE_DICT['run']) + ')-'
856  regex += '(' + '|'.join(VARIANT_TYPE_DICT['prebuild']) + ')-'
857  regex += '(' + '|'.join(VARIANT_TYPE_DICT['compiler']) + ')-'
858  regex += '(' + '|'.join(VARIANT_TYPE_DICT['relocate']) + ')-'
859  regex += '(' + '|'.join(VARIANT_TYPE_DICT['trace']) + ')-'
860  regex += '(' + '|'.join(VARIANT_TYPE_DICT['gc']) + ')-'
861  regex += '(' + '|'.join(VARIANT_TYPE_DICT['jni']) + ')-'
862  regex += '(' + '|'.join(VARIANT_TYPE_DICT['image']) + ')-'
863  regex += '(' + '|'.join(VARIANT_TYPE_DICT['debuggable']) + ')-'
864  regex += '(' + '|'.join(VARIANT_TYPE_DICT['jvmti']) + ')-'
865  regex += '(' + '|'.join(VARIANT_TYPE_DICT['cdex_level']) + ')-'
866  regex += '(' + '|'.join(RUN_TEST_SET) + ')'
867  regex += '(' + '|'.join(VARIANT_TYPE_DICT['address_sizes']) + ')$'
868  match = re.match(regex, test_name)
869  if match:
870    _user_input_variants['target'].add(match.group(1))
871    _user_input_variants['run'].add(match.group(2))
872    _user_input_variants['prebuild'].add(match.group(3))
873    _user_input_variants['compiler'].add(match.group(4))
874    _user_input_variants['relocate'].add(match.group(5))
875    _user_input_variants['trace'].add(match.group(6))
876    _user_input_variants['gc'].add(match.group(7))
877    _user_input_variants['jni'].add(match.group(8))
878    _user_input_variants['image'].add(match.group(9))
879    _user_input_variants['debuggable'].add(match.group(10))
880    _user_input_variants['jvmti'].add(match.group(11))
881    _user_input_variants['cdex_level'].add(match.group(12))
882    _user_input_variants['address_sizes'].add(match.group(14))
883    return {match.group(13)}
884  raise ValueError(test_name + " is not a valid test")
885
886
887def setup_env_for_build_target(build_target, parser, options):
888  """Setup environment for the build target
889
890  The method setup environment for the master-art-host targets.
891  """
892  os.environ.update(build_target['env'])
893  os.environ['SOONG_ALLOW_MISSING_DEPENDENCIES'] = 'true'
894  print_text('%s\n' % (str(os.environ)))
895
896  target_options = vars(parser.parse_args(build_target['flags']))
897  target_options['host'] = True
898  target_options['verbose'] = True
899  target_options['build'] = True
900  target_options['n_thread'] = options['n_thread']
901  target_options['dry_run'] = options['dry_run']
902
903  return target_options
904
905def get_default_threads(target):
906  if target is 'target':
907    adb_command = 'adb shell cat /sys/devices/system/cpu/present'
908    cpu_info_proc = subprocess.Popen(adb_command.split(), stdout=subprocess.PIPE)
909    cpu_info = cpu_info_proc.stdout.read()
910    if type(cpu_info) is bytes:
911      cpu_info = cpu_info.decode('utf-8')
912    cpu_info_regex = '\d*-(\d*)'
913    match = re.match(cpu_info_regex, cpu_info)
914    if match:
915      return int(match.group(1))
916    else:
917      raise ValueError('Unable to predict the concurrency for the target. '
918                       'Is device connected?')
919  else:
920    return multiprocessing.cpu_count()
921
922def parse_option():
923  global verbose
924  global dry_run
925  global ignore_skips
926  global n_thread
927  global build
928  global gdb
929  global gdb_arg
930  global runtime_option
931  global run_test_option
932  global timeout
933  global dex2oat_jobs
934  global run_all_configs
935  global with_agent
936  global zipapex_loc
937
938  parser = argparse.ArgumentParser(description="Runs all or a subset of the ART test suite.")
939  parser.add_argument('-t', '--test', action='append', dest='tests', help='name(s) of the test(s)')
940  global_group = parser.add_argument_group('Global options',
941                                           'Options that affect all tests being run')
942  global_group.add_argument('-j', type=int, dest='n_thread')
943  global_group.add_argument('--timeout', default=timeout, type=int, dest='timeout')
944  global_group.add_argument('--verbose', '-v', action='store_true', dest='verbose')
945  global_group.add_argument('--dry-run', action='store_true', dest='dry_run')
946  global_group.add_argument("--skip", action='append', dest="skips", default=[],
947                            help="Skip the given test in all circumstances.")
948  global_group.add_argument("--no-skips", dest="ignore_skips", action='store_true', default=False,
949                            help="""Don't skip any run-test configurations listed in
950                            knownfailures.json.""")
951  global_group.add_argument('--no-build-dependencies',
952                            action='store_false', dest='build',
953                            help="""Don't build dependencies under any circumstances. This is the
954                            behavior if ART_TEST_RUN_TEST_ALWAYS_BUILD is not set to 'true'.""")
955  global_group.add_argument('-b', '--build-dependencies',
956                            action='store_true', dest='build',
957                            help="""Build dependencies under all circumstances. By default we will
958                            not build dependencies unless ART_TEST_RUN_TEST_BUILD=true.""")
959  global_group.add_argument('--build-target', dest='build_target', help='master-art-host targets')
960  global_group.set_defaults(build = env.ART_TEST_RUN_TEST_BUILD)
961  global_group.add_argument('--gdb', action='store_true', dest='gdb')
962  global_group.add_argument('--gdb-arg', dest='gdb_arg')
963  global_group.add_argument('--run-test-option', action='append', dest='run_test_option',
964                            default=[],
965                            help="""Pass an option, unaltered, to the run-test script.
966                            This should be enclosed in single-quotes to allow for spaces. The option
967                            will be split using shlex.split() prior to invoking run-test.
968                            Example \"--run-test-option='--with-agent libtifast.so=MethodExit'\"""")
969  global_group.add_argument('--with-agent', action='append', dest='with_agent',
970                            help="""Pass an agent to be attached to the runtime""")
971  global_group.add_argument('--runtime-option', action='append', dest='runtime_option',
972                            help="""Pass an option to the runtime. Runtime options
973                            starting with a '-' must be separated by a '=', for
974                            example '--runtime-option=-Xjitthreshold:0'.""")
975  global_group.add_argument('--dex2oat-jobs', type=int, dest='dex2oat_jobs',
976                            help='Number of dex2oat jobs')
977  global_group.add_argument('--runtime-zipapex', dest='runtime_zipapex', default=None,
978                            help='Location for runtime zipapex.')
979  global_group.add_argument('-a', '--all', action='store_true', dest='run_all',
980                            help="Run all the possible configurations for the input test set")
981  for variant_type, variant_set in VARIANT_TYPE_DICT.items():
982    var_group = parser.add_argument_group(
983        '{}-type Options'.format(variant_type),
984        "Options that control the '{}' variants.".format(variant_type))
985    for variant in variant_set:
986      flag = '--' + variant
987      var_group.add_argument(flag, action='store_true', dest=variant)
988
989  options = vars(parser.parse_args())
990  if options['build_target']:
991    options = setup_env_for_build_target(target_config[options['build_target']],
992                                         parser, options)
993
994  tests = None
995  env.EXTRA_DISABLED_TESTS.update(set(options['skips']))
996  if options['tests']:
997    tests = set()
998    for test_name in options['tests']:
999      tests |= parse_test_name(test_name)
1000
1001  for variant_type in VARIANT_TYPE_DICT:
1002    for variant in VARIANT_TYPE_DICT[variant_type]:
1003      if options.get(variant):
1004        _user_input_variants[variant_type].add(variant)
1005
1006  if options['verbose']:
1007    verbose = True
1008  if options['n_thread']:
1009    n_thread = max(1, options['n_thread'])
1010  ignore_skips = options['ignore_skips']
1011  if options['dry_run']:
1012    dry_run = True
1013    verbose = True
1014  build = options['build']
1015  if options['gdb']:
1016    n_thread = 1
1017    gdb = True
1018    if options['gdb_arg']:
1019      gdb_arg = options['gdb_arg']
1020  runtime_option = options['runtime_option'];
1021  with_agent = options['with_agent'];
1022  run_test_option = sum(map(shlex.split, options['run_test_option']), [])
1023  zipapex_loc = options['runtime_zipapex']
1024
1025  timeout = options['timeout']
1026  if options['dex2oat_jobs']:
1027    dex2oat_jobs = options['dex2oat_jobs']
1028  if options['run_all']:
1029    run_all_configs = True
1030
1031  return tests
1032
1033def main():
1034  gather_test_info()
1035  user_requested_tests = parse_option()
1036  setup_test_env()
1037  if build:
1038    build_targets = ''
1039    if 'host' in _user_input_variants['target']:
1040      build_targets += 'test-art-host-run-test-dependencies '
1041    if 'target' in _user_input_variants['target']:
1042      build_targets += 'test-art-target-run-test-dependencies '
1043    if 'jvm' in _user_input_variants['target']:
1044      build_targets += 'test-art-host-run-test-dependencies '
1045    build_command = env.ANDROID_BUILD_TOP + '/build/soong/soong_ui.bash --make-mode'
1046    build_command += ' DX='
1047    build_command += ' ' + build_targets
1048    if subprocess.call(build_command.split()):
1049      # Debugging for b/62653020
1050      if env.DIST_DIR:
1051        shutil.copyfile(env.SOONG_OUT_DIR + '/build.ninja', env.DIST_DIR + '/soong.ninja')
1052      sys.exit(1)
1053  if user_requested_tests:
1054    test_runner_thread = threading.Thread(target=run_tests, args=(user_requested_tests,))
1055  else:
1056    test_runner_thread = threading.Thread(target=run_tests, args=(RUN_TEST_SET,))
1057  test_runner_thread.daemon = True
1058  try:
1059    test_runner_thread.start()
1060    # This loops waits for all the threads to finish, unless
1061    # stop_testrunner is set to True. When ART_TEST_KEEP_GOING
1062    # is set to false, stop_testrunner is set to True as soon as
1063    # a test fails to signal the parent thread  to stop
1064    # the execution of the testrunner.
1065    while threading.active_count() > 1 and not stop_testrunner:
1066      time.sleep(0.1)
1067    print_analysis()
1068  except Exception as e:
1069    print_analysis()
1070    print_text(str(e))
1071    sys.exit(1)
1072  if failed_tests:
1073    sys.exit(1)
1074  sys.exit(0)
1075
1076if __name__ == '__main__':
1077  main()
1078