1#!/usr/bin/env python3
2#
3# Copyright 2017, The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#     http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17"""ART Run-Test TestRunner
18
19The testrunner runs the ART run-tests by simply invoking the script.
20It fetches the list of eligible tests from art/test directory, and list of
21disabled tests from art/test/knownfailures.json. It runs the tests by
22invoking art/test/run-test script and checks the exit value to decide if the
23test passed or failed.
24
25Before invoking the script, first build all the tests dependencies.
26There are two major build targets for building target and host tests
27dependencies:
281) test-art-host-run-test
292) test-art-target-run-test
30
31There are various options to invoke the script which are:
32-t: Either the test name as in art/test or the test name including the variant
33    information. Eg, "-t 001-HelloWorld",
34    "-t test-art-host-run-test-debug-prebuild-optimizing-relocate-ntrace-cms-checkjni-picimage-npictest-ndebuggable-001-HelloWorld32"
35-j: Number of thread workers to be used. Eg - "-j64"
36--dry-run: Instead of running the test name, just print its name.
37--verbose
38-b / --build-dependencies: to build the dependencies before running the test
39
40To specify any specific variants for the test, use --<<variant-name>>.
41For eg, for compiler type as optimizing, use --optimizing.
42
43
44In the end, the script will print the failed and skipped tests if any.
45
46"""
47import argparse
48import fnmatch
49import itertools
50import json
51import multiprocessing
52import os
53import re
54import subprocess
55import sys
56import tempfile
57import threading
58import time
59
60import env
61from target_config import target_config
62
63TARGET_TYPES = set()
64RUN_TYPES = set()
65PREBUILD_TYPES = set()
66COMPILER_TYPES = set()
67RELOCATE_TYPES = set()
68TRACE_TYPES = set()
69GC_TYPES = set()
70JNI_TYPES = set()
71IMAGE_TYPES = set()
72PICTEST_TYPES = set()
73DEBUGGABLE_TYPES = set()
74ADDRESS_SIZES = set()
75OPTIMIZING_COMPILER_TYPES = set()
76JVMTI_TYPES = set()
77ADDRESS_SIZES_TARGET = {'host': set(), 'target': set()}
78# timeout for individual tests.
79# TODO: make it adjustable per tests and for buildbots
80timeout = 3000 # 50 minutes
81
82# DISABLED_TEST_CONTAINER holds information about the disabled tests. It is a map
83# that has key as the test name (like 001-HelloWorld), and value as set of
84# variants that the test is disabled for.
85DISABLED_TEST_CONTAINER = {}
86
87# The Dict contains the list of all possible variants for a given type. For example,
88# for key TARGET, the value would be target and host. The list is used to parse
89# the test name given as the argument to run.
90VARIANT_TYPE_DICT = {}
91
92# The set contains all the variants of each time.
93TOTAL_VARIANTS_SET = set()
94
95# The colors are used in the output. When a test passes, COLOR_PASS is used,
96# and so on.
97COLOR_ERROR = '\033[91m'
98COLOR_PASS = '\033[92m'
99COLOR_SKIP = '\033[93m'
100COLOR_NORMAL = '\033[0m'
101
102# The mutex object is used by the threads for exclusive access of test_count
103# to make any changes in its value.
104test_count_mutex = threading.Lock()
105
106# The set contains the list of all the possible run tests that are in art/test
107# directory.
108RUN_TEST_SET = set()
109
110# The semaphore object is used by the testrunner to limit the number of
111# threads to the user requested concurrency value.
112semaphore = threading.Semaphore(1)
113
114# The mutex object is used to provide exclusive access to a thread to print
115# its output.
116print_mutex = threading.Lock()
117failed_tests = []
118skipped_tests = []
119
120# Flags
121n_thread = -1
122test_count = 0
123total_test_count = 0
124verbose = False
125dry_run = False
126build = False
127gdb = False
128gdb_arg = ''
129stop_testrunner = False
130
131def gather_test_info():
132  """The method gathers test information about the test to be run which includes
133  generating the list of total tests from the art/test directory and the list
134  of disabled test. It also maps various variants to types.
135  """
136  global TOTAL_VARIANTS_SET
137  global DISABLED_TEST_CONTAINER
138  # TODO: Avoid duplication of the variant names in different lists.
139  VARIANT_TYPE_DICT['pictest'] = {'pictest', 'npictest'}
140  VARIANT_TYPE_DICT['run'] = {'ndebug', 'debug'}
141  VARIANT_TYPE_DICT['target'] = {'target', 'host'}
142  VARIANT_TYPE_DICT['trace'] = {'trace', 'ntrace', 'stream'}
143  VARIANT_TYPE_DICT['image'] = {'picimage', 'no-image', 'multipicimage'}
144  VARIANT_TYPE_DICT['debuggable'] = {'ndebuggable', 'debuggable'}
145  VARIANT_TYPE_DICT['gc'] = {'gcstress', 'gcverify', 'cms'}
146  VARIANT_TYPE_DICT['prebuild'] = {'no-prebuild', 'no-dex2oat', 'prebuild'}
147  VARIANT_TYPE_DICT['relocate'] = {'relocate-npatchoat', 'relocate', 'no-relocate'}
148  VARIANT_TYPE_DICT['jni'] = {'jni', 'forcecopy', 'checkjni'}
149  VARIANT_TYPE_DICT['address_sizes'] = {'64', '32'}
150  VARIANT_TYPE_DICT['jvmti'] = {'no-jvmti', 'jvmti-stress'}
151  VARIANT_TYPE_DICT['compiler'] = {'interp-ac', 'interpreter', 'jit', 'optimizing',
152                              'regalloc_gc', 'speed-profile'}
153
154  for v_type in VARIANT_TYPE_DICT:
155    TOTAL_VARIANTS_SET = TOTAL_VARIANTS_SET.union(VARIANT_TYPE_DICT.get(v_type))
156
157  test_dir = env.ANDROID_BUILD_TOP + '/art/test'
158  for f in os.listdir(test_dir):
159    if fnmatch.fnmatch(f, '[0-9]*'):
160      RUN_TEST_SET.add(f)
161  DISABLED_TEST_CONTAINER = get_disabled_test_info()
162
163
164def setup_test_env():
165  """The method sets default value for the various variants of the tests if they
166  are already not set.
167  """
168  if env.ART_TEST_BISECTION:
169    env.ART_TEST_RUN_TEST_NO_PREBUILD = True
170    env.ART_TEST_RUN_TEST_PREBUILD = False
171    # Bisection search writes to standard output.
172    env.ART_TEST_QUIET = False
173
174  if not TARGET_TYPES:
175    TARGET_TYPES.add('host')
176    TARGET_TYPES.add('target')
177
178  if env.ART_TEST_RUN_TEST_NO_PREBUILD:
179    PREBUILD_TYPES.add('no-prebuild')
180  if env.ART_TEST_RUN_TEST_NO_DEX2OAT:
181    PREBUILD_TYPES.add('no-dex2oat')
182  if env.ART_TEST_RUN_TEST_PREBUILD or not PREBUILD_TYPES: # Default
183    PREBUILD_TYPES.add('prebuild')
184
185  if env.ART_TEST_INTERPRETER_ACCESS_CHECKS:
186    COMPILER_TYPES.add('interp-ac')
187  if env.ART_TEST_INTERPRETER:
188    COMPILER_TYPES.add('interpreter')
189  if env.ART_TEST_JIT:
190    COMPILER_TYPES.add('jit')
191  if env.ART_TEST_OPTIMIZING_GRAPH_COLOR:
192    COMPILER_TYPES.add('regalloc_gc')
193    OPTIMIZING_COMPILER_TYPES.add('regalloc_gc')
194  if env.ART_TEST_OPTIMIZING:
195    COMPILER_TYPES.add('optimizing')
196    OPTIMIZING_COMPILER_TYPES.add('optimizing')
197  if env.ART_TEST_SPEED_PROFILE:
198    COMPILER_TYPES.add('speed-profile')
199
200  # By default only run without jvmti
201  if not JVMTI_TYPES:
202    JVMTI_TYPES.add('no-jvmti')
203
204  # By default we run all 'compiler' variants.
205  if not COMPILER_TYPES:
206    COMPILER_TYPES.add('optimizing')
207    COMPILER_TYPES.add('jit')
208    COMPILER_TYPES.add('interpreter')
209    COMPILER_TYPES.add('interp-ac')
210    COMPILER_TYPES.add('speed-profile')
211    OPTIMIZING_COMPILER_TYPES.add('optimizing')
212
213  if env.ART_TEST_RUN_TEST_RELOCATE:
214    RELOCATE_TYPES.add('relocate')
215  if env.ART_TEST_RUN_TEST_RELOCATE_NO_PATCHOAT:
216    RELOCATE_TYPES.add('relocate-npatchoat')
217  if not RELOCATE_TYPES: # Default
218    RELOCATE_TYPES.add('no-relocate')
219
220  if env.ART_TEST_TRACE:
221    TRACE_TYPES.add('trace')
222  if env.ART_TEST_TRACE_STREAM:
223    TRACE_TYPES.add('stream')
224  if not TRACE_TYPES: # Default
225    TRACE_TYPES.add('ntrace')
226
227  if env.ART_TEST_GC_STRESS:
228    GC_TYPES.add('gcstress')
229  if env.ART_TEST_GC_VERIFY:
230    GC_TYPES.add('gcverify')
231  if not GC_TYPES: # Default
232    GC_TYPES.add('cms')
233
234  if env.ART_TEST_JNI_FORCECOPY:
235    JNI_TYPES.add('forcecopy')
236  if not JNI_TYPES: # Default
237    JNI_TYPES.add('checkjni')
238
239  if env.ART_TEST_RUN_TEST_NO_IMAGE:
240    IMAGE_TYPES.add('no-image')
241  if env.ART_TEST_RUN_TEST_MULTI_IMAGE:
242    IMAGE_TYPES.add('multipicimage')
243  if env.ART_TEST_RUN_TEST_IMAGE or not IMAGE_TYPES: # Default
244    IMAGE_TYPES.add('picimage')
245
246  if env.ART_TEST_PIC_TEST:
247    PICTEST_TYPES.add('pictest')
248  if not PICTEST_TYPES: # Default
249    PICTEST_TYPES.add('npictest')
250
251  if env.ART_TEST_RUN_TEST_NDEBUG:
252    RUN_TYPES.add('ndebug')
253  if env.ART_TEST_RUN_TEST_DEBUG or not RUN_TYPES: # Default
254    RUN_TYPES.add('debug')
255
256  if env.ART_TEST_RUN_TEST_DEBUGGABLE:
257    DEBUGGABLE_TYPES.add('debuggable')
258  if not DEBUGGABLE_TYPES: # Default
259    DEBUGGABLE_TYPES.add('ndebuggable')
260
261  if not ADDRESS_SIZES:
262    ADDRESS_SIZES_TARGET['target'].add(env.ART_PHONY_TEST_TARGET_SUFFIX)
263    ADDRESS_SIZES_TARGET['host'].add(env.ART_PHONY_TEST_HOST_SUFFIX)
264    if env.ART_TEST_RUN_TEST_2ND_ARCH:
265      ADDRESS_SIZES_TARGET['host'].add(env.ART_2ND_PHONY_TEST_HOST_SUFFIX)
266      ADDRESS_SIZES_TARGET['target'].add(env.ART_2ND_PHONY_TEST_TARGET_SUFFIX)
267  else:
268    ADDRESS_SIZES_TARGET['host'] = ADDRESS_SIZES_TARGET['host'].union(ADDRESS_SIZES)
269    ADDRESS_SIZES_TARGET['target'] = ADDRESS_SIZES_TARGET['target'].union(ADDRESS_SIZES)
270
271  global n_thread
272  if n_thread is -1:
273    if 'target' in TARGET_TYPES:
274      n_thread = get_default_threads('target')
275    else:
276      n_thread = get_default_threads('host')
277
278  global semaphore
279  semaphore = threading.Semaphore(n_thread)
280
281  if not sys.stdout.isatty():
282    global COLOR_ERROR
283    global COLOR_PASS
284    global COLOR_SKIP
285    global COLOR_NORMAL
286    COLOR_ERROR = ''
287    COLOR_PASS = ''
288    COLOR_SKIP = ''
289    COLOR_NORMAL = ''
290
291
292def run_tests(tests):
293  """Creates thread workers to run the tests.
294
295  The method generates command and thread worker to run the tests. Depending on
296  the user input for the number of threads to be used, the method uses a
297  semaphore object to keep a count in control for the thread workers. When a new
298  worker is created, it acquires the semaphore object, and when the number of
299  workers reaches the maximum allowed concurrency, the method wait for an
300  existing thread worker to release the semaphore object. Worker releases the
301  semaphore object when they finish printing the output.
302
303  Args:
304    tests: The set of tests to be run.
305  """
306  options_all = ''
307  global total_test_count
308  total_test_count = len(tests)
309  total_test_count *= len(RUN_TYPES)
310  total_test_count *= len(PREBUILD_TYPES)
311  total_test_count *= len(RELOCATE_TYPES)
312  total_test_count *= len(TRACE_TYPES)
313  total_test_count *= len(GC_TYPES)
314  total_test_count *= len(JNI_TYPES)
315  total_test_count *= len(IMAGE_TYPES)
316  total_test_count *= len(PICTEST_TYPES)
317  total_test_count *= len(DEBUGGABLE_TYPES)
318  total_test_count *= len(COMPILER_TYPES)
319  total_test_count *= len(JVMTI_TYPES)
320  target_address_combinations = 0
321  for target in TARGET_TYPES:
322    for address_size in ADDRESS_SIZES_TARGET[target]:
323      target_address_combinations += 1
324  total_test_count *= target_address_combinations
325
326  if env.ART_TEST_WITH_STRACE:
327    options_all += ' --strace'
328
329  if env.ART_TEST_RUN_TEST_ALWAYS_CLEAN:
330    options_all += ' --always-clean'
331
332  if env.ART_TEST_BISECTION:
333    options_all += ' --bisection-search'
334
335  if env.ART_TEST_ANDROID_ROOT:
336    options_all += ' --android-root ' + env.ART_TEST_ANDROID_ROOT
337
338  if gdb:
339    options_all += ' --gdb'
340    if gdb_arg:
341      options_all += ' --gdb-arg ' + gdb_arg
342
343  config = itertools.product(tests, TARGET_TYPES, RUN_TYPES, PREBUILD_TYPES,
344                             COMPILER_TYPES, RELOCATE_TYPES, TRACE_TYPES,
345                             GC_TYPES, JNI_TYPES, IMAGE_TYPES, PICTEST_TYPES,
346                             DEBUGGABLE_TYPES, JVMTI_TYPES)
347
348  for test, target, run, prebuild, compiler, relocate, trace, gc, \
349      jni, image, pictest, debuggable, jvmti in config:
350    for address_size in ADDRESS_SIZES_TARGET[target]:
351      if stop_testrunner:
352        # When ART_TEST_KEEP_GOING is set to false, then as soon as a test
353        # fails, stop_testrunner is set to True. When this happens, the method
354        # stops creating any any thread and wait for all the exising threads
355        # to end.
356        while threading.active_count() > 2:
357          time.sleep(0.1)
358          return
359      test_name = 'test-art-'
360      test_name += target + '-run-test-'
361      test_name += run + '-'
362      test_name += prebuild + '-'
363      test_name += compiler + '-'
364      test_name += relocate + '-'
365      test_name += trace + '-'
366      test_name += gc + '-'
367      test_name += jni + '-'
368      test_name += image + '-'
369      test_name += pictest + '-'
370      test_name += debuggable + '-'
371      test_name += jvmti + '-'
372      test_name += test
373      test_name += address_size
374
375      variant_set = {target, run, prebuild, compiler, relocate, trace, gc, jni,
376                     image, pictest, debuggable, jvmti, address_size}
377
378      options_test = options_all
379
380      if target == 'host':
381        options_test += ' --host'
382
383      if run == 'ndebug':
384        options_test += ' -O'
385
386      if prebuild == 'prebuild':
387        options_test += ' --prebuild'
388      elif prebuild == 'no-prebuild':
389        options_test += ' --no-prebuild'
390      elif prebuild == 'no-dex2oat':
391        options_test += ' --no-prebuild --no-dex2oat'
392
393      if compiler == 'optimizing':
394        options_test += ' --optimizing'
395      elif compiler == 'regalloc_gc':
396        options_test += ' --optimizing -Xcompiler-option --register-allocation-strategy=graph-color'
397      elif compiler == 'interpreter':
398        options_test += ' --interpreter'
399      elif compiler == 'interp-ac':
400        options_test += ' --interpreter --verify-soft-fail'
401      elif compiler == 'jit':
402        options_test += ' --jit'
403      elif compiler == 'speed-profile':
404        options_test += ' --random-profile'
405
406      if relocate == 'relocate':
407        options_test += ' --relocate'
408      elif relocate == 'no-relocate':
409        options_test += ' --no-relocate'
410      elif relocate == 'relocate-npatchoat':
411        options_test += ' --relocate --no-patchoat'
412
413      if trace == 'trace':
414        options_test += ' --trace'
415      elif trace == 'stream':
416        options_test += ' --trace --stream'
417
418      if gc == 'gcverify':
419        options_test += ' --gcverify'
420      elif gc == 'gcstress':
421        options_test += ' --gcstress'
422
423      if jni == 'forcecopy':
424        options_test += ' --runtime-option -Xjniopts:forcecopy'
425      elif jni == 'checkjni':
426        options_test += ' --runtime-option -Xcheck:jni'
427
428      if image == 'no-image':
429        options_test += ' --no-image'
430      elif image == 'multipicimage':
431        options_test += ' --multi-image'
432
433      if pictest == 'pictest':
434        options_test += ' --pic-test'
435
436      if debuggable == 'debuggable':
437        options_test += ' --debuggable'
438
439      if jvmti == 'jvmti-stress':
440        options_test += ' --jvmti-stress'
441
442      if address_size == '64':
443        options_test += ' --64'
444
445        if env.DEX2OAT_HOST_INSTRUCTION_SET_FEATURES:
446          options_test += ' --instruction-set-features' + env.DEX2OAT_HOST_INSTRUCTION_SET_FEATURES
447
448      elif address_size == '32':
449        if env.HOST_2ND_ARCH_PREFIX_DEX2OAT_HOST_INSTRUCTION_SET_FEATURES:
450          options_test += ' --instruction-set-features ' + \
451                          env.HOST_2ND_ARCH_PREFIX_DEX2OAT_HOST_INSTRUCTION_SET_FEATURES
452
453      # TODO(http://36039166): This is a temporary solution to
454      # fix build breakages.
455      options_test = (' --output-path %s') % (
456          tempfile.mkdtemp(dir=env.ART_HOST_TEST_DIR)) + options_test
457
458      run_test_sh = env.ANDROID_BUILD_TOP + '/art/test/run-test'
459      command = run_test_sh + ' ' + options_test + ' ' + test
460
461      semaphore.acquire()
462      worker = threading.Thread(target=run_test, args=(command, test, variant_set, test_name))
463      worker.daemon = True
464      worker.start()
465
466  while threading.active_count() > 2:
467    time.sleep(0.1)
468
469
470def run_test(command, test, test_variant, test_name):
471  """Runs the test.
472
473  It invokes art/test/run-test script to run the test. The output of the script
474  is checked, and if it ends with "Succeeded!", it assumes that the tests
475  passed, otherwise, put it in the list of failed test. Before actually running
476  the test, it also checks if the test is placed in the list of disabled tests,
477  and if yes, it skips running it, and adds the test in the list of skipped
478  tests. The method uses print_text method to actually print the output. After
479  successfully running and capturing the output for the test, it releases the
480  semaphore object.
481
482  Args:
483    command: The command to be used to invoke the script
484    test: The name of the test without the variant information.
485    test_variant: The set of variant for the test.
486    test_name: The name of the test along with the variants.
487  """
488  global stop_testrunner
489  try:
490    if is_test_disabled(test, test_variant):
491      test_skipped = True
492    else:
493      test_skipped = False
494      proc = subprocess.Popen(command.split(), stderr=subprocess.STDOUT, stdout=subprocess.PIPE, universal_newlines=True)
495      script_output = proc.communicate(timeout=timeout)[0]
496      test_passed = not proc.wait()
497
498    if not test_skipped:
499      if test_passed:
500        print_test_info(test_name, 'PASS')
501      else:
502        failed_tests.append((test_name, script_output))
503        if not env.ART_TEST_KEEP_GOING:
504          stop_testrunner = True
505        print_test_info(test_name, 'FAIL', ('%s\n%s') % (
506          command, script_output))
507    elif not dry_run:
508      print_test_info(test_name, 'SKIP')
509      skipped_tests.append(test_name)
510    else:
511      print_test_info(test_name, '')
512  except subprocess.TimeoutExpired as e:
513    failed_tests.append((test_name, 'Timed out in %d seconds' % timeout))
514    print_test_info(test_name, 'TIMEOUT', 'Timed out in %d seconds\n%s' % (
515        timeout, command))
516  except Exception as e:
517    failed_tests.append((test_name, str(e)))
518    print_test_info(test_name, 'FAIL',
519    ('%s\n%s\n\n') % (command, str(e)))
520  finally:
521    semaphore.release()
522
523
524def print_test_info(test_name, result, failed_test_info=""):
525  """Print the continous test information
526
527  If verbose is set to True, it continuously prints test status information
528  on a new line.
529  If verbose is set to False, it keeps on erasing test
530  information by overriding it with the latest test information. Also,
531  in this case it stictly makes sure that the information length doesn't
532  exceed the console width. It does so by shortening the test_name.
533
534  When a test fails, it prints the output of the run-test script and
535  command used to invoke the script. It doesn't override the failing
536  test information in either of the cases.
537  """
538
539  global test_count
540  info = ''
541  if not verbose:
542    # Without --verbose, the testrunner erases passing test info. It
543    # does that by overriding the printed text with white spaces all across
544    # the console width.
545    console_width = int(os.popen('stty size', 'r').read().split()[1])
546    info = '\r' + ' ' * console_width + '\r'
547  try:
548    print_mutex.acquire()
549    test_count += 1
550    percent = (test_count * 100) / total_test_count
551    progress_info = ('[ %d%% %d/%d ]') % (
552      percent,
553      test_count,
554      total_test_count)
555
556    if result == 'FAIL' or result == 'TIMEOUT':
557      info += ('%s %s %s\n%s\n') % (
558        progress_info,
559        test_name,
560        COLOR_ERROR + result + COLOR_NORMAL,
561        failed_test_info)
562    else:
563      result_text = ''
564      if result == 'PASS':
565        result_text += COLOR_PASS + 'PASS' + COLOR_NORMAL
566      elif result == 'SKIP':
567        result_text += COLOR_SKIP + 'SKIP' + COLOR_NORMAL
568
569      if verbose:
570        info += ('%s %s %s\n') % (
571          progress_info,
572          test_name,
573          result_text)
574      else:
575        total_output_length = 2 # Two spaces
576        total_output_length += len(progress_info)
577        total_output_length += len(result)
578        allowed_test_length = console_width - total_output_length
579        test_name_len = len(test_name)
580        if allowed_test_length < test_name_len:
581          test_name = ('...%s') % (
582            test_name[-(allowed_test_length - 3):])
583        info += ('%s %s %s') % (
584          progress_info,
585          test_name,
586          result_text)
587    print_text(info)
588  except Exception as e:
589    print_text(('%s\n%s\n') % (test_name, str(e)))
590    failed_tests.append(test_name)
591  finally:
592    print_mutex.release()
593
594def verify_knownfailure_entry(entry):
595  supported_field = {
596      'tests' : (list, str),
597      'description' : (list, str),
598      'bug' : (str,),
599      'variant' : (str,),
600      'env_vars' : (dict,),
601  }
602  for field in entry:
603    field_type = type(entry[field])
604    if field_type not in supported_field[field]:
605      raise ValueError('%s is not supported type for %s\n%s' % (
606          str(field_type),
607          field,
608          str(entry)))
609
610def get_disabled_test_info():
611  """Generate set of known failures.
612
613  It parses the art/test/knownfailures.json file to generate the list of
614  disabled tests.
615
616  Returns:
617    The method returns a dict of tests mapped to the variants list
618    for which the test should not be run.
619  """
620  known_failures_file = env.ANDROID_BUILD_TOP + '/art/test/knownfailures.json'
621  with open(known_failures_file) as known_failures_json:
622    known_failures_info = json.loads(known_failures_json.read())
623
624  disabled_test_info = {}
625  for failure in known_failures_info:
626    verify_knownfailure_entry(failure)
627    tests = failure.get('tests', [])
628    if isinstance(tests, str):
629      tests = [tests]
630    variants = parse_variants(failure.get('variant'))
631    env_vars = failure.get('env_vars')
632
633    if check_env_vars(env_vars):
634      for test in tests:
635        if test not in RUN_TEST_SET:
636          raise ValueError('%s is not a valid run-test' % (
637              test))
638        if test in disabled_test_info:
639          disabled_test_info[test] = disabled_test_info[test].union(variants)
640        else:
641          disabled_test_info[test] = variants
642  return disabled_test_info
643
644
645def check_env_vars(env_vars):
646  """Checks if the env variables are set as required to run the test.
647
648  Returns:
649    True if all the env variables are set as required, otherwise False.
650  """
651
652  if not env_vars:
653    return True
654  for key in env_vars:
655    if env.get_env(key) != env_vars.get(key):
656      return False
657  return True
658
659
660def is_test_disabled(test, variant_set):
661  """Checks if the test along with the variant_set is disabled.
662
663  Args:
664    test: The name of the test as in art/test directory.
665    variant_set: Variants to be used for the test.
666  Returns:
667    True, if the test is disabled.
668  """
669  if dry_run:
670    return True
671  if test in env.EXTRA_DISABLED_TESTS:
672    return True
673  variants_list = DISABLED_TEST_CONTAINER.get(test, {})
674  for variants in variants_list:
675    variants_present = True
676    for variant in variants:
677      if variant not in variant_set:
678        variants_present = False
679        break
680    if variants_present:
681      return True
682  return False
683
684
685def parse_variants(variants):
686  """Parse variants fetched from art/test/knownfailures.json.
687  """
688  if not variants:
689    variants = ''
690    for variant in TOTAL_VARIANTS_SET:
691      variants += variant
692      variants += '|'
693    variants = variants[:-1]
694  variant_list = set()
695  or_variants = variants.split('|')
696  for or_variant in or_variants:
697    and_variants = or_variant.split('&')
698    variant = set()
699    for and_variant in and_variants:
700      and_variant = and_variant.strip()
701      if and_variant not in TOTAL_VARIANTS_SET:
702        raise ValueError('%s is not a valid variant' % (
703            and_variant))
704      variant.add(and_variant)
705    variant_list.add(frozenset(variant))
706  return variant_list
707
708def print_text(output):
709  sys.stdout.write(output)
710  sys.stdout.flush()
711
712def print_analysis():
713  if not verbose:
714    # Without --verbose, the testrunner erases passing test info. It
715    # does that by overriding the printed text with white spaces all across
716    # the console width.
717    console_width = int(os.popen('stty size', 'r').read().split()[1])
718    eraser_text = '\r' + ' ' * console_width + '\r'
719    print_text(eraser_text)
720
721  # Prints information about the total tests run.
722  # E.g., "2/38 (5%) tests passed".
723  passed_test_count = total_test_count - len(skipped_tests) - len(failed_tests)
724  passed_test_information = ('%d/%d (%d%%) %s passed.\n') % (
725      passed_test_count,
726      total_test_count,
727      (passed_test_count*100)/total_test_count,
728      'tests' if passed_test_count > 1 else 'test')
729  print_text(passed_test_information)
730
731  # Prints the list of skipped tests, if any.
732  if skipped_tests:
733    print_text(COLOR_SKIP + 'SKIPPED TESTS: ' + COLOR_NORMAL + '\n')
734    for test in skipped_tests:
735      print_text(test + '\n')
736    print_text('\n')
737
738  # Prints the list of failed tests, if any.
739  if failed_tests:
740    print_text(COLOR_ERROR + 'FAILED: ' + COLOR_NORMAL + '\n')
741    for test_info in failed_tests:
742      print_text(('%s\n%s\n' % (test_info[0], test_info[1])))
743
744
745def parse_test_name(test_name):
746  """Parses the testname provided by the user.
747  It supports two types of test_name:
748  1) Like 001-HelloWorld. In this case, it will just verify if the test actually
749  exists and if it does, it returns the testname.
750  2) Like test-art-host-run-test-debug-prebuild-interpreter-no-relocate-ntrace-cms-checkjni-picimage-npictest-ndebuggable-001-HelloWorld32
751  In this case, it will parse all the variants and check if they are placed
752  correctly. If yes, it will set the various VARIANT_TYPES to use the
753  variants required to run the test. Again, it returns the test_name
754  without the variant information like 001-HelloWorld.
755  """
756  test_set = set()
757  for test in RUN_TEST_SET:
758    if test.startswith(test_name):
759      test_set.add(test)
760  if test_set:
761    return test_set
762
763  regex = '^test-art-'
764  regex += '(' + '|'.join(VARIANT_TYPE_DICT['target']) + ')-'
765  regex += 'run-test-'
766  regex += '(' + '|'.join(VARIANT_TYPE_DICT['run']) + ')-'
767  regex += '(' + '|'.join(VARIANT_TYPE_DICT['prebuild']) + ')-'
768  regex += '(' + '|'.join(VARIANT_TYPE_DICT['compiler']) + ')-'
769  regex += '(' + '|'.join(VARIANT_TYPE_DICT['relocate']) + ')-'
770  regex += '(' + '|'.join(VARIANT_TYPE_DICT['trace']) + ')-'
771  regex += '(' + '|'.join(VARIANT_TYPE_DICT['gc']) + ')-'
772  regex += '(' + '|'.join(VARIANT_TYPE_DICT['jni']) + ')-'
773  regex += '(' + '|'.join(VARIANT_TYPE_DICT['image']) + ')-'
774  regex += '(' + '|'.join(VARIANT_TYPE_DICT['pictest']) + ')-'
775  regex += '(' + '|'.join(VARIANT_TYPE_DICT['debuggable']) + ')-'
776  regex += '(' + '|'.join(VARIANT_TYPE_DICT['jvmti']) + ')-'
777  regex += '(' + '|'.join(RUN_TEST_SET) + ')'
778  regex += '(' + '|'.join(VARIANT_TYPE_DICT['address_sizes']) + ')$'
779  match = re.match(regex, test_name)
780  if match:
781    TARGET_TYPES.add(match.group(1))
782    RUN_TYPES.add(match.group(2))
783    PREBUILD_TYPES.add(match.group(3))
784    COMPILER_TYPES.add(match.group(4))
785    RELOCATE_TYPES.add(match.group(5))
786    TRACE_TYPES.add(match.group(6))
787    GC_TYPES.add(match.group(7))
788    JNI_TYPES.add(match.group(8))
789    IMAGE_TYPES.add(match.group(9))
790    PICTEST_TYPES.add(match.group(10))
791    DEBUGGABLE_TYPES.add(match.group(11))
792    JVMTI_TYPES.add(match.group(12))
793    ADDRESS_SIZES.add(match.group(14))
794    return {match.group(13)}
795  raise ValueError(test_name + " is not a valid test")
796
797
798def setup_env_for_build_target(build_target, parser, options):
799  """Setup environment for the build target
800
801  The method setup environment for the master-art-host targets.
802  """
803  os.environ.update(build_target['env'])
804  os.environ['SOONG_ALLOW_MISSING_DEPENDENCIES'] = 'true'
805  print_text('%s\n' % (str(os.environ)))
806
807  target_options = vars(parser.parse_args(build_target['flags']))
808  target_options['host'] = True
809  target_options['verbose'] = True
810  target_options['build'] = True
811  target_options['n_thread'] = options['n_thread']
812  target_options['dry_run'] = options['dry_run']
813
814  return target_options
815
816def get_default_threads(target):
817  if target is 'target':
818    adb_command = 'adb shell cat /sys/devices/system/cpu/present'
819    cpu_info_proc = subprocess.Popen(adb_command.split(), stdout=subprocess.PIPE)
820    cpu_info = cpu_info_proc.stdout.read()
821    return int(cpu_info.split('-')[1])
822  else:
823    return multiprocessing.cpu_count()
824
825def parse_option():
826  global verbose
827  global dry_run
828  global n_thread
829  global build
830  global gdb
831  global gdb_arg
832  global timeout
833
834  parser = argparse.ArgumentParser(description="Runs all or a subset of the ART test suite.")
835  parser.add_argument('-t', '--test', dest='test', help='name of the test')
836  parser.add_argument('-j', type=int, dest='n_thread')
837  parser.add_argument('--timeout', default=timeout, type=int, dest='timeout')
838  for variant in TOTAL_VARIANTS_SET:
839    flag = '--' + variant
840    flag_dest = variant.replace('-', '_')
841    if variant == '32' or variant == '64':
842      flag_dest = 'n' + flag_dest
843    parser.add_argument(flag, action='store_true', dest=flag_dest)
844  parser.add_argument('--verbose', '-v', action='store_true', dest='verbose')
845  parser.add_argument('--dry-run', action='store_true', dest='dry_run')
846  parser.add_argument("--skip", action="append", dest="skips", default=[],
847                      help="Skip the given test in all circumstances.")
848  parser.add_argument('--no-build-dependencies',
849                      action='store_false', dest='build',
850                      help="Don't build dependencies under any circumstances. This is the " +
851                           "behavior if ART_TEST_RUN_TEST_ALWAYS_BUILD is not set to 'true'.")
852  parser.add_argument('-b', '--build-dependencies',
853                      action='store_true', dest='build',
854                      help="Build dependencies under all circumstances. By default we will " +
855                           "not build dependencies unless ART_TEST_RUN_TEST_BUILD=true.")
856  parser.add_argument('--build-target', dest='build_target', help='master-art-host targets')
857  parser.set_defaults(build = env.ART_TEST_RUN_TEST_BUILD)
858  parser.add_argument('--gdb', action='store_true', dest='gdb')
859  parser.add_argument('--gdb-arg', dest='gdb_arg')
860
861  options = vars(parser.parse_args())
862  if options['build_target']:
863    options = setup_env_for_build_target(target_config[options['build_target']],
864                                         parser, options)
865
866  test = ''
867  env.EXTRA_DISABLED_TESTS.update(set(options['skips']))
868  if options['test']:
869    test = parse_test_name(options['test'])
870  if options['pictest']:
871    PICTEST_TYPES.add('pictest')
872  if options['ndebug']:
873    RUN_TYPES.add('ndebug')
874  if options['interp_ac']:
875    COMPILER_TYPES.add('interp-ac')
876  if options['picimage']:
877    IMAGE_TYPES.add('picimage')
878  if options['n64']:
879    ADDRESS_SIZES.add('64')
880  if options['interpreter']:
881    COMPILER_TYPES.add('interpreter')
882  if options['jni']:
883    JNI_TYPES.add('jni')
884  if options['relocate_npatchoat']:
885    RELOCATE_TYPES.add('relocate-npatchoat')
886  if options['no_prebuild']:
887    PREBUILD_TYPES.add('no-prebuild')
888  if options['npictest']:
889    PICTEST_TYPES.add('npictest')
890  if options['no_dex2oat']:
891    PREBUILD_TYPES.add('no-dex2oat')
892  if options['jit']:
893    COMPILER_TYPES.add('jit')
894  if options['relocate']:
895    RELOCATE_TYPES.add('relocate')
896  if options['ndebuggable']:
897    DEBUGGABLE_TYPES.add('ndebuggable')
898  if options['no_image']:
899    IMAGE_TYPES.add('no-image')
900  if options['optimizing']:
901    COMPILER_TYPES.add('optimizing')
902  if options['speed_profile']:
903    COMPILER_TYPES.add('speed-profile')
904  if options['trace']:
905    TRACE_TYPES.add('trace')
906  if options['gcstress']:
907    GC_TYPES.add('gcstress')
908  if options['no_relocate']:
909    RELOCATE_TYPES.add('no-relocate')
910  if options['target']:
911    TARGET_TYPES.add('target')
912  if options['forcecopy']:
913    JNI_TYPES.add('forcecopy')
914  if options['n32']:
915    ADDRESS_SIZES.add('32')
916  if options['host']:
917    TARGET_TYPES.add('host')
918  if options['gcverify']:
919    GC_TYPES.add('gcverify')
920  if options['debuggable']:
921    DEBUGGABLE_TYPES.add('debuggable')
922  if options['prebuild']:
923    PREBUILD_TYPES.add('prebuild')
924  if options['debug']:
925    RUN_TYPES.add('debug')
926  if options['checkjni']:
927    JNI_TYPES.add('checkjni')
928  if options['ntrace']:
929    TRACE_TYPES.add('ntrace')
930  if options['cms']:
931    GC_TYPES.add('cms')
932  if options['multipicimage']:
933    IMAGE_TYPES.add('multipicimage')
934  if options['jvmti_stress']:
935    JVMTI_TYPES.add('jvmti-stress')
936  if options['no_jvmti']:
937    JVMTI_TYPES.add('no-jvmti')
938  if options['verbose']:
939    verbose = True
940  if options['n_thread']:
941    n_thread = max(1, options['n_thread'])
942  if options['dry_run']:
943    dry_run = True
944    verbose = True
945  build = options['build']
946  if options['gdb']:
947    n_thread = 1
948    gdb = True
949    if options['gdb_arg']:
950      gdb_arg = options['gdb_arg']
951  timeout = options['timeout']
952
953  return test
954
955def main():
956  gather_test_info()
957  user_requested_test = parse_option()
958  setup_test_env()
959  if build:
960    build_targets = ''
961    if 'host' in TARGET_TYPES:
962      build_targets += 'test-art-host-run-test-dependencies'
963    if 'target' in TARGET_TYPES:
964      build_targets += 'test-art-target-run-test-dependencies'
965    build_command = 'make'
966    build_command += ' -j'
967    build_command += ' -C ' + env.ANDROID_BUILD_TOP
968    build_command += ' ' + build_targets
969    # Add 'dist' to avoid Jack issues b/36169180.
970    build_command += ' dist'
971    if subprocess.call(build_command.split()):
972      sys.exit(1)
973  if user_requested_test:
974    test_runner_thread = threading.Thread(target=run_tests, args=(user_requested_test,))
975  else:
976    test_runner_thread = threading.Thread(target=run_tests, args=(RUN_TEST_SET,))
977  test_runner_thread.daemon = True
978  try:
979    test_runner_thread.start()
980    while threading.active_count() > 1:
981      time.sleep(0.1)
982    print_analysis()
983  except Exception as e:
984    print_analysis()
985    print_text(str(e))
986    sys.exit(1)
987  if failed_tests:
988    sys.exit(1)
989  sys.exit(0)
990
991if __name__ == '__main__':
992  main()
993