1#!/usr/bin/env python2.7
2
3# Copyright 2015, VIXL authors
4# All rights reserved.
5#
6# Redistribution and use in source and binary forms, with or without
7# modification, are permitted provided that the following conditions are met:
8#
9#   * Redistributions of source code must retain the above copyright notice,
10#     this list of conditions and the following disclaimer.
11#   * Redistributions in binary form must reproduce the above copyright notice,
12#     this list of conditions and the following disclaimer in the documentation
13#     and/or other materials provided with the distribution.
14#   * Neither the name of ARM Limited nor the names of its contributors may be
15#     used to endorse or promote products derived from this software without
16#     specific prior written permission.
17#
18# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
19# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
20# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
21# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
22# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
24# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
25# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
26# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29import argparse
30import fcntl
31import git
32import itertools
33import multiprocessing
34import os
35from os.path import join
36import platform
37import re
38import subprocess
39import sys
40import time
41
42import config
43import clang_format
44import lint
45import printer
46import test
47import threaded_tests
48import util
49
50
51dir_root = config.dir_root
52
53def Optionify(name):
54  return '--' + name
55
56
57# The options that can be tested are abstracted to provide an easy way to add
58# new ones.
59# Environment options influence the environment. They can be used for example to
60# set the compiler used.
61# Build options are options passed to scons, with a syntax like `scons opt=val`
62# Runtime options are options passed to the test program.
63# See the definition of `test_options` below.
64
65# 'all' is a special value for the options. If specified, all other values of
66# the option are tested.
67class TestOption(object):
68  type_environment = 'type_environment'
69  type_build = 'type_build'
70  type_run = 'type_run'
71
72  def __init__(self, option_type, name, help,
73               val_test_choices, val_test_default = None,
74               # If unset, the user can pass any value.
75               strict_choices = True, test_independently = False):
76    self.name = name
77    self.option_type = option_type
78    self.help = help
79    self.val_test_choices = val_test_choices
80    self.strict_choices = strict_choices
81    self.test_independently = test_independently
82    if val_test_default is not None:
83      self.val_test_default = val_test_default
84    else:
85      self.val_test_default = val_test_choices[0]
86
87  def ArgList(self, to_test):
88    res = []
89    if to_test == 'all':
90      for value in self.val_test_choices:
91        if value != 'all':
92          res.append(self.GetOptionString(value))
93    else:
94      for value in to_test:
95        res.append(self.GetOptionString(value))
96    return res
97
98class EnvironmentOption(TestOption):
99  option_type = TestOption.type_environment
100  def __init__(self, name, environment_variable_name, help,
101               val_test_choices, val_test_default = None,
102               strict_choices = True):
103    super(EnvironmentOption, self).__init__(EnvironmentOption.option_type,
104                                      name,
105                                      help,
106                                      val_test_choices,
107                                      val_test_default,
108                                      strict_choices = strict_choices)
109    self.environment_variable_name = environment_variable_name
110
111  def GetOptionString(self, value):
112    return self.environment_variable_name + '=' + value
113
114
115class BuildOption(TestOption):
116  option_type = TestOption.type_build
117  def __init__(self, name, help,
118               val_test_choices, val_test_default = None,
119               strict_choices = True, test_independently = False):
120    super(BuildOption, self).__init__(BuildOption.option_type,
121                                      name,
122                                      help,
123                                      val_test_choices,
124                                      val_test_default,
125                                      strict_choices = strict_choices,
126                                      test_independently = test_independently)
127  def GetOptionString(self, value):
128    return self.name + '=' + value
129
130
131class RuntimeOption(TestOption):
132  option_type = TestOption.type_run
133  def __init__(self, name, help,
134               val_test_choices, val_test_default = None):
135    super(RuntimeOption, self).__init__(RuntimeOption.option_type,
136                                        name,
137                                        help,
138                                        val_test_choices,
139                                        val_test_default)
140  def GetOptionString(self, value):
141    if value == 'on':
142      return Optionify(self.name)
143    else:
144      return None
145
146
147
148environment_option_compiler = \
149  EnvironmentOption('compiler', 'CXX', 'Test for the specified compilers.',
150                    val_test_choices=['all'] + config.tested_compilers,
151                    strict_choices = False)
152test_environment_options = [
153  environment_option_compiler
154]
155
156build_option_mode = \
157  BuildOption('mode', 'Test with the specified build modes.',
158              val_test_choices=['all'] + config.build_options_modes)
159build_option_standard = \
160  BuildOption('std', 'Test with the specified C++ standard.',
161              val_test_choices=['all'] + config.tested_cpp_standards,
162              strict_choices = False)
163build_option_target = \
164  BuildOption('target', 'Test with the specified isa enabled.',
165              val_test_choices=['all'] + config.build_options_target,
166              strict_choices = False, test_independently = True)
167build_option_negative_testing = \
168  BuildOption('negative_testing', 'Test with negative testing enabled.',
169              val_test_choices=['all'] + config.build_options_negative_testing,
170              strict_choices = False, test_independently = True)
171test_build_options = [
172  build_option_mode,
173  build_option_standard,
174  build_option_target,
175  build_option_negative_testing
176]
177
178runtime_option_debugger = \
179  RuntimeOption('debugger',
180                '''Test with the specified configurations for the debugger.
181                Note that this is only tested if we are using the simulator.''',
182                val_test_choices=['all', 'on', 'off'])
183test_runtime_options = [
184  runtime_option_debugger
185]
186
187test_options = \
188  test_environment_options + test_build_options + test_runtime_options
189
190
191def BuildOptions():
192  args = argparse.ArgumentParser(
193    description =
194    '''This tool runs all tests matching the specified filters for multiple
195    environment, build options, and runtime options configurations.''',
196    # Print default values.
197    formatter_class=argparse.ArgumentDefaultsHelpFormatter)
198
199  args.add_argument('filters', metavar='filter', nargs='*',
200                    help='Run tests matching all of the (regexp) filters.')
201
202  # We automatically build the script options from the options to be tested.
203  test_arguments = args.add_argument_group(
204    'Test options',
205    'These options indicate what should be tested')
206  for option in test_options:
207    choices = option.val_test_choices if option.strict_choices else None
208    help = option.help
209    if not option.strict_choices:
210      help += ' Supported values: {' + ','.join(option.val_test_choices) + '}'
211    test_arguments.add_argument(Optionify(option.name),
212                                nargs='+',
213                                choices=choices,
214                                default=option.val_test_default,
215                                help=help,
216                                action='store')
217
218  general_arguments = args.add_argument_group('General options')
219  general_arguments.add_argument('--fast', action='store_true',
220                                 help='''Skip the lint and clang-format tests,
221                                 and run only with one compiler, in one mode,
222                                 with one C++ standard, and with an appropriate
223                                 default for runtime options.''')
224  general_arguments.add_argument('--dry-run', action='store_true',
225                                 help='''Don't actually build or run anything,
226                                 but print the configurations that would be
227                                 tested.''')
228  general_arguments.add_argument(
229    '--jobs', '-j', metavar='N', type=int, nargs='?',
230    default=multiprocessing.cpu_count(),
231    const=multiprocessing.cpu_count(),
232    help='''Runs the tests using N jobs. If the option is set but no value is
233    provided, the script will use as many jobs as it thinks useful.''')
234  general_arguments.add_argument('--nobench', action='store_true',
235                                 help='Do not run benchmarks.')
236  general_arguments.add_argument('--nolint', action='store_true',
237                                 help='Do not run the linter.')
238  general_arguments.add_argument('--noclang-format', action='store_true',
239                                 help='Do not run clang-format.')
240  general_arguments.add_argument('--notest', action='store_true',
241                                 help='Do not run tests.')
242  general_arguments.add_argument('--fail-early', action='store_true',
243                                 help='Exit as soon as a test fails.')
244  sim_default = 'none' if platform.machine() == 'aarch64' else 'aarch64'
245  general_arguments.add_argument(
246    '--simulator', action='store', choices=['aarch64', 'none'],
247    default=sim_default,
248    help='Explicitly enable or disable the simulator.')
249  general_arguments.add_argument(
250    '--under_valgrind', action='store_true',
251    help='''Run the test-runner commands under Valgrind.
252            Note that a few tests are known to fail because of
253            issues in Valgrind''')
254  return args.parse_args()
255
256
257def RunCommand(command, environment_options = None):
258  # Create a copy of the environment. We do not want to pollute the environment
259  # of future commands run.
260  environment = os.environ
261  # Configure the environment.
262  # TODO: We currently pass the options as strings, so we need to parse them. We
263  # should instead pass them as a data structure and build the string option
264  # later. `environment_options` looks like `['CXX=compiler', 'OPT=val']`.
265  if environment_options:
266    for option in environment_options:
267      opt, val = option.split('=')
268      environment[opt] = val
269
270  printable_command = ''
271  if environment_options:
272    printable_command += ' '.join(environment_options) + ' '
273  printable_command += ' '.join(command)
274
275  printable_command_orange = \
276    printer.COLOUR_ORANGE + printable_command + printer.NO_COLOUR
277  printer.PrintOverwritableLine(printable_command_orange)
278  sys.stdout.flush()
279
280  # Start a process for the command.
281  # Interleave `stderr` and `stdout`.
282  p = subprocess.Popen(command,
283                       stdout=subprocess.PIPE,
284                       stderr=subprocess.STDOUT,
285                       env=environment)
286
287  # We want to be able to display a continuously updated 'work indicator' while
288  # the process is running. Since the process can hang if the `stdout` pipe is
289  # full, we need to pull from it regularly. We cannot do so via the
290  # `readline()` function because it is blocking, and would thus cause the
291  # indicator to not be updated properly. So use file control mechanisms
292  # instead.
293  indicator = ' (still working: %d seconds elapsed)'
294
295  # Mark the process output as non-blocking.
296  flags = fcntl.fcntl(p.stdout, fcntl.F_GETFL)
297  fcntl.fcntl(p.stdout, fcntl.F_SETFL, flags | os.O_NONBLOCK)
298
299  t_start = time.time()
300  t_last_indication = t_start
301  process_output = ''
302
303  # Keep looping as long as the process is running.
304  while p.poll() is None:
305    # Avoid polling too often.
306    time.sleep(0.1)
307    # Update the progress indicator.
308    t_current = time.time()
309    if (t_current - t_start >= 2) and (t_current - t_last_indication >= 1):
310      printer.PrintOverwritableLine(
311        printable_command_orange + indicator % int(t_current - t_start))
312      sys.stdout.flush()
313      t_last_indication = t_current
314    # Pull from the process output.
315    while True:
316      try:
317        line = os.read(p.stdout.fileno(), 1024)
318      except OSError:
319        line = ''
320        break
321      if line == '': break
322      process_output += line
323
324  # The process has exited. Don't forget to retrieve the rest of its output.
325  out, err = p.communicate()
326  rc = p.poll()
327  process_output += out
328
329  if rc == 0:
330    printer.Print(printer.COLOUR_GREEN + printable_command + printer.NO_COLOUR)
331  else:
332    printer.Print(printer.COLOUR_RED + printable_command + printer.NO_COLOUR)
333    printer.Print(process_output)
334  return rc
335
336
337def RunLinter():
338  rc, default_tracked_files = lint.GetDefaultFilesToLint()
339  if rc:
340    return rc
341  return lint.RunLinter(map(lambda x: join(dir_root, x), default_tracked_files),
342                        jobs = args.jobs, progress_prefix = 'cpp lint: ')
343
344
345def RunClangFormat():
346  return clang_format.ClangFormatFiles(clang_format.GetCppSourceFilesToFormat(),
347                                       jobs = args.jobs,
348                                       progress_prefix = 'clang-format: ')
349
350
351
352def BuildAll(build_options, jobs):
353  scons_command = ["scons", "-C", dir_root, 'all', '-j', str(jobs)]
354  scons_command += list(build_options)
355  return RunCommand(scons_command, list(environment_options))
356
357
358# Work out if the given options or args allow to run on the specified arch.
359#  * arches is a list of ISA/architecture (a64, aarch32, etc)
360#  * options are test.py's command line options if any.
361#  * args are the arguments given to the build script.
362def CanRunOn(arches, options, args):
363  # First we check in the build specific options.
364  for option in options:
365    if 'target' in option:
366      # The option format is 'target=x,y,z'.
367      for target in (option.split('='))[1].split(','):
368        if target in arches:
369          return True
370
371      # There was a target build option but it didn't include the target arch.
372      return False
373
374  # No specific build option, check the script arguments.
375  # The meaning of 'all' will depend on the platform, e.g. 32-bit compilers
376  # cannot handle Aarch64 while 64-bit compiler can handle Aarch32. To avoid
377  # any issues no benchmarks are run for target='all'.
378  if args.target == 'all': return False
379
380  for target in args.target[0].split(','):
381    if target in arches:
382      return True
383
384  return False
385
386
387def CanRunAarch64(options, args):
388  return CanRunOn(['aarch64', 'a64'], options, args)
389
390
391def CanRunAarch32(options, args):
392  return CanRunOn(['aarch32', 'a32', 't32'], options, args)
393
394
395def RunBenchmarks(options, args):
396  rc = 0
397  if CanRunAarch32(options, args):
398    benchmark_names = util.ListCCFilesWithoutExt(config.dir_aarch32_benchmarks)
399    for bench in benchmark_names:
400      rc |= RunCommand(
401        [os.path.realpath(
402          join(config.dir_build_latest, 'benchmarks/aarch32', bench))])
403  if CanRunAarch64(options, args):
404    benchmark_names = util.ListCCFilesWithoutExt(config.dir_aarch64_benchmarks)
405    for bench in benchmark_names:
406      rc |= RunCommand(
407        [util.relrealpath(
408            join(config.dir_build_latest, 'benchmarks/aarch64', bench))])
409  return rc
410
411
412def PrintStatus(success):
413  printer.Print('\n$ ' + ' '.join(sys.argv))
414  if success:
415    printer.Print('SUCCESS')
416  else:
417    printer.Print('FAILURE')
418
419
420
421if __name__ == '__main__':
422  util.require_program('scons')
423  rc = 0
424
425  args = BuildOptions()
426
427  def MaybeExitEarly(rc):
428    if args.fail_early and rc != 0:
429      PrintStatus(rc == 0)
430      sys.exit(rc)
431
432  if args.under_valgrind:
433    util.require_program('valgrind')
434
435  if args.fast:
436    def SetFast(option, specified, default):
437      option.val_test_choices = \
438        [default if specified == 'all' else specified[0]]
439    # `g++` is very slow to compile a few aarch32 test files.
440    SetFast(environment_option_compiler, args.compiler, 'clang++')
441    SetFast(build_option_standard, args.std, 'c++98')
442    SetFast(build_option_mode, args.mode, 'debug')
443    SetFast(runtime_option_debugger, args.debugger, 'on')
444
445  if not args.nolint and not (args.fast or args.dry_run):
446    rc |= RunLinter()
447    MaybeExitEarly(rc)
448
449  if not args.noclang_format and not (args.fast or args.dry_run):
450    rc |= RunClangFormat()
451    MaybeExitEarly(rc)
452
453  # Don't try to test the debugger if we are not running with the simulator.
454  if not args.simulator:
455    test_runtime_options = \
456      filter(lambda x: x.name != 'debugger', test_runtime_options)
457
458  # List all combinations of options that will be tested.
459  def ListCombinations(args, options):
460    opts_list = [
461        opt.ArgList(args.__dict__[opt.name])
462        for opt in options
463        if not opt.test_independently
464    ]
465    return list(itertools.product(*opts_list))
466  # List combinations of options that should only be tested independently.
467  def ListIndependentCombinations(args, options, base):
468    n = []
469    for opt in options:
470      if opt.test_independently:
471        for o in opt.ArgList(args.__dict__[opt.name]):
472          n.append(base + (o,))
473    return n
474  # TODO: We should refine the configurations we test by default, instead of
475  #       always testing all possible combinations.
476  test_env_combinations = ListCombinations(args, test_environment_options)
477  test_build_combinations = ListCombinations(args, test_build_options)
478  if not args.fast:
479    test_build_combinations.extend(
480        ListIndependentCombinations(args,
481                                    test_build_options,
482                                    test_build_combinations[0]))
483  test_runtime_combinations = ListCombinations(args, test_runtime_options)
484
485  for environment_options in test_env_combinations:
486    for build_options in test_build_combinations:
487      if (args.dry_run):
488        for runtime_options in test_runtime_combinations:
489          print(' '.join(filter(None, environment_options)) + ', ' +
490                ' '.join(filter(None, build_options)) + ', ' +
491                ' '.join(filter(None, runtime_options)))
492        continue
493
494      # Avoid going through the build stage if we are not using the build
495      # result.
496      if not (args.notest and args.nobench):
497        build_rc = BuildAll(build_options, args.jobs)
498        # Don't run the tests for this configuration if the build failed.
499        if build_rc != 0:
500          rc |= build_rc
501          MaybeExitEarly(rc)
502          continue
503
504      # Use the realpath of the test executable so that the commands printed
505      # can be copy-pasted and run.
506      test_executable = util.relrealpath(
507        join(config.dir_build_latest, 'test', 'test-runner'))
508
509      if not args.notest:
510        printer.Print(test_executable)
511
512      for runtime_options in test_runtime_combinations:
513        if not args.notest:
514          runtime_options = [x for x in runtime_options if x is not None]
515          prefix = '  ' + ' '.join(runtime_options) + '  '
516          rc |= threaded_tests.RunTests(test_executable,
517                                        args.filters,
518                                        list(runtime_options),
519                                        args.under_valgrind,
520                                        jobs = args.jobs, prefix = prefix)
521          MaybeExitEarly(rc)
522
523      if not args.nobench:
524        rc |= RunBenchmarks(build_options, args)
525        MaybeExitEarly(rc)
526
527  if not args.dry_run:
528    PrintStatus(rc == 0)
529
530  sys.exit(rc)
531