1#!/usr/bin/env python2.7
2
3# Copyright 2013, ARM Limited
4# All rights reserved.
5#
6# Redistribution and use in source and binary forms, with or without
7# modification, are permitted provided that the following conditions are met:
8#
9#   * Redistributions of source code must retain the above copyright notice,
10#     this list of conditions and the following disclaimer.
11#   * Redistributions in binary form must reproduce the above copyright notice,
12#     this list of conditions and the following disclaimer in the documentation
13#     and/or other materials provided with the distribution.
14#   * Neither the name of ARM Limited nor the names of its contributors may be
15#     used to endorse or promote products derived from this software without
16#     specific prior written permission.
17#
18# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
19# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
20# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
21# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
22# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
24# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
25# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
26# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29import os
30import sys
31import argparse
32import re
33import subprocess
34import time
35import util
36
37
38from printer import EnsureNewLine, Print, UpdateProgress
39
40
41def BuildOptions():
42  result = argparse.ArgumentParser(description =
43      '''This tool runs each test reported by $CCTEST --list (and filtered as
44         specified). A summary will be printed, and detailed test output will be
45         stored in log/$CCTEST.''')
46  result.add_argument('filters', metavar='filter', nargs='*',
47                      help='Run tests matching all of the (regexp) filters.')
48  result.add_argument('--cctest', action='store', required=True,
49                      help='The cctest executable to run.')
50  result.add_argument('--coloured_trace', action='store_true',
51                      help='''Pass --coloured_trace to cctest. This will put
52                              colour codes in the log files. The coloured output
53                              can be viewed by "less -R", for example.''')
54  result.add_argument('--coverage', action='store_true',
55                      help='Run coverage tests.')
56  result.add_argument('--debugger', action='store_true',
57                      help='''Pass --debugger to cctest, so that the debugger is
58                              used instead of the simulator. This has no effect
59                              when running natively.''')
60  result.add_argument('--verbose', action='store_true',
61                      help='Print verbose output.')
62  return result.parse_args()
63
64
65def VerbosePrint(string):
66  if args.verbose:
67    Print(string)
68
69
70# A class representing an individual test.
71class Test:
72  def __init__(self, name):
73    self.name = name
74    self.logpath = os.path.join('log', os.path.basename(args.cctest))
75    if args.debugger:
76      basename = name + '_debugger'
77    else:
78      basename = name
79    self.logout = os.path.join(self.logpath, basename + '.stdout')
80    self.logerr = os.path.join(self.logpath, basename + '.stderr')
81
82  # Run the test.
83  # Use a thread to be able to control the test.
84  def Run(self):
85    command = [args.cctest, '--trace_sim', '--trace_reg', self.name]
86    if args.coloured_trace:
87      command.append('--coloured_trace')
88
89    VerbosePrint('==== Running ' + self.name + '... ====')
90    sys.stdout.flush()
91
92    process = subprocess.Popen(command,
93                               stdout=subprocess.PIPE,
94                               stderr=subprocess.PIPE)
95    # Get the output and return status of the test.
96    stdout, stderr = process.communicate()
97    retcode = process.poll()
98
99    # Write stdout and stderr to the log.
100    if not os.path.exists(self.logpath): os.makedirs(self.logpath)
101    with open(self.logout, 'w') as f: f.write(stdout)
102    with open(self.logerr, 'w') as f: f.write(stderr)
103
104    if retcode == 0:
105      # Success.
106      # We normally only print the command on failure, but with --verbose we
107      # should also print it on success.
108      VerbosePrint('COMMAND: ' + ' '.join(command))
109      VerbosePrint('LOG (stdout): ' + self.logout)
110      VerbosePrint('LOG (stderr): ' + self.logerr + '\n')
111    else:
112      # Failure.
113      Print('--- FAILED ' + self.name + ' ---')
114      Print('COMMAND: ' + ' '.join(command))
115      Print('LOG (stdout): ' + self.logout)
116      Print('LOG (stderr): ' + self.logerr + '\n')
117
118    return retcode
119
120
121# Scan matching tests and return a test manifest.
122def ReadManifest(filters):
123  status, output = util.getstatusoutput(args.cctest +  ' --list')
124  if status != 0: util.abort('Failed to list all tests')
125
126  names = output.split()
127  for f in filters:
128    names = filter(re.compile(f).search, names)
129
130  return map(Test, names)
131
132
133# Run all tests in the manifest.
134def RunTests(manifest):
135  count = len(manifest)
136  passed = 0
137  failed = 0
138
139  if count == 0:
140    Print('No tests to run.')
141    return 0
142
143  Print('Running %d tests...' % (count))
144  start_time = time.time()
145
146  for test in manifest:
147    # Update the progress counter with the name of the test we're about to run.
148    UpdateProgress(start_time, passed, failed, count, args.verbose, test.name)
149    retcode = test.Run()
150    # Update the counters and progress indicator.
151    if retcode == 0:
152      passed += 1
153    else:
154      failed += 1
155  UpdateProgress(start_time, passed, failed, count, args.verbose, '== Done ==')
156
157  return failed     # 0 indicates success.
158
159
160if __name__ == '__main__':
161  # $ROOT/tools/test.py
162  root_dir = os.path.dirname(os.path.dirname(os.path.abspath(sys.argv[0])))
163
164  # Parse the arguments.
165  args = BuildOptions()
166
167  # Find a valid path to args.cctest (in case it doesn't begin with './').
168  args.cctest = os.path.join('.', args.cctest)
169
170  if not os.access(args.cctest, os.X_OK):
171    print "'" + args.cctest + "' is not executable or does not exist."
172    sys.exit(1)
173
174  # List all matching tests.
175  manifest = ReadManifest(args.filters)
176
177  # Delete coverage data files.
178  if args.coverage:
179    status, output = util.getstatusoutput('find obj/coverage -name "*.gcda" -exec rm {} \;')
180
181  # Run the tests.
182  status = RunTests(manifest)
183  EnsureNewLine()
184
185  # Print coverage information.
186  if args.coverage:
187    cmd = 'tggcov -R summary_all,untested_functions_per_file obj/coverage/src/a64'
188    p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE,
189                         stderr=subprocess.PIPE)
190    stdout, stderr = p.communicate()
191    print(stdout)
192
193  sys.exit(status)
194
195