1# Copyright 2008 Google Inc. All Rights Reserved.
2#
3# Redistribution and use in source and binary forms, with or without
4# modification, are permitted provided that the following conditions are
5# met:
6#
7#     * Redistributions of source code must retain the above copyright
8# notice, this list of conditions and the following disclaimer.
9#     * Redistributions in binary form must reproduce the above
10# copyright notice, this list of conditions and the following disclaimer
11# in the documentation and/or other materials provided with the
12# distribution.
13#     * Neither the name of Google Inc. nor the names of its
14# contributors may be used to endorse or promote products derived from
15# this software without specific prior written permission.
16#
17# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29"""Provides facilities for running SCons-built Google Test/Mock tests."""
30
31
32import optparse
33import os
34import re
35import sets
36import sys
37
38try:
39  # subrocess module is a preferable way to invoke subprocesses but it may
40  # not be available on MacOS X 10.4.
41  # Suppresses the 'Import not at the top of the file' lint complaint.
42  # pylint: disable-msg=C6204
43  import subprocess
44except ImportError:
45  subprocess = None
46
47HELP_MSG = """Runs the specified tests for %(proj)s.
48
49SYNOPSIS
50       run_tests.py [OPTION]... [BUILD_DIR]... [TEST]...
51
52DESCRIPTION
53       Runs the specified tests (either binary or Python), and prints a
54       summary of the results. BUILD_DIRS will be used to search for the
55       binaries. If no TESTs are specified, all binary tests found in
56       BUILD_DIRs and all Python tests found in the directory test/ (in the
57       %(proj)s root) are run.
58
59       TEST is a name of either a binary or a Python test. A binary test is
60       an executable file named *_test or *_unittest (with the .exe
61       extension on Windows) A Python test is a script named *_test.py or
62       *_unittest.py.
63
64OPTIONS
65       -h, --help
66              Print this help message.
67       -c CONFIGURATIONS
68              Specify build directories via build configurations.
69              CONFIGURATIONS is either a comma-separated list of build
70              configurations or 'all'. Each configuration is equivalent to
71              adding 'scons/build/<configuration>/%(proj)s/scons' to BUILD_DIRs.
72              Specifying -c=all is equivalent to providing all directories
73              listed in KNOWN BUILD DIRECTORIES section below.
74       -a
75              Equivalent to -c=all
76       -b
77              Equivalent to -c=all with the exception that the script will not
78              fail if some of the KNOWN BUILD DIRECTORIES do not exists; the
79              script will simply not run the tests there. 'b' stands for
80              'built directories'.
81
82RETURN VALUE
83       Returns 0 if all tests are successful; otherwise returns 1.
84
85EXAMPLES
86       run_tests.py
87              Runs all tests for the default build configuration.
88       run_tests.py -a
89              Runs all tests with binaries in KNOWN BUILD DIRECTORIES.
90       run_tests.py -b
91              Runs all tests in KNOWN BUILD DIRECTORIES that have been
92              built.
93       run_tests.py foo/
94              Runs all tests in the foo/ directory and all Python tests in
95              the directory test. The Python tests are instructed to look
96              for binaries in foo/.
97       run_tests.py bar_test.exe test/baz_test.exe foo/ bar/
98              Runs foo/bar_test.exe, bar/bar_test.exe, foo/baz_test.exe, and
99              bar/baz_test.exe.
100       run_tests.py foo bar test/foo_test.py
101              Runs test/foo_test.py twice instructing it to look for its
102              test binaries in the directories foo and bar,
103              correspondingly.
104
105KNOWN BUILD DIRECTORIES
106      run_tests.py knows about directories where the SCons build script
107      deposits its products. These are the directories where run_tests.py
108      will be looking for its binaries. Currently, %(proj)s's SConstruct file
109      defines them as follows (the default build directory is the first one
110      listed in each group):
111      On Windows:
112              <%(proj)s root>/scons/build/win-dbg8/%(proj)s/scons/
113              <%(proj)s root>/scons/build/win-opt8/%(proj)s/scons/
114      On Mac:
115              <%(proj)s root>/scons/build/mac-dbg/%(proj)s/scons/
116              <%(proj)s root>/scons/build/mac-opt/%(proj)s/scons/
117      On other platforms:
118              <%(proj)s root>/scons/build/dbg/%(proj)s/scons/
119              <%(proj)s root>/scons/build/opt/%(proj)s/scons/"""
120
121IS_WINDOWS = os.name == 'nt'
122IS_MAC = os.name == 'posix' and os.uname()[0] == 'Darwin'
123IS_CYGWIN = os.name == 'posix' and 'CYGWIN' in os.uname()[0]
124
125# Definition of CONFIGS must match that of the build directory names in the
126# SConstruct script. The first list item is the default build configuration.
127if IS_WINDOWS:
128  CONFIGS = ('win-dbg8', 'win-opt8')
129elif IS_MAC:
130  CONFIGS = ('mac-dbg', 'mac-opt')
131else:
132  CONFIGS = ('dbg', 'opt')
133
134if IS_WINDOWS or IS_CYGWIN:
135  PYTHON_TEST_REGEX = re.compile(r'_(unit)?test\.py$', re.IGNORECASE)
136  BINARY_TEST_REGEX = re.compile(r'_(unit)?test(\.exe)?$', re.IGNORECASE)
137  BINARY_TEST_SEARCH_REGEX = re.compile(r'_(unit)?test\.exe$', re.IGNORECASE)
138else:
139  PYTHON_TEST_REGEX = re.compile(r'_(unit)?test\.py$')
140  BINARY_TEST_REGEX = re.compile(r'_(unit)?test$')
141  BINARY_TEST_SEARCH_REGEX = BINARY_TEST_REGEX
142
143
144def _GetGtestBuildDir(injected_os, script_dir, config):
145  """Calculates path to the Google Test SCons build directory."""
146
147  return injected_os.path.normpath(injected_os.path.join(script_dir,
148                                                         'scons/build',
149                                                         config,
150                                                         'gtest/scons'))
151
152
153def _GetConfigFromBuildDir(build_dir):
154  """Extracts the configuration name from the build directory."""
155
156  # We don't want to depend on build_dir containing the correct path
157  # separators.
158  m = re.match(r'.*[\\/]([^\\/]+)[\\/][^\\/]+[\\/]scons[\\/]?$', build_dir)
159  if m:
160    return m.group(1)
161  else:
162    print >>sys.stderr, ('%s is an invalid build directory that does not '
163                         'correspond to any configuration.' % (build_dir,))
164    return ''
165
166
167# All paths in this script are either absolute or relative to the current
168# working directory, unless otherwise specified.
169class TestRunner(object):
170  """Provides facilities for running Python and binary tests for Google Test."""
171
172  def __init__(self,
173               script_dir,
174               build_dir_var_name='GTEST_BUILD_DIR',
175               injected_os=os,
176               injected_subprocess=subprocess,
177               injected_build_dir_finder=_GetGtestBuildDir):
178    """Initializes a TestRunner instance.
179
180    Args:
181      script_dir:                File path to the calling script.
182      build_dir_var_name:        Name of the env variable used to pass the
183                                 the build directory path to the invoked
184                                 tests.
185      injected_os:               standard os module or a mock/stub for
186                                 testing.
187      injected_subprocess:       standard subprocess module or a mock/stub
188                                 for testing
189      injected_build_dir_finder: function that determines the path to
190                                 the build directory.
191    """
192
193    self.os = injected_os
194    self.subprocess = injected_subprocess
195    self.build_dir_finder = injected_build_dir_finder
196    self.build_dir_var_name = build_dir_var_name
197    self.script_dir = script_dir
198
199  def _GetBuildDirForConfig(self, config):
200    """Returns the build directory for a given configuration."""
201
202    return self.build_dir_finder(self.os, self.script_dir, config)
203
204  def _Run(self, args):
205    """Runs the executable with given args (args[0] is the executable name).
206
207    Args:
208      args: Command line arguments for the process.
209
210    Returns:
211      Process's exit code if it exits normally, or -signal if the process is
212      killed by a signal.
213    """
214
215    if self.subprocess:
216      return self.subprocess.Popen(args).wait()
217    else:
218      return self.os.spawnv(self.os.P_WAIT, args[0], args)
219
220  def _RunBinaryTest(self, test):
221    """Runs the binary test given its path.
222
223    Args:
224      test: Path to the test binary.
225
226    Returns:
227      Process's exit code if it exits normally, or -signal if the process is
228      killed by a signal.
229    """
230
231    return self._Run([test])
232
233  def _RunPythonTest(self, test, build_dir):
234    """Runs the Python test script with the specified build directory.
235
236    Args:
237      test: Path to the test's Python script.
238      build_dir: Path to the directory where the test binary is to be found.
239
240    Returns:
241      Process's exit code if it exits normally, or -signal if the process is
242      killed by a signal.
243    """
244
245    old_build_dir = self.os.environ.get(self.build_dir_var_name)
246
247    try:
248      self.os.environ[self.build_dir_var_name] = build_dir
249
250      # If this script is run on a Windows machine that has no association
251      # between the .py extension and a python interpreter, simply passing
252      # the script name into subprocess.Popen/os.spawn will not work.
253      print 'Running %s . . .' % (test,)
254      return self._Run([sys.executable, test])
255
256    finally:
257      if old_build_dir is None:
258        del self.os.environ[self.build_dir_var_name]
259      else:
260        self.os.environ[self.build_dir_var_name] = old_build_dir
261
262  def _FindFilesByRegex(self, directory, regex):
263    """Returns files in a directory whose names match a regular expression.
264
265    Args:
266      directory: Path to the directory to search for files.
267      regex: Regular expression to filter file names.
268
269    Returns:
270      The list of the paths to the files in the directory.
271    """
272
273    return [self.os.path.join(directory, file_name)
274            for file_name in self.os.listdir(directory)
275            if re.search(regex, file_name)]
276
277  # TODO(vladl@google.com): Implement parsing of scons/SConscript to run all
278  # tests defined there when no tests are specified.
279  # TODO(vladl@google.com): Update the docstring after the code is changed to
280  # try to test all builds defined in scons/SConscript.
281  def GetTestsToRun(self,
282                    args,
283                    named_configurations,
284                    built_configurations,
285                    available_configurations=CONFIGS,
286                    python_tests_to_skip=None):
287    """Determines what tests should be run.
288
289    Args:
290      args: The list of non-option arguments from the command line.
291      named_configurations: The list of configurations specified via -c or -a.
292      built_configurations: True if -b has been specified.
293      available_configurations: a list of configurations available on the
294                            current platform, injectable for testing.
295      python_tests_to_skip: a collection of (configuration, python test name)s
296                            that need to be skipped.
297
298    Returns:
299      A tuple with 2 elements: the list of Python tests to run and the list of
300      binary tests to run.
301    """
302
303    if named_configurations == 'all':
304      named_configurations = ','.join(available_configurations)
305
306    normalized_args = [self.os.path.normpath(arg) for arg in args]
307
308    # A final list of build directories which will be searched for the test
309    # binaries. First, add directories specified directly on the command
310    # line.
311    build_dirs = filter(self.os.path.isdir, normalized_args)
312
313    # Adds build directories specified via their build configurations using
314    # the -c or -a options.
315    if named_configurations:
316      build_dirs += [self._GetBuildDirForConfig(config)
317                     for config in named_configurations.split(',')]
318
319    # Adds KNOWN BUILD DIRECTORIES if -b is specified.
320    if built_configurations:
321      build_dirs += [self._GetBuildDirForConfig(config)
322                     for config in available_configurations
323                     if self.os.path.isdir(self._GetBuildDirForConfig(config))]
324
325    # If no directories were specified either via -a, -b, -c, or directly, use
326    # the default configuration.
327    elif not build_dirs:
328      build_dirs = [self._GetBuildDirForConfig(available_configurations[0])]
329
330    # Makes sure there are no duplications.
331    build_dirs = sets.Set(build_dirs)
332
333    errors_found = False
334    listed_python_tests = []  # All Python tests listed on the command line.
335    listed_binary_tests = []  # All binary tests listed on the command line.
336
337    test_dir = self.os.path.normpath(self.os.path.join(self.script_dir, 'test'))
338
339    # Sifts through non-directory arguments fishing for any Python or binary
340    # tests and detecting errors.
341    for argument in sets.Set(normalized_args) - build_dirs:
342      if re.search(PYTHON_TEST_REGEX, argument):
343        python_path = self.os.path.join(test_dir,
344                                        self.os.path.basename(argument))
345        if self.os.path.isfile(python_path):
346          listed_python_tests.append(python_path)
347        else:
348          sys.stderr.write('Unable to find Python test %s' % argument)
349          errors_found = True
350      elif re.search(BINARY_TEST_REGEX, argument):
351        # This script also accepts binary test names prefixed with test/ for
352        # the convenience of typing them (can use path completions in the
353        # shell).  Strips test/ prefix from the binary test names.
354        listed_binary_tests.append(self.os.path.basename(argument))
355      else:
356        sys.stderr.write('%s is neither test nor build directory' % argument)
357        errors_found = True
358
359    if errors_found:
360      return None
361
362    user_has_listed_tests = listed_python_tests or listed_binary_tests
363
364    if user_has_listed_tests:
365      selected_python_tests = listed_python_tests
366    else:
367      selected_python_tests = self._FindFilesByRegex(test_dir,
368                                                     PYTHON_TEST_REGEX)
369
370    # TODO(vladl@google.com): skip unbuilt Python tests when -b is specified.
371    python_test_pairs = []
372    for directory in build_dirs:
373      for test in selected_python_tests:
374        config = _GetConfigFromBuildDir(directory)
375        file_name = os.path.basename(test)
376        if python_tests_to_skip and (config, file_name) in python_tests_to_skip:
377          print ('NOTE: %s is skipped for configuration %s, as it does not '
378                 'work there.' % (file_name, config))
379        else:
380          python_test_pairs.append((directory, test))
381
382    binary_test_pairs = []
383    for directory in build_dirs:
384      if user_has_listed_tests:
385        binary_test_pairs.extend(
386            [(directory, self.os.path.join(directory, test))
387             for test in listed_binary_tests])
388      else:
389        tests = self._FindFilesByRegex(directory, BINARY_TEST_SEARCH_REGEX)
390        binary_test_pairs.extend([(directory, test) for test in tests])
391
392    return (python_test_pairs, binary_test_pairs)
393
394  def RunTests(self, python_tests, binary_tests):
395    """Runs Python and binary tests and reports results to the standard output.
396
397    Args:
398      python_tests: List of Python tests to run in the form of tuples
399                    (build directory, Python test script).
400      binary_tests: List of binary tests to run in the form of tuples
401                    (build directory, binary file).
402
403    Returns:
404      The exit code the program should pass into sys.exit().
405    """
406
407    if python_tests or binary_tests:
408      results = []
409      for directory, test in python_tests:
410        results.append((directory,
411                        test,
412                        self._RunPythonTest(test, directory) == 0))
413      for directory, test in binary_tests:
414        results.append((directory,
415                        self.os.path.basename(test),
416                        self._RunBinaryTest(test) == 0))
417
418      failed = [(directory, test)
419                for (directory, test, success) in results
420                if not success]
421      print
422      print '%d tests run.' % len(results)
423      if failed:
424        print 'The following %d tests failed:' % len(failed)
425        for (directory, test) in failed:
426          print '%s in %s' % (test, directory)
427        return 1
428      else:
429        print 'All tests passed!'
430    else:  # No tests defined
431      print 'Nothing to test - no tests specified!'
432
433    return 0
434
435
436def ParseArgs(project_name, argv=None, help_callback=None):
437  """Parses the options run_tests.py uses."""
438
439  # Suppresses lint warning on unused arguments.  These arguments are
440  # required by optparse, even though they are unused.
441  # pylint: disable-msg=W0613
442  def PrintHelp(option, opt, value, parser):
443    print HELP_MSG % {'proj': project_name}
444    sys.exit(1)
445
446  parser = optparse.OptionParser()
447  parser.add_option('-c',
448                    action='store',
449                    dest='configurations',
450                    default=None)
451  parser.add_option('-a',
452                    action='store_const',
453                    dest='configurations',
454                    default=None,
455                    const='all')
456  parser.add_option('-b',
457                    action='store_const',
458                    dest='built_configurations',
459                    default=False,
460                    const=True)
461  # Replaces the built-in help with ours.
462  parser.remove_option('-h')
463  parser.add_option('-h', '--help',
464                    action='callback',
465                    callback=help_callback or PrintHelp)
466  return parser.parse_args(argv)
467