1import argparse
2import os
3import sys
4from test import support
5
6
7USAGE = """\
8python -m test [options] [test_name1 [test_name2 ...]]
9python path/to/Lib/test/regrtest.py [options] [test_name1 [test_name2 ...]]
10"""
11
12DESCRIPTION = """\
13Run Python regression tests.
14
15If no arguments or options are provided, finds all files matching
16the pattern "test_*" in the Lib/test subdirectory and runs
17them in alphabetical order (but see -M and -u, below, for exceptions).
18
19For more rigorous testing, it is useful to use the following
20command line:
21
22python -E -Wd -m test [options] [test_name1 ...]
23"""
24
25EPILOG = """\
26Additional option details:
27
28-r randomizes test execution order. You can use --randseed=int to provide an
29int seed value for the randomizer; this is useful for reproducing troublesome
30test orders.
31
32-s On the first invocation of regrtest using -s, the first test file found
33or the first test file given on the command line is run, and the name of
34the next test is recorded in a file named pynexttest.  If run from the
35Python build directory, pynexttest is located in the 'build' subdirectory,
36otherwise it is located in tempfile.gettempdir().  On subsequent runs,
37the test in pynexttest is run, and the next test is written to pynexttest.
38When the last test has been run, pynexttest is deleted.  In this way it
39is possible to single step through the test files.  This is useful when
40doing memory analysis on the Python interpreter, which process tends to
41consume too many resources to run the full regression test non-stop.
42
43-S is used to continue running tests after an aborted run.  It will
44maintain the order a standard run (ie, this assumes -r is not used).
45This is useful after the tests have prematurely stopped for some external
46reason and you want to start running from where you left off rather
47than starting from the beginning.
48
49-f reads the names of tests from the file given as f's argument, one
50or more test names per line.  Whitespace is ignored.  Blank lines and
51lines beginning with '#' are ignored.  This is especially useful for
52whittling down failures involving interactions among tests.
53
54-L causes the leaks(1) command to be run just before exit if it exists.
55leaks(1) is available on Mac OS X and presumably on some other
56FreeBSD-derived systems.
57
58-R runs each test several times and examines sys.gettotalrefcount() to
59see if the test appears to be leaking references.  The argument should
60be of the form stab:run:fname where 'stab' is the number of times the
61test is run to let gettotalrefcount settle down, 'run' is the number
62of times further it is run and 'fname' is the name of the file the
63reports are written to.  These parameters all have defaults (5, 4 and
64"reflog.txt" respectively), and the minimal invocation is '-R :'.
65
66-M runs tests that require an exorbitant amount of memory. These tests
67typically try to ascertain containers keep working when containing more than
682 billion objects, which only works on 64-bit systems. There are also some
69tests that try to exhaust the address space of the process, which only makes
70sense on 32-bit systems with at least 2Gb of memory. The passed-in memlimit,
71which is a string in the form of '2.5Gb', determines how much memory the
72tests will limit themselves to (but they may go slightly over.) The number
73shouldn't be more memory than the machine has (including swap memory). You
74should also keep in mind that swap memory is generally much, much slower
75than RAM, and setting memlimit to all available RAM or higher will heavily
76tax the machine. On the other hand, it is no use running these tests with a
77limit of less than 2.5Gb, and many require more than 20Gb. Tests that expect
78to use more than memlimit memory will be skipped. The big-memory tests
79generally run very, very long.
80
81-u is used to specify which special resource intensive tests to run,
82such as those requiring large file support or network connectivity.
83The argument is a comma-separated list of words indicating the
84resources to test.  Currently only the following are defined:
85
86    all -       Enable all special resources.
87
88    none -      Disable all special resources (this is the default).
89
90    audio -     Tests that use the audio device.  (There are known
91                cases of broken audio drivers that can crash Python or
92                even the Linux kernel.)
93
94    curses -    Tests that use curses and will modify the terminal's
95                state and output modes.
96
97    largefile - It is okay to run some test that may create huge
98                files.  These tests can take a long time and may
99                consume >2 GiB of disk space temporarily.
100
101    network -   It is okay to run tests that use external network
102                resource, e.g. testing SSL support for sockets.
103
104    decimal -   Test the decimal module against a large suite that
105                verifies compliance with standards.
106
107    cpu -       Used for certain CPU-heavy tests.
108
109    subprocess  Run all tests for the subprocess module.
110
111    urlfetch -  It is okay to download files required on testing.
112
113    gui -       Run tests that require a running GUI.
114
115    tzdata -    Run tests that require timezone data.
116
117To enable all resources except one, use '-uall,-<resource>'.  For
118example, to run all the tests except for the gui tests, give the
119option '-uall,-gui'.
120
121--matchfile filters tests using a text file, one pattern per line.
122Pattern examples:
123
124- test method: test_stat_attributes
125- test class: FileTests
126- test identifier: test_os.FileTests.test_stat_attributes
127"""
128
129
130ALL_RESOURCES = ('audio', 'curses', 'largefile', 'network',
131                 'decimal', 'cpu', 'subprocess', 'urlfetch', 'gui')
132
133# Other resources excluded from --use=all:
134#
135# - extralagefile (ex: test_zipfile64): really too slow to be enabled
136#   "by default"
137# - tzdata: while needed to validate fully test_datetime, it makes
138#   test_datetime too slow (15-20 min on some buildbots) and so is disabled by
139#   default (see bpo-30822).
140RESOURCE_NAMES = ALL_RESOURCES + ('extralargefile', 'tzdata')
141
142class _ArgParser(argparse.ArgumentParser):
143
144    def error(self, message):
145        super().error(message + "\nPass -h or --help for complete help.")
146
147
148def _create_parser():
149    # Set prog to prevent the uninformative "__main__.py" from displaying in
150    # error messages when using "python -m test ...".
151    parser = _ArgParser(prog='regrtest.py',
152                        usage=USAGE,
153                        description=DESCRIPTION,
154                        epilog=EPILOG,
155                        add_help=False,
156                        formatter_class=argparse.RawDescriptionHelpFormatter)
157
158    # Arguments with this clause added to its help are described further in
159    # the epilog's "Additional option details" section.
160    more_details = '  See the section at bottom for more details.'
161
162    group = parser.add_argument_group('General options')
163    # We add help explicitly to control what argument group it renders under.
164    group.add_argument('-h', '--help', action='help',
165                       help='show this help message and exit')
166    group.add_argument('--timeout', metavar='TIMEOUT', type=float,
167                        help='dump the traceback and exit if a test takes '
168                             'more than TIMEOUT seconds; disabled if TIMEOUT '
169                             'is negative or equals to zero')
170    group.add_argument('--wait', action='store_true',
171                       help='wait for user input, e.g., allow a debugger '
172                            'to be attached')
173    group.add_argument('--worker-args', metavar='ARGS')
174    group.add_argument('-S', '--start', metavar='START',
175                       help='the name of the test at which to start.' +
176                            more_details)
177
178    group = parser.add_argument_group('Verbosity')
179    group.add_argument('-v', '--verbose', action='count',
180                       help='run tests in verbose mode with output to stdout')
181    group.add_argument('-w', '--verbose2', action='store_true',
182                       help='re-run failed tests in verbose mode')
183    group.add_argument('-W', '--verbose3', action='store_true',
184                       help='display test output on failure')
185    group.add_argument('-q', '--quiet', action='store_true',
186                       help='no output unless one or more tests fail')
187    group.add_argument('-o', '--slowest', action='store_true', dest='print_slow',
188                       help='print the slowest 10 tests')
189    group.add_argument('--header', action='store_true',
190                       help='print header with interpreter info')
191
192    group = parser.add_argument_group('Selecting tests')
193    group.add_argument('-r', '--randomize', action='store_true',
194                       help='randomize test execution order.' + more_details)
195    group.add_argument('--randseed', metavar='SEED',
196                       dest='random_seed', type=int,
197                       help='pass a random seed to reproduce a previous '
198                            'random run')
199    group.add_argument('-f', '--fromfile', metavar='FILE',
200                       help='read names of tests to run from a file.' +
201                            more_details)
202    group.add_argument('-x', '--exclude', action='store_true',
203                       help='arguments are tests to *exclude*')
204    group.add_argument('-s', '--single', action='store_true',
205                       help='single step through a set of tests.' +
206                            more_details)
207    group.add_argument('-m', '--match', metavar='PAT',
208                       dest='match_tests', action='append',
209                       help='match test cases and methods with glob pattern PAT')
210    group.add_argument('--matchfile', metavar='FILENAME',
211                       dest='match_filename',
212                       help='similar to --match but get patterns from a '
213                            'text file, one pattern per line')
214    group.add_argument('-G', '--failfast', action='store_true',
215                       help='fail as soon as a test fails (only with -v or -W)')
216    group.add_argument('-u', '--use', metavar='RES1,RES2,...',
217                       action='append', type=resources_list,
218                       help='specify which special resource intensive tests '
219                            'to run.' + more_details)
220    group.add_argument('-M', '--memlimit', metavar='LIMIT',
221                       help='run very large memory-consuming tests.' +
222                            more_details)
223    group.add_argument('--testdir', metavar='DIR',
224                       type=relative_filename,
225                       help='execute test files in the specified directory '
226                            '(instead of the Python stdlib test suite)')
227
228    group = parser.add_argument_group('Special runs')
229    group.add_argument('-l', '--findleaks', action='store_true',
230                       help='if GC is available detect tests that leak memory')
231    group.add_argument('-L', '--runleaks', action='store_true',
232                       help='run the leaks(1) command just before exit.' +
233                            more_details)
234    group.add_argument('-R', '--huntrleaks', metavar='RUNCOUNTS',
235                       type=huntrleaks,
236                       help='search for reference leaks (needs debug build, '
237                            'very slow).' + more_details)
238    group.add_argument('-j', '--multiprocess', metavar='PROCESSES',
239                       dest='use_mp', type=int,
240                       help='run PROCESSES processes at once')
241    group.add_argument('-T', '--coverage', action='store_true',
242                       dest='trace',
243                       help='turn on code coverage tracing using the trace '
244                            'module')
245    group.add_argument('-D', '--coverdir', metavar='DIR',
246                       type=relative_filename,
247                       help='directory where coverage files are put')
248    group.add_argument('-N', '--nocoverdir',
249                       action='store_const', const=None, dest='coverdir',
250                       help='put coverage files alongside modules')
251    group.add_argument('-t', '--threshold', metavar='THRESHOLD',
252                       type=int,
253                       help='call gc.set_threshold(THRESHOLD)')
254    group.add_argument('-n', '--nowindows', action='store_true',
255                       help='suppress error message boxes on Windows')
256    group.add_argument('-F', '--forever', action='store_true',
257                       help='run the specified tests in a loop, until an '
258                            'error happens')
259    group.add_argument('--list-tests', action='store_true',
260                       help="only write the name of tests that will be run, "
261                            "don't execute them")
262    group.add_argument('--list-cases', action='store_true',
263                       help='only write the name of test cases that will be run'
264                            ' , don\'t execute them')
265    group.add_argument('-P', '--pgo', dest='pgo', action='store_true',
266                       help='enable Profile Guided Optimization training')
267    group.add_argument('--fail-env-changed', action='store_true',
268                       help='if a test file alters the environment, mark '
269                            'the test as failed')
270
271    group.add_argument('--junit-xml', dest='xmlpath', metavar='FILENAME',
272                       help='writes JUnit-style XML results to the specified '
273                            'file')
274    group.add_argument('--tempdir', dest='tempdir', metavar='PATH',
275                       help='override the working directory for the test run')
276    return parser
277
278
279def relative_filename(string):
280    # CWD is replaced with a temporary dir before calling main(), so we
281    # join it with the saved CWD so it ends up where the user expects.
282    return os.path.join(support.SAVEDCWD, string)
283
284
285def huntrleaks(string):
286    args = string.split(':')
287    if len(args) not in (2, 3):
288        raise argparse.ArgumentTypeError(
289            'needs 2 or 3 colon-separated arguments')
290    nwarmup = int(args[0]) if args[0] else 5
291    ntracked = int(args[1]) if args[1] else 4
292    fname = args[2] if len(args) > 2 and args[2] else 'reflog.txt'
293    return nwarmup, ntracked, fname
294
295
296def resources_list(string):
297    u = [x.lower() for x in string.split(',')]
298    for r in u:
299        if r == 'all' or r == 'none':
300            continue
301        if r[0] == '-':
302            r = r[1:]
303        if r not in RESOURCE_NAMES:
304            raise argparse.ArgumentTypeError('invalid resource: ' + r)
305    return u
306
307
308def _parse_args(args, **kwargs):
309    # Defaults
310    ns = argparse.Namespace(testdir=None, verbose=0, quiet=False,
311         exclude=False, single=False, randomize=False, fromfile=None,
312         findleaks=False, use_resources=None, trace=False, coverdir='coverage',
313         runleaks=False, huntrleaks=False, verbose2=False, print_slow=False,
314         random_seed=None, use_mp=None, verbose3=False, forever=False,
315         header=False, failfast=False, match_tests=None, pgo=False)
316    for k, v in kwargs.items():
317        if not hasattr(ns, k):
318            raise TypeError('%r is an invalid keyword argument '
319                            'for this function' % k)
320        setattr(ns, k, v)
321    if ns.use_resources is None:
322        ns.use_resources = []
323
324    parser = _create_parser()
325    # Issue #14191: argparse doesn't support "intermixed" positional and
326    # optional arguments. Use parse_known_args() as workaround.
327    ns.args = parser.parse_known_args(args=args, namespace=ns)[1]
328    for arg in ns.args:
329        if arg.startswith('-'):
330            parser.error("unrecognized arguments: %s" % arg)
331            sys.exit(1)
332
333    if ns.single and ns.fromfile:
334        parser.error("-s and -f don't go together!")
335    if ns.use_mp is not None and ns.trace:
336        parser.error("-T and -j don't go together!")
337    if ns.use_mp is not None and ns.findleaks:
338        parser.error("-l and -j don't go together!")
339    if ns.failfast and not (ns.verbose or ns.verbose3):
340        parser.error("-G/--failfast needs either -v or -W")
341    if ns.pgo and (ns.verbose or ns.verbose2 or ns.verbose3):
342        parser.error("--pgo/-v don't go together!")
343
344    if ns.nowindows:
345        print("Warning: the --nowindows (-n) option is deprecated. "
346              "Use -vv to display assertions in stderr.", file=sys.stderr)
347
348    if ns.quiet:
349        ns.verbose = 0
350    if ns.timeout is not None:
351        if ns.timeout <= 0:
352            ns.timeout = None
353    if ns.use_mp is not None:
354        if ns.use_mp <= 0:
355            # Use all cores + extras for tests that like to sleep
356            ns.use_mp = 2 + (os.cpu_count() or 1)
357    if ns.use:
358        for a in ns.use:
359            for r in a:
360                if r == 'all':
361                    ns.use_resources[:] = ALL_RESOURCES
362                    continue
363                if r == 'none':
364                    del ns.use_resources[:]
365                    continue
366                remove = False
367                if r[0] == '-':
368                    remove = True
369                    r = r[1:]
370                if remove:
371                    if r in ns.use_resources:
372                        ns.use_resources.remove(r)
373                elif r not in ns.use_resources:
374                    ns.use_resources.append(r)
375    if ns.random_seed is not None:
376        ns.randomize = True
377    if ns.verbose:
378        ns.header = True
379    if ns.huntrleaks and ns.verbose3:
380        ns.verbose3 = False
381        print("WARNING: Disable --verbose3 because it's incompatible with "
382              "--huntrleaks: see http://bugs.python.org/issue27103",
383              file=sys.stderr)
384    if ns.match_filename:
385        if ns.match_tests is None:
386            ns.match_tests = []
387        with open(ns.match_filename) as fp:
388            for line in fp:
389                ns.match_tests.append(line.strip())
390
391    return ns
392