1#!/usr/bin/env python3
2"""
3Command line tool to bisect failing CPython tests.
4
5Find the test_os test method which alters the environment:
6
7    ./python -m test.bisect_cmd --fail-env-changed test_os
8
9Find a reference leak in "test_os", write the list of failing tests into the
10"bisect" file:
11
12    ./python -m test.bisect_cmd -o bisect -R 3:3 test_os
13
14Load an existing list of tests from a file using -i option:
15
16    ./python -m test --list-cases -m FileTests test_os > tests
17    ./python -m test.bisect_cmd -i tests test_os
18"""
19
20import argparse
21import datetime
22import os.path
23import math
24import random
25import subprocess
26import sys
27import tempfile
28import time
29
30
31def write_tests(filename, tests):
32    with open(filename, "w") as fp:
33        for name in tests:
34            print(name, file=fp)
35        fp.flush()
36
37
38def write_output(filename, tests):
39    if not filename:
40        return
41    print("Writing %s tests into %s" % (len(tests), filename))
42    write_tests(filename, tests)
43    return filename
44
45
46def format_shell_args(args):
47    return ' '.join(args)
48
49
50def python_cmd():
51    cmd = [sys.executable]
52    cmd.extend(subprocess._args_from_interpreter_flags())
53    cmd.extend(subprocess._optim_args_from_interpreter_flags())
54    return cmd
55
56
57def list_cases(args):
58    cmd = python_cmd()
59    cmd.extend(['-m', 'test', '--list-cases'])
60    cmd.extend(args.test_args)
61    proc = subprocess.run(cmd,
62                          stdout=subprocess.PIPE,
63                          universal_newlines=True)
64    exitcode = proc.returncode
65    if exitcode:
66        cmd = format_shell_args(cmd)
67        print("Failed to list tests: %s failed with exit code %s"
68              % (cmd, exitcode))
69        sys.exit(exitcode)
70    tests = proc.stdout.splitlines()
71    return tests
72
73
74def run_tests(args, tests, huntrleaks=None):
75    tmp = tempfile.mktemp()
76    try:
77        write_tests(tmp, tests)
78
79        cmd = python_cmd()
80        cmd.extend(['-m', 'test', '--matchfile', tmp])
81        cmd.extend(args.test_args)
82        print("+ %s" % format_shell_args(cmd))
83        proc = subprocess.run(cmd)
84        return proc.returncode
85    finally:
86        if os.path.exists(tmp):
87            os.unlink(tmp)
88
89
90def parse_args():
91    parser = argparse.ArgumentParser()
92    parser.add_argument('-i', '--input',
93                        help='Test names produced by --list-tests written '
94                             'into a file. If not set, run --list-tests')
95    parser.add_argument('-o', '--output',
96                        help='Result of the bisection')
97    parser.add_argument('-n', '--max-tests', type=int, default=1,
98                        help='Maximum number of tests to stop the bisection '
99                             '(default: 1)')
100    parser.add_argument('-N', '--max-iter', type=int, default=100,
101                        help='Maximum number of bisection iterations '
102                             '(default: 100)')
103    # FIXME: document that following arguments are test arguments
104
105    args, test_args = parser.parse_known_args()
106    args.test_args = test_args
107    return args
108
109
110def main():
111    args = parse_args()
112    if '-w' in args.test_args or '--verbose2' in args.test_args:
113        print("WARNING: -w/--verbose2 option should not be used to bisect!")
114        print()
115
116    if args.input:
117        with open(args.input) as fp:
118            tests = [line.strip() for line in fp]
119    else:
120        tests = list_cases(args)
121
122    print("Start bisection with %s tests" % len(tests))
123    print("Test arguments: %s" % format_shell_args(args.test_args))
124    print("Bisection will stop when getting %s or less tests "
125          "(-n/--max-tests option), or after %s iterations "
126          "(-N/--max-iter option)"
127          % (args.max_tests, args.max_iter))
128    output = write_output(args.output, tests)
129    print()
130
131    start_time = time.monotonic()
132    iteration = 1
133    try:
134        while len(tests) > args.max_tests and iteration <= args.max_iter:
135            ntest = len(tests)
136            ntest = max(ntest // 2, 1)
137            subtests = random.sample(tests, ntest)
138
139            print("[+] Iteration %s: run %s tests/%s"
140                  % (iteration, len(subtests), len(tests)))
141            print()
142
143            exitcode = run_tests(args, subtests)
144
145            print("ran %s tests/%s" % (ntest, len(tests)))
146            print("exit", exitcode)
147            if exitcode:
148                print("Tests failed: continuing with this subtest")
149                tests = subtests
150                output = write_output(args.output, tests)
151            else:
152                print("Tests succeeded: skipping this subtest, trying a new subset")
153            print()
154            iteration += 1
155    except KeyboardInterrupt:
156        print()
157        print("Bisection interrupted!")
158        print()
159
160    print("Tests (%s):" % len(tests))
161    for test in tests:
162        print("* %s" % test)
163    print()
164
165    if output:
166        print("Output written into %s" % output)
167
168    dt = math.ceil(time.monotonic() - start_time)
169    if len(tests) <= args.max_tests:
170        print("Bisection completed in %s iterations and %s"
171              % (iteration, datetime.timedelta(seconds=dt)))
172        sys.exit(1)
173    else:
174        print("Bisection failed after %s iterations and %s"
175              % (iteration, datetime.timedelta(seconds=dt)))
176
177
178if __name__ == "__main__":
179    main()
180