1import errno
2import os
3import re
4import sys
5import warnings
6from inspect import isabstract
7from test import support
8
9
10try:
11    MAXFD = os.sysconf("SC_OPEN_MAX")
12except Exception:
13    MAXFD = 256
14
15
16def fd_count():
17    """Count the number of open file descriptors"""
18    if sys.platform.startswith(('linux', 'freebsd')):
19        try:
20            names = os.listdir("/proc/self/fd")
21            return len(names)
22        except FileNotFoundError:
23            pass
24
25    count = 0
26    for fd in range(MAXFD):
27        try:
28            # Prefer dup() over fstat(). fstat() can require input/output
29            # whereas dup() doesn't.
30            fd2 = os.dup(fd)
31        except OSError as e:
32            if e.errno != errno.EBADF:
33                raise
34        else:
35            os.close(fd2)
36            count += 1
37    return count
38
39
40def dash_R(the_module, test, indirect_test, huntrleaks):
41    """Run a test multiple times, looking for reference leaks.
42
43    Returns:
44        False if the test didn't leak references; True if we detected refleaks.
45    """
46    # This code is hackish and inelegant, but it seems to do the job.
47    import copyreg
48    import collections.abc
49
50    if not hasattr(sys, 'gettotalrefcount'):
51        raise Exception("Tracking reference leaks requires a debug build "
52                        "of Python")
53
54    # Save current values for dash_R_cleanup() to restore.
55    fs = warnings.filters[:]
56    ps = copyreg.dispatch_table.copy()
57    pic = sys.path_importer_cache.copy()
58    try:
59        import zipimport
60    except ImportError:
61        zdc = None # Run unmodified on platforms without zipimport support
62    else:
63        zdc = zipimport._zip_directory_cache.copy()
64    abcs = {}
65    for abc in [getattr(collections.abc, a) for a in collections.abc.__all__]:
66        if not isabstract(abc):
67            continue
68        for obj in abc.__subclasses__() + [abc]:
69            abcs[obj] = obj._abc_registry.copy()
70
71    nwarmup, ntracked, fname = huntrleaks
72    fname = os.path.join(support.SAVEDCWD, fname)
73    repcount = nwarmup + ntracked
74    rc_deltas = [0] * repcount
75    alloc_deltas = [0] * repcount
76    fd_deltas = [0] * repcount
77
78    print("beginning", repcount, "repetitions", file=sys.stderr)
79    print(("1234567890"*(repcount//10 + 1))[:repcount], file=sys.stderr,
80          flush=True)
81    # initialize variables to make pyflakes quiet
82    rc_before = alloc_before = fd_before = 0
83    for i in range(repcount):
84        indirect_test()
85        alloc_after, rc_after, fd_after = dash_R_cleanup(fs, ps, pic, zdc,
86                                                         abcs)
87        print('.', end='', flush=True)
88        if i >= nwarmup:
89            rc_deltas[i] = rc_after - rc_before
90            alloc_deltas[i] = alloc_after - alloc_before
91            fd_deltas[i] = fd_after - fd_before
92        alloc_before = alloc_after
93        rc_before = rc_after
94        fd_before = fd_after
95    print(file=sys.stderr)
96    # These checkers return False on success, True on failure
97    def check_rc_deltas(deltas):
98        return any(deltas)
99    def check_alloc_deltas(deltas):
100        # At least 1/3rd of 0s
101        if 3 * deltas.count(0) < len(deltas):
102            return True
103        # Nothing else than 1s, 0s and -1s
104        if not set(deltas) <= {1,0,-1}:
105            return True
106        return False
107    failed = False
108    for deltas, item_name, checker in [
109        (rc_deltas, 'references', check_rc_deltas),
110        (alloc_deltas, 'memory blocks', check_alloc_deltas),
111        (fd_deltas, 'file descriptors', check_rc_deltas)]:
112        if checker(deltas):
113            msg = '%s leaked %s %s, sum=%s' % (
114                test, deltas[nwarmup:], item_name, sum(deltas))
115            print(msg, file=sys.stderr, flush=True)
116            with open(fname, "a") as refrep:
117                print(msg, file=refrep)
118                refrep.flush()
119            failed = True
120    return failed
121
122
123def dash_R_cleanup(fs, ps, pic, zdc, abcs):
124    import gc, copyreg
125    import collections.abc
126    from weakref import WeakSet
127
128    # Restore some original values.
129    warnings.filters[:] = fs
130    copyreg.dispatch_table.clear()
131    copyreg.dispatch_table.update(ps)
132    sys.path_importer_cache.clear()
133    sys.path_importer_cache.update(pic)
134    try:
135        import zipimport
136    except ImportError:
137        pass # Run unmodified on platforms without zipimport support
138    else:
139        zipimport._zip_directory_cache.clear()
140        zipimport._zip_directory_cache.update(zdc)
141
142    # clear type cache
143    sys._clear_type_cache()
144
145    # Clear ABC registries, restoring previously saved ABC registries.
146    for abc in [getattr(collections.abc, a) for a in collections.abc.__all__]:
147        if not isabstract(abc):
148            continue
149        for obj in abc.__subclasses__() + [abc]:
150            obj._abc_registry = abcs.get(obj, WeakSet()).copy()
151            obj._abc_cache.clear()
152            obj._abc_negative_cache.clear()
153
154    clear_caches()
155
156    # Collect cyclic trash and read memory statistics immediately after.
157    func1 = sys.getallocatedblocks
158    func2 = sys.gettotalrefcount
159    gc.collect()
160    return func1(), func2(), fd_count()
161
162
163def clear_caches():
164    import gc
165
166    # Clear the warnings registry, so they can be displayed again
167    for mod in sys.modules.values():
168        if hasattr(mod, '__warningregistry__'):
169            del mod.__warningregistry__
170
171    # Flush standard output, so that buffered data is sent to the OS and
172    # associated Python objects are reclaimed.
173    for stream in (sys.stdout, sys.stderr, sys.__stdout__, sys.__stderr__):
174        if stream is not None:
175            stream.flush()
176
177    # Clear assorted module caches.
178    # Don't worry about resetting the cache if the module is not loaded
179    try:
180        distutils_dir_util = sys.modules['distutils.dir_util']
181    except KeyError:
182        pass
183    else:
184        distutils_dir_util._path_created.clear()
185    re.purge()
186
187    try:
188        _strptime = sys.modules['_strptime']
189    except KeyError:
190        pass
191    else:
192        _strptime._regex_cache.clear()
193
194    try:
195        urllib_parse = sys.modules['urllib.parse']
196    except KeyError:
197        pass
198    else:
199        urllib_parse.clear_cache()
200
201    try:
202        urllib_request = sys.modules['urllib.request']
203    except KeyError:
204        pass
205    else:
206        urllib_request.urlcleanup()
207
208    try:
209        linecache = sys.modules['linecache']
210    except KeyError:
211        pass
212    else:
213        linecache.clearcache()
214
215    try:
216        mimetypes = sys.modules['mimetypes']
217    except KeyError:
218        pass
219    else:
220        mimetypes._default_mime_types()
221
222    try:
223        filecmp = sys.modules['filecmp']
224    except KeyError:
225        pass
226    else:
227        filecmp._cache.clear()
228
229    try:
230        struct = sys.modules['struct']
231    except KeyError:
232        pass
233    else:
234        struct._clearcache()
235
236    try:
237        doctest = sys.modules['doctest']
238    except KeyError:
239        pass
240    else:
241        doctest.master = None
242
243    try:
244        ctypes = sys.modules['ctypes']
245    except KeyError:
246        pass
247    else:
248        ctypes._reset_cache()
249
250    try:
251        typing = sys.modules['typing']
252    except KeyError:
253        pass
254    else:
255        for f in typing._cleanups:
256            f()
257
258    gc.collect()
259
260
261def warm_caches():
262    # char cache
263    s = bytes(range(256))
264    for i in range(256):
265        s[i:i+1]
266    # unicode cache
267    [chr(i) for i in range(256)]
268    # int cache
269    list(range(-5, 257))
270