1"""
2Test harness for the venv module.
3
4Copyright (C) 2011-2012 Vinay Sajip.
5Licensed to the PSF under a contributor agreement.
6"""
7
8import ensurepip
9import os
10import os.path
11import re
12import shutil
13import struct
14import subprocess
15import sys
16import tempfile
17from test.support import (captured_stdout, captured_stderr, requires_zlib,
18                          can_symlink, EnvironmentVarGuard, rmtree,
19                          import_module,
20                          skip_if_broken_multiprocessing_synchronize)
21import unittest
22import venv
23from unittest.mock import patch
24
25try:
26    import ctypes
27except ImportError:
28    ctypes = None
29
30# Platforms that set sys._base_executable can create venvs from within
31# another venv, so no need to skip tests that require venv.create().
32requireVenvCreate = unittest.skipUnless(
33    sys.prefix == sys.base_prefix
34    or sys._base_executable != sys.executable,
35    'cannot run venv.create from within a venv on this platform')
36
37def check_output(cmd, encoding=None):
38    p = subprocess.Popen(cmd,
39        stdout=subprocess.PIPE,
40        stderr=subprocess.PIPE,
41        encoding=encoding)
42    out, err = p.communicate()
43    if p.returncode:
44        raise subprocess.CalledProcessError(
45            p.returncode, cmd, out, err)
46    return out, err
47
48class BaseTest(unittest.TestCase):
49    """Base class for venv tests."""
50    maxDiff = 80 * 50
51
52    def setUp(self):
53        self.env_dir = os.path.realpath(tempfile.mkdtemp())
54        if os.name == 'nt':
55            self.bindir = 'Scripts'
56            self.lib = ('Lib',)
57            self.include = 'Include'
58        else:
59            self.bindir = 'bin'
60            self.lib = ('lib', 'python%d.%d' % sys.version_info[:2])
61            self.include = 'include'
62        executable = sys._base_executable
63        self.exe = os.path.split(executable)[-1]
64        if (sys.platform == 'win32'
65            and os.path.lexists(executable)
66            and not os.path.exists(executable)):
67            self.cannot_link_exe = True
68        else:
69            self.cannot_link_exe = False
70
71    def tearDown(self):
72        rmtree(self.env_dir)
73
74    def run_with_capture(self, func, *args, **kwargs):
75        with captured_stdout() as output:
76            with captured_stderr() as error:
77                func(*args, **kwargs)
78        return output.getvalue(), error.getvalue()
79
80    def get_env_file(self, *args):
81        return os.path.join(self.env_dir, *args)
82
83    def get_text_file_contents(self, *args, encoding='utf-8'):
84        with open(self.get_env_file(*args), 'r', encoding=encoding) as f:
85            result = f.read()
86        return result
87
88class BasicTest(BaseTest):
89    """Test venv module functionality."""
90
91    def isdir(self, *args):
92        fn = self.get_env_file(*args)
93        self.assertTrue(os.path.isdir(fn))
94
95    def test_defaults(self):
96        """
97        Test the create function with default arguments.
98        """
99        rmtree(self.env_dir)
100        self.run_with_capture(venv.create, self.env_dir)
101        self.isdir(self.bindir)
102        self.isdir(self.include)
103        self.isdir(*self.lib)
104        # Issue 21197
105        p = self.get_env_file('lib64')
106        conditions = ((struct.calcsize('P') == 8) and (os.name == 'posix') and
107                      (sys.platform != 'darwin'))
108        if conditions:
109            self.assertTrue(os.path.islink(p))
110        else:
111            self.assertFalse(os.path.exists(p))
112        data = self.get_text_file_contents('pyvenv.cfg')
113        executable = sys._base_executable
114        path = os.path.dirname(executable)
115        self.assertIn('home = %s' % path, data)
116        fn = self.get_env_file(self.bindir, self.exe)
117        if not os.path.exists(fn):  # diagnostics for Windows buildbot failures
118            bd = self.get_env_file(self.bindir)
119            print('Contents of %r:' % bd)
120            print('    %r' % os.listdir(bd))
121        self.assertTrue(os.path.exists(fn), 'File %r should exist.' % fn)
122
123    def test_prompt(self):
124        env_name = os.path.split(self.env_dir)[1]
125
126        rmtree(self.env_dir)
127        builder = venv.EnvBuilder()
128        self.run_with_capture(builder.create, self.env_dir)
129        context = builder.ensure_directories(self.env_dir)
130        data = self.get_text_file_contents('pyvenv.cfg')
131        self.assertEqual(context.prompt, '(%s) ' % env_name)
132        self.assertNotIn("prompt = ", data)
133
134        rmtree(self.env_dir)
135        builder = venv.EnvBuilder(prompt='My prompt')
136        self.run_with_capture(builder.create, self.env_dir)
137        context = builder.ensure_directories(self.env_dir)
138        data = self.get_text_file_contents('pyvenv.cfg')
139        self.assertEqual(context.prompt, '(My prompt) ')
140        self.assertIn("prompt = 'My prompt'\n", data)
141
142        rmtree(self.env_dir)
143        builder = venv.EnvBuilder(prompt='.')
144        cwd = os.path.basename(os.getcwd())
145        self.run_with_capture(builder.create, self.env_dir)
146        context = builder.ensure_directories(self.env_dir)
147        data = self.get_text_file_contents('pyvenv.cfg')
148        self.assertEqual(context.prompt, '(%s) ' % cwd)
149        self.assertIn("prompt = '%s'\n" % cwd, data)
150
151    def test_upgrade_dependencies(self):
152        builder = venv.EnvBuilder()
153        bin_path = 'Scripts' if sys.platform == 'win32' else 'bin'
154        python_exe = 'python.exe' if sys.platform == 'win32' else 'python'
155        with tempfile.TemporaryDirectory() as fake_env_dir:
156
157            def pip_cmd_checker(cmd):
158                self.assertEqual(
159                    cmd,
160                    [
161                        os.path.join(fake_env_dir, bin_path, python_exe),
162                        '-m',
163                        'pip',
164                        'install',
165                        '--upgrade',
166                        'pip',
167                        'setuptools'
168                    ]
169                )
170
171            fake_context = builder.ensure_directories(fake_env_dir)
172            with patch('venv.subprocess.check_call', pip_cmd_checker):
173                builder.upgrade_dependencies(fake_context)
174
175    @requireVenvCreate
176    def test_prefixes(self):
177        """
178        Test that the prefix values are as expected.
179        """
180        # check a venv's prefixes
181        rmtree(self.env_dir)
182        self.run_with_capture(venv.create, self.env_dir)
183        envpy = os.path.join(self.env_dir, self.bindir, self.exe)
184        cmd = [envpy, '-c', None]
185        for prefix, expected in (
186            ('prefix', self.env_dir),
187            ('exec_prefix', self.env_dir),
188            ('base_prefix', sys.base_prefix),
189            ('base_exec_prefix', sys.base_exec_prefix)):
190            cmd[2] = 'import sys; print(sys.%s)' % prefix
191            out, err = check_output(cmd)
192            self.assertEqual(out.strip(), expected.encode())
193
194    if sys.platform == 'win32':
195        ENV_SUBDIRS = (
196            ('Scripts',),
197            ('Include',),
198            ('Lib',),
199            ('Lib', 'site-packages'),
200        )
201    else:
202        ENV_SUBDIRS = (
203            ('bin',),
204            ('include',),
205            ('lib',),
206            ('lib', 'python%d.%d' % sys.version_info[:2]),
207            ('lib', 'python%d.%d' % sys.version_info[:2], 'site-packages'),
208        )
209
210    def create_contents(self, paths, filename):
211        """
212        Create some files in the environment which are unrelated
213        to the virtual environment.
214        """
215        for subdirs in paths:
216            d = os.path.join(self.env_dir, *subdirs)
217            os.mkdir(d)
218            fn = os.path.join(d, filename)
219            with open(fn, 'wb') as f:
220                f.write(b'Still here?')
221
222    def test_overwrite_existing(self):
223        """
224        Test creating environment in an existing directory.
225        """
226        self.create_contents(self.ENV_SUBDIRS, 'foo')
227        venv.create(self.env_dir)
228        for subdirs in self.ENV_SUBDIRS:
229            fn = os.path.join(self.env_dir, *(subdirs + ('foo',)))
230            self.assertTrue(os.path.exists(fn))
231            with open(fn, 'rb') as f:
232                self.assertEqual(f.read(), b'Still here?')
233
234        builder = venv.EnvBuilder(clear=True)
235        builder.create(self.env_dir)
236        for subdirs in self.ENV_SUBDIRS:
237            fn = os.path.join(self.env_dir, *(subdirs + ('foo',)))
238            self.assertFalse(os.path.exists(fn))
239
240    def clear_directory(self, path):
241        for fn in os.listdir(path):
242            fn = os.path.join(path, fn)
243            if os.path.islink(fn) or os.path.isfile(fn):
244                os.remove(fn)
245            elif os.path.isdir(fn):
246                rmtree(fn)
247
248    def test_unoverwritable_fails(self):
249        #create a file clashing with directories in the env dir
250        for paths in self.ENV_SUBDIRS[:3]:
251            fn = os.path.join(self.env_dir, *paths)
252            with open(fn, 'wb') as f:
253                f.write(b'')
254            self.assertRaises((ValueError, OSError), venv.create, self.env_dir)
255            self.clear_directory(self.env_dir)
256
257    def test_upgrade(self):
258        """
259        Test upgrading an existing environment directory.
260        """
261        # See Issue #21643: the loop needs to run twice to ensure
262        # that everything works on the upgrade (the first run just creates
263        # the venv).
264        for upgrade in (False, True):
265            builder = venv.EnvBuilder(upgrade=upgrade)
266            self.run_with_capture(builder.create, self.env_dir)
267            self.isdir(self.bindir)
268            self.isdir(self.include)
269            self.isdir(*self.lib)
270            fn = self.get_env_file(self.bindir, self.exe)
271            if not os.path.exists(fn):
272                # diagnostics for Windows buildbot failures
273                bd = self.get_env_file(self.bindir)
274                print('Contents of %r:' % bd)
275                print('    %r' % os.listdir(bd))
276            self.assertTrue(os.path.exists(fn), 'File %r should exist.' % fn)
277
278    def test_isolation(self):
279        """
280        Test isolation from system site-packages
281        """
282        for ssp, s in ((True, 'true'), (False, 'false')):
283            builder = venv.EnvBuilder(clear=True, system_site_packages=ssp)
284            builder.create(self.env_dir)
285            data = self.get_text_file_contents('pyvenv.cfg')
286            self.assertIn('include-system-site-packages = %s\n' % s, data)
287
288    @unittest.skipUnless(can_symlink(), 'Needs symlinks')
289    def test_symlinking(self):
290        """
291        Test symlinking works as expected
292        """
293        for usl in (False, True):
294            builder = venv.EnvBuilder(clear=True, symlinks=usl)
295            builder.create(self.env_dir)
296            fn = self.get_env_file(self.bindir, self.exe)
297            # Don't test when False, because e.g. 'python' is always
298            # symlinked to 'python3.3' in the env, even when symlinking in
299            # general isn't wanted.
300            if usl:
301                if self.cannot_link_exe:
302                    # Symlinking is skipped when our executable is already a
303                    # special app symlink
304                    self.assertFalse(os.path.islink(fn))
305                else:
306                    self.assertTrue(os.path.islink(fn))
307
308    # If a venv is created from a source build and that venv is used to
309    # run the test, the pyvenv.cfg in the venv created in the test will
310    # point to the venv being used to run the test, and we lose the link
311    # to the source build - so Python can't initialise properly.
312    @requireVenvCreate
313    def test_executable(self):
314        """
315        Test that the sys.executable value is as expected.
316        """
317        rmtree(self.env_dir)
318        self.run_with_capture(venv.create, self.env_dir)
319        envpy = os.path.join(os.path.realpath(self.env_dir),
320                             self.bindir, self.exe)
321        out, err = check_output([envpy, '-c',
322            'import sys; print(sys.executable)'])
323        self.assertEqual(out.strip(), envpy.encode())
324
325    @unittest.skipUnless(can_symlink(), 'Needs symlinks')
326    def test_executable_symlinks(self):
327        """
328        Test that the sys.executable value is as expected.
329        """
330        rmtree(self.env_dir)
331        builder = venv.EnvBuilder(clear=True, symlinks=True)
332        builder.create(self.env_dir)
333        envpy = os.path.join(os.path.realpath(self.env_dir),
334                             self.bindir, self.exe)
335        out, err = check_output([envpy, '-c',
336            'import sys; print(sys.executable)'])
337        self.assertEqual(out.strip(), envpy.encode())
338
339    @unittest.skipUnless(os.name == 'nt', 'only relevant on Windows')
340    def test_unicode_in_batch_file(self):
341        """
342        Test handling of Unicode paths
343        """
344        rmtree(self.env_dir)
345        env_dir = os.path.join(os.path.realpath(self.env_dir), 'ϼўТλФЙ')
346        builder = venv.EnvBuilder(clear=True)
347        builder.create(env_dir)
348        activate = os.path.join(env_dir, self.bindir, 'activate.bat')
349        envpy = os.path.join(env_dir, self.bindir, self.exe)
350        out, err = check_output(
351            [activate, '&', self.exe, '-c', 'print(0)'],
352            encoding='oem',
353        )
354        self.assertEqual(out.strip(), '0')
355
356    @requireVenvCreate
357    def test_multiprocessing(self):
358        """
359        Test that the multiprocessing is able to spawn.
360        """
361        # bpo-36342: Instantiation of a Pool object imports the
362        # multiprocessing.synchronize module. Skip the test if this module
363        # cannot be imported.
364        skip_if_broken_multiprocessing_synchronize()
365
366        rmtree(self.env_dir)
367        self.run_with_capture(venv.create, self.env_dir)
368        envpy = os.path.join(os.path.realpath(self.env_dir),
369                             self.bindir, self.exe)
370        out, err = check_output([envpy, '-c',
371            'from multiprocessing import Pool; '
372            'pool = Pool(1); '
373            'print(pool.apply_async("Python".lower).get(3)); '
374            'pool.terminate()'])
375        self.assertEqual(out.strip(), "python".encode())
376
377    @unittest.skipIf(os.name == 'nt', 'not relevant on Windows')
378    def test_deactivate_with_strict_bash_opts(self):
379        bash = shutil.which("bash")
380        if bash is None:
381            self.skipTest("bash required for this test")
382        rmtree(self.env_dir)
383        builder = venv.EnvBuilder(clear=True)
384        builder.create(self.env_dir)
385        activate = os.path.join(self.env_dir, self.bindir, "activate")
386        test_script = os.path.join(self.env_dir, "test_strict.sh")
387        with open(test_script, "w") as f:
388            f.write("set -euo pipefail\n"
389                    f"source {activate}\n"
390                    "deactivate\n")
391        out, err = check_output([bash, test_script])
392        self.assertEqual(out, "".encode())
393        self.assertEqual(err, "".encode())
394
395
396    @unittest.skipUnless(sys.platform == 'darwin', 'only relevant on macOS')
397    def test_macos_env(self):
398        rmtree(self.env_dir)
399        builder = venv.EnvBuilder()
400        builder.create(self.env_dir)
401
402        envpy = os.path.join(os.path.realpath(self.env_dir),
403                             self.bindir, self.exe)
404        out, err = check_output([envpy, '-c',
405            'import os; print("__PYVENV_LAUNCHER__" in os.environ)'])
406        self.assertEqual(out.strip(), 'False'.encode())
407
408@requireVenvCreate
409class EnsurePipTest(BaseTest):
410    """Test venv module installation of pip."""
411    def assert_pip_not_installed(self):
412        envpy = os.path.join(os.path.realpath(self.env_dir),
413                             self.bindir, self.exe)
414        out, err = check_output([envpy, '-c',
415            'try:\n import pip\nexcept ImportError:\n print("OK")'])
416        # We force everything to text, so unittest gives the detailed diff
417        # if we get unexpected results
418        err = err.decode("latin-1") # Force to text, prevent decoding errors
419        self.assertEqual(err, "")
420        out = out.decode("latin-1") # Force to text, prevent decoding errors
421        self.assertEqual(out.strip(), "OK")
422
423
424    def test_no_pip_by_default(self):
425        rmtree(self.env_dir)
426        self.run_with_capture(venv.create, self.env_dir)
427        self.assert_pip_not_installed()
428
429    def test_explicit_no_pip(self):
430        rmtree(self.env_dir)
431        self.run_with_capture(venv.create, self.env_dir, with_pip=False)
432        self.assert_pip_not_installed()
433
434    def test_devnull(self):
435        # Fix for issue #20053 uses os.devnull to force a config file to
436        # appear empty. However http://bugs.python.org/issue20541 means
437        # that doesn't currently work properly on Windows. Once that is
438        # fixed, the "win_location" part of test_with_pip should be restored
439        with open(os.devnull, "rb") as f:
440            self.assertEqual(f.read(), b"")
441
442        self.assertTrue(os.path.exists(os.devnull))
443
444    def do_test_with_pip(self, system_site_packages):
445        rmtree(self.env_dir)
446        with EnvironmentVarGuard() as envvars:
447            # pip's cross-version compatibility may trigger deprecation
448            # warnings in current versions of Python. Ensure related
449            # environment settings don't cause venv to fail.
450            envvars["PYTHONWARNINGS"] = "e"
451            # ensurepip is different enough from a normal pip invocation
452            # that we want to ensure it ignores the normal pip environment
453            # variable settings. We set PIP_NO_INSTALL here specifically
454            # to check that ensurepip (and hence venv) ignores it.
455            # See http://bugs.python.org/issue19734
456            envvars["PIP_NO_INSTALL"] = "1"
457            # Also check that we ignore the pip configuration file
458            # See http://bugs.python.org/issue20053
459            with tempfile.TemporaryDirectory() as home_dir:
460                envvars["HOME"] = home_dir
461                bad_config = "[global]\nno-install=1"
462                # Write to both config file names on all platforms to reduce
463                # cross-platform variation in test code behaviour
464                win_location = ("pip", "pip.ini")
465                posix_location = (".pip", "pip.conf")
466                # Skips win_location due to http://bugs.python.org/issue20541
467                for dirname, fname in (posix_location,):
468                    dirpath = os.path.join(home_dir, dirname)
469                    os.mkdir(dirpath)
470                    fpath = os.path.join(dirpath, fname)
471                    with open(fpath, 'w') as f:
472                        f.write(bad_config)
473
474                # Actually run the create command with all that unhelpful
475                # config in place to ensure we ignore it
476                try:
477                    self.run_with_capture(venv.create, self.env_dir,
478                                          system_site_packages=system_site_packages,
479                                          with_pip=True)
480                except subprocess.CalledProcessError as exc:
481                    # The output this produces can be a little hard to read,
482                    # but at least it has all the details
483                    details = exc.output.decode(errors="replace")
484                    msg = "{}\n\n**Subprocess Output**\n{}"
485                    self.fail(msg.format(exc, details))
486        # Ensure pip is available in the virtual environment
487        envpy = os.path.join(os.path.realpath(self.env_dir), self.bindir, self.exe)
488        # Ignore DeprecationWarning since pip code is not part of Python
489        out, err = check_output([envpy, '-W', 'ignore::DeprecationWarning', '-I',
490               '-m', 'pip', '--version'])
491        # We force everything to text, so unittest gives the detailed diff
492        # if we get unexpected results
493        err = err.decode("latin-1") # Force to text, prevent decoding errors
494        self.assertEqual(err, "")
495        out = out.decode("latin-1") # Force to text, prevent decoding errors
496        expected_version = "pip {}".format(ensurepip.version())
497        self.assertEqual(out[:len(expected_version)], expected_version)
498        env_dir = os.fsencode(self.env_dir).decode("latin-1")
499        self.assertIn(env_dir, out)
500
501        # http://bugs.python.org/issue19728
502        # Check the private uninstall command provided for the Windows
503        # installers works (at least in a virtual environment)
504        with EnvironmentVarGuard() as envvars:
505            out, err = check_output([envpy,
506                '-W', 'ignore::DeprecationWarning', '-I',
507                '-m', 'ensurepip._uninstall'])
508        # We force everything to text, so unittest gives the detailed diff
509        # if we get unexpected results
510        err = err.decode("latin-1") # Force to text, prevent decoding errors
511        # Ignore the warning:
512        #   "The directory '$HOME/.cache/pip/http' or its parent directory
513        #    is not owned by the current user and the cache has been disabled.
514        #    Please check the permissions and owner of that directory. If
515        #    executing pip with sudo, you may want sudo's -H flag."
516        # where $HOME is replaced by the HOME environment variable.
517        err = re.sub("^(WARNING: )?The directory .* or its parent directory "
518                     "is not owned or is not writable by the current user.*$", "",
519                     err, flags=re.MULTILINE)
520        self.assertEqual(err.rstrip(), "")
521        # Being fairly specific regarding the expected behaviour for the
522        # initial bundling phase in Python 3.4. If the output changes in
523        # future pip versions, this test can likely be relaxed further.
524        out = out.decode("latin-1") # Force to text, prevent decoding errors
525        self.assertIn("Successfully uninstalled pip", out)
526        self.assertIn("Successfully uninstalled setuptools", out)
527        # Check pip is now gone from the virtual environment. This only
528        # applies in the system_site_packages=False case, because in the
529        # other case, pip may still be available in the system site-packages
530        if not system_site_packages:
531            self.assert_pip_not_installed()
532
533    # Issue #26610: pip/pep425tags.py requires ctypes
534    @unittest.skipUnless(ctypes, 'pip requires ctypes')
535    @requires_zlib()
536    def test_with_pip(self):
537        self.do_test_with_pip(False)
538        self.do_test_with_pip(True)
539
540if __name__ == "__main__":
541    unittest.main()
542