1""" 2Test harness for the venv module. 3 4Copyright (C) 2011-2012 Vinay Sajip. 5Licensed to the PSF under a contributor agreement. 6""" 7 8import ensurepip 9import os 10import os.path 11import re 12import shutil 13import struct 14import subprocess 15import sys 16import tempfile 17from test.support import (captured_stdout, captured_stderr, requires_zlib, 18 can_symlink, EnvironmentVarGuard, rmtree, 19 import_module, 20 skip_if_broken_multiprocessing_synchronize) 21import unittest 22import venv 23from unittest.mock import patch 24 25try: 26 import ctypes 27except ImportError: 28 ctypes = None 29 30# Platforms that set sys._base_executable can create venvs from within 31# another venv, so no need to skip tests that require venv.create(). 32requireVenvCreate = unittest.skipUnless( 33 sys.prefix == sys.base_prefix 34 or sys._base_executable != sys.executable, 35 'cannot run venv.create from within a venv on this platform') 36 37def check_output(cmd, encoding=None): 38 p = subprocess.Popen(cmd, 39 stdout=subprocess.PIPE, 40 stderr=subprocess.PIPE, 41 encoding=encoding) 42 out, err = p.communicate() 43 if p.returncode: 44 raise subprocess.CalledProcessError( 45 p.returncode, cmd, out, err) 46 return out, err 47 48class BaseTest(unittest.TestCase): 49 """Base class for venv tests.""" 50 maxDiff = 80 * 50 51 52 def setUp(self): 53 self.env_dir = os.path.realpath(tempfile.mkdtemp()) 54 if os.name == 'nt': 55 self.bindir = 'Scripts' 56 self.lib = ('Lib',) 57 self.include = 'Include' 58 else: 59 self.bindir = 'bin' 60 self.lib = ('lib', 'python%d.%d' % sys.version_info[:2]) 61 self.include = 'include' 62 executable = sys._base_executable 63 self.exe = os.path.split(executable)[-1] 64 if (sys.platform == 'win32' 65 and os.path.lexists(executable) 66 and not os.path.exists(executable)): 67 self.cannot_link_exe = True 68 else: 69 self.cannot_link_exe = False 70 71 def tearDown(self): 72 rmtree(self.env_dir) 73 74 def run_with_capture(self, func, *args, **kwargs): 75 with captured_stdout() as output: 76 with captured_stderr() as error: 77 func(*args, **kwargs) 78 return output.getvalue(), error.getvalue() 79 80 def get_env_file(self, *args): 81 return os.path.join(self.env_dir, *args) 82 83 def get_text_file_contents(self, *args, encoding='utf-8'): 84 with open(self.get_env_file(*args), 'r', encoding=encoding) as f: 85 result = f.read() 86 return result 87 88class BasicTest(BaseTest): 89 """Test venv module functionality.""" 90 91 def isdir(self, *args): 92 fn = self.get_env_file(*args) 93 self.assertTrue(os.path.isdir(fn)) 94 95 def test_defaults(self): 96 """ 97 Test the create function with default arguments. 98 """ 99 rmtree(self.env_dir) 100 self.run_with_capture(venv.create, self.env_dir) 101 self.isdir(self.bindir) 102 self.isdir(self.include) 103 self.isdir(*self.lib) 104 # Issue 21197 105 p = self.get_env_file('lib64') 106 conditions = ((struct.calcsize('P') == 8) and (os.name == 'posix') and 107 (sys.platform != 'darwin')) 108 if conditions: 109 self.assertTrue(os.path.islink(p)) 110 else: 111 self.assertFalse(os.path.exists(p)) 112 data = self.get_text_file_contents('pyvenv.cfg') 113 executable = sys._base_executable 114 path = os.path.dirname(executable) 115 self.assertIn('home = %s' % path, data) 116 fn = self.get_env_file(self.bindir, self.exe) 117 if not os.path.exists(fn): # diagnostics for Windows buildbot failures 118 bd = self.get_env_file(self.bindir) 119 print('Contents of %r:' % bd) 120 print(' %r' % os.listdir(bd)) 121 self.assertTrue(os.path.exists(fn), 'File %r should exist.' % fn) 122 123 def test_prompt(self): 124 env_name = os.path.split(self.env_dir)[1] 125 126 rmtree(self.env_dir) 127 builder = venv.EnvBuilder() 128 self.run_with_capture(builder.create, self.env_dir) 129 context = builder.ensure_directories(self.env_dir) 130 data = self.get_text_file_contents('pyvenv.cfg') 131 self.assertEqual(context.prompt, '(%s) ' % env_name) 132 self.assertNotIn("prompt = ", data) 133 134 rmtree(self.env_dir) 135 builder = venv.EnvBuilder(prompt='My prompt') 136 self.run_with_capture(builder.create, self.env_dir) 137 context = builder.ensure_directories(self.env_dir) 138 data = self.get_text_file_contents('pyvenv.cfg') 139 self.assertEqual(context.prompt, '(My prompt) ') 140 self.assertIn("prompt = 'My prompt'\n", data) 141 142 rmtree(self.env_dir) 143 builder = venv.EnvBuilder(prompt='.') 144 cwd = os.path.basename(os.getcwd()) 145 self.run_with_capture(builder.create, self.env_dir) 146 context = builder.ensure_directories(self.env_dir) 147 data = self.get_text_file_contents('pyvenv.cfg') 148 self.assertEqual(context.prompt, '(%s) ' % cwd) 149 self.assertIn("prompt = '%s'\n" % cwd, data) 150 151 def test_upgrade_dependencies(self): 152 builder = venv.EnvBuilder() 153 bin_path = 'Scripts' if sys.platform == 'win32' else 'bin' 154 python_exe = 'python.exe' if sys.platform == 'win32' else 'python' 155 with tempfile.TemporaryDirectory() as fake_env_dir: 156 157 def pip_cmd_checker(cmd): 158 self.assertEqual( 159 cmd, 160 [ 161 os.path.join(fake_env_dir, bin_path, python_exe), 162 '-m', 163 'pip', 164 'install', 165 '--upgrade', 166 'pip', 167 'setuptools' 168 ] 169 ) 170 171 fake_context = builder.ensure_directories(fake_env_dir) 172 with patch('venv.subprocess.check_call', pip_cmd_checker): 173 builder.upgrade_dependencies(fake_context) 174 175 @requireVenvCreate 176 def test_prefixes(self): 177 """ 178 Test that the prefix values are as expected. 179 """ 180 # check a venv's prefixes 181 rmtree(self.env_dir) 182 self.run_with_capture(venv.create, self.env_dir) 183 envpy = os.path.join(self.env_dir, self.bindir, self.exe) 184 cmd = [envpy, '-c', None] 185 for prefix, expected in ( 186 ('prefix', self.env_dir), 187 ('exec_prefix', self.env_dir), 188 ('base_prefix', sys.base_prefix), 189 ('base_exec_prefix', sys.base_exec_prefix)): 190 cmd[2] = 'import sys; print(sys.%s)' % prefix 191 out, err = check_output(cmd) 192 self.assertEqual(out.strip(), expected.encode()) 193 194 if sys.platform == 'win32': 195 ENV_SUBDIRS = ( 196 ('Scripts',), 197 ('Include',), 198 ('Lib',), 199 ('Lib', 'site-packages'), 200 ) 201 else: 202 ENV_SUBDIRS = ( 203 ('bin',), 204 ('include',), 205 ('lib',), 206 ('lib', 'python%d.%d' % sys.version_info[:2]), 207 ('lib', 'python%d.%d' % sys.version_info[:2], 'site-packages'), 208 ) 209 210 def create_contents(self, paths, filename): 211 """ 212 Create some files in the environment which are unrelated 213 to the virtual environment. 214 """ 215 for subdirs in paths: 216 d = os.path.join(self.env_dir, *subdirs) 217 os.mkdir(d) 218 fn = os.path.join(d, filename) 219 with open(fn, 'wb') as f: 220 f.write(b'Still here?') 221 222 def test_overwrite_existing(self): 223 """ 224 Test creating environment in an existing directory. 225 """ 226 self.create_contents(self.ENV_SUBDIRS, 'foo') 227 venv.create(self.env_dir) 228 for subdirs in self.ENV_SUBDIRS: 229 fn = os.path.join(self.env_dir, *(subdirs + ('foo',))) 230 self.assertTrue(os.path.exists(fn)) 231 with open(fn, 'rb') as f: 232 self.assertEqual(f.read(), b'Still here?') 233 234 builder = venv.EnvBuilder(clear=True) 235 builder.create(self.env_dir) 236 for subdirs in self.ENV_SUBDIRS: 237 fn = os.path.join(self.env_dir, *(subdirs + ('foo',))) 238 self.assertFalse(os.path.exists(fn)) 239 240 def clear_directory(self, path): 241 for fn in os.listdir(path): 242 fn = os.path.join(path, fn) 243 if os.path.islink(fn) or os.path.isfile(fn): 244 os.remove(fn) 245 elif os.path.isdir(fn): 246 rmtree(fn) 247 248 def test_unoverwritable_fails(self): 249 #create a file clashing with directories in the env dir 250 for paths in self.ENV_SUBDIRS[:3]: 251 fn = os.path.join(self.env_dir, *paths) 252 with open(fn, 'wb') as f: 253 f.write(b'') 254 self.assertRaises((ValueError, OSError), venv.create, self.env_dir) 255 self.clear_directory(self.env_dir) 256 257 def test_upgrade(self): 258 """ 259 Test upgrading an existing environment directory. 260 """ 261 # See Issue #21643: the loop needs to run twice to ensure 262 # that everything works on the upgrade (the first run just creates 263 # the venv). 264 for upgrade in (False, True): 265 builder = venv.EnvBuilder(upgrade=upgrade) 266 self.run_with_capture(builder.create, self.env_dir) 267 self.isdir(self.bindir) 268 self.isdir(self.include) 269 self.isdir(*self.lib) 270 fn = self.get_env_file(self.bindir, self.exe) 271 if not os.path.exists(fn): 272 # diagnostics for Windows buildbot failures 273 bd = self.get_env_file(self.bindir) 274 print('Contents of %r:' % bd) 275 print(' %r' % os.listdir(bd)) 276 self.assertTrue(os.path.exists(fn), 'File %r should exist.' % fn) 277 278 def test_isolation(self): 279 """ 280 Test isolation from system site-packages 281 """ 282 for ssp, s in ((True, 'true'), (False, 'false')): 283 builder = venv.EnvBuilder(clear=True, system_site_packages=ssp) 284 builder.create(self.env_dir) 285 data = self.get_text_file_contents('pyvenv.cfg') 286 self.assertIn('include-system-site-packages = %s\n' % s, data) 287 288 @unittest.skipUnless(can_symlink(), 'Needs symlinks') 289 def test_symlinking(self): 290 """ 291 Test symlinking works as expected 292 """ 293 for usl in (False, True): 294 builder = venv.EnvBuilder(clear=True, symlinks=usl) 295 builder.create(self.env_dir) 296 fn = self.get_env_file(self.bindir, self.exe) 297 # Don't test when False, because e.g. 'python' is always 298 # symlinked to 'python3.3' in the env, even when symlinking in 299 # general isn't wanted. 300 if usl: 301 if self.cannot_link_exe: 302 # Symlinking is skipped when our executable is already a 303 # special app symlink 304 self.assertFalse(os.path.islink(fn)) 305 else: 306 self.assertTrue(os.path.islink(fn)) 307 308 # If a venv is created from a source build and that venv is used to 309 # run the test, the pyvenv.cfg in the venv created in the test will 310 # point to the venv being used to run the test, and we lose the link 311 # to the source build - so Python can't initialise properly. 312 @requireVenvCreate 313 def test_executable(self): 314 """ 315 Test that the sys.executable value is as expected. 316 """ 317 rmtree(self.env_dir) 318 self.run_with_capture(venv.create, self.env_dir) 319 envpy = os.path.join(os.path.realpath(self.env_dir), 320 self.bindir, self.exe) 321 out, err = check_output([envpy, '-c', 322 'import sys; print(sys.executable)']) 323 self.assertEqual(out.strip(), envpy.encode()) 324 325 @unittest.skipUnless(can_symlink(), 'Needs symlinks') 326 def test_executable_symlinks(self): 327 """ 328 Test that the sys.executable value is as expected. 329 """ 330 rmtree(self.env_dir) 331 builder = venv.EnvBuilder(clear=True, symlinks=True) 332 builder.create(self.env_dir) 333 envpy = os.path.join(os.path.realpath(self.env_dir), 334 self.bindir, self.exe) 335 out, err = check_output([envpy, '-c', 336 'import sys; print(sys.executable)']) 337 self.assertEqual(out.strip(), envpy.encode()) 338 339 @unittest.skipUnless(os.name == 'nt', 'only relevant on Windows') 340 def test_unicode_in_batch_file(self): 341 """ 342 Test handling of Unicode paths 343 """ 344 rmtree(self.env_dir) 345 env_dir = os.path.join(os.path.realpath(self.env_dir), 'ϼўТλФЙ') 346 builder = venv.EnvBuilder(clear=True) 347 builder.create(env_dir) 348 activate = os.path.join(env_dir, self.bindir, 'activate.bat') 349 envpy = os.path.join(env_dir, self.bindir, self.exe) 350 out, err = check_output( 351 [activate, '&', self.exe, '-c', 'print(0)'], 352 encoding='oem', 353 ) 354 self.assertEqual(out.strip(), '0') 355 356 @requireVenvCreate 357 def test_multiprocessing(self): 358 """ 359 Test that the multiprocessing is able to spawn. 360 """ 361 # bpo-36342: Instantiation of a Pool object imports the 362 # multiprocessing.synchronize module. Skip the test if this module 363 # cannot be imported. 364 skip_if_broken_multiprocessing_synchronize() 365 366 rmtree(self.env_dir) 367 self.run_with_capture(venv.create, self.env_dir) 368 envpy = os.path.join(os.path.realpath(self.env_dir), 369 self.bindir, self.exe) 370 out, err = check_output([envpy, '-c', 371 'from multiprocessing import Pool; ' 372 'pool = Pool(1); ' 373 'print(pool.apply_async("Python".lower).get(3)); ' 374 'pool.terminate()']) 375 self.assertEqual(out.strip(), "python".encode()) 376 377 @unittest.skipIf(os.name == 'nt', 'not relevant on Windows') 378 def test_deactivate_with_strict_bash_opts(self): 379 bash = shutil.which("bash") 380 if bash is None: 381 self.skipTest("bash required for this test") 382 rmtree(self.env_dir) 383 builder = venv.EnvBuilder(clear=True) 384 builder.create(self.env_dir) 385 activate = os.path.join(self.env_dir, self.bindir, "activate") 386 test_script = os.path.join(self.env_dir, "test_strict.sh") 387 with open(test_script, "w") as f: 388 f.write("set -euo pipefail\n" 389 f"source {activate}\n" 390 "deactivate\n") 391 out, err = check_output([bash, test_script]) 392 self.assertEqual(out, "".encode()) 393 self.assertEqual(err, "".encode()) 394 395 396 @unittest.skipUnless(sys.platform == 'darwin', 'only relevant on macOS') 397 def test_macos_env(self): 398 rmtree(self.env_dir) 399 builder = venv.EnvBuilder() 400 builder.create(self.env_dir) 401 402 envpy = os.path.join(os.path.realpath(self.env_dir), 403 self.bindir, self.exe) 404 out, err = check_output([envpy, '-c', 405 'import os; print("__PYVENV_LAUNCHER__" in os.environ)']) 406 self.assertEqual(out.strip(), 'False'.encode()) 407 408@requireVenvCreate 409class EnsurePipTest(BaseTest): 410 """Test venv module installation of pip.""" 411 def assert_pip_not_installed(self): 412 envpy = os.path.join(os.path.realpath(self.env_dir), 413 self.bindir, self.exe) 414 out, err = check_output([envpy, '-c', 415 'try:\n import pip\nexcept ImportError:\n print("OK")']) 416 # We force everything to text, so unittest gives the detailed diff 417 # if we get unexpected results 418 err = err.decode("latin-1") # Force to text, prevent decoding errors 419 self.assertEqual(err, "") 420 out = out.decode("latin-1") # Force to text, prevent decoding errors 421 self.assertEqual(out.strip(), "OK") 422 423 424 def test_no_pip_by_default(self): 425 rmtree(self.env_dir) 426 self.run_with_capture(venv.create, self.env_dir) 427 self.assert_pip_not_installed() 428 429 def test_explicit_no_pip(self): 430 rmtree(self.env_dir) 431 self.run_with_capture(venv.create, self.env_dir, with_pip=False) 432 self.assert_pip_not_installed() 433 434 def test_devnull(self): 435 # Fix for issue #20053 uses os.devnull to force a config file to 436 # appear empty. However http://bugs.python.org/issue20541 means 437 # that doesn't currently work properly on Windows. Once that is 438 # fixed, the "win_location" part of test_with_pip should be restored 439 with open(os.devnull, "rb") as f: 440 self.assertEqual(f.read(), b"") 441 442 self.assertTrue(os.path.exists(os.devnull)) 443 444 def do_test_with_pip(self, system_site_packages): 445 rmtree(self.env_dir) 446 with EnvironmentVarGuard() as envvars: 447 # pip's cross-version compatibility may trigger deprecation 448 # warnings in current versions of Python. Ensure related 449 # environment settings don't cause venv to fail. 450 envvars["PYTHONWARNINGS"] = "e" 451 # ensurepip is different enough from a normal pip invocation 452 # that we want to ensure it ignores the normal pip environment 453 # variable settings. We set PIP_NO_INSTALL here specifically 454 # to check that ensurepip (and hence venv) ignores it. 455 # See http://bugs.python.org/issue19734 456 envvars["PIP_NO_INSTALL"] = "1" 457 # Also check that we ignore the pip configuration file 458 # See http://bugs.python.org/issue20053 459 with tempfile.TemporaryDirectory() as home_dir: 460 envvars["HOME"] = home_dir 461 bad_config = "[global]\nno-install=1" 462 # Write to both config file names on all platforms to reduce 463 # cross-platform variation in test code behaviour 464 win_location = ("pip", "pip.ini") 465 posix_location = (".pip", "pip.conf") 466 # Skips win_location due to http://bugs.python.org/issue20541 467 for dirname, fname in (posix_location,): 468 dirpath = os.path.join(home_dir, dirname) 469 os.mkdir(dirpath) 470 fpath = os.path.join(dirpath, fname) 471 with open(fpath, 'w') as f: 472 f.write(bad_config) 473 474 # Actually run the create command with all that unhelpful 475 # config in place to ensure we ignore it 476 try: 477 self.run_with_capture(venv.create, self.env_dir, 478 system_site_packages=system_site_packages, 479 with_pip=True) 480 except subprocess.CalledProcessError as exc: 481 # The output this produces can be a little hard to read, 482 # but at least it has all the details 483 details = exc.output.decode(errors="replace") 484 msg = "{}\n\n**Subprocess Output**\n{}" 485 self.fail(msg.format(exc, details)) 486 # Ensure pip is available in the virtual environment 487 envpy = os.path.join(os.path.realpath(self.env_dir), self.bindir, self.exe) 488 # Ignore DeprecationWarning since pip code is not part of Python 489 out, err = check_output([envpy, '-W', 'ignore::DeprecationWarning', '-I', 490 '-m', 'pip', '--version']) 491 # We force everything to text, so unittest gives the detailed diff 492 # if we get unexpected results 493 err = err.decode("latin-1") # Force to text, prevent decoding errors 494 self.assertEqual(err, "") 495 out = out.decode("latin-1") # Force to text, prevent decoding errors 496 expected_version = "pip {}".format(ensurepip.version()) 497 self.assertEqual(out[:len(expected_version)], expected_version) 498 env_dir = os.fsencode(self.env_dir).decode("latin-1") 499 self.assertIn(env_dir, out) 500 501 # http://bugs.python.org/issue19728 502 # Check the private uninstall command provided for the Windows 503 # installers works (at least in a virtual environment) 504 with EnvironmentVarGuard() as envvars: 505 out, err = check_output([envpy, 506 '-W', 'ignore::DeprecationWarning', '-I', 507 '-m', 'ensurepip._uninstall']) 508 # We force everything to text, so unittest gives the detailed diff 509 # if we get unexpected results 510 err = err.decode("latin-1") # Force to text, prevent decoding errors 511 # Ignore the warning: 512 # "The directory '$HOME/.cache/pip/http' or its parent directory 513 # is not owned by the current user and the cache has been disabled. 514 # Please check the permissions and owner of that directory. If 515 # executing pip with sudo, you may want sudo's -H flag." 516 # where $HOME is replaced by the HOME environment variable. 517 err = re.sub("^(WARNING: )?The directory .* or its parent directory " 518 "is not owned or is not writable by the current user.*$", "", 519 err, flags=re.MULTILINE) 520 self.assertEqual(err.rstrip(), "") 521 # Being fairly specific regarding the expected behaviour for the 522 # initial bundling phase in Python 3.4. If the output changes in 523 # future pip versions, this test can likely be relaxed further. 524 out = out.decode("latin-1") # Force to text, prevent decoding errors 525 self.assertIn("Successfully uninstalled pip", out) 526 self.assertIn("Successfully uninstalled setuptools", out) 527 # Check pip is now gone from the virtual environment. This only 528 # applies in the system_site_packages=False case, because in the 529 # other case, pip may still be available in the system site-packages 530 if not system_site_packages: 531 self.assert_pip_not_installed() 532 533 # Issue #26610: pip/pep425tags.py requires ctypes 534 @unittest.skipUnless(ctypes, 'pip requires ctypes') 535 @requires_zlib() 536 def test_with_pip(self): 537 self.do_test_with_pip(False) 538 self.do_test_with_pip(True) 539 540if __name__ == "__main__": 541 unittest.main() 542