1# subprocess - Subprocesses with accessible I/O streams 2# 3# For more information about this module, see PEP 324. 4# 5# Copyright (c) 2003-2005 by Peter Astrand <astrand@lysator.liu.se> 6# 7# Licensed to PSF under a Contributor Agreement. 8# See http://www.python.org/2.4/license for licensing details. 9 10r"""Subprocesses with accessible I/O streams 11 12This module allows you to spawn processes, connect to their 13input/output/error pipes, and obtain their return codes. 14 15For a complete description of this module see the Python documentation. 16 17Main API 18======== 19call(...): Runs a command, waits for it to complete, then returns 20 the return code. 21check_call(...): Same as call() but raises CalledProcessError() 22 if return code is not 0 23check_output(...): Same as check_call() but returns the contents of 24 stdout instead of a return code 25Popen(...): A class for flexibly executing a command in a new process 26 27Constants 28--------- 29PIPE: Special value that indicates a pipe should be created 30STDOUT: Special value that indicates that stderr should go to stdout 31""" 32 33import sys 34mswindows = (sys.platform == "win32") 35 36import os 37import types 38import traceback 39import gc 40import signal 41import errno 42 43# Exception classes used by this module. 44class CalledProcessError(Exception): 45 """This exception is raised when a process run by check_call() or 46 check_output() returns a non-zero exit status. 47 48 Attributes: 49 cmd, returncode, output 50 """ 51 def __init__(self, returncode, cmd, output=None): 52 self.returncode = returncode 53 self.cmd = cmd 54 self.output = output 55 def __str__(self): 56 return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode) 57 58 59if mswindows: 60 import threading 61 import msvcrt 62 import _subprocess 63 class STARTUPINFO: 64 dwFlags = 0 65 hStdInput = None 66 hStdOutput = None 67 hStdError = None 68 wShowWindow = 0 69 class pywintypes: 70 error = IOError 71else: 72 import select 73 _has_poll = hasattr(select, 'poll') 74 try: 75 import threading 76 except ImportError: 77 threading = None 78 import fcntl 79 import pickle 80 81 # When select or poll has indicated that the file is writable, 82 # we can write up to _PIPE_BUF bytes without risk of blocking. 83 # POSIX defines PIPE_BUF as >= 512. 84 _PIPE_BUF = getattr(select, 'PIPE_BUF', 512) 85 86 87__all__ = ["Popen", "PIPE", "STDOUT", "call", "check_call", 88 "check_output", "CalledProcessError"] 89 90if mswindows: 91 from _subprocess import (CREATE_NEW_CONSOLE, CREATE_NEW_PROCESS_GROUP, 92 STD_INPUT_HANDLE, STD_OUTPUT_HANDLE, 93 STD_ERROR_HANDLE, SW_HIDE, 94 STARTF_USESTDHANDLES, STARTF_USESHOWWINDOW) 95 96 __all__.extend(["CREATE_NEW_CONSOLE", "CREATE_NEW_PROCESS_GROUP", 97 "STD_INPUT_HANDLE", "STD_OUTPUT_HANDLE", 98 "STD_ERROR_HANDLE", "SW_HIDE", 99 "STARTF_USESTDHANDLES", "STARTF_USESHOWWINDOW"]) 100try: 101 MAXFD = os.sysconf("SC_OPEN_MAX") 102except: 103 MAXFD = 256 104 105_active = [] 106 107def _cleanup(): 108 for inst in _active[:]: 109 res = inst._internal_poll(_deadstate=sys.maxint) 110 if res is not None: 111 try: 112 _active.remove(inst) 113 except ValueError: 114 # This can happen if two threads create a new Popen instance. 115 # It's harmless that it was already removed, so ignore. 116 pass 117 118PIPE = -1 119STDOUT = -2 120 121 122def _eintr_retry_call(func, *args): 123 while True: 124 try: 125 return func(*args) 126 except (OSError, IOError) as e: 127 if e.errno == errno.EINTR: 128 continue 129 raise 130 131 132# XXX This function is only used by multiprocessing and the test suite, 133# but it's here so that it can be imported when Python is compiled without 134# threads. 135 136def _args_from_interpreter_flags(): 137 """Return a list of command-line arguments reproducing the current 138 settings in sys.flags and sys.warnoptions.""" 139 flag_opt_map = { 140 'debug': 'd', 141 # 'inspect': 'i', 142 # 'interactive': 'i', 143 'optimize': 'O', 144 'dont_write_bytecode': 'B', 145 'no_user_site': 's', 146 'no_site': 'S', 147 'ignore_environment': 'E', 148 'verbose': 'v', 149 'bytes_warning': 'b', 150 'py3k_warning': '3', 151 } 152 args = [] 153 for flag, opt in flag_opt_map.items(): 154 v = getattr(sys.flags, flag) 155 if v > 0: 156 args.append('-' + opt * v) 157 if getattr(sys.flags, 'hash_randomization') != 0: 158 args.append('-R') 159 for opt in sys.warnoptions: 160 args.append('-W' + opt) 161 return args 162 163 164def call(*popenargs, **kwargs): 165 """Run command with arguments. Wait for command to complete, then 166 return the returncode attribute. 167 168 The arguments are the same as for the Popen constructor. Example: 169 170 retcode = call(["ls", "-l"]) 171 """ 172 return Popen(*popenargs, **kwargs).wait() 173 174 175def check_call(*popenargs, **kwargs): 176 """Run command with arguments. Wait for command to complete. If 177 the exit code was zero then return, otherwise raise 178 CalledProcessError. The CalledProcessError object will have the 179 return code in the returncode attribute. 180 181 The arguments are the same as for the Popen constructor. Example: 182 183 check_call(["ls", "-l"]) 184 """ 185 retcode = call(*popenargs, **kwargs) 186 if retcode: 187 cmd = kwargs.get("args") 188 if cmd is None: 189 cmd = popenargs[0] 190 raise CalledProcessError(retcode, cmd) 191 return 0 192 193 194def check_output(*popenargs, **kwargs): 195 r"""Run command with arguments and return its output as a byte string. 196 197 If the exit code was non-zero it raises a CalledProcessError. The 198 CalledProcessError object will have the return code in the returncode 199 attribute and output in the output attribute. 200 201 The arguments are the same as for the Popen constructor. Example: 202 203 >>> check_output(["ls", "-l", "/dev/null"]) 204 'crw-rw-rw- 1 root root 1, 3 Oct 18 2007 /dev/null\n' 205 206 The stdout argument is not allowed as it is used internally. 207 To capture standard error in the result, use stderr=STDOUT. 208 209 >>> check_output(["/bin/sh", "-c", 210 ... "ls -l non_existent_file ; exit 0"], 211 ... stderr=STDOUT) 212 'ls: non_existent_file: No such file or directory\n' 213 """ 214 if 'stdout' in kwargs: 215 raise ValueError('stdout argument not allowed, it will be overridden.') 216 process = Popen(stdout=PIPE, *popenargs, **kwargs) 217 output, unused_err = process.communicate() 218 retcode = process.poll() 219 if retcode: 220 cmd = kwargs.get("args") 221 if cmd is None: 222 cmd = popenargs[0] 223 raise CalledProcessError(retcode, cmd, output=output) 224 return output 225 226 227def list2cmdline(seq): 228 """ 229 Translate a sequence of arguments into a command line 230 string, using the same rules as the MS C runtime: 231 232 1) Arguments are delimited by white space, which is either a 233 space or a tab. 234 235 2) A string surrounded by double quotation marks is 236 interpreted as a single argument, regardless of white space 237 contained within. A quoted string can be embedded in an 238 argument. 239 240 3) A double quotation mark preceded by a backslash is 241 interpreted as a literal double quotation mark. 242 243 4) Backslashes are interpreted literally, unless they 244 immediately precede a double quotation mark. 245 246 5) If backslashes immediately precede a double quotation mark, 247 every pair of backslashes is interpreted as a literal 248 backslash. If the number of backslashes is odd, the last 249 backslash escapes the next double quotation mark as 250 described in rule 3. 251 """ 252 253 # See 254 # http://msdn.microsoft.com/en-us/library/17w5ykft.aspx 255 # or search http://msdn.microsoft.com for 256 # "Parsing C++ Command-Line Arguments" 257 result = [] 258 needquote = False 259 for arg in seq: 260 bs_buf = [] 261 262 # Add a space to separate this argument from the others 263 if result: 264 result.append(' ') 265 266 needquote = (" " in arg) or ("\t" in arg) or not arg 267 if needquote: 268 result.append('"') 269 270 for c in arg: 271 if c == '\\': 272 # Don't know if we need to double yet. 273 bs_buf.append(c) 274 elif c == '"': 275 # Double backslashes. 276 result.append('\\' * len(bs_buf)*2) 277 bs_buf = [] 278 result.append('\\"') 279 else: 280 # Normal char 281 if bs_buf: 282 result.extend(bs_buf) 283 bs_buf = [] 284 result.append(c) 285 286 # Add remaining backslashes, if any. 287 if bs_buf: 288 result.extend(bs_buf) 289 290 if needquote: 291 result.extend(bs_buf) 292 result.append('"') 293 294 return ''.join(result) 295 296 297class Popen(object): 298 """ Execute a child program in a new process. 299 300 For a complete description of the arguments see the Python documentation. 301 302 Arguments: 303 args: A string, or a sequence of program arguments. 304 305 bufsize: supplied as the buffering argument to the open() function when 306 creating the stdin/stdout/stderr pipe file objects 307 308 executable: A replacement program to execute. 309 310 stdin, stdout and stderr: These specify the executed programs' standard 311 input, standard output and standard error file handles, respectively. 312 313 preexec_fn: (POSIX only) An object to be called in the child process 314 just before the child is executed. 315 316 close_fds: Controls closing or inheriting of file descriptors. 317 318 shell: If true, the command will be executed through the shell. 319 320 cwd: Sets the current directory before the child is executed. 321 322 env: Defines the environment variables for the new process. 323 324 universal_newlines: If true, use universal line endings for file 325 objects stdin, stdout and stderr. 326 327 startupinfo and creationflags (Windows only) 328 329 Attributes: 330 stdin, stdout, stderr, pid, returncode 331 """ 332 _child_created = False # Set here since __del__ checks it 333 334 def __init__(self, args, bufsize=0, executable=None, 335 stdin=None, stdout=None, stderr=None, 336 preexec_fn=None, close_fds=False, shell=False, 337 cwd=None, env=None, universal_newlines=False, 338 startupinfo=None, creationflags=0): 339 """Create new Popen instance.""" 340 _cleanup() 341 342 if not isinstance(bufsize, (int, long)): 343 raise TypeError("bufsize must be an integer") 344 345 if mswindows: 346 if preexec_fn is not None: 347 raise ValueError("preexec_fn is not supported on Windows " 348 "platforms") 349 if close_fds and (stdin is not None or stdout is not None or 350 stderr is not None): 351 raise ValueError("close_fds is not supported on Windows " 352 "platforms if you redirect stdin/stdout/stderr") 353 else: 354 # POSIX 355 if startupinfo is not None: 356 raise ValueError("startupinfo is only supported on Windows " 357 "platforms") 358 if creationflags != 0: 359 raise ValueError("creationflags is only supported on Windows " 360 "platforms") 361 362 self.stdin = None 363 self.stdout = None 364 self.stderr = None 365 self.pid = None 366 self.returncode = None 367 self.universal_newlines = universal_newlines 368 369 # Input and output objects. The general principle is like 370 # this: 371 # 372 # Parent Child 373 # ------ ----- 374 # p2cwrite ---stdin---> p2cread 375 # c2pread <--stdout--- c2pwrite 376 # errread <--stderr--- errwrite 377 # 378 # On POSIX, the child objects are file descriptors. On 379 # Windows, these are Windows file handles. The parent objects 380 # are file descriptors on both platforms. The parent objects 381 # are None when not using PIPEs. The child objects are None 382 # when not redirecting. 383 384 (p2cread, p2cwrite, 385 c2pread, c2pwrite, 386 errread, errwrite), to_close = self._get_handles(stdin, stdout, stderr) 387 388 try: 389 self._execute_child(args, executable, preexec_fn, close_fds, 390 cwd, env, universal_newlines, 391 startupinfo, creationflags, shell, to_close, 392 p2cread, p2cwrite, 393 c2pread, c2pwrite, 394 errread, errwrite) 395 except Exception: 396 # Preserve original exception in case os.close raises. 397 exc_type, exc_value, exc_trace = sys.exc_info() 398 399 for fd in to_close: 400 try: 401 if mswindows: 402 fd.Close() 403 else: 404 os.close(fd) 405 except EnvironmentError: 406 pass 407 408 raise exc_type, exc_value, exc_trace 409 410 if mswindows: 411 if p2cwrite is not None: 412 p2cwrite = msvcrt.open_osfhandle(p2cwrite.Detach(), 0) 413 if c2pread is not None: 414 c2pread = msvcrt.open_osfhandle(c2pread.Detach(), 0) 415 if errread is not None: 416 errread = msvcrt.open_osfhandle(errread.Detach(), 0) 417 418 if p2cwrite is not None: 419 self.stdin = os.fdopen(p2cwrite, 'wb', bufsize) 420 if c2pread is not None: 421 if universal_newlines: 422 self.stdout = os.fdopen(c2pread, 'rU', bufsize) 423 else: 424 self.stdout = os.fdopen(c2pread, 'rb', bufsize) 425 if errread is not None: 426 if universal_newlines: 427 self.stderr = os.fdopen(errread, 'rU', bufsize) 428 else: 429 self.stderr = os.fdopen(errread, 'rb', bufsize) 430 431 432 def _translate_newlines(self, data): 433 data = data.replace("\r\n", "\n") 434 data = data.replace("\r", "\n") 435 return data 436 437 438 def __del__(self, _maxint=sys.maxint): 439 # If __init__ hasn't had a chance to execute (e.g. if it 440 # was passed an undeclared keyword argument), we don't 441 # have a _child_created attribute at all. 442 if not self._child_created: 443 # We didn't get to successfully create a child process. 444 return 445 # In case the child hasn't been waited on, check if it's done. 446 self._internal_poll(_deadstate=_maxint) 447 if self.returncode is None and _active is not None: 448 # Child is still running, keep us alive until we can wait on it. 449 _active.append(self) 450 451 452 def communicate(self, input=None): 453 """Interact with process: Send data to stdin. Read data from 454 stdout and stderr, until end-of-file is reached. Wait for 455 process to terminate. The optional input argument should be a 456 string to be sent to the child process, or None, if no data 457 should be sent to the child. 458 459 communicate() returns a tuple (stdout, stderr).""" 460 461 # Optimization: If we are only using one pipe, or no pipe at 462 # all, using select() or threads is unnecessary. 463 if [self.stdin, self.stdout, self.stderr].count(None) >= 2: 464 stdout = None 465 stderr = None 466 if self.stdin: 467 if input: 468 try: 469 self.stdin.write(input) 470 except IOError as e: 471 if e.errno != errno.EPIPE and e.errno != errno.EINVAL: 472 raise 473 self.stdin.close() 474 elif self.stdout: 475 stdout = _eintr_retry_call(self.stdout.read) 476 self.stdout.close() 477 elif self.stderr: 478 stderr = _eintr_retry_call(self.stderr.read) 479 self.stderr.close() 480 self.wait() 481 return (stdout, stderr) 482 483 return self._communicate(input) 484 485 486 def poll(self): 487 """Check if child process has terminated. Set and return returncode 488 attribute.""" 489 return self._internal_poll() 490 491 492 if mswindows: 493 # 494 # Windows methods 495 # 496 def _get_handles(self, stdin, stdout, stderr): 497 """Construct and return tuple with IO objects: 498 p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite 499 """ 500 to_close = set() 501 if stdin is None and stdout is None and stderr is None: 502 return (None, None, None, None, None, None), to_close 503 504 p2cread, p2cwrite = None, None 505 c2pread, c2pwrite = None, None 506 errread, errwrite = None, None 507 508 if stdin is None: 509 p2cread = _subprocess.GetStdHandle(_subprocess.STD_INPUT_HANDLE) 510 if p2cread is None: 511 p2cread, _ = _subprocess.CreatePipe(None, 0) 512 elif stdin == PIPE: 513 p2cread, p2cwrite = _subprocess.CreatePipe(None, 0) 514 elif isinstance(stdin, (int, long)): 515 p2cread = msvcrt.get_osfhandle(stdin) 516 else: 517 # Assuming file-like object 518 p2cread = msvcrt.get_osfhandle(stdin.fileno()) 519 p2cread = self._make_inheritable(p2cread) 520 # We just duplicated the handle, it has to be closed at the end 521 to_close.add(p2cread) 522 if stdin == PIPE: 523 to_close.add(p2cwrite) 524 525 if stdout is None: 526 c2pwrite = _subprocess.GetStdHandle(_subprocess.STD_OUTPUT_HANDLE) 527 if c2pwrite is None: 528 _, c2pwrite = _subprocess.CreatePipe(None, 0) 529 elif stdout == PIPE: 530 c2pread, c2pwrite = _subprocess.CreatePipe(None, 0) 531 elif isinstance(stdout, (int, long)): 532 c2pwrite = msvcrt.get_osfhandle(stdout) 533 else: 534 # Assuming file-like object 535 c2pwrite = msvcrt.get_osfhandle(stdout.fileno()) 536 c2pwrite = self._make_inheritable(c2pwrite) 537 # We just duplicated the handle, it has to be closed at the end 538 to_close.add(c2pwrite) 539 if stdout == PIPE: 540 to_close.add(c2pread) 541 542 if stderr is None: 543 errwrite = _subprocess.GetStdHandle(_subprocess.STD_ERROR_HANDLE) 544 if errwrite is None: 545 _, errwrite = _subprocess.CreatePipe(None, 0) 546 elif stderr == PIPE: 547 errread, errwrite = _subprocess.CreatePipe(None, 0) 548 elif stderr == STDOUT: 549 errwrite = c2pwrite 550 elif isinstance(stderr, (int, long)): 551 errwrite = msvcrt.get_osfhandle(stderr) 552 else: 553 # Assuming file-like object 554 errwrite = msvcrt.get_osfhandle(stderr.fileno()) 555 errwrite = self._make_inheritable(errwrite) 556 # We just duplicated the handle, it has to be closed at the end 557 to_close.add(errwrite) 558 if stderr == PIPE: 559 to_close.add(errread) 560 561 return (p2cread, p2cwrite, 562 c2pread, c2pwrite, 563 errread, errwrite), to_close 564 565 566 def _make_inheritable(self, handle): 567 """Return a duplicate of handle, which is inheritable""" 568 return _subprocess.DuplicateHandle(_subprocess.GetCurrentProcess(), 569 handle, _subprocess.GetCurrentProcess(), 0, 1, 570 _subprocess.DUPLICATE_SAME_ACCESS) 571 572 573 def _find_w9xpopen(self): 574 """Find and return absolut path to w9xpopen.exe""" 575 w9xpopen = os.path.join( 576 os.path.dirname(_subprocess.GetModuleFileName(0)), 577 "w9xpopen.exe") 578 if not os.path.exists(w9xpopen): 579 # Eeek - file-not-found - possibly an embedding 580 # situation - see if we can locate it in sys.exec_prefix 581 w9xpopen = os.path.join(os.path.dirname(sys.exec_prefix), 582 "w9xpopen.exe") 583 if not os.path.exists(w9xpopen): 584 raise RuntimeError("Cannot locate w9xpopen.exe, which is " 585 "needed for Popen to work with your " 586 "shell or platform.") 587 return w9xpopen 588 589 590 def _execute_child(self, args, executable, preexec_fn, close_fds, 591 cwd, env, universal_newlines, 592 startupinfo, creationflags, shell, to_close, 593 p2cread, p2cwrite, 594 c2pread, c2pwrite, 595 errread, errwrite): 596 """Execute program (MS Windows version)""" 597 598 if not isinstance(args, types.StringTypes): 599 args = list2cmdline(args) 600 601 # Process startup details 602 if startupinfo is None: 603 startupinfo = STARTUPINFO() 604 if None not in (p2cread, c2pwrite, errwrite): 605 startupinfo.dwFlags |= _subprocess.STARTF_USESTDHANDLES 606 startupinfo.hStdInput = p2cread 607 startupinfo.hStdOutput = c2pwrite 608 startupinfo.hStdError = errwrite 609 610 if shell: 611 startupinfo.dwFlags |= _subprocess.STARTF_USESHOWWINDOW 612 startupinfo.wShowWindow = _subprocess.SW_HIDE 613 comspec = os.environ.get("COMSPEC", "cmd.exe") 614 args = '{} /c "{}"'.format (comspec, args) 615 if (_subprocess.GetVersion() >= 0x80000000 or 616 os.path.basename(comspec).lower() == "command.com"): 617 # Win9x, or using command.com on NT. We need to 618 # use the w9xpopen intermediate program. For more 619 # information, see KB Q150956 620 # (http://web.archive.org/web/20011105084002/http://support.microsoft.com/support/kb/articles/Q150/9/56.asp) 621 w9xpopen = self._find_w9xpopen() 622 args = '"%s" %s' % (w9xpopen, args) 623 # Not passing CREATE_NEW_CONSOLE has been known to 624 # cause random failures on win9x. Specifically a 625 # dialog: "Your program accessed mem currently in 626 # use at xxx" and a hopeful warning about the 627 # stability of your system. Cost is Ctrl+C wont 628 # kill children. 629 creationflags |= _subprocess.CREATE_NEW_CONSOLE 630 631 def _close_in_parent(fd): 632 fd.Close() 633 to_close.remove(fd) 634 635 # Start the process 636 try: 637 hp, ht, pid, tid = _subprocess.CreateProcess(executable, args, 638 # no special security 639 None, None, 640 int(not close_fds), 641 creationflags, 642 env, 643 cwd, 644 startupinfo) 645 except pywintypes.error, e: 646 # Translate pywintypes.error to WindowsError, which is 647 # a subclass of OSError. FIXME: We should really 648 # translate errno using _sys_errlist (or similar), but 649 # how can this be done from Python? 650 raise WindowsError(*e.args) 651 finally: 652 # Child is launched. Close the parent's copy of those pipe 653 # handles that only the child should have open. You need 654 # to make sure that no handles to the write end of the 655 # output pipe are maintained in this process or else the 656 # pipe will not close when the child process exits and the 657 # ReadFile will hang. 658 if p2cread is not None: 659 _close_in_parent(p2cread) 660 if c2pwrite is not None: 661 _close_in_parent(c2pwrite) 662 if errwrite is not None: 663 _close_in_parent(errwrite) 664 665 # Retain the process handle, but close the thread handle 666 self._child_created = True 667 self._handle = hp 668 self.pid = pid 669 ht.Close() 670 671 def _internal_poll(self, _deadstate=None, 672 _WaitForSingleObject=_subprocess.WaitForSingleObject, 673 _WAIT_OBJECT_0=_subprocess.WAIT_OBJECT_0, 674 _GetExitCodeProcess=_subprocess.GetExitCodeProcess): 675 """Check if child process has terminated. Returns returncode 676 attribute. 677 678 This method is called by __del__, so it can only refer to objects 679 in its local scope. 680 681 """ 682 if self.returncode is None: 683 if _WaitForSingleObject(self._handle, 0) == _WAIT_OBJECT_0: 684 self.returncode = _GetExitCodeProcess(self._handle) 685 return self.returncode 686 687 688 def wait(self): 689 """Wait for child process to terminate. Returns returncode 690 attribute.""" 691 if self.returncode is None: 692 _subprocess.WaitForSingleObject(self._handle, 693 _subprocess.INFINITE) 694 self.returncode = _subprocess.GetExitCodeProcess(self._handle) 695 return self.returncode 696 697 698 def _readerthread(self, fh, buffer): 699 buffer.append(fh.read()) 700 701 702 def _communicate(self, input): 703 stdout = None # Return 704 stderr = None # Return 705 706 if self.stdout: 707 stdout = [] 708 stdout_thread = threading.Thread(target=self._readerthread, 709 args=(self.stdout, stdout)) 710 stdout_thread.setDaemon(True) 711 stdout_thread.start() 712 if self.stderr: 713 stderr = [] 714 stderr_thread = threading.Thread(target=self._readerthread, 715 args=(self.stderr, stderr)) 716 stderr_thread.setDaemon(True) 717 stderr_thread.start() 718 719 if self.stdin: 720 if input is not None: 721 try: 722 self.stdin.write(input) 723 except IOError as e: 724 if e.errno == errno.EPIPE: 725 # communicate() should ignore broken pipe error 726 pass 727 elif e.errno == errno.EINVAL: 728 # bpo-19612, bpo-30418: On Windows, stdin.write() 729 # fails with EINVAL if the child process exited or 730 # if the child process is still running but closed 731 # the pipe. 732 pass 733 else: 734 raise 735 self.stdin.close() 736 737 if self.stdout: 738 stdout_thread.join() 739 if self.stderr: 740 stderr_thread.join() 741 742 # All data exchanged. Translate lists into strings. 743 if stdout is not None: 744 stdout = stdout[0] 745 if stderr is not None: 746 stderr = stderr[0] 747 748 # Translate newlines, if requested. We cannot let the file 749 # object do the translation: It is based on stdio, which is 750 # impossible to combine with select (unless forcing no 751 # buffering). 752 if self.universal_newlines and hasattr(file, 'newlines'): 753 if stdout: 754 stdout = self._translate_newlines(stdout) 755 if stderr: 756 stderr = self._translate_newlines(stderr) 757 758 self.wait() 759 return (stdout, stderr) 760 761 def send_signal(self, sig): 762 """Send a signal to the process 763 """ 764 if sig == signal.SIGTERM: 765 self.terminate() 766 elif sig == signal.CTRL_C_EVENT: 767 os.kill(self.pid, signal.CTRL_C_EVENT) 768 elif sig == signal.CTRL_BREAK_EVENT: 769 os.kill(self.pid, signal.CTRL_BREAK_EVENT) 770 else: 771 raise ValueError("Unsupported signal: {}".format(sig)) 772 773 def terminate(self): 774 """Terminates the process 775 """ 776 try: 777 _subprocess.TerminateProcess(self._handle, 1) 778 except OSError as e: 779 # ERROR_ACCESS_DENIED (winerror 5) is received when the 780 # process already died. 781 if e.winerror != 5: 782 raise 783 rc = _subprocess.GetExitCodeProcess(self._handle) 784 if rc == _subprocess.STILL_ACTIVE: 785 raise 786 self.returncode = rc 787 788 kill = terminate 789 790 else: 791 # 792 # POSIX methods 793 # 794 def _get_handles(self, stdin, stdout, stderr): 795 """Construct and return tuple with IO objects: 796 p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite 797 """ 798 to_close = set() 799 p2cread, p2cwrite = None, None 800 c2pread, c2pwrite = None, None 801 errread, errwrite = None, None 802 803 if stdin is None: 804 pass 805 elif stdin == PIPE: 806 p2cread, p2cwrite = self.pipe_cloexec() 807 to_close.update((p2cread, p2cwrite)) 808 elif isinstance(stdin, (int, long)): 809 p2cread = stdin 810 else: 811 # Assuming file-like object 812 p2cread = stdin.fileno() 813 814 if stdout is None: 815 pass 816 elif stdout == PIPE: 817 c2pread, c2pwrite = self.pipe_cloexec() 818 to_close.update((c2pread, c2pwrite)) 819 elif isinstance(stdout, (int, long)): 820 c2pwrite = stdout 821 else: 822 # Assuming file-like object 823 c2pwrite = stdout.fileno() 824 825 if stderr is None: 826 pass 827 elif stderr == PIPE: 828 errread, errwrite = self.pipe_cloexec() 829 to_close.update((errread, errwrite)) 830 elif stderr == STDOUT: 831 if c2pwrite is not None: 832 errwrite = c2pwrite 833 else: # child's stdout is not set, use parent's stdout 834 errwrite = sys.__stdout__.fileno() 835 elif isinstance(stderr, (int, long)): 836 errwrite = stderr 837 else: 838 # Assuming file-like object 839 errwrite = stderr.fileno() 840 841 return (p2cread, p2cwrite, 842 c2pread, c2pwrite, 843 errread, errwrite), to_close 844 845 846 def _set_cloexec_flag(self, fd, cloexec=True): 847 try: 848 cloexec_flag = fcntl.FD_CLOEXEC 849 except AttributeError: 850 cloexec_flag = 1 851 852 old = fcntl.fcntl(fd, fcntl.F_GETFD) 853 if cloexec: 854 fcntl.fcntl(fd, fcntl.F_SETFD, old | cloexec_flag) 855 else: 856 fcntl.fcntl(fd, fcntl.F_SETFD, old & ~cloexec_flag) 857 858 859 def pipe_cloexec(self): 860 """Create a pipe with FDs set CLOEXEC.""" 861 # Pipes' FDs are set CLOEXEC by default because we don't want them 862 # to be inherited by other subprocesses: the CLOEXEC flag is removed 863 # from the child's FDs by _dup2(), between fork() and exec(). 864 # This is not atomic: we would need the pipe2() syscall for that. 865 r, w = os.pipe() 866 self._set_cloexec_flag(r) 867 self._set_cloexec_flag(w) 868 return r, w 869 870 871 def _close_fds(self, but): 872 if hasattr(os, 'closerange'): 873 os.closerange(3, but) 874 os.closerange(but + 1, MAXFD) 875 else: 876 for i in xrange(3, MAXFD): 877 if i == but: 878 continue 879 try: 880 os.close(i) 881 except: 882 pass 883 884 885 # Used as a bandaid workaround for https://bugs.python.org/issue27448 886 # to prevent multiple simultaneous subprocess launches from interfering 887 # with one another and leaving gc disabled. 888 if threading: 889 _disabling_gc_lock = threading.Lock() 890 else: 891 class _noop_context_manager(object): 892 # A dummy context manager that does nothing for the rare 893 # user of a --without-threads build. 894 def __enter__(self): pass 895 def __exit__(self, *args): pass 896 897 _disabling_gc_lock = _noop_context_manager() 898 899 900 def _execute_child(self, args, executable, preexec_fn, close_fds, 901 cwd, env, universal_newlines, 902 startupinfo, creationflags, shell, to_close, 903 p2cread, p2cwrite, 904 c2pread, c2pwrite, 905 errread, errwrite): 906 """Execute program (POSIX version)""" 907 908 if isinstance(args, types.StringTypes): 909 args = [args] 910 else: 911 args = list(args) 912 913 if shell: 914 args = ["/bin/sh", "-c"] + args 915 if executable: 916 args[0] = executable 917 918 if executable is None: 919 executable = args[0] 920 921 def _close_in_parent(fd): 922 os.close(fd) 923 to_close.remove(fd) 924 925 # For transferring possible exec failure from child to parent 926 # The first char specifies the exception type: 0 means 927 # OSError, 1 means some other error. 928 errpipe_read, errpipe_write = self.pipe_cloexec() 929 try: 930 try: 931 with self._disabling_gc_lock: 932 gc_was_enabled = gc.isenabled() 933 # Disable gc to avoid bug where gc -> file_dealloc -> 934 # write to stderr -> hang. 935 # https://bugs.python.org/issue1336 936 gc.disable() 937 try: 938 self.pid = os.fork() 939 except: 940 if gc_was_enabled: 941 gc.enable() 942 raise 943 self._child_created = True 944 if self.pid == 0: 945 # Child 946 try: 947 # Close parent's pipe ends 948 if p2cwrite is not None: 949 os.close(p2cwrite) 950 if c2pread is not None: 951 os.close(c2pread) 952 if errread is not None: 953 os.close(errread) 954 os.close(errpipe_read) 955 956 # When duping fds, if there arises a situation 957 # where one of the fds is either 0, 1 or 2, it 958 # is possible that it is overwritten (#12607). 959 if c2pwrite == 0: 960 c2pwrite = os.dup(c2pwrite) 961 if errwrite == 0 or errwrite == 1: 962 errwrite = os.dup(errwrite) 963 964 # Dup fds for child 965 def _dup2(a, b): 966 # dup2() removes the CLOEXEC flag but 967 # we must do it ourselves if dup2() 968 # would be a no-op (issue #10806). 969 if a == b: 970 self._set_cloexec_flag(a, False) 971 elif a is not None: 972 os.dup2(a, b) 973 _dup2(p2cread, 0) 974 _dup2(c2pwrite, 1) 975 _dup2(errwrite, 2) 976 977 # Close pipe fds. Make sure we don't close the 978 # same fd more than once, or standard fds. 979 closed = { None } 980 for fd in [p2cread, c2pwrite, errwrite]: 981 if fd not in closed and fd > 2: 982 os.close(fd) 983 closed.add(fd) 984 985 if cwd is not None: 986 os.chdir(cwd) 987 988 if preexec_fn: 989 preexec_fn() 990 991 # Close all other fds, if asked for - after 992 # preexec_fn(), which may open FDs. 993 if close_fds: 994 self._close_fds(but=errpipe_write) 995 996 if env is None: 997 os.execvp(executable, args) 998 else: 999 os.execvpe(executable, args, env) 1000 1001 except: 1002 exc_type, exc_value, tb = sys.exc_info() 1003 # Save the traceback and attach it to the exception object 1004 exc_lines = traceback.format_exception(exc_type, 1005 exc_value, 1006 tb) 1007 exc_value.child_traceback = ''.join(exc_lines) 1008 os.write(errpipe_write, pickle.dumps(exc_value)) 1009 1010 finally: 1011 # This exitcode won't be reported to applications, so it 1012 # really doesn't matter what we return. 1013 os._exit(255) 1014 1015 # Parent 1016 if gc_was_enabled: 1017 gc.enable() 1018 finally: 1019 # be sure the FD is closed no matter what 1020 os.close(errpipe_write) 1021 1022 # Wait for exec to fail or succeed; possibly raising exception 1023 data = _eintr_retry_call(os.read, errpipe_read, 1048576) 1024 pickle_bits = [] 1025 while data: 1026 pickle_bits.append(data) 1027 data = _eintr_retry_call(os.read, errpipe_read, 1048576) 1028 data = "".join(pickle_bits) 1029 finally: 1030 if p2cread is not None and p2cwrite is not None: 1031 _close_in_parent(p2cread) 1032 if c2pwrite is not None and c2pread is not None: 1033 _close_in_parent(c2pwrite) 1034 if errwrite is not None and errread is not None: 1035 _close_in_parent(errwrite) 1036 1037 # be sure the FD is closed no matter what 1038 os.close(errpipe_read) 1039 1040 if data != "": 1041 try: 1042 _eintr_retry_call(os.waitpid, self.pid, 0) 1043 except OSError as e: 1044 if e.errno != errno.ECHILD: 1045 raise 1046 child_exception = pickle.loads(data) 1047 raise child_exception 1048 1049 1050 def _handle_exitstatus(self, sts, _WIFSIGNALED=os.WIFSIGNALED, 1051 _WTERMSIG=os.WTERMSIG, _WIFEXITED=os.WIFEXITED, 1052 _WEXITSTATUS=os.WEXITSTATUS, _WIFSTOPPED=os.WIFSTOPPED, 1053 _WSTOPSIG=os.WSTOPSIG): 1054 # This method is called (indirectly) by __del__, so it cannot 1055 # refer to anything outside of its local scope. 1056 if _WIFSIGNALED(sts): 1057 self.returncode = -_WTERMSIG(sts) 1058 elif _WIFEXITED(sts): 1059 self.returncode = _WEXITSTATUS(sts) 1060 elif _WIFSTOPPED(sts): 1061 self.returncode = -_WSTOPSIG(sts) 1062 else: 1063 # Should never happen 1064 raise RuntimeError("Unknown child exit status!") 1065 1066 1067 def _internal_poll(self, _deadstate=None, _waitpid=os.waitpid, 1068 _WNOHANG=os.WNOHANG, _os_error=os.error, _ECHILD=errno.ECHILD): 1069 """Check if child process has terminated. Returns returncode 1070 attribute. 1071 1072 This method is called by __del__, so it cannot reference anything 1073 outside of the local scope (nor can any methods it calls). 1074 1075 """ 1076 if self.returncode is None: 1077 try: 1078 pid, sts = _waitpid(self.pid, _WNOHANG) 1079 if pid == self.pid: 1080 self._handle_exitstatus(sts) 1081 except _os_error as e: 1082 if _deadstate is not None: 1083 self.returncode = _deadstate 1084 if e.errno == _ECHILD: 1085 # This happens if SIGCLD is set to be ignored or 1086 # waiting for child processes has otherwise been 1087 # disabled for our process. This child is dead, we 1088 # can't get the status. 1089 # http://bugs.python.org/issue15756 1090 self.returncode = 0 1091 return self.returncode 1092 1093 1094 def wait(self): 1095 """Wait for child process to terminate. Returns returncode 1096 attribute.""" 1097 while self.returncode is None: 1098 try: 1099 pid, sts = _eintr_retry_call(os.waitpid, self.pid, 0) 1100 except OSError as e: 1101 if e.errno != errno.ECHILD: 1102 raise 1103 # This happens if SIGCLD is set to be ignored or waiting 1104 # for child processes has otherwise been disabled for our 1105 # process. This child is dead, we can't get the status. 1106 pid = self.pid 1107 sts = 0 1108 # Check the pid and loop as waitpid has been known to return 1109 # 0 even without WNOHANG in odd situations. issue14396. 1110 if pid == self.pid: 1111 self._handle_exitstatus(sts) 1112 return self.returncode 1113 1114 1115 def _communicate(self, input): 1116 if self.stdin: 1117 # Flush stdio buffer. This might block, if the user has 1118 # been writing to .stdin in an uncontrolled fashion. 1119 self.stdin.flush() 1120 if not input: 1121 self.stdin.close() 1122 1123 if _has_poll: 1124 stdout, stderr = self._communicate_with_poll(input) 1125 else: 1126 stdout, stderr = self._communicate_with_select(input) 1127 1128 # All data exchanged. Translate lists into strings. 1129 if stdout is not None: 1130 stdout = ''.join(stdout) 1131 if stderr is not None: 1132 stderr = ''.join(stderr) 1133 1134 # Translate newlines, if requested. We cannot let the file 1135 # object do the translation: It is based on stdio, which is 1136 # impossible to combine with select (unless forcing no 1137 # buffering). 1138 if self.universal_newlines and hasattr(file, 'newlines'): 1139 if stdout: 1140 stdout = self._translate_newlines(stdout) 1141 if stderr: 1142 stderr = self._translate_newlines(stderr) 1143 1144 self.wait() 1145 return (stdout, stderr) 1146 1147 1148 def _communicate_with_poll(self, input): 1149 stdout = None # Return 1150 stderr = None # Return 1151 fd2file = {} 1152 fd2output = {} 1153 1154 poller = select.poll() 1155 def register_and_append(file_obj, eventmask): 1156 poller.register(file_obj.fileno(), eventmask) 1157 fd2file[file_obj.fileno()] = file_obj 1158 1159 def close_unregister_and_remove(fd): 1160 poller.unregister(fd) 1161 fd2file[fd].close() 1162 fd2file.pop(fd) 1163 1164 if self.stdin and input: 1165 register_and_append(self.stdin, select.POLLOUT) 1166 1167 select_POLLIN_POLLPRI = select.POLLIN | select.POLLPRI 1168 if self.stdout: 1169 register_and_append(self.stdout, select_POLLIN_POLLPRI) 1170 fd2output[self.stdout.fileno()] = stdout = [] 1171 if self.stderr: 1172 register_and_append(self.stderr, select_POLLIN_POLLPRI) 1173 fd2output[self.stderr.fileno()] = stderr = [] 1174 1175 input_offset = 0 1176 while fd2file: 1177 try: 1178 ready = poller.poll() 1179 except select.error, e: 1180 if e.args[0] == errno.EINTR: 1181 continue 1182 raise 1183 1184 for fd, mode in ready: 1185 if mode & select.POLLOUT: 1186 chunk = input[input_offset : input_offset + _PIPE_BUF] 1187 try: 1188 input_offset += os.write(fd, chunk) 1189 except OSError as e: 1190 if e.errno == errno.EPIPE: 1191 close_unregister_and_remove(fd) 1192 else: 1193 raise 1194 else: 1195 if input_offset >= len(input): 1196 close_unregister_and_remove(fd) 1197 elif mode & select_POLLIN_POLLPRI: 1198 data = os.read(fd, 4096) 1199 if not data: 1200 close_unregister_and_remove(fd) 1201 fd2output[fd].append(data) 1202 else: 1203 # Ignore hang up or errors. 1204 close_unregister_and_remove(fd) 1205 1206 return (stdout, stderr) 1207 1208 1209 def _communicate_with_select(self, input): 1210 read_set = [] 1211 write_set = [] 1212 stdout = None # Return 1213 stderr = None # Return 1214 1215 if self.stdin and input: 1216 write_set.append(self.stdin) 1217 if self.stdout: 1218 read_set.append(self.stdout) 1219 stdout = [] 1220 if self.stderr: 1221 read_set.append(self.stderr) 1222 stderr = [] 1223 1224 input_offset = 0 1225 while read_set or write_set: 1226 try: 1227 rlist, wlist, xlist = select.select(read_set, write_set, []) 1228 except select.error, e: 1229 if e.args[0] == errno.EINTR: 1230 continue 1231 raise 1232 1233 if self.stdin in wlist: 1234 chunk = input[input_offset : input_offset + _PIPE_BUF] 1235 try: 1236 bytes_written = os.write(self.stdin.fileno(), chunk) 1237 except OSError as e: 1238 if e.errno == errno.EPIPE: 1239 self.stdin.close() 1240 write_set.remove(self.stdin) 1241 else: 1242 raise 1243 else: 1244 input_offset += bytes_written 1245 if input_offset >= len(input): 1246 self.stdin.close() 1247 write_set.remove(self.stdin) 1248 1249 if self.stdout in rlist: 1250 data = os.read(self.stdout.fileno(), 1024) 1251 if data == "": 1252 self.stdout.close() 1253 read_set.remove(self.stdout) 1254 stdout.append(data) 1255 1256 if self.stderr in rlist: 1257 data = os.read(self.stderr.fileno(), 1024) 1258 if data == "": 1259 self.stderr.close() 1260 read_set.remove(self.stderr) 1261 stderr.append(data) 1262 1263 return (stdout, stderr) 1264 1265 1266 def send_signal(self, sig): 1267 """Send a signal to the process 1268 """ 1269 os.kill(self.pid, sig) 1270 1271 def terminate(self): 1272 """Terminate the process with SIGTERM 1273 """ 1274 self.send_signal(signal.SIGTERM) 1275 1276 def kill(self): 1277 """Kill the process with SIGKILL 1278 """ 1279 self.send_signal(signal.SIGKILL) 1280 1281 1282def _demo_posix(): 1283 # 1284 # Example 1: Simple redirection: Get process list 1285 # 1286 plist = Popen(["ps"], stdout=PIPE).communicate()[0] 1287 print "Process list:" 1288 print plist 1289 1290 # 1291 # Example 2: Change uid before executing child 1292 # 1293 if os.getuid() == 0: 1294 p = Popen(["id"], preexec_fn=lambda: os.setuid(100)) 1295 p.wait() 1296 1297 # 1298 # Example 3: Connecting several subprocesses 1299 # 1300 print "Looking for 'hda'..." 1301 p1 = Popen(["dmesg"], stdout=PIPE) 1302 p2 = Popen(["grep", "hda"], stdin=p1.stdout, stdout=PIPE) 1303 print repr(p2.communicate()[0]) 1304 1305 # 1306 # Example 4: Catch execution error 1307 # 1308 print 1309 print "Trying a weird file..." 1310 try: 1311 print Popen(["/this/path/does/not/exist"]).communicate() 1312 except OSError, e: 1313 if e.errno == errno.ENOENT: 1314 print "The file didn't exist. I thought so..." 1315 print "Child traceback:" 1316 print e.child_traceback 1317 else: 1318 print "Error", e.errno 1319 else: 1320 print >>sys.stderr, "Gosh. No error." 1321 1322 1323def _demo_windows(): 1324 # 1325 # Example 1: Connecting several subprocesses 1326 # 1327 print "Looking for 'PROMPT' in set output..." 1328 p1 = Popen("set", stdout=PIPE, shell=True) 1329 p2 = Popen('find "PROMPT"', stdin=p1.stdout, stdout=PIPE) 1330 print repr(p2.communicate()[0]) 1331 1332 # 1333 # Example 2: Simple execution of program 1334 # 1335 print "Executing calc..." 1336 p = Popen("calc") 1337 p.wait() 1338 1339 1340if __name__ == "__main__": 1341 if mswindows: 1342 _demo_windows() 1343 else: 1344 _demo_posix() 1345