1"""Open an arbitrary URL.
2
3See the following document for more info on URLs:
4"Names and Addresses, URIs, URLs, URNs, URCs", at
5http://www.w3.org/pub/WWW/Addressing/Overview.html
6
7See also the HTTP spec (from which the error codes are derived):
8"HTTP - Hypertext Transfer Protocol", at
9http://www.w3.org/pub/WWW/Protocols/
10
11Related standards and specs:
12- RFC1808: the "relative URL" spec. (authoritative status)
13- RFC1738 - the "URL standard". (authoritative status)
14- RFC1630 - the "URI spec". (informational status)
15
16The object returned by URLopener().open(file) will differ per
17protocol.  All you know is that is has methods read(), readline(),
18readlines(), fileno(), close() and info().  The read*(), fileno()
19and close() methods work like those of open files.
20The info() method returns a mimetools.Message object which can be
21used to query various info about the object, if available.
22(mimetools.Message objects are queried with the getheader() method.)
23"""
24
25import string
26import socket
27import os
28import time
29import sys
30from urlparse import urljoin as basejoin
31
32__all__ = ["urlopen", "URLopener", "FancyURLopener", "urlretrieve",
33           "urlcleanup", "quote", "quote_plus", "unquote", "unquote_plus",
34           "urlencode", "url2pathname", "pathname2url", "splittag",
35           "localhost", "thishost", "ftperrors", "basejoin", "unwrap",
36           "splittype", "splithost", "splituser", "splitpasswd", "splitport",
37           "splitnport", "splitquery", "splitattr", "splitvalue",
38           "getproxies"]
39
40__version__ = '1.17'    # XXX This version is not always updated :-(
41
42MAXFTPCACHE = 10        # Trim the ftp cache beyond this size
43
44# Helper for non-unix systems
45if os.name == 'nt':
46    from nturl2path import url2pathname, pathname2url
47elif os.name == 'riscos':
48    from rourl2path import url2pathname, pathname2url
49else:
50    def url2pathname(pathname):
51        """OS-specific conversion from a relative URL of the 'file' scheme
52        to a file system path; not recommended for general use."""
53        return unquote(pathname)
54
55    def pathname2url(pathname):
56        """OS-specific conversion from a file system path to a relative URL
57        of the 'file' scheme; not recommended for general use."""
58        return quote(pathname)
59
60# This really consists of two pieces:
61# (1) a class which handles opening of all sorts of URLs
62#     (plus assorted utilities etc.)
63# (2) a set of functions for parsing URLs
64# XXX Should these be separated out into different modules?
65
66
67# Shortcut for basic usage
68_urlopener = None
69def urlopen(url, data=None, proxies=None):
70    """Create a file-like object for the specified URL to read from."""
71    from warnings import warnpy3k
72    warnpy3k("urllib.urlopen() has been removed in Python 3.0 in "
73             "favor of urllib2.urlopen()", stacklevel=2)
74
75    global _urlopener
76    if proxies is not None:
77        opener = FancyURLopener(proxies=proxies)
78    elif not _urlopener:
79        opener = FancyURLopener()
80        _urlopener = opener
81    else:
82        opener = _urlopener
83    if data is None:
84        return opener.open(url)
85    else:
86        return opener.open(url, data)
87def urlretrieve(url, filename=None, reporthook=None, data=None):
88    global _urlopener
89    if not _urlopener:
90        _urlopener = FancyURLopener()
91    return _urlopener.retrieve(url, filename, reporthook, data)
92def urlcleanup():
93    if _urlopener:
94        _urlopener.cleanup()
95    _safe_quoters.clear()
96    ftpcache.clear()
97
98# check for SSL
99try:
100    import ssl
101except:
102    _have_ssl = False
103else:
104    _have_ssl = True
105
106# exception raised when downloaded size does not match content-length
107class ContentTooShortError(IOError):
108    def __init__(self, message, content):
109        IOError.__init__(self, message)
110        self.content = content
111
112ftpcache = {}
113class URLopener:
114    """Class to open URLs.
115    This is a class rather than just a subroutine because we may need
116    more than one set of global protocol-specific options.
117    Note -- this is a base class for those who don't want the
118    automatic handling of errors type 302 (relocated) and 401
119    (authorization needed)."""
120
121    __tempfiles = None
122
123    version = "Python-urllib/%s" % __version__
124
125    # Constructor
126    def __init__(self, proxies=None, **x509):
127        if proxies is None:
128            proxies = getproxies()
129        assert hasattr(proxies, 'has_key'), "proxies must be a mapping"
130        self.proxies = proxies
131        self.key_file = x509.get('key_file')
132        self.cert_file = x509.get('cert_file')
133        self.addheaders = [('User-Agent', self.version)]
134        self.__tempfiles = []
135        self.__unlink = os.unlink # See cleanup()
136        self.tempcache = None
137        # Undocumented feature: if you assign {} to tempcache,
138        # it is used to cache files retrieved with
139        # self.retrieve().  This is not enabled by default
140        # since it does not work for changing documents (and I
141        # haven't got the logic to check expiration headers
142        # yet).
143        self.ftpcache = ftpcache
144        # Undocumented feature: you can use a different
145        # ftp cache by assigning to the .ftpcache member;
146        # in case you want logically independent URL openers
147        # XXX This is not threadsafe.  Bah.
148
149    def __del__(self):
150        self.close()
151
152    def close(self):
153        self.cleanup()
154
155    def cleanup(self):
156        # This code sometimes runs when the rest of this module
157        # has already been deleted, so it can't use any globals
158        # or import anything.
159        if self.__tempfiles:
160            for file in self.__tempfiles:
161                try:
162                    self.__unlink(file)
163                except OSError:
164                    pass
165            del self.__tempfiles[:]
166        if self.tempcache:
167            self.tempcache.clear()
168
169    def addheader(self, *args):
170        """Add a header to be used by the HTTP interface only
171        e.g. u.addheader('Accept', 'sound/basic')"""
172        self.addheaders.append(args)
173
174    # External interface
175    def open(self, fullurl, data=None):
176        """Use URLopener().open(file) instead of open(file, 'r')."""
177        fullurl = unwrap(toBytes(fullurl))
178        # percent encode url, fixing lame server errors for e.g, like space
179        # within url paths.
180        fullurl = quote(fullurl, safe="%/:=&?~#+!$,;'@()*[]|")
181        if self.tempcache and fullurl in self.tempcache:
182            filename, headers = self.tempcache[fullurl]
183            fp = open(filename, 'rb')
184            return addinfourl(fp, headers, fullurl)
185        urltype, url = splittype(fullurl)
186        if not urltype:
187            urltype = 'file'
188        if urltype in self.proxies:
189            proxy = self.proxies[urltype]
190            urltype, proxyhost = splittype(proxy)
191            host, selector = splithost(proxyhost)
192            url = (host, fullurl) # Signal special case to open_*()
193        else:
194            proxy = None
195        name = 'open_' + urltype
196        self.type = urltype
197        name = name.replace('-', '_')
198        if not hasattr(self, name):
199            if proxy:
200                return self.open_unknown_proxy(proxy, fullurl, data)
201            else:
202                return self.open_unknown(fullurl, data)
203        try:
204            if data is None:
205                return getattr(self, name)(url)
206            else:
207                return getattr(self, name)(url, data)
208        except socket.error, msg:
209            raise IOError, ('socket error', msg), sys.exc_info()[2]
210
211    def open_unknown(self, fullurl, data=None):
212        """Overridable interface to open unknown URL type."""
213        type, url = splittype(fullurl)
214        raise IOError, ('url error', 'unknown url type', type)
215
216    def open_unknown_proxy(self, proxy, fullurl, data=None):
217        """Overridable interface to open unknown URL type."""
218        type, url = splittype(fullurl)
219        raise IOError, ('url error', 'invalid proxy for %s' % type, proxy)
220
221    # External interface
222    def retrieve(self, url, filename=None, reporthook=None, data=None):
223        """retrieve(url) returns (filename, headers) for a local object
224        or (tempfilename, headers) for a remote object."""
225        url = unwrap(toBytes(url))
226        if self.tempcache and url in self.tempcache:
227            return self.tempcache[url]
228        type, url1 = splittype(url)
229        if filename is None and (not type or type == 'file'):
230            try:
231                fp = self.open_local_file(url1)
232                hdrs = fp.info()
233                fp.close()
234                return url2pathname(splithost(url1)[1]), hdrs
235            except IOError:
236                pass
237        fp = self.open(url, data)
238        try:
239            headers = fp.info()
240            if filename:
241                tfp = open(filename, 'wb')
242            else:
243                import tempfile
244                garbage, path = splittype(url)
245                garbage, path = splithost(path or "")
246                path, garbage = splitquery(path or "")
247                path, garbage = splitattr(path or "")
248                suffix = os.path.splitext(path)[1]
249                (fd, filename) = tempfile.mkstemp(suffix)
250                self.__tempfiles.append(filename)
251                tfp = os.fdopen(fd, 'wb')
252            try:
253                result = filename, headers
254                if self.tempcache is not None:
255                    self.tempcache[url] = result
256                bs = 1024*8
257                size = -1
258                read = 0
259                blocknum = 0
260                if reporthook:
261                    if "content-length" in headers:
262                        size = int(headers["Content-Length"])
263                    reporthook(blocknum, bs, size)
264                while 1:
265                    block = fp.read(bs)
266                    if block == "":
267                        break
268                    read += len(block)
269                    tfp.write(block)
270                    blocknum += 1
271                    if reporthook:
272                        reporthook(blocknum, bs, size)
273            finally:
274                tfp.close()
275        finally:
276            fp.close()
277
278        # raise exception if actual size does not match content-length header
279        if size >= 0 and read < size:
280            raise ContentTooShortError("retrieval incomplete: got only %i out "
281                                       "of %i bytes" % (read, size), result)
282
283        return result
284
285    # Each method named open_<type> knows how to open that type of URL
286
287    def open_http(self, url, data=None):
288        """Use HTTP protocol."""
289        import httplib
290        user_passwd = None
291        proxy_passwd= None
292        if isinstance(url, str):
293            host, selector = splithost(url)
294            if host:
295                user_passwd, host = splituser(host)
296                host = unquote(host)
297            realhost = host
298        else:
299            host, selector = url
300            # check whether the proxy contains authorization information
301            proxy_passwd, host = splituser(host)
302            # now we proceed with the url we want to obtain
303            urltype, rest = splittype(selector)
304            url = rest
305            user_passwd = None
306            if urltype.lower() != 'http':
307                realhost = None
308            else:
309                realhost, rest = splithost(rest)
310                if realhost:
311                    user_passwd, realhost = splituser(realhost)
312                if user_passwd:
313                    selector = "%s://%s%s" % (urltype, realhost, rest)
314                if proxy_bypass(realhost):
315                    host = realhost
316
317            #print "proxy via http:", host, selector
318        if not host: raise IOError, ('http error', 'no host given')
319
320        if proxy_passwd:
321            import base64
322            proxy_auth = base64.b64encode(proxy_passwd).strip()
323        else:
324            proxy_auth = None
325
326        if user_passwd:
327            import base64
328            auth = base64.b64encode(user_passwd).strip()
329        else:
330            auth = None
331        h = httplib.HTTP(host)
332        if data is not None:
333            h.putrequest('POST', selector)
334            h.putheader('Content-Type', 'application/x-www-form-urlencoded')
335            h.putheader('Content-Length', '%d' % len(data))
336        else:
337            h.putrequest('GET', selector)
338        if proxy_auth: h.putheader('Proxy-Authorization', 'Basic %s' % proxy_auth)
339        if auth: h.putheader('Authorization', 'Basic %s' % auth)
340        if realhost: h.putheader('Host', realhost)
341        for args in self.addheaders: h.putheader(*args)
342        h.endheaders(data)
343        errcode, errmsg, headers = h.getreply()
344        fp = h.getfile()
345        if errcode == -1:
346            if fp: fp.close()
347            # something went wrong with the HTTP status line
348            raise IOError, ('http protocol error', 0,
349                            'got a bad status line', None)
350        # According to RFC 2616, "2xx" code indicates that the client's
351        # request was successfully received, understood, and accepted.
352        if (200 <= errcode < 300):
353            return addinfourl(fp, headers, "http:" + url, errcode)
354        else:
355            if data is None:
356                return self.http_error(url, fp, errcode, errmsg, headers)
357            else:
358                return self.http_error(url, fp, errcode, errmsg, headers, data)
359
360    def http_error(self, url, fp, errcode, errmsg, headers, data=None):
361        """Handle http errors.
362        Derived class can override this, or provide specific handlers
363        named http_error_DDD where DDD is the 3-digit error code."""
364        # First check if there's a specific handler for this error
365        name = 'http_error_%d' % errcode
366        if hasattr(self, name):
367            method = getattr(self, name)
368            if data is None:
369                result = method(url, fp, errcode, errmsg, headers)
370            else:
371                result = method(url, fp, errcode, errmsg, headers, data)
372            if result: return result
373        return self.http_error_default(url, fp, errcode, errmsg, headers)
374
375    def http_error_default(self, url, fp, errcode, errmsg, headers):
376        """Default error handler: close the connection and raise IOError."""
377        void = fp.read()
378        fp.close()
379        raise IOError, ('http error', errcode, errmsg, headers)
380
381    if _have_ssl:
382        def open_https(self, url, data=None):
383            """Use HTTPS protocol."""
384
385            import httplib
386            user_passwd = None
387            proxy_passwd = None
388            if isinstance(url, str):
389                host, selector = splithost(url)
390                if host:
391                    user_passwd, host = splituser(host)
392                    host = unquote(host)
393                realhost = host
394            else:
395                host, selector = url
396                # here, we determine, whether the proxy contains authorization information
397                proxy_passwd, host = splituser(host)
398                urltype, rest = splittype(selector)
399                url = rest
400                user_passwd = None
401                if urltype.lower() != 'https':
402                    realhost = None
403                else:
404                    realhost, rest = splithost(rest)
405                    if realhost:
406                        user_passwd, realhost = splituser(realhost)
407                    if user_passwd:
408                        selector = "%s://%s%s" % (urltype, realhost, rest)
409                #print "proxy via https:", host, selector
410            if not host: raise IOError, ('https error', 'no host given')
411            if proxy_passwd:
412                import base64
413                proxy_auth = base64.b64encode(proxy_passwd).strip()
414            else:
415                proxy_auth = None
416            if user_passwd:
417                import base64
418                auth = base64.b64encode(user_passwd).strip()
419            else:
420                auth = None
421            h = httplib.HTTPS(host, 0,
422                              key_file=self.key_file,
423                              cert_file=self.cert_file)
424            if data is not None:
425                h.putrequest('POST', selector)
426                h.putheader('Content-Type',
427                            'application/x-www-form-urlencoded')
428                h.putheader('Content-Length', '%d' % len(data))
429            else:
430                h.putrequest('GET', selector)
431            if proxy_auth: h.putheader('Proxy-Authorization', 'Basic %s' % proxy_auth)
432            if auth: h.putheader('Authorization', 'Basic %s' % auth)
433            if realhost: h.putheader('Host', realhost)
434            for args in self.addheaders: h.putheader(*args)
435            h.endheaders(data)
436            errcode, errmsg, headers = h.getreply()
437            fp = h.getfile()
438            if errcode == -1:
439                if fp: fp.close()
440                # something went wrong with the HTTP status line
441                raise IOError, ('http protocol error', 0,
442                                'got a bad status line', None)
443            # According to RFC 2616, "2xx" code indicates that the client's
444            # request was successfully received, understood, and accepted.
445            if (200 <= errcode < 300):
446                return addinfourl(fp, headers, "https:" + url, errcode)
447            else:
448                if data is None:
449                    return self.http_error(url, fp, errcode, errmsg, headers)
450                else:
451                    return self.http_error(url, fp, errcode, errmsg, headers,
452                                           data)
453
454    def open_file(self, url):
455        """Use local file or FTP depending on form of URL."""
456        if not isinstance(url, str):
457            raise IOError, ('file error', 'proxy support for file protocol currently not implemented')
458        if url[:2] == '//' and url[2:3] != '/' and url[2:12].lower() != 'localhost/':
459            return self.open_ftp(url)
460        else:
461            return self.open_local_file(url)
462
463    def open_local_file(self, url):
464        """Use local file."""
465        import mimetypes, mimetools, email.utils
466        try:
467            from cStringIO import StringIO
468        except ImportError:
469            from StringIO import StringIO
470        host, file = splithost(url)
471        localname = url2pathname(file)
472        try:
473            stats = os.stat(localname)
474        except OSError, e:
475            raise IOError(e.errno, e.strerror, e.filename)
476        size = stats.st_size
477        modified = email.utils.formatdate(stats.st_mtime, usegmt=True)
478        mtype = mimetypes.guess_type(url)[0]
479        headers = mimetools.Message(StringIO(
480            'Content-Type: %s\nContent-Length: %d\nLast-modified: %s\n' %
481            (mtype or 'text/plain', size, modified)))
482        if not host:
483            urlfile = file
484            if file[:1] == '/':
485                urlfile = 'file://' + file
486            return addinfourl(open(localname, 'rb'),
487                              headers, urlfile)
488        host, port = splitport(host)
489        if not port \
490           and socket.gethostbyname(host) in (localhost(), thishost()):
491            urlfile = file
492            if file[:1] == '/':
493                urlfile = 'file://' + file
494            return addinfourl(open(localname, 'rb'),
495                              headers, urlfile)
496        raise IOError, ('local file error', 'not on local host')
497
498    def open_ftp(self, url):
499        """Use FTP protocol."""
500        if not isinstance(url, str):
501            raise IOError, ('ftp error', 'proxy support for ftp protocol currently not implemented')
502        import mimetypes, mimetools
503        try:
504            from cStringIO import StringIO
505        except ImportError:
506            from StringIO import StringIO
507        host, path = splithost(url)
508        if not host: raise IOError, ('ftp error', 'no host given')
509        host, port = splitport(host)
510        user, host = splituser(host)
511        if user: user, passwd = splitpasswd(user)
512        else: passwd = None
513        host = unquote(host)
514        user = user or ''
515        passwd = passwd or ''
516        host = socket.gethostbyname(host)
517        if not port:
518            import ftplib
519            port = ftplib.FTP_PORT
520        else:
521            port = int(port)
522        path, attrs = splitattr(path)
523        path = unquote(path)
524        dirs = path.split('/')
525        dirs, file = dirs[:-1], dirs[-1]
526        if dirs and not dirs[0]: dirs = dirs[1:]
527        if dirs and not dirs[0]: dirs[0] = '/'
528        key = user, host, port, '/'.join(dirs)
529        # XXX thread unsafe!
530        if len(self.ftpcache) > MAXFTPCACHE:
531            # Prune the cache, rather arbitrarily
532            for k in self.ftpcache.keys():
533                if k != key:
534                    v = self.ftpcache[k]
535                    del self.ftpcache[k]
536                    v.close()
537        try:
538            if not key in self.ftpcache:
539                self.ftpcache[key] = \
540                    ftpwrapper(user, passwd, host, port, dirs)
541            if not file: type = 'D'
542            else: type = 'I'
543            for attr in attrs:
544                attr, value = splitvalue(attr)
545                if attr.lower() == 'type' and \
546                   value in ('a', 'A', 'i', 'I', 'd', 'D'):
547                    type = value.upper()
548            (fp, retrlen) = self.ftpcache[key].retrfile(file, type)
549            mtype = mimetypes.guess_type("ftp:" + url)[0]
550            headers = ""
551            if mtype:
552                headers += "Content-Type: %s\n" % mtype
553            if retrlen is not None and retrlen >= 0:
554                headers += "Content-Length: %d\n" % retrlen
555            headers = mimetools.Message(StringIO(headers))
556            return addinfourl(fp, headers, "ftp:" + url)
557        except ftperrors(), msg:
558            raise IOError, ('ftp error', msg), sys.exc_info()[2]
559
560    def open_data(self, url, data=None):
561        """Use "data" URL."""
562        if not isinstance(url, str):
563            raise IOError, ('data error', 'proxy support for data protocol currently not implemented')
564        # ignore POSTed data
565        #
566        # syntax of data URLs:
567        # dataurl   := "data:" [ mediatype ] [ ";base64" ] "," data
568        # mediatype := [ type "/" subtype ] *( ";" parameter )
569        # data      := *urlchar
570        # parameter := attribute "=" value
571        import mimetools
572        try:
573            from cStringIO import StringIO
574        except ImportError:
575            from StringIO import StringIO
576        try:
577            [type, data] = url.split(',', 1)
578        except ValueError:
579            raise IOError, ('data error', 'bad data URL')
580        if not type:
581            type = 'text/plain;charset=US-ASCII'
582        semi = type.rfind(';')
583        if semi >= 0 and '=' not in type[semi:]:
584            encoding = type[semi+1:]
585            type = type[:semi]
586        else:
587            encoding = ''
588        msg = []
589        msg.append('Date: %s'%time.strftime('%a, %d %b %Y %H:%M:%S GMT',
590                                            time.gmtime(time.time())))
591        msg.append('Content-type: %s' % type)
592        if encoding == 'base64':
593            import base64
594            data = base64.decodestring(data)
595        else:
596            data = unquote(data)
597        msg.append('Content-Length: %d' % len(data))
598        msg.append('')
599        msg.append(data)
600        msg = '\n'.join(msg)
601        f = StringIO(msg)
602        headers = mimetools.Message(f, 0)
603        #f.fileno = None     # needed for addinfourl
604        return addinfourl(f, headers, url)
605
606
607class FancyURLopener(URLopener):
608    """Derived class with handlers for errors we can handle (perhaps)."""
609
610    def __init__(self, *args, **kwargs):
611        URLopener.__init__(self, *args, **kwargs)
612        self.auth_cache = {}
613        self.tries = 0
614        self.maxtries = 10
615
616    def http_error_default(self, url, fp, errcode, errmsg, headers):
617        """Default error handling -- don't raise an exception."""
618        return addinfourl(fp, headers, "http:" + url, errcode)
619
620    def http_error_302(self, url, fp, errcode, errmsg, headers, data=None):
621        """Error 302 -- relocated (temporarily)."""
622        self.tries += 1
623        if self.maxtries and self.tries >= self.maxtries:
624            if hasattr(self, "http_error_500"):
625                meth = self.http_error_500
626            else:
627                meth = self.http_error_default
628            self.tries = 0
629            return meth(url, fp, 500,
630                        "Internal Server Error: Redirect Recursion", headers)
631        result = self.redirect_internal(url, fp, errcode, errmsg, headers,
632                                        data)
633        self.tries = 0
634        return result
635
636    def redirect_internal(self, url, fp, errcode, errmsg, headers, data):
637        if 'location' in headers:
638            newurl = headers['location']
639        elif 'uri' in headers:
640            newurl = headers['uri']
641        else:
642            return
643        void = fp.read()
644        fp.close()
645        # In case the server sent a relative URL, join with original:
646        newurl = basejoin(self.type + ":" + url, newurl)
647
648        # For security reasons we do not allow redirects to protocols
649        # other than HTTP, HTTPS or FTP.
650        newurl_lower = newurl.lower()
651        if not (newurl_lower.startswith('http://') or
652                newurl_lower.startswith('https://') or
653                newurl_lower.startswith('ftp://')):
654            raise IOError('redirect error', errcode,
655                          errmsg + " - Redirection to url '%s' is not allowed" %
656                          newurl,
657                          headers)
658
659        return self.open(newurl)
660
661    def http_error_301(self, url, fp, errcode, errmsg, headers, data=None):
662        """Error 301 -- also relocated (permanently)."""
663        return self.http_error_302(url, fp, errcode, errmsg, headers, data)
664
665    def http_error_303(self, url, fp, errcode, errmsg, headers, data=None):
666        """Error 303 -- also relocated (essentially identical to 302)."""
667        return self.http_error_302(url, fp, errcode, errmsg, headers, data)
668
669    def http_error_307(self, url, fp, errcode, errmsg, headers, data=None):
670        """Error 307 -- relocated, but turn POST into error."""
671        if data is None:
672            return self.http_error_302(url, fp, errcode, errmsg, headers, data)
673        else:
674            return self.http_error_default(url, fp, errcode, errmsg, headers)
675
676    def http_error_401(self, url, fp, errcode, errmsg, headers, data=None):
677        """Error 401 -- authentication required.
678        This function supports Basic authentication only."""
679        if not 'www-authenticate' in headers:
680            URLopener.http_error_default(self, url, fp,
681                                         errcode, errmsg, headers)
682        stuff = headers['www-authenticate']
683        import re
684        match = re.match('[ \t]*([^ \t]+)[ \t]+realm="([^"]*)"', stuff)
685        if not match:
686            URLopener.http_error_default(self, url, fp,
687                                         errcode, errmsg, headers)
688        scheme, realm = match.groups()
689        if scheme.lower() != 'basic':
690            URLopener.http_error_default(self, url, fp,
691                                         errcode, errmsg, headers)
692        name = 'retry_' + self.type + '_basic_auth'
693        if data is None:
694            return getattr(self,name)(url, realm)
695        else:
696            return getattr(self,name)(url, realm, data)
697
698    def http_error_407(self, url, fp, errcode, errmsg, headers, data=None):
699        """Error 407 -- proxy authentication required.
700        This function supports Basic authentication only."""
701        if not 'proxy-authenticate' in headers:
702            URLopener.http_error_default(self, url, fp,
703                                         errcode, errmsg, headers)
704        stuff = headers['proxy-authenticate']
705        import re
706        match = re.match('[ \t]*([^ \t]+)[ \t]+realm="([^"]*)"', stuff)
707        if not match:
708            URLopener.http_error_default(self, url, fp,
709                                         errcode, errmsg, headers)
710        scheme, realm = match.groups()
711        if scheme.lower() != 'basic':
712            URLopener.http_error_default(self, url, fp,
713                                         errcode, errmsg, headers)
714        name = 'retry_proxy_' + self.type + '_basic_auth'
715        if data is None:
716            return getattr(self,name)(url, realm)
717        else:
718            return getattr(self,name)(url, realm, data)
719
720    def retry_proxy_http_basic_auth(self, url, realm, data=None):
721        host, selector = splithost(url)
722        newurl = 'http://' + host + selector
723        proxy = self.proxies['http']
724        urltype, proxyhost = splittype(proxy)
725        proxyhost, proxyselector = splithost(proxyhost)
726        i = proxyhost.find('@') + 1
727        proxyhost = proxyhost[i:]
728        user, passwd = self.get_user_passwd(proxyhost, realm, i)
729        if not (user or passwd): return None
730        proxyhost = quote(user, safe='') + ':' + quote(passwd, safe='') + '@' + proxyhost
731        self.proxies['http'] = 'http://' + proxyhost + proxyselector
732        if data is None:
733            return self.open(newurl)
734        else:
735            return self.open(newurl, data)
736
737    def retry_proxy_https_basic_auth(self, url, realm, data=None):
738        host, selector = splithost(url)
739        newurl = 'https://' + host + selector
740        proxy = self.proxies['https']
741        urltype, proxyhost = splittype(proxy)
742        proxyhost, proxyselector = splithost(proxyhost)
743        i = proxyhost.find('@') + 1
744        proxyhost = proxyhost[i:]
745        user, passwd = self.get_user_passwd(proxyhost, realm, i)
746        if not (user or passwd): return None
747        proxyhost = quote(user, safe='') + ':' + quote(passwd, safe='') + '@' + proxyhost
748        self.proxies['https'] = 'https://' + proxyhost + proxyselector
749        if data is None:
750            return self.open(newurl)
751        else:
752            return self.open(newurl, data)
753
754    def retry_http_basic_auth(self, url, realm, data=None):
755        host, selector = splithost(url)
756        i = host.find('@') + 1
757        host = host[i:]
758        user, passwd = self.get_user_passwd(host, realm, i)
759        if not (user or passwd): return None
760        host = quote(user, safe='') + ':' + quote(passwd, safe='') + '@' + host
761        newurl = 'http://' + host + selector
762        if data is None:
763            return self.open(newurl)
764        else:
765            return self.open(newurl, data)
766
767    def retry_https_basic_auth(self, url, realm, data=None):
768        host, selector = splithost(url)
769        i = host.find('@') + 1
770        host = host[i:]
771        user, passwd = self.get_user_passwd(host, realm, i)
772        if not (user or passwd): return None
773        host = quote(user, safe='') + ':' + quote(passwd, safe='') + '@' + host
774        newurl = 'https://' + host + selector
775        if data is None:
776            return self.open(newurl)
777        else:
778            return self.open(newurl, data)
779
780    def get_user_passwd(self, host, realm, clear_cache=0):
781        key = realm + '@' + host.lower()
782        if key in self.auth_cache:
783            if clear_cache:
784                del self.auth_cache[key]
785            else:
786                return self.auth_cache[key]
787        user, passwd = self.prompt_user_passwd(host, realm)
788        if user or passwd: self.auth_cache[key] = (user, passwd)
789        return user, passwd
790
791    def prompt_user_passwd(self, host, realm):
792        """Override this in a GUI environment!"""
793        import getpass
794        try:
795            user = raw_input("Enter username for %s at %s: " % (realm,
796                                                                host))
797            passwd = getpass.getpass("Enter password for %s in %s at %s: " %
798                (user, realm, host))
799            return user, passwd
800        except KeyboardInterrupt:
801            print
802            return None, None
803
804
805# Utility functions
806
807_localhost = None
808def localhost():
809    """Return the IP address of the magic hostname 'localhost'."""
810    global _localhost
811    if _localhost is None:
812        _localhost = socket.gethostbyname('localhost')
813    return _localhost
814
815_thishost = None
816def thishost():
817    """Return the IP address of the current host."""
818    global _thishost
819    if _thishost is None:
820        _thishost = socket.gethostbyname(socket.gethostname())
821    return _thishost
822
823_ftperrors = None
824def ftperrors():
825    """Return the set of errors raised by the FTP class."""
826    global _ftperrors
827    if _ftperrors is None:
828        import ftplib
829        _ftperrors = ftplib.all_errors
830    return _ftperrors
831
832_noheaders = None
833def noheaders():
834    """Return an empty mimetools.Message object."""
835    global _noheaders
836    if _noheaders is None:
837        import mimetools
838        try:
839            from cStringIO import StringIO
840        except ImportError:
841            from StringIO import StringIO
842        _noheaders = mimetools.Message(StringIO(), 0)
843        _noheaders.fp.close()   # Recycle file descriptor
844    return _noheaders
845
846
847# Utility classes
848
849class ftpwrapper:
850    """Class used by open_ftp() for cache of open FTP connections."""
851
852    def __init__(self, user, passwd, host, port, dirs,
853                 timeout=socket._GLOBAL_DEFAULT_TIMEOUT):
854        self.user = user
855        self.passwd = passwd
856        self.host = host
857        self.port = port
858        self.dirs = dirs
859        self.timeout = timeout
860        self.init()
861
862    def init(self):
863        import ftplib
864        self.busy = 0
865        self.ftp = ftplib.FTP()
866        self.ftp.connect(self.host, self.port, self.timeout)
867        self.ftp.login(self.user, self.passwd)
868        for dir in self.dirs:
869            self.ftp.cwd(dir)
870
871    def retrfile(self, file, type):
872        import ftplib
873        self.endtransfer()
874        if type in ('d', 'D'): cmd = 'TYPE A'; isdir = 1
875        else: cmd = 'TYPE ' + type; isdir = 0
876        try:
877            self.ftp.voidcmd(cmd)
878        except ftplib.all_errors:
879            self.init()
880            self.ftp.voidcmd(cmd)
881        conn = None
882        if file and not isdir:
883            # Try to retrieve as a file
884            try:
885                cmd = 'RETR ' + file
886                conn = self.ftp.ntransfercmd(cmd)
887            except ftplib.error_perm, reason:
888                if str(reason)[:3] != '550':
889                    raise IOError, ('ftp error', reason), sys.exc_info()[2]
890        if not conn:
891            # Set transfer mode to ASCII!
892            self.ftp.voidcmd('TYPE A')
893            # Try a directory listing. Verify that directory exists.
894            if file:
895                pwd = self.ftp.pwd()
896                try:
897                    try:
898                        self.ftp.cwd(file)
899                    except ftplib.error_perm, reason:
900                        raise IOError, ('ftp error', reason), sys.exc_info()[2]
901                finally:
902                    self.ftp.cwd(pwd)
903                cmd = 'LIST ' + file
904            else:
905                cmd = 'LIST'
906            conn = self.ftp.ntransfercmd(cmd)
907        self.busy = 1
908        # Pass back both a suitably decorated object and a retrieval length
909        return (addclosehook(conn[0].makefile('rb'),
910                             self.endtransfer), conn[1])
911    def endtransfer(self):
912        if not self.busy:
913            return
914        self.busy = 0
915        try:
916            self.ftp.voidresp()
917        except ftperrors():
918            pass
919
920    def close(self):
921        self.endtransfer()
922        try:
923            self.ftp.close()
924        except ftperrors():
925            pass
926
927class addbase:
928    """Base class for addinfo and addclosehook."""
929
930    def __init__(self, fp):
931        self.fp = fp
932        self.read = self.fp.read
933        self.readline = self.fp.readline
934        if hasattr(self.fp, "readlines"): self.readlines = self.fp.readlines
935        if hasattr(self.fp, "fileno"):
936            self.fileno = self.fp.fileno
937        else:
938            self.fileno = lambda: None
939        if hasattr(self.fp, "__iter__"):
940            self.__iter__ = self.fp.__iter__
941            if hasattr(self.fp, "next"):
942                self.next = self.fp.next
943
944    def __repr__(self):
945        return '<%s at %r whose fp = %r>' % (self.__class__.__name__,
946                                             id(self), self.fp)
947
948    def close(self):
949        self.read = None
950        self.readline = None
951        self.readlines = None
952        self.fileno = None
953        if self.fp: self.fp.close()
954        self.fp = None
955
956class addclosehook(addbase):
957    """Class to add a close hook to an open file."""
958
959    def __init__(self, fp, closehook, *hookargs):
960        addbase.__init__(self, fp)
961        self.closehook = closehook
962        self.hookargs = hookargs
963
964    def close(self):
965        addbase.close(self)
966        if self.closehook:
967            self.closehook(*self.hookargs)
968            self.closehook = None
969            self.hookargs = None
970
971class addinfo(addbase):
972    """class to add an info() method to an open file."""
973
974    def __init__(self, fp, headers):
975        addbase.__init__(self, fp)
976        self.headers = headers
977
978    def info(self):
979        return self.headers
980
981class addinfourl(addbase):
982    """class to add info() and geturl() methods to an open file."""
983
984    def __init__(self, fp, headers, url, code=None):
985        addbase.__init__(self, fp)
986        self.headers = headers
987        self.url = url
988        self.code = code
989
990    def info(self):
991        return self.headers
992
993    def getcode(self):
994        return self.code
995
996    def geturl(self):
997        return self.url
998
999
1000# Utilities to parse URLs (most of these return None for missing parts):
1001# unwrap('<URL:type://host/path>') --> 'type://host/path'
1002# splittype('type:opaquestring') --> 'type', 'opaquestring'
1003# splithost('//host[:port]/path') --> 'host[:port]', '/path'
1004# splituser('user[:passwd]@host[:port]') --> 'user[:passwd]', 'host[:port]'
1005# splitpasswd('user:passwd') -> 'user', 'passwd'
1006# splitport('host:port') --> 'host', 'port'
1007# splitquery('/path?query') --> '/path', 'query'
1008# splittag('/path#tag') --> '/path', 'tag'
1009# splitattr('/path;attr1=value1;attr2=value2;...') ->
1010#   '/path', ['attr1=value1', 'attr2=value2', ...]
1011# splitvalue('attr=value') --> 'attr', 'value'
1012# unquote('abc%20def') -> 'abc def'
1013# quote('abc def') -> 'abc%20def')
1014
1015try:
1016    unicode
1017except NameError:
1018    def _is_unicode(x):
1019        return 0
1020else:
1021    def _is_unicode(x):
1022        return isinstance(x, unicode)
1023
1024def toBytes(url):
1025    """toBytes(u"URL") --> 'URL'."""
1026    # Most URL schemes require ASCII. If that changes, the conversion
1027    # can be relaxed
1028    if _is_unicode(url):
1029        try:
1030            url = url.encode("ASCII")
1031        except UnicodeError:
1032            raise UnicodeError("URL " + repr(url) +
1033                               " contains non-ASCII characters")
1034    return url
1035
1036def unwrap(url):
1037    """unwrap('<URL:type://host/path>') --> 'type://host/path'."""
1038    url = url.strip()
1039    if url[:1] == '<' and url[-1:] == '>':
1040        url = url[1:-1].strip()
1041    if url[:4] == 'URL:': url = url[4:].strip()
1042    return url
1043
1044_typeprog = None
1045def splittype(url):
1046    """splittype('type:opaquestring') --> 'type', 'opaquestring'."""
1047    global _typeprog
1048    if _typeprog is None:
1049        import re
1050        _typeprog = re.compile('^([^/:]+):')
1051
1052    match = _typeprog.match(url)
1053    if match:
1054        scheme = match.group(1)
1055        return scheme.lower(), url[len(scheme) + 1:]
1056    return None, url
1057
1058_hostprog = None
1059def splithost(url):
1060    """splithost('//host[:port]/path') --> 'host[:port]', '/path'."""
1061    global _hostprog
1062    if _hostprog is None:
1063        import re
1064        _hostprog = re.compile('^//([^/?]*)(.*)$')
1065
1066    match = _hostprog.match(url)
1067    if match:
1068        host_port = match.group(1)
1069        path = match.group(2)
1070        if path and not path.startswith('/'):
1071            path = '/' + path
1072        return host_port, path
1073    return None, url
1074
1075_userprog = None
1076def splituser(host):
1077    """splituser('user[:passwd]@host[:port]') --> 'user[:passwd]', 'host[:port]'."""
1078    global _userprog
1079    if _userprog is None:
1080        import re
1081        _userprog = re.compile('^(.*)@(.*)$')
1082
1083    match = _userprog.match(host)
1084    if match: return match.group(1, 2)
1085    return None, host
1086
1087_passwdprog = None
1088def splitpasswd(user):
1089    """splitpasswd('user:passwd') -> 'user', 'passwd'."""
1090    global _passwdprog
1091    if _passwdprog is None:
1092        import re
1093        _passwdprog = re.compile('^([^:]*):(.*)$',re.S)
1094
1095    match = _passwdprog.match(user)
1096    if match: return match.group(1, 2)
1097    return user, None
1098
1099# splittag('/path#tag') --> '/path', 'tag'
1100_portprog = None
1101def splitport(host):
1102    """splitport('host:port') --> 'host', 'port'."""
1103    global _portprog
1104    if _portprog is None:
1105        import re
1106        _portprog = re.compile('^(.*):([0-9]+)$')
1107
1108    match = _portprog.match(host)
1109    if match: return match.group(1, 2)
1110    return host, None
1111
1112_nportprog = None
1113def splitnport(host, defport=-1):
1114    """Split host and port, returning numeric port.
1115    Return given default port if no ':' found; defaults to -1.
1116    Return numerical port if a valid number are found after ':'.
1117    Return None if ':' but not a valid number."""
1118    global _nportprog
1119    if _nportprog is None:
1120        import re
1121        _nportprog = re.compile('^(.*):(.*)$')
1122
1123    match = _nportprog.match(host)
1124    if match:
1125        host, port = match.group(1, 2)
1126        try:
1127            if not port: raise ValueError, "no digits"
1128            nport = int(port)
1129        except ValueError:
1130            nport = None
1131        return host, nport
1132    return host, defport
1133
1134_queryprog = None
1135def splitquery(url):
1136    """splitquery('/path?query') --> '/path', 'query'."""
1137    global _queryprog
1138    if _queryprog is None:
1139        import re
1140        _queryprog = re.compile('^(.*)\?([^?]*)$')
1141
1142    match = _queryprog.match(url)
1143    if match: return match.group(1, 2)
1144    return url, None
1145
1146_tagprog = None
1147def splittag(url):
1148    """splittag('/path#tag') --> '/path', 'tag'."""
1149    global _tagprog
1150    if _tagprog is None:
1151        import re
1152        _tagprog = re.compile('^(.*)#([^#]*)$')
1153
1154    match = _tagprog.match(url)
1155    if match: return match.group(1, 2)
1156    return url, None
1157
1158def splitattr(url):
1159    """splitattr('/path;attr1=value1;attr2=value2;...') ->
1160        '/path', ['attr1=value1', 'attr2=value2', ...]."""
1161    words = url.split(';')
1162    return words[0], words[1:]
1163
1164_valueprog = None
1165def splitvalue(attr):
1166    """splitvalue('attr=value') --> 'attr', 'value'."""
1167    global _valueprog
1168    if _valueprog is None:
1169        import re
1170        _valueprog = re.compile('^([^=]*)=(.*)$')
1171
1172    match = _valueprog.match(attr)
1173    if match: return match.group(1, 2)
1174    return attr, None
1175
1176# urlparse contains a duplicate of this method to avoid a circular import.  If
1177# you update this method, also update the copy in urlparse.  This code
1178# duplication does not exist in Python3.
1179
1180_hexdig = '0123456789ABCDEFabcdef'
1181_hextochr = dict((a + b, chr(int(a + b, 16)))
1182                 for a in _hexdig for b in _hexdig)
1183
1184def unquote(s):
1185    """unquote('abc%20def') -> 'abc def'."""
1186    res = s.split('%')
1187    # fastpath
1188    if len(res) == 1:
1189        return s
1190    s = res[0]
1191    for item in res[1:]:
1192        try:
1193            s += _hextochr[item[:2]] + item[2:]
1194        except KeyError:
1195            s += '%' + item
1196        except UnicodeDecodeError:
1197            s += unichr(int(item[:2], 16)) + item[2:]
1198    return s
1199
1200def unquote_plus(s):
1201    """unquote('%7e/abc+def') -> '~/abc def'"""
1202    s = s.replace('+', ' ')
1203    return unquote(s)
1204
1205always_safe = ('ABCDEFGHIJKLMNOPQRSTUVWXYZ'
1206               'abcdefghijklmnopqrstuvwxyz'
1207               '0123456789' '_.-')
1208_safe_map = {}
1209for i, c in zip(xrange(256), str(bytearray(xrange(256)))):
1210    _safe_map[c] = c if (i < 128 and c in always_safe) else '%{:02X}'.format(i)
1211_safe_quoters = {}
1212
1213def quote(s, safe='/'):
1214    """quote('abc def') -> 'abc%20def'
1215
1216    Each part of a URL, e.g. the path info, the query, etc., has a
1217    different set of reserved characters that must be quoted.
1218
1219    RFC 2396 Uniform Resource Identifiers (URI): Generic Syntax lists
1220    the following reserved characters.
1221
1222    reserved    = ";" | "/" | "?" | ":" | "@" | "&" | "=" | "+" |
1223                  "$" | ","
1224
1225    Each of these characters is reserved in some component of a URL,
1226    but not necessarily in all of them.
1227
1228    By default, the quote function is intended for quoting the path
1229    section of a URL.  Thus, it will not encode '/'.  This character
1230    is reserved, but in typical usage the quote function is being
1231    called on a path where the existing slash characters are used as
1232    reserved characters.
1233    """
1234    # fastpath
1235    if not s:
1236        if s is None:
1237            raise TypeError('None object cannot be quoted')
1238        return s
1239    cachekey = (safe, always_safe)
1240    try:
1241        (quoter, safe) = _safe_quoters[cachekey]
1242    except KeyError:
1243        safe_map = _safe_map.copy()
1244        safe_map.update([(c, c) for c in safe])
1245        quoter = safe_map.__getitem__
1246        safe = always_safe + safe
1247        _safe_quoters[cachekey] = (quoter, safe)
1248    if not s.rstrip(safe):
1249        return s
1250    return ''.join(map(quoter, s))
1251
1252def quote_plus(s, safe=''):
1253    """Quote the query fragment of a URL; replacing ' ' with '+'"""
1254    if ' ' in s:
1255        s = quote(s, safe + ' ')
1256        return s.replace(' ', '+')
1257    return quote(s, safe)
1258
1259def urlencode(query, doseq=0):
1260    """Encode a sequence of two-element tuples or dictionary into a URL query string.
1261
1262    If any values in the query arg are sequences and doseq is true, each
1263    sequence element is converted to a separate parameter.
1264
1265    If the query arg is a sequence of two-element tuples, the order of the
1266    parameters in the output will match the order of parameters in the
1267    input.
1268    """
1269
1270    if hasattr(query,"items"):
1271        # mapping objects
1272        query = query.items()
1273    else:
1274        # it's a bother at times that strings and string-like objects are
1275        # sequences...
1276        try:
1277            # non-sequence items should not work with len()
1278            # non-empty strings will fail this
1279            if len(query) and not isinstance(query[0], tuple):
1280                raise TypeError
1281            # zero-length sequences of all types will get here and succeed,
1282            # but that's a minor nit - since the original implementation
1283            # allowed empty dicts that type of behavior probably should be
1284            # preserved for consistency
1285        except TypeError:
1286            ty,va,tb = sys.exc_info()
1287            raise TypeError, "not a valid non-string sequence or mapping object", tb
1288
1289    l = []
1290    if not doseq:
1291        # preserve old behavior
1292        for k, v in query:
1293            k = quote_plus(str(k))
1294            v = quote_plus(str(v))
1295            l.append(k + '=' + v)
1296    else:
1297        for k, v in query:
1298            k = quote_plus(str(k))
1299            if isinstance(v, str):
1300                v = quote_plus(v)
1301                l.append(k + '=' + v)
1302            elif _is_unicode(v):
1303                # is there a reasonable way to convert to ASCII?
1304                # encode generates a string, but "replace" or "ignore"
1305                # lose information and "strict" can raise UnicodeError
1306                v = quote_plus(v.encode("ASCII","replace"))
1307                l.append(k + '=' + v)
1308            else:
1309                try:
1310                    # is this a sufficient test for sequence-ness?
1311                    len(v)
1312                except TypeError:
1313                    # not a sequence
1314                    v = quote_plus(str(v))
1315                    l.append(k + '=' + v)
1316                else:
1317                    # loop over the sequence
1318                    for elt in v:
1319                        l.append(k + '=' + quote_plus(str(elt)))
1320    return '&'.join(l)
1321
1322# Proxy handling
1323def getproxies_environment():
1324    """Return a dictionary of scheme -> proxy server URL mappings.
1325
1326    Scan the environment for variables named <scheme>_proxy;
1327    this seems to be the standard convention.  If you need a
1328    different way, you can pass a proxies dictionary to the
1329    [Fancy]URLopener constructor.
1330
1331    """
1332    proxies = {}
1333    for name, value in os.environ.items():
1334        name = name.lower()
1335        if value and name[-6:] == '_proxy':
1336            proxies[name[:-6]] = value
1337    return proxies
1338
1339def proxy_bypass_environment(host):
1340    """Test if proxies should not be used for a particular host.
1341
1342    Checks the environment for a variable named no_proxy, which should
1343    be a list of DNS suffixes separated by commas, or '*' for all hosts.
1344    """
1345    no_proxy = os.environ.get('no_proxy', '') or os.environ.get('NO_PROXY', '')
1346    # '*' is special case for always bypass
1347    if no_proxy == '*':
1348        return 1
1349    # strip port off host
1350    hostonly, port = splitport(host)
1351    # check if the host ends with any of the DNS suffixes
1352    for name in no_proxy.split(','):
1353        if name and (hostonly.endswith(name) or host.endswith(name)):
1354            return 1
1355    # otherwise, don't bypass
1356    return 0
1357
1358
1359if sys.platform == 'darwin':
1360    from _scproxy import _get_proxy_settings, _get_proxies
1361
1362    def proxy_bypass_macosx_sysconf(host):
1363        """
1364        Return True iff this host shouldn't be accessed using a proxy
1365
1366        This function uses the MacOSX framework SystemConfiguration
1367        to fetch the proxy information.
1368        """
1369        import re
1370        import socket
1371        from fnmatch import fnmatch
1372
1373        hostonly, port = splitport(host)
1374
1375        def ip2num(ipAddr):
1376            parts = ipAddr.split('.')
1377            parts = map(int, parts)
1378            if len(parts) != 4:
1379                parts = (parts + [0, 0, 0, 0])[:4]
1380            return (parts[0] << 24) | (parts[1] << 16) | (parts[2] << 8) | parts[3]
1381
1382        proxy_settings = _get_proxy_settings()
1383
1384        # Check for simple host names:
1385        if '.' not in host:
1386            if proxy_settings['exclude_simple']:
1387                return True
1388
1389        hostIP = None
1390
1391        for value in proxy_settings.get('exceptions', ()):
1392            # Items in the list are strings like these: *.local, 169.254/16
1393            if not value: continue
1394
1395            m = re.match(r"(\d+(?:\.\d+)*)(/\d+)?", value)
1396            if m is not None:
1397                if hostIP is None:
1398                    try:
1399                        hostIP = socket.gethostbyname(hostonly)
1400                        hostIP = ip2num(hostIP)
1401                    except socket.error:
1402                        continue
1403
1404                base = ip2num(m.group(1))
1405                mask = m.group(2)
1406                if mask is None:
1407                    mask = 8 * (m.group(1).count('.') + 1)
1408
1409                else:
1410                    mask = int(mask[1:])
1411                mask = 32 - mask
1412
1413                if (hostIP >> mask) == (base >> mask):
1414                    return True
1415
1416            elif fnmatch(host, value):
1417                return True
1418
1419        return False
1420
1421    def getproxies_macosx_sysconf():
1422        """Return a dictionary of scheme -> proxy server URL mappings.
1423
1424        This function uses the MacOSX framework SystemConfiguration
1425        to fetch the proxy information.
1426        """
1427        return _get_proxies()
1428
1429    def proxy_bypass(host):
1430        if getproxies_environment():
1431            return proxy_bypass_environment(host)
1432        else:
1433            return proxy_bypass_macosx_sysconf(host)
1434
1435    def getproxies():
1436        return getproxies_environment() or getproxies_macosx_sysconf()
1437
1438elif os.name == 'nt':
1439    def getproxies_registry():
1440        """Return a dictionary of scheme -> proxy server URL mappings.
1441
1442        Win32 uses the registry to store proxies.
1443
1444        """
1445        proxies = {}
1446        try:
1447            import _winreg
1448        except ImportError:
1449            # Std module, so should be around - but you never know!
1450            return proxies
1451        try:
1452            internetSettings = _winreg.OpenKey(_winreg.HKEY_CURRENT_USER,
1453                r'Software\Microsoft\Windows\CurrentVersion\Internet Settings')
1454            proxyEnable = _winreg.QueryValueEx(internetSettings,
1455                                               'ProxyEnable')[0]
1456            if proxyEnable:
1457                # Returned as Unicode but problems if not converted to ASCII
1458                proxyServer = str(_winreg.QueryValueEx(internetSettings,
1459                                                       'ProxyServer')[0])
1460                if '=' in proxyServer:
1461                    # Per-protocol settings
1462                    for p in proxyServer.split(';'):
1463                        protocol, address = p.split('=', 1)
1464                        # See if address has a type:// prefix
1465                        import re
1466                        if not re.match('^([^/:]+)://', address):
1467                            address = '%s://%s' % (protocol, address)
1468                        proxies[protocol] = address
1469                else:
1470                    # Use one setting for all protocols
1471                    if proxyServer[:5] == 'http:':
1472                        proxies['http'] = proxyServer
1473                    else:
1474                        proxies['http'] = 'http://%s' % proxyServer
1475                        proxies['https'] = 'https://%s' % proxyServer
1476                        proxies['ftp'] = 'ftp://%s' % proxyServer
1477            internetSettings.Close()
1478        except (WindowsError, ValueError, TypeError):
1479            # Either registry key not found etc, or the value in an
1480            # unexpected format.
1481            # proxies already set up to be empty so nothing to do
1482            pass
1483        return proxies
1484
1485    def getproxies():
1486        """Return a dictionary of scheme -> proxy server URL mappings.
1487
1488        Returns settings gathered from the environment, if specified,
1489        or the registry.
1490
1491        """
1492        return getproxies_environment() or getproxies_registry()
1493
1494    def proxy_bypass_registry(host):
1495        try:
1496            import _winreg
1497            import re
1498        except ImportError:
1499            # Std modules, so should be around - but you never know!
1500            return 0
1501        try:
1502            internetSettings = _winreg.OpenKey(_winreg.HKEY_CURRENT_USER,
1503                r'Software\Microsoft\Windows\CurrentVersion\Internet Settings')
1504            proxyEnable = _winreg.QueryValueEx(internetSettings,
1505                                               'ProxyEnable')[0]
1506            proxyOverride = str(_winreg.QueryValueEx(internetSettings,
1507                                                     'ProxyOverride')[0])
1508            # ^^^^ Returned as Unicode but problems if not converted to ASCII
1509        except WindowsError:
1510            return 0
1511        if not proxyEnable or not proxyOverride:
1512            return 0
1513        # try to make a host list from name and IP address.
1514        rawHost, port = splitport(host)
1515        host = [rawHost]
1516        try:
1517            addr = socket.gethostbyname(rawHost)
1518            if addr != rawHost:
1519                host.append(addr)
1520        except socket.error:
1521            pass
1522        try:
1523            fqdn = socket.getfqdn(rawHost)
1524            if fqdn != rawHost:
1525                host.append(fqdn)
1526        except socket.error:
1527            pass
1528        # make a check value list from the registry entry: replace the
1529        # '<local>' string by the localhost entry and the corresponding
1530        # canonical entry.
1531        proxyOverride = proxyOverride.split(';')
1532        # now check if we match one of the registry values.
1533        for test in proxyOverride:
1534            if test == '<local>':
1535                if '.' not in rawHost:
1536                    return 1
1537            test = test.replace(".", r"\.")     # mask dots
1538            test = test.replace("*", r".*")     # change glob sequence
1539            test = test.replace("?", r".")      # change glob char
1540            for val in host:
1541                # print "%s <--> %s" %( test, val )
1542                if re.match(test, val, re.I):
1543                    return 1
1544        return 0
1545
1546    def proxy_bypass(host):
1547        """Return a dictionary of scheme -> proxy server URL mappings.
1548
1549        Returns settings gathered from the environment, if specified,
1550        or the registry.
1551
1552        """
1553        if getproxies_environment():
1554            return proxy_bypass_environment(host)
1555        else:
1556            return proxy_bypass_registry(host)
1557
1558else:
1559    # By default use environment variables
1560    getproxies = getproxies_environment
1561    proxy_bypass = proxy_bypass_environment
1562
1563# Test and time quote() and unquote()
1564def test1():
1565    s = ''
1566    for i in range(256): s = s + chr(i)
1567    s = s*4
1568    t0 = time.time()
1569    qs = quote(s)
1570    uqs = unquote(qs)
1571    t1 = time.time()
1572    if uqs != s:
1573        print 'Wrong!'
1574    print repr(s)
1575    print repr(qs)
1576    print repr(uqs)
1577    print round(t1 - t0, 3), 'sec'
1578
1579
1580def reporthook(blocknum, blocksize, totalsize):
1581    # Report during remote transfers
1582    print "Block number: %d, Block size: %d, Total size: %d" % (
1583        blocknum, blocksize, totalsize)
1584
1585# Test program
1586def test(args=[]):
1587    if not args:
1588        args = [
1589            '/etc/passwd',
1590            'file:/etc/passwd',
1591            'file://localhost/etc/passwd',
1592            'ftp://ftp.gnu.org/pub/README',
1593            'http://www.python.org/index.html',
1594            ]
1595        if hasattr(URLopener, "open_https"):
1596            args.append('https://synergy.as.cmu.edu/~geek/')
1597    try:
1598        for url in args:
1599            print '-'*10, url, '-'*10
1600            fn, h = urlretrieve(url, None, reporthook)
1601            print fn
1602            if h:
1603                print '======'
1604                for k in h.keys(): print k + ':', h[k]
1605                print '======'
1606            with open(fn, 'rb') as fp:
1607                data = fp.read()
1608            if '\r' in data:
1609                table = string.maketrans("", "")
1610                data = data.translate(table, "\r")
1611            print data
1612            fn, h = None, None
1613        print '-'*40
1614    finally:
1615        urlcleanup()
1616
1617def main():
1618    import getopt, sys
1619    try:
1620        opts, args = getopt.getopt(sys.argv[1:], "th")
1621    except getopt.error, msg:
1622        print msg
1623        print "Use -h for help"
1624        return
1625    t = 0
1626    for o, a in opts:
1627        if o == '-t':
1628            t = t + 1
1629        if o == '-h':
1630            print "Usage: python urllib.py [-t] [url ...]"
1631            print "-t runs self-test;",
1632            print "otherwise, contents of urls are printed"
1633            return
1634    if t:
1635        if t > 1:
1636            test1()
1637        test(args)
1638    else:
1639        if not args:
1640            print "Use -h for help"
1641        for url in args:
1642            print urlopen(url).read(),
1643
1644# Run test program when run as a script
1645if __name__ == '__main__':
1646    main()
1647