1import unittest
2from test import test_support
3
4import socket
5import urllib
6import sys
7import os
8import time
9
10try:
11    import ssl
12except ImportError:
13    ssl = None
14
15here = os.path.dirname(__file__)
16# Self-signed cert file for self-signed.pythontest.net
17CERT_selfsigned_pythontestdotnet = os.path.join(here, 'selfsigned_pythontestdotnet.pem')
18
19mimetools = test_support.import_module("mimetools", deprecated=True)
20
21
22def _open_with_retry(func, host, *args, **kwargs):
23    # Connecting to remote hosts is flaky.  Make it more robust
24    # by retrying the connection several times.
25    for i in range(3):
26        try:
27            return func(host, *args, **kwargs)
28        except IOError, last_exc:
29            continue
30        except:
31            raise
32    raise last_exc
33
34
35class URLTimeoutTest(unittest.TestCase):
36
37    TIMEOUT = 10.0
38
39    def setUp(self):
40        socket.setdefaulttimeout(self.TIMEOUT)
41
42    def tearDown(self):
43        socket.setdefaulttimeout(None)
44
45    def testURLread(self):
46        f = _open_with_retry(urllib.urlopen, "http://www.example.com/")
47        x = f.read()
48
49class urlopenNetworkTests(unittest.TestCase):
50    """Tests urllib.urlopen using the network.
51
52    These tests are not exhaustive.  Assuming that testing using files does a
53    good job overall of some of the basic interface features.  There are no
54    tests exercising the optional 'data' and 'proxies' arguments.  No tests
55    for transparent redirection have been written.
56
57    setUp is not used for always constructing a connection to
58    http://www.example.com/ since there a few tests that don't use that address
59    and making a connection is expensive enough to warrant minimizing unneeded
60    connections.
61
62    """
63
64    def urlopen(self, *args):
65        return _open_with_retry(urllib.urlopen, *args)
66
67    def test_basic(self):
68        # Simple test expected to pass.
69        open_url = self.urlopen("http://www.example.com/")
70        for attr in ("read", "readline", "readlines", "fileno", "close",
71                     "info", "geturl"):
72            self.assertTrue(hasattr(open_url, attr), "object returned from "
73                            "urlopen lacks the %s attribute" % attr)
74        try:
75            self.assertTrue(open_url.read(), "calling 'read' failed")
76        finally:
77            open_url.close()
78
79    def test_readlines(self):
80        # Test both readline and readlines.
81        open_url = self.urlopen("http://www.example.com/")
82        try:
83            self.assertIsInstance(open_url.readline(), basestring,
84                                  "readline did not return a string")
85            self.assertIsInstance(open_url.readlines(), list,
86                                  "readlines did not return a list")
87        finally:
88            open_url.close()
89
90    def test_info(self):
91        # Test 'info'.
92        open_url = self.urlopen("http://www.example.com/")
93        try:
94            info_obj = open_url.info()
95        finally:
96            open_url.close()
97            self.assertIsInstance(info_obj, mimetools.Message,
98                                  "object returned by 'info' is not an "
99                                  "instance of mimetools.Message")
100            self.assertEqual(info_obj.getsubtype(), "html")
101
102    def test_geturl(self):
103        # Make sure same URL as opened is returned by geturl.
104        URL = "http://www.example.com/"
105        open_url = self.urlopen(URL)
106        try:
107            gotten_url = open_url.geturl()
108        finally:
109            open_url.close()
110        self.assertEqual(gotten_url, URL)
111
112    def test_getcode(self):
113        # test getcode() with the fancy opener to get 404 error codes
114        URL = "http://www.pythontest.net/XXXinvalidXXX"
115        open_url = urllib.FancyURLopener().open(URL)
116        try:
117            code = open_url.getcode()
118        finally:
119            open_url.close()
120        self.assertEqual(code, 404)
121
122    @unittest.skipIf(sys.platform in ('win32',), 'not appropriate for Windows')
123    @unittest.skipUnless(hasattr(os, 'fdopen'), 'os.fdopen not available')
124    def test_fileno(self):
125        # Make sure fd returned by fileno is valid.
126        open_url = self.urlopen("http://www.example.com/")
127        fd = open_url.fileno()
128        FILE = os.fdopen(fd)
129        try:
130            self.assertTrue(FILE.read(), "reading from file created using fd "
131                                      "returned by fileno failed")
132        finally:
133            FILE.close()
134
135    def test_bad_address(self):
136        # Make sure proper exception is raised when connecting to a bogus
137        # address.
138        bogus_domain = "sadflkjsasf.i.nvali.d"
139        try:
140            socket.gethostbyname(bogus_domain)
141        except socket.gaierror:
142            pass
143        else:
144            # This happens with some overzealous DNS providers such as OpenDNS
145            self.skipTest("%r should not resolve for test to work" % bogus_domain)
146        self.assertRaises(IOError,
147                          # SF patch 809915:  In Sep 2003, VeriSign started
148                          # highjacking invalid .com and .net addresses to
149                          # boost traffic to their own site.  This test
150                          # started failing then.  One hopes the .invalid
151                          # domain will be spared to serve its defined
152                          # purpose.
153                          # urllib.urlopen, "http://www.sadflkjsasadf.com/")
154                          urllib.urlopen, "http://sadflkjsasf.i.nvali.d/")
155
156class urlretrieveNetworkTests(unittest.TestCase):
157    """Tests urllib.urlretrieve using the network."""
158
159    def urlretrieve(self, *args):
160        return _open_with_retry(urllib.urlretrieve, *args)
161
162    def test_basic(self):
163        # Test basic functionality.
164        file_location,info = self.urlretrieve("http://www.example.com/")
165        self.assertTrue(os.path.exists(file_location), "file location returned by"
166                        " urlretrieve is not a valid path")
167        FILE = file(file_location)
168        try:
169            self.assertTrue(FILE.read(), "reading from the file location returned"
170                         " by urlretrieve failed")
171        finally:
172            FILE.close()
173            os.unlink(file_location)
174
175    def test_specified_path(self):
176        # Make sure that specifying the location of the file to write to works.
177        file_location,info = self.urlretrieve("http://www.example.com/",
178                                              test_support.TESTFN)
179        self.assertEqual(file_location, test_support.TESTFN)
180        self.assertTrue(os.path.exists(file_location))
181        FILE = file(file_location)
182        try:
183            self.assertTrue(FILE.read(), "reading from temporary file failed")
184        finally:
185            FILE.close()
186            os.unlink(file_location)
187
188    def test_header(self):
189        # Make sure header returned as 2nd value from urlretrieve is good.
190        file_location, header = self.urlretrieve("http://www.example.com/")
191        os.unlink(file_location)
192        self.assertIsInstance(header, mimetools.Message,
193                              "header is not an instance of mimetools.Message")
194
195    def test_data_header(self):
196        logo = "http://www.example.com/"
197        file_location, fileheaders = self.urlretrieve(logo)
198        os.unlink(file_location)
199        datevalue = fileheaders.getheader('Date')
200        dateformat = '%a, %d %b %Y %H:%M:%S GMT'
201        try:
202            time.strptime(datevalue, dateformat)
203        except ValueError:
204            self.fail('Date value not in %r format', dateformat)
205
206
207@unittest.skipIf(ssl is None, "requires ssl")
208class urlopen_HttpsTests(unittest.TestCase):
209
210    def test_context_argument(self):
211        context = ssl.create_default_context(cafile=CERT_selfsigned_pythontestdotnet)
212        response = urllib.urlopen("https://self-signed.pythontest.net", context=context)
213        self.assertIn("Python", response.read())
214
215
216def test_main():
217    test_support.requires('network')
218    with test_support.check_py3k_warnings(
219            ("urllib.urlopen.. has been removed", DeprecationWarning)):
220        test_support.run_unittest(URLTimeoutTest,
221                                  urlopenNetworkTests,
222                                  urlretrieveNetworkTests,
223                                  urlopen_HttpsTests)
224
225if __name__ == "__main__":
226    test_main()
227