1import unittest 2from test import support 3from test import test_urllib 4 5import os 6import io 7import socket 8import array 9import sys 10import tempfile 11import subprocess 12 13import urllib.request 14# The proxy bypass method imported below has logic specific to the OSX 15# proxy config data structure but is testable on all platforms. 16from urllib.request import (Request, OpenerDirector, HTTPBasicAuthHandler, 17 HTTPPasswordMgrWithPriorAuth, _parse_proxy, 18 _proxy_bypass_macosx_sysconf, 19 AbstractDigestAuthHandler) 20from urllib.parse import urlparse 21import urllib.error 22import http.client 23 24# XXX 25# Request 26# CacheFTPHandler (hard to write) 27# parse_keqv_list, parse_http_list, HTTPDigestAuthHandler 28 29 30class TrivialTests(unittest.TestCase): 31 32 def test___all__(self): 33 # Verify which names are exposed 34 for module in 'request', 'response', 'parse', 'error', 'robotparser': 35 context = {} 36 exec('from urllib.%s import *' % module, context) 37 del context['__builtins__'] 38 if module == 'request' and os.name == 'nt': 39 u, p = context.pop('url2pathname'), context.pop('pathname2url') 40 self.assertEqual(u.__module__, 'nturl2path') 41 self.assertEqual(p.__module__, 'nturl2path') 42 for k, v in context.items(): 43 self.assertEqual(v.__module__, 'urllib.%s' % module, 44 "%r is exposed in 'urllib.%s' but defined in %r" % 45 (k, module, v.__module__)) 46 47 def test_trivial(self): 48 # A couple trivial tests 49 50 self.assertRaises(ValueError, urllib.request.urlopen, 'bogus url') 51 52 # XXX Name hacking to get this to work on Windows. 53 fname = os.path.abspath(urllib.request.__file__).replace(os.sep, '/') 54 55 if os.name == 'nt': 56 file_url = "file:///%s" % fname 57 else: 58 file_url = "file://%s" % fname 59 60 f = urllib.request.urlopen(file_url) 61 62 f.read() 63 f.close() 64 65 def test_parse_http_list(self): 66 tests = [ 67 ('a,b,c', ['a', 'b', 'c']), 68 ('path"o,l"og"i"cal, example', ['path"o,l"og"i"cal', 'example']), 69 ('a, b, "c", "d", "e,f", g, h', 70 ['a', 'b', '"c"', '"d"', '"e,f"', 'g', 'h']), 71 ('a="b\\"c", d="e\\,f", g="h\\\\i"', 72 ['a="b"c"', 'd="e,f"', 'g="h\\i"'])] 73 for string, list in tests: 74 self.assertEqual(urllib.request.parse_http_list(string), list) 75 76 def test_URLError_reasonstr(self): 77 err = urllib.error.URLError('reason') 78 self.assertIn(err.reason, str(err)) 79 80 81class RequestHdrsTests(unittest.TestCase): 82 83 def test_request_headers_dict(self): 84 """ 85 The Request.headers dictionary is not a documented interface. It 86 should stay that way, because the complete set of headers are only 87 accessible through the .get_header(), .has_header(), .header_items() 88 interface. However, .headers pre-dates those methods, and so real code 89 will be using the dictionary. 90 91 The introduction in 2.4 of those methods was a mistake for the same 92 reason: code that previously saw all (urllib2 user)-provided headers in 93 .headers now sees only a subset. 94 95 """ 96 url = "http://example.com" 97 self.assertEqual(Request(url, 98 headers={"Spam-eggs": "blah"} 99 ).headers["Spam-eggs"], "blah") 100 self.assertEqual(Request(url, 101 headers={"spam-EggS": "blah"} 102 ).headers["Spam-eggs"], "blah") 103 104 def test_request_headers_methods(self): 105 """ 106 Note the case normalization of header names here, to 107 .capitalize()-case. This should be preserved for 108 backwards-compatibility. (In the HTTP case, normalization to 109 .title()-case is done by urllib2 before sending headers to 110 http.client). 111 112 Note that e.g. r.has_header("spam-EggS") is currently False, and 113 r.get_header("spam-EggS") returns None, but that could be changed in 114 future. 115 116 Method r.remove_header should remove items both from r.headers and 117 r.unredirected_hdrs dictionaries 118 """ 119 url = "http://example.com" 120 req = Request(url, headers={"Spam-eggs": "blah"}) 121 self.assertTrue(req.has_header("Spam-eggs")) 122 self.assertEqual(req.header_items(), [('Spam-eggs', 'blah')]) 123 124 req.add_header("Foo-Bar", "baz") 125 self.assertEqual(sorted(req.header_items()), 126 [('Foo-bar', 'baz'), ('Spam-eggs', 'blah')]) 127 self.assertFalse(req.has_header("Not-there")) 128 self.assertIsNone(req.get_header("Not-there")) 129 self.assertEqual(req.get_header("Not-there", "default"), "default") 130 131 req.remove_header("Spam-eggs") 132 self.assertFalse(req.has_header("Spam-eggs")) 133 134 req.add_unredirected_header("Unredirected-spam", "Eggs") 135 self.assertTrue(req.has_header("Unredirected-spam")) 136 137 req.remove_header("Unredirected-spam") 138 self.assertFalse(req.has_header("Unredirected-spam")) 139 140 def test_password_manager(self): 141 mgr = urllib.request.HTTPPasswordMgr() 142 add = mgr.add_password 143 find_user_pass = mgr.find_user_password 144 add("Some Realm", "http://example.com/", "joe", "password") 145 add("Some Realm", "http://example.com/ni", "ni", "ni") 146 add("c", "http://example.com/foo", "foo", "ni") 147 add("c", "http://example.com/bar", "bar", "nini") 148 add("b", "http://example.com/", "first", "blah") 149 add("b", "http://example.com/", "second", "spam") 150 add("a", "http://example.com", "1", "a") 151 add("Some Realm", "http://c.example.com:3128", "3", "c") 152 add("Some Realm", "d.example.com", "4", "d") 153 add("Some Realm", "e.example.com:3128", "5", "e") 154 155 self.assertEqual(find_user_pass("Some Realm", "example.com"), 156 ('joe', 'password')) 157 158 #self.assertEqual(find_user_pass("Some Realm", "http://example.com/ni"), 159 # ('ni', 'ni')) 160 161 self.assertEqual(find_user_pass("Some Realm", "http://example.com"), 162 ('joe', 'password')) 163 self.assertEqual(find_user_pass("Some Realm", "http://example.com/"), 164 ('joe', 'password')) 165 self.assertEqual( 166 find_user_pass("Some Realm", "http://example.com/spam"), 167 ('joe', 'password')) 168 self.assertEqual( 169 find_user_pass("Some Realm", "http://example.com/spam/spam"), 170 ('joe', 'password')) 171 self.assertEqual(find_user_pass("c", "http://example.com/foo"), 172 ('foo', 'ni')) 173 self.assertEqual(find_user_pass("c", "http://example.com/bar"), 174 ('bar', 'nini')) 175 self.assertEqual(find_user_pass("b", "http://example.com/"), 176 ('second', 'spam')) 177 178 # No special relationship between a.example.com and example.com: 179 180 self.assertEqual(find_user_pass("a", "http://example.com/"), 181 ('1', 'a')) 182 self.assertEqual(find_user_pass("a", "http://a.example.com/"), 183 (None, None)) 184 185 # Ports: 186 187 self.assertEqual(find_user_pass("Some Realm", "c.example.com"), 188 (None, None)) 189 self.assertEqual(find_user_pass("Some Realm", "c.example.com:3128"), 190 ('3', 'c')) 191 self.assertEqual( 192 find_user_pass("Some Realm", "http://c.example.com:3128"), 193 ('3', 'c')) 194 self.assertEqual(find_user_pass("Some Realm", "d.example.com"), 195 ('4', 'd')) 196 self.assertEqual(find_user_pass("Some Realm", "e.example.com:3128"), 197 ('5', 'e')) 198 199 def test_password_manager_default_port(self): 200 """ 201 The point to note here is that we can't guess the default port if 202 there's no scheme. This applies to both add_password and 203 find_user_password. 204 """ 205 mgr = urllib.request.HTTPPasswordMgr() 206 add = mgr.add_password 207 find_user_pass = mgr.find_user_password 208 add("f", "http://g.example.com:80", "10", "j") 209 add("g", "http://h.example.com", "11", "k") 210 add("h", "i.example.com:80", "12", "l") 211 add("i", "j.example.com", "13", "m") 212 self.assertEqual(find_user_pass("f", "g.example.com:100"), 213 (None, None)) 214 self.assertEqual(find_user_pass("f", "g.example.com:80"), 215 ('10', 'j')) 216 self.assertEqual(find_user_pass("f", "g.example.com"), 217 (None, None)) 218 self.assertEqual(find_user_pass("f", "http://g.example.com:100"), 219 (None, None)) 220 self.assertEqual(find_user_pass("f", "http://g.example.com:80"), 221 ('10', 'j')) 222 self.assertEqual(find_user_pass("f", "http://g.example.com"), 223 ('10', 'j')) 224 self.assertEqual(find_user_pass("g", "h.example.com"), ('11', 'k')) 225 self.assertEqual(find_user_pass("g", "h.example.com:80"), ('11', 'k')) 226 self.assertEqual(find_user_pass("g", "http://h.example.com:80"), 227 ('11', 'k')) 228 self.assertEqual(find_user_pass("h", "i.example.com"), (None, None)) 229 self.assertEqual(find_user_pass("h", "i.example.com:80"), ('12', 'l')) 230 self.assertEqual(find_user_pass("h", "http://i.example.com:80"), 231 ('12', 'l')) 232 self.assertEqual(find_user_pass("i", "j.example.com"), ('13', 'm')) 233 self.assertEqual(find_user_pass("i", "j.example.com:80"), 234 (None, None)) 235 self.assertEqual(find_user_pass("i", "http://j.example.com"), 236 ('13', 'm')) 237 self.assertEqual(find_user_pass("i", "http://j.example.com:80"), 238 (None, None)) 239 240 241class MockOpener: 242 addheaders = [] 243 244 def open(self, req, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT): 245 self.req, self.data, self.timeout = req, data, timeout 246 247 def error(self, proto, *args): 248 self.proto, self.args = proto, args 249 250 251class MockFile: 252 def read(self, count=None): 253 pass 254 255 def readline(self, count=None): 256 pass 257 258 def close(self): 259 pass 260 261 262class MockHeaders(dict): 263 def getheaders(self, name): 264 return list(self.values()) 265 266 267class MockResponse(io.StringIO): 268 def __init__(self, code, msg, headers, data, url=None): 269 io.StringIO.__init__(self, data) 270 self.code, self.msg, self.headers, self.url = code, msg, headers, url 271 272 def info(self): 273 return self.headers 274 275 def geturl(self): 276 return self.url 277 278 279class MockCookieJar: 280 def add_cookie_header(self, request): 281 self.ach_req = request 282 283 def extract_cookies(self, response, request): 284 self.ec_req, self.ec_r = request, response 285 286 287class FakeMethod: 288 def __init__(self, meth_name, action, handle): 289 self.meth_name = meth_name 290 self.handle = handle 291 self.action = action 292 293 def __call__(self, *args): 294 return self.handle(self.meth_name, self.action, *args) 295 296 297class MockHTTPResponse(io.IOBase): 298 def __init__(self, fp, msg, status, reason): 299 self.fp = fp 300 self.msg = msg 301 self.status = status 302 self.reason = reason 303 self.code = 200 304 305 def read(self): 306 return '' 307 308 def info(self): 309 return {} 310 311 def geturl(self): 312 return self.url 313 314 315class MockHTTPClass: 316 def __init__(self): 317 self.level = 0 318 self.req_headers = [] 319 self.data = None 320 self.raise_on_endheaders = False 321 self.sock = None 322 self._tunnel_headers = {} 323 324 def __call__(self, host, timeout=socket._GLOBAL_DEFAULT_TIMEOUT): 325 self.host = host 326 self.timeout = timeout 327 return self 328 329 def set_debuglevel(self, level): 330 self.level = level 331 332 def set_tunnel(self, host, port=None, headers=None): 333 self._tunnel_host = host 334 self._tunnel_port = port 335 if headers: 336 self._tunnel_headers = headers 337 else: 338 self._tunnel_headers.clear() 339 340 def request(self, method, url, body=None, headers=None, *, 341 encode_chunked=False): 342 self.method = method 343 self.selector = url 344 if headers is not None: 345 self.req_headers += headers.items() 346 self.req_headers.sort() 347 if body: 348 self.data = body 349 self.encode_chunked = encode_chunked 350 if self.raise_on_endheaders: 351 raise OSError() 352 353 def getresponse(self): 354 return MockHTTPResponse(MockFile(), {}, 200, "OK") 355 356 def close(self): 357 pass 358 359 360class MockHandler: 361 # useful for testing handler machinery 362 # see add_ordered_mock_handlers() docstring 363 handler_order = 500 364 365 def __init__(self, methods): 366 self._define_methods(methods) 367 368 def _define_methods(self, methods): 369 for spec in methods: 370 if len(spec) == 2: 371 name, action = spec 372 else: 373 name, action = spec, None 374 meth = FakeMethod(name, action, self.handle) 375 setattr(self.__class__, name, meth) 376 377 def handle(self, fn_name, action, *args, **kwds): 378 self.parent.calls.append((self, fn_name, args, kwds)) 379 if action is None: 380 return None 381 elif action == "return self": 382 return self 383 elif action == "return response": 384 res = MockResponse(200, "OK", {}, "") 385 return res 386 elif action == "return request": 387 return Request("http://blah/") 388 elif action.startswith("error"): 389 code = action[action.rfind(" ")+1:] 390 try: 391 code = int(code) 392 except ValueError: 393 pass 394 res = MockResponse(200, "OK", {}, "") 395 return self.parent.error("http", args[0], res, code, "", {}) 396 elif action == "raise": 397 raise urllib.error.URLError("blah") 398 assert False 399 400 def close(self): 401 pass 402 403 def add_parent(self, parent): 404 self.parent = parent 405 self.parent.calls = [] 406 407 def __lt__(self, other): 408 if not hasattr(other, "handler_order"): 409 # No handler_order, leave in original order. Yuck. 410 return True 411 return self.handler_order < other.handler_order 412 413 414def add_ordered_mock_handlers(opener, meth_spec): 415 """Create MockHandlers and add them to an OpenerDirector. 416 417 meth_spec: list of lists of tuples and strings defining methods to define 418 on handlers. eg: 419 420 [["http_error", "ftp_open"], ["http_open"]] 421 422 defines methods .http_error() and .ftp_open() on one handler, and 423 .http_open() on another. These methods just record their arguments and 424 return None. Using a tuple instead of a string causes the method to 425 perform some action (see MockHandler.handle()), eg: 426 427 [["http_error"], [("http_open", "return request")]] 428 429 defines .http_error() on one handler (which simply returns None), and 430 .http_open() on another handler, which returns a Request object. 431 432 """ 433 handlers = [] 434 count = 0 435 for meths in meth_spec: 436 class MockHandlerSubclass(MockHandler): 437 pass 438 439 h = MockHandlerSubclass(meths) 440 h.handler_order += count 441 h.add_parent(opener) 442 count = count + 1 443 handlers.append(h) 444 opener.add_handler(h) 445 return handlers 446 447 448def build_test_opener(*handler_instances): 449 opener = OpenerDirector() 450 for h in handler_instances: 451 opener.add_handler(h) 452 return opener 453 454 455class MockHTTPHandler(urllib.request.BaseHandler): 456 # useful for testing redirections and auth 457 # sends supplied headers and code as first response 458 # sends 200 OK as second response 459 def __init__(self, code, headers): 460 self.code = code 461 self.headers = headers 462 self.reset() 463 464 def reset(self): 465 self._count = 0 466 self.requests = [] 467 468 def http_open(self, req): 469 import email, copy 470 self.requests.append(copy.deepcopy(req)) 471 if self._count == 0: 472 self._count = self._count + 1 473 name = http.client.responses[self.code] 474 msg = email.message_from_string(self.headers) 475 return self.parent.error( 476 "http", req, MockFile(), self.code, name, msg) 477 else: 478 self.req = req 479 msg = email.message_from_string("\r\n\r\n") 480 return MockResponse(200, "OK", msg, "", req.get_full_url()) 481 482 483class MockHTTPSHandler(urllib.request.AbstractHTTPHandler): 484 # Useful for testing the Proxy-Authorization request by verifying the 485 # properties of httpcon 486 487 def __init__(self, debuglevel=0): 488 urllib.request.AbstractHTTPHandler.__init__(self, debuglevel=debuglevel) 489 self.httpconn = MockHTTPClass() 490 491 def https_open(self, req): 492 return self.do_open(self.httpconn, req) 493 494 495class MockHTTPHandlerCheckAuth(urllib.request.BaseHandler): 496 # useful for testing auth 497 # sends supplied code response 498 # checks if auth header is specified in request 499 def __init__(self, code): 500 self.code = code 501 self.has_auth_header = False 502 503 def reset(self): 504 self.has_auth_header = False 505 506 def http_open(self, req): 507 if req.has_header('Authorization'): 508 self.has_auth_header = True 509 name = http.client.responses[self.code] 510 return MockResponse(self.code, name, MockFile(), "", req.get_full_url()) 511 512 513 514class MockPasswordManager: 515 def add_password(self, realm, uri, user, password): 516 self.realm = realm 517 self.url = uri 518 self.user = user 519 self.password = password 520 521 def find_user_password(self, realm, authuri): 522 self.target_realm = realm 523 self.target_url = authuri 524 return self.user, self.password 525 526 527class OpenerDirectorTests(unittest.TestCase): 528 529 def test_add_non_handler(self): 530 class NonHandler(object): 531 pass 532 self.assertRaises(TypeError, 533 OpenerDirector().add_handler, NonHandler()) 534 535 def test_badly_named_methods(self): 536 # test work-around for three methods that accidentally follow the 537 # naming conventions for handler methods 538 # (*_open() / *_request() / *_response()) 539 540 # These used to call the accidentally-named methods, causing a 541 # TypeError in real code; here, returning self from these mock 542 # methods would either cause no exception, or AttributeError. 543 544 from urllib.error import URLError 545 546 o = OpenerDirector() 547 meth_spec = [ 548 [("do_open", "return self"), ("proxy_open", "return self")], 549 [("redirect_request", "return self")], 550 ] 551 add_ordered_mock_handlers(o, meth_spec) 552 o.add_handler(urllib.request.UnknownHandler()) 553 for scheme in "do", "proxy", "redirect": 554 self.assertRaises(URLError, o.open, scheme+"://example.com/") 555 556 def test_handled(self): 557 # handler returning non-None means no more handlers will be called 558 o = OpenerDirector() 559 meth_spec = [ 560 ["http_open", "ftp_open", "http_error_302"], 561 ["ftp_open"], 562 [("http_open", "return self")], 563 [("http_open", "return self")], 564 ] 565 handlers = add_ordered_mock_handlers(o, meth_spec) 566 567 req = Request("http://example.com/") 568 r = o.open(req) 569 # Second .http_open() gets called, third doesn't, since second returned 570 # non-None. Handlers without .http_open() never get any methods called 571 # on them. 572 # In fact, second mock handler defining .http_open() returns self 573 # (instead of response), which becomes the OpenerDirector's return 574 # value. 575 self.assertEqual(r, handlers[2]) 576 calls = [(handlers[0], "http_open"), (handlers[2], "http_open")] 577 for expected, got in zip(calls, o.calls): 578 handler, name, args, kwds = got 579 self.assertEqual((handler, name), expected) 580 self.assertEqual(args, (req,)) 581 582 def test_handler_order(self): 583 o = OpenerDirector() 584 handlers = [] 585 for meths, handler_order in [([("http_open", "return self")], 500), 586 (["http_open"], 0)]: 587 class MockHandlerSubclass(MockHandler): 588 pass 589 590 h = MockHandlerSubclass(meths) 591 h.handler_order = handler_order 592 handlers.append(h) 593 o.add_handler(h) 594 595 o.open("http://example.com/") 596 # handlers called in reverse order, thanks to their sort order 597 self.assertEqual(o.calls[0][0], handlers[1]) 598 self.assertEqual(o.calls[1][0], handlers[0]) 599 600 def test_raise(self): 601 # raising URLError stops processing of request 602 o = OpenerDirector() 603 meth_spec = [ 604 [("http_open", "raise")], 605 [("http_open", "return self")], 606 ] 607 handlers = add_ordered_mock_handlers(o, meth_spec) 608 609 req = Request("http://example.com/") 610 self.assertRaises(urllib.error.URLError, o.open, req) 611 self.assertEqual(o.calls, [(handlers[0], "http_open", (req,), {})]) 612 613 def test_http_error(self): 614 # XXX http_error_default 615 # http errors are a special case 616 o = OpenerDirector() 617 meth_spec = [ 618 [("http_open", "error 302")], 619 [("http_error_400", "raise"), "http_open"], 620 [("http_error_302", "return response"), "http_error_303", 621 "http_error"], 622 [("http_error_302")], 623 ] 624 handlers = add_ordered_mock_handlers(o, meth_spec) 625 626 class Unknown: 627 def __eq__(self, other): 628 return True 629 630 req = Request("http://example.com/") 631 o.open(req) 632 assert len(o.calls) == 2 633 calls = [(handlers[0], "http_open", (req,)), 634 (handlers[2], "http_error_302", 635 (req, Unknown(), 302, "", {}))] 636 for expected, got in zip(calls, o.calls): 637 handler, method_name, args = expected 638 self.assertEqual((handler, method_name), got[:2]) 639 self.assertEqual(args, got[2]) 640 641 def test_processors(self): 642 # *_request / *_response methods get called appropriately 643 o = OpenerDirector() 644 meth_spec = [ 645 [("http_request", "return request"), 646 ("http_response", "return response")], 647 [("http_request", "return request"), 648 ("http_response", "return response")], 649 ] 650 handlers = add_ordered_mock_handlers(o, meth_spec) 651 652 req = Request("http://example.com/") 653 o.open(req) 654 # processor methods are called on *all* handlers that define them, 655 # not just the first handler that handles the request 656 calls = [ 657 (handlers[0], "http_request"), (handlers[1], "http_request"), 658 (handlers[0], "http_response"), (handlers[1], "http_response")] 659 660 for i, (handler, name, args, kwds) in enumerate(o.calls): 661 if i < 2: 662 # *_request 663 self.assertEqual((handler, name), calls[i]) 664 self.assertEqual(len(args), 1) 665 self.assertIsInstance(args[0], Request) 666 else: 667 # *_response 668 self.assertEqual((handler, name), calls[i]) 669 self.assertEqual(len(args), 2) 670 self.assertIsInstance(args[0], Request) 671 # response from opener.open is None, because there's no 672 # handler that defines http_open to handle it 673 if args[1] is not None: 674 self.assertIsInstance(args[1], MockResponse) 675 676 677def sanepathname2url(path): 678 try: 679 path.encode("utf-8") 680 except UnicodeEncodeError: 681 raise unittest.SkipTest("path is not encodable to utf8") 682 urlpath = urllib.request.pathname2url(path) 683 if os.name == "nt" and urlpath.startswith("///"): 684 urlpath = urlpath[2:] 685 # XXX don't ask me about the mac... 686 return urlpath 687 688 689class HandlerTests(unittest.TestCase): 690 691 def test_ftp(self): 692 class MockFTPWrapper: 693 def __init__(self, data): 694 self.data = data 695 696 def retrfile(self, filename, filetype): 697 self.filename, self.filetype = filename, filetype 698 return io.StringIO(self.data), len(self.data) 699 700 def close(self): 701 pass 702 703 class NullFTPHandler(urllib.request.FTPHandler): 704 def __init__(self, data): 705 self.data = data 706 707 def connect_ftp(self, user, passwd, host, port, dirs, 708 timeout=socket._GLOBAL_DEFAULT_TIMEOUT): 709 self.user, self.passwd = user, passwd 710 self.host, self.port = host, port 711 self.dirs = dirs 712 self.ftpwrapper = MockFTPWrapper(self.data) 713 return self.ftpwrapper 714 715 import ftplib 716 data = "rheum rhaponicum" 717 h = NullFTPHandler(data) 718 h.parent = MockOpener() 719 720 for url, host, port, user, passwd, type_, dirs, filename, mimetype in [ 721 ("ftp://localhost/foo/bar/baz.html", 722 "localhost", ftplib.FTP_PORT, "", "", "I", 723 ["foo", "bar"], "baz.html", "text/html"), 724 ("ftp://parrot@localhost/foo/bar/baz.html", 725 "localhost", ftplib.FTP_PORT, "parrot", "", "I", 726 ["foo", "bar"], "baz.html", "text/html"), 727 ("ftp://%25parrot@localhost/foo/bar/baz.html", 728 "localhost", ftplib.FTP_PORT, "%parrot", "", "I", 729 ["foo", "bar"], "baz.html", "text/html"), 730 ("ftp://%2542parrot@localhost/foo/bar/baz.html", 731 "localhost", ftplib.FTP_PORT, "%42parrot", "", "I", 732 ["foo", "bar"], "baz.html", "text/html"), 733 ("ftp://localhost:80/foo/bar/", 734 "localhost", 80, "", "", "D", 735 ["foo", "bar"], "", None), 736 ("ftp://localhost/baz.gif;type=a", 737 "localhost", ftplib.FTP_PORT, "", "", "A", 738 [], "baz.gif", None), # XXX really this should guess image/gif 739 ]: 740 req = Request(url) 741 req.timeout = None 742 r = h.ftp_open(req) 743 # ftp authentication not yet implemented by FTPHandler 744 self.assertEqual(h.user, user) 745 self.assertEqual(h.passwd, passwd) 746 self.assertEqual(h.host, socket.gethostbyname(host)) 747 self.assertEqual(h.port, port) 748 self.assertEqual(h.dirs, dirs) 749 self.assertEqual(h.ftpwrapper.filename, filename) 750 self.assertEqual(h.ftpwrapper.filetype, type_) 751 headers = r.info() 752 self.assertEqual(headers.get("Content-type"), mimetype) 753 self.assertEqual(int(headers["Content-length"]), len(data)) 754 755 def test_file(self): 756 import email.utils 757 h = urllib.request.FileHandler() 758 o = h.parent = MockOpener() 759 760 TESTFN = support.TESTFN 761 urlpath = sanepathname2url(os.path.abspath(TESTFN)) 762 towrite = b"hello, world\n" 763 urls = [ 764 "file://localhost%s" % urlpath, 765 "file://%s" % urlpath, 766 "file://%s%s" % (socket.gethostbyname('localhost'), urlpath), 767 ] 768 try: 769 localaddr = socket.gethostbyname(socket.gethostname()) 770 except socket.gaierror: 771 localaddr = '' 772 if localaddr: 773 urls.append("file://%s%s" % (localaddr, urlpath)) 774 775 for url in urls: 776 f = open(TESTFN, "wb") 777 try: 778 try: 779 f.write(towrite) 780 finally: 781 f.close() 782 783 r = h.file_open(Request(url)) 784 try: 785 data = r.read() 786 headers = r.info() 787 respurl = r.geturl() 788 finally: 789 r.close() 790 stats = os.stat(TESTFN) 791 modified = email.utils.formatdate(stats.st_mtime, usegmt=True) 792 finally: 793 os.remove(TESTFN) 794 self.assertEqual(data, towrite) 795 self.assertEqual(headers["Content-type"], "text/plain") 796 self.assertEqual(headers["Content-length"], "13") 797 self.assertEqual(headers["Last-modified"], modified) 798 self.assertEqual(respurl, url) 799 800 for url in [ 801 "file://localhost:80%s" % urlpath, 802 "file:///file_does_not_exist.txt", 803 "file://not-a-local-host.com//dir/file.txt", 804 "file://%s:80%s/%s" % (socket.gethostbyname('localhost'), 805 os.getcwd(), TESTFN), 806 "file://somerandomhost.ontheinternet.com%s/%s" % 807 (os.getcwd(), TESTFN), 808 ]: 809 try: 810 f = open(TESTFN, "wb") 811 try: 812 f.write(towrite) 813 finally: 814 f.close() 815 816 self.assertRaises(urllib.error.URLError, 817 h.file_open, Request(url)) 818 finally: 819 os.remove(TESTFN) 820 821 h = urllib.request.FileHandler() 822 o = h.parent = MockOpener() 823 # XXXX why does // mean ftp (and /// mean not ftp!), and where 824 # is file: scheme specified? I think this is really a bug, and 825 # what was intended was to distinguish between URLs like: 826 # file:/blah.txt (a file) 827 # file://localhost/blah.txt (a file) 828 # file:///blah.txt (a file) 829 # file://ftp.example.com/blah.txt (an ftp URL) 830 for url, ftp in [ 831 ("file://ftp.example.com//foo.txt", False), 832 ("file://ftp.example.com///foo.txt", False), 833# XXXX bug: fails with OSError, should be URLError 834 ("file://ftp.example.com/foo.txt", False), 835 ("file://somehost//foo/something.txt", False), 836 ("file://localhost//foo/something.txt", False), 837 ]: 838 req = Request(url) 839 try: 840 h.file_open(req) 841 # XXXX remove OSError when bug fixed 842 except (urllib.error.URLError, OSError): 843 self.assertFalse(ftp) 844 else: 845 self.assertIs(o.req, req) 846 self.assertEqual(req.type, "ftp") 847 self.assertEqual(req.type == "ftp", ftp) 848 849 def test_http(self): 850 851 h = urllib.request.AbstractHTTPHandler() 852 o = h.parent = MockOpener() 853 854 url = "http://example.com/" 855 for method, data in [("GET", None), ("POST", b"blah")]: 856 req = Request(url, data, {"Foo": "bar"}) 857 req.timeout = None 858 req.add_unredirected_header("Spam", "eggs") 859 http = MockHTTPClass() 860 r = h.do_open(http, req) 861 862 # result attributes 863 r.read; r.readline # wrapped MockFile methods 864 r.info; r.geturl # addinfourl methods 865 r.code, r.msg == 200, "OK" # added from MockHTTPClass.getreply() 866 hdrs = r.info() 867 hdrs.get; hdrs.__contains__ # r.info() gives dict from .getreply() 868 self.assertEqual(r.geturl(), url) 869 870 self.assertEqual(http.host, "example.com") 871 self.assertEqual(http.level, 0) 872 self.assertEqual(http.method, method) 873 self.assertEqual(http.selector, "/") 874 self.assertEqual(http.req_headers, 875 [("Connection", "close"), 876 ("Foo", "bar"), ("Spam", "eggs")]) 877 self.assertEqual(http.data, data) 878 879 # check OSError converted to URLError 880 http.raise_on_endheaders = True 881 self.assertRaises(urllib.error.URLError, h.do_open, http, req) 882 883 # Check for TypeError on POST data which is str. 884 req = Request("http://example.com/","badpost") 885 self.assertRaises(TypeError, h.do_request_, req) 886 887 # check adding of standard headers 888 o.addheaders = [("Spam", "eggs")] 889 for data in b"", None: # POST, GET 890 req = Request("http://example.com/", data) 891 r = MockResponse(200, "OK", {}, "") 892 newreq = h.do_request_(req) 893 if data is None: # GET 894 self.assertNotIn("Content-length", req.unredirected_hdrs) 895 self.assertNotIn("Content-type", req.unredirected_hdrs) 896 else: # POST 897 self.assertEqual(req.unredirected_hdrs["Content-length"], "0") 898 self.assertEqual(req.unredirected_hdrs["Content-type"], 899 "application/x-www-form-urlencoded") 900 # XXX the details of Host could be better tested 901 self.assertEqual(req.unredirected_hdrs["Host"], "example.com") 902 self.assertEqual(req.unredirected_hdrs["Spam"], "eggs") 903 904 # don't clobber existing headers 905 req.add_unredirected_header("Content-length", "foo") 906 req.add_unredirected_header("Content-type", "bar") 907 req.add_unredirected_header("Host", "baz") 908 req.add_unredirected_header("Spam", "foo") 909 newreq = h.do_request_(req) 910 self.assertEqual(req.unredirected_hdrs["Content-length"], "foo") 911 self.assertEqual(req.unredirected_hdrs["Content-type"], "bar") 912 self.assertEqual(req.unredirected_hdrs["Host"], "baz") 913 self.assertEqual(req.unredirected_hdrs["Spam"], "foo") 914 915 def test_http_body_file(self): 916 # A regular file - chunked encoding is used unless Content Length is 917 # already set. 918 919 h = urllib.request.AbstractHTTPHandler() 920 o = h.parent = MockOpener() 921 922 file_obj = tempfile.NamedTemporaryFile(mode='w+b', delete=False) 923 file_path = file_obj.name 924 file_obj.close() 925 self.addCleanup(os.unlink, file_path) 926 927 with open(file_path, "rb") as f: 928 req = Request("http://example.com/", f, {}) 929 newreq = h.do_request_(req) 930 te = newreq.get_header('Transfer-encoding') 931 self.assertEqual(te, "chunked") 932 self.assertFalse(newreq.has_header('Content-length')) 933 934 with open(file_path, "rb") as f: 935 req = Request("http://example.com/", f, {"Content-Length": 30}) 936 newreq = h.do_request_(req) 937 self.assertEqual(int(newreq.get_header('Content-length')), 30) 938 self.assertFalse(newreq.has_header("Transfer-encoding")) 939 940 def test_http_body_fileobj(self): 941 # A file object - chunked encoding is used 942 # unless Content Length is already set. 943 # (Note that there are some subtle differences to a regular 944 # file, that is why we are testing both cases.) 945 946 h = urllib.request.AbstractHTTPHandler() 947 o = h.parent = MockOpener() 948 file_obj = io.BytesIO() 949 950 req = Request("http://example.com/", file_obj, {}) 951 newreq = h.do_request_(req) 952 self.assertEqual(newreq.get_header('Transfer-encoding'), 'chunked') 953 self.assertFalse(newreq.has_header('Content-length')) 954 955 headers = {"Content-Length": 30} 956 req = Request("http://example.com/", file_obj, headers) 957 newreq = h.do_request_(req) 958 self.assertEqual(int(newreq.get_header('Content-length')), 30) 959 self.assertFalse(newreq.has_header("Transfer-encoding")) 960 961 file_obj.close() 962 963 def test_http_body_pipe(self): 964 # A file reading from a pipe. 965 # A pipe cannot be seek'ed. There is no way to determine the 966 # content length up front. Thus, do_request_() should fall 967 # back to Transfer-encoding chunked. 968 969 h = urllib.request.AbstractHTTPHandler() 970 o = h.parent = MockOpener() 971 972 cmd = [sys.executable, "-c", r"pass"] 973 for headers in {}, {"Content-Length": 30}: 974 with subprocess.Popen(cmd, stdout=subprocess.PIPE) as proc: 975 req = Request("http://example.com/", proc.stdout, headers) 976 newreq = h.do_request_(req) 977 if not headers: 978 self.assertEqual(newreq.get_header('Content-length'), None) 979 self.assertEqual(newreq.get_header('Transfer-encoding'), 980 'chunked') 981 else: 982 self.assertEqual(int(newreq.get_header('Content-length')), 983 30) 984 985 def test_http_body_iterable(self): 986 # Generic iterable. There is no way to determine the content 987 # length up front. Fall back to Transfer-encoding chunked. 988 989 h = urllib.request.AbstractHTTPHandler() 990 o = h.parent = MockOpener() 991 992 def iterable_body(): 993 yield b"one" 994 995 for headers in {}, {"Content-Length": 11}: 996 req = Request("http://example.com/", iterable_body(), headers) 997 newreq = h.do_request_(req) 998 if not headers: 999 self.assertEqual(newreq.get_header('Content-length'), None) 1000 self.assertEqual(newreq.get_header('Transfer-encoding'), 1001 'chunked') 1002 else: 1003 self.assertEqual(int(newreq.get_header('Content-length')), 11) 1004 1005 def test_http_body_empty_seq(self): 1006 # Zero-length iterable body should be treated like any other iterable 1007 h = urllib.request.AbstractHTTPHandler() 1008 h.parent = MockOpener() 1009 req = h.do_request_(Request("http://example.com/", ())) 1010 self.assertEqual(req.get_header("Transfer-encoding"), "chunked") 1011 self.assertFalse(req.has_header("Content-length")) 1012 1013 def test_http_body_array(self): 1014 # array.array Iterable - Content Length is calculated 1015 1016 h = urllib.request.AbstractHTTPHandler() 1017 o = h.parent = MockOpener() 1018 1019 iterable_array = array.array("I",[1,2,3,4]) 1020 1021 for headers in {}, {"Content-Length": 16}: 1022 req = Request("http://example.com/", iterable_array, headers) 1023 newreq = h.do_request_(req) 1024 self.assertEqual(int(newreq.get_header('Content-length')),16) 1025 1026 def test_http_handler_debuglevel(self): 1027 o = OpenerDirector() 1028 h = MockHTTPSHandler(debuglevel=1) 1029 o.add_handler(h) 1030 o.open("https://www.example.com") 1031 self.assertEqual(h._debuglevel, 1) 1032 1033 def test_http_doubleslash(self): 1034 # Checks the presence of any unnecessary double slash in url does not 1035 # break anything. Previously, a double slash directly after the host 1036 # could cause incorrect parsing. 1037 h = urllib.request.AbstractHTTPHandler() 1038 h.parent = MockOpener() 1039 1040 data = b"" 1041 ds_urls = [ 1042 "http://example.com/foo/bar/baz.html", 1043 "http://example.com//foo/bar/baz.html", 1044 "http://example.com/foo//bar/baz.html", 1045 "http://example.com/foo/bar//baz.html" 1046 ] 1047 1048 for ds_url in ds_urls: 1049 ds_req = Request(ds_url, data) 1050 1051 # Check whether host is determined correctly if there is no proxy 1052 np_ds_req = h.do_request_(ds_req) 1053 self.assertEqual(np_ds_req.unredirected_hdrs["Host"], "example.com") 1054 1055 # Check whether host is determined correctly if there is a proxy 1056 ds_req.set_proxy("someproxy:3128", None) 1057 p_ds_req = h.do_request_(ds_req) 1058 self.assertEqual(p_ds_req.unredirected_hdrs["Host"], "example.com") 1059 1060 def test_full_url_setter(self): 1061 # Checks to ensure that components are set correctly after setting the 1062 # full_url of a Request object 1063 1064 urls = [ 1065 'http://example.com?foo=bar#baz', 1066 'http://example.com?foo=bar&spam=eggs#bash', 1067 'http://example.com', 1068 ] 1069 1070 # testing a reusable request instance, but the url parameter is 1071 # required, so just use a dummy one to instantiate 1072 r = Request('http://example.com') 1073 for url in urls: 1074 r.full_url = url 1075 parsed = urlparse(url) 1076 1077 self.assertEqual(r.get_full_url(), url) 1078 # full_url setter uses splittag to split into components. 1079 # splittag sets the fragment as None while urlparse sets it to '' 1080 self.assertEqual(r.fragment or '', parsed.fragment) 1081 self.assertEqual(urlparse(r.get_full_url()).query, parsed.query) 1082 1083 def test_full_url_deleter(self): 1084 r = Request('http://www.example.com') 1085 del r.full_url 1086 self.assertIsNone(r.full_url) 1087 self.assertIsNone(r.fragment) 1088 self.assertEqual(r.selector, '') 1089 1090 def test_fixpath_in_weirdurls(self): 1091 # Issue4493: urllib2 to supply '/' when to urls where path does not 1092 # start with'/' 1093 1094 h = urllib.request.AbstractHTTPHandler() 1095 h.parent = MockOpener() 1096 1097 weird_url = 'http://www.python.org?getspam' 1098 req = Request(weird_url) 1099 newreq = h.do_request_(req) 1100 self.assertEqual(newreq.host, 'www.python.org') 1101 self.assertEqual(newreq.selector, '/?getspam') 1102 1103 url_without_path = 'http://www.python.org' 1104 req = Request(url_without_path) 1105 newreq = h.do_request_(req) 1106 self.assertEqual(newreq.host, 'www.python.org') 1107 self.assertEqual(newreq.selector, '') 1108 1109 def test_errors(self): 1110 h = urllib.request.HTTPErrorProcessor() 1111 o = h.parent = MockOpener() 1112 1113 url = "http://example.com/" 1114 req = Request(url) 1115 # all 2xx are passed through 1116 r = MockResponse(200, "OK", {}, "", url) 1117 newr = h.http_response(req, r) 1118 self.assertIs(r, newr) 1119 self.assertFalse(hasattr(o, "proto")) # o.error not called 1120 r = MockResponse(202, "Accepted", {}, "", url) 1121 newr = h.http_response(req, r) 1122 self.assertIs(r, newr) 1123 self.assertFalse(hasattr(o, "proto")) # o.error not called 1124 r = MockResponse(206, "Partial content", {}, "", url) 1125 newr = h.http_response(req, r) 1126 self.assertIs(r, newr) 1127 self.assertFalse(hasattr(o, "proto")) # o.error not called 1128 # anything else calls o.error (and MockOpener returns None, here) 1129 r = MockResponse(502, "Bad gateway", {}, "", url) 1130 self.assertIsNone(h.http_response(req, r)) 1131 self.assertEqual(o.proto, "http") # o.error called 1132 self.assertEqual(o.args, (req, r, 502, "Bad gateway", {})) 1133 1134 def test_cookies(self): 1135 cj = MockCookieJar() 1136 h = urllib.request.HTTPCookieProcessor(cj) 1137 h.parent = MockOpener() 1138 1139 req = Request("http://example.com/") 1140 r = MockResponse(200, "OK", {}, "") 1141 newreq = h.http_request(req) 1142 self.assertIs(cj.ach_req, req) 1143 self.assertIs(cj.ach_req, newreq) 1144 self.assertEqual(req.origin_req_host, "example.com") 1145 self.assertFalse(req.unverifiable) 1146 newr = h.http_response(req, r) 1147 self.assertIs(cj.ec_req, req) 1148 self.assertIs(cj.ec_r, r) 1149 self.assertIs(r, newr) 1150 1151 def test_redirect(self): 1152 from_url = "http://example.com/a.html" 1153 to_url = "http://example.com/b.html" 1154 h = urllib.request.HTTPRedirectHandler() 1155 o = h.parent = MockOpener() 1156 1157 # ordinary redirect behaviour 1158 for code in 301, 302, 303, 307: 1159 for data in None, "blah\nblah\n": 1160 method = getattr(h, "http_error_%s" % code) 1161 req = Request(from_url, data) 1162 req.timeout = socket._GLOBAL_DEFAULT_TIMEOUT 1163 req.add_header("Nonsense", "viking=withhold") 1164 if data is not None: 1165 req.add_header("Content-Length", str(len(data))) 1166 req.add_unredirected_header("Spam", "spam") 1167 try: 1168 method(req, MockFile(), code, "Blah", 1169 MockHeaders({"location": to_url})) 1170 except urllib.error.HTTPError: 1171 # 307 in response to POST requires user OK 1172 self.assertEqual(code, 307) 1173 self.assertIsNotNone(data) 1174 self.assertEqual(o.req.get_full_url(), to_url) 1175 try: 1176 self.assertEqual(o.req.get_method(), "GET") 1177 except AttributeError: 1178 self.assertFalse(o.req.data) 1179 1180 # now it's a GET, there should not be headers regarding content 1181 # (possibly dragged from before being a POST) 1182 headers = [x.lower() for x in o.req.headers] 1183 self.assertNotIn("content-length", headers) 1184 self.assertNotIn("content-type", headers) 1185 1186 self.assertEqual(o.req.headers["Nonsense"], 1187 "viking=withhold") 1188 self.assertNotIn("Spam", o.req.headers) 1189 self.assertNotIn("Spam", o.req.unredirected_hdrs) 1190 1191 # loop detection 1192 req = Request(from_url) 1193 req.timeout = socket._GLOBAL_DEFAULT_TIMEOUT 1194 1195 def redirect(h, req, url=to_url): 1196 h.http_error_302(req, MockFile(), 302, "Blah", 1197 MockHeaders({"location": url})) 1198 # Note that the *original* request shares the same record of 1199 # redirections with the sub-requests caused by the redirections. 1200 1201 # detect infinite loop redirect of a URL to itself 1202 req = Request(from_url, origin_req_host="example.com") 1203 count = 0 1204 req.timeout = socket._GLOBAL_DEFAULT_TIMEOUT 1205 try: 1206 while 1: 1207 redirect(h, req, "http://example.com/") 1208 count = count + 1 1209 except urllib.error.HTTPError: 1210 # don't stop until max_repeats, because cookies may introduce state 1211 self.assertEqual(count, urllib.request.HTTPRedirectHandler.max_repeats) 1212 1213 # detect endless non-repeating chain of redirects 1214 req = Request(from_url, origin_req_host="example.com") 1215 count = 0 1216 req.timeout = socket._GLOBAL_DEFAULT_TIMEOUT 1217 try: 1218 while 1: 1219 redirect(h, req, "http://example.com/%d" % count) 1220 count = count + 1 1221 except urllib.error.HTTPError: 1222 self.assertEqual(count, 1223 urllib.request.HTTPRedirectHandler.max_redirections) 1224 1225 def test_invalid_redirect(self): 1226 from_url = "http://example.com/a.html" 1227 valid_schemes = ['http','https','ftp'] 1228 invalid_schemes = ['file','imap','ldap'] 1229 schemeless_url = "example.com/b.html" 1230 h = urllib.request.HTTPRedirectHandler() 1231 o = h.parent = MockOpener() 1232 req = Request(from_url) 1233 req.timeout = socket._GLOBAL_DEFAULT_TIMEOUT 1234 1235 for scheme in invalid_schemes: 1236 invalid_url = scheme + '://' + schemeless_url 1237 self.assertRaises(urllib.error.HTTPError, h.http_error_302, 1238 req, MockFile(), 302, "Security Loophole", 1239 MockHeaders({"location": invalid_url})) 1240 1241 for scheme in valid_schemes: 1242 valid_url = scheme + '://' + schemeless_url 1243 h.http_error_302(req, MockFile(), 302, "That's fine", 1244 MockHeaders({"location": valid_url})) 1245 self.assertEqual(o.req.get_full_url(), valid_url) 1246 1247 def test_relative_redirect(self): 1248 from_url = "http://example.com/a.html" 1249 relative_url = "/b.html" 1250 h = urllib.request.HTTPRedirectHandler() 1251 o = h.parent = MockOpener() 1252 req = Request(from_url) 1253 req.timeout = socket._GLOBAL_DEFAULT_TIMEOUT 1254 1255 valid_url = urllib.parse.urljoin(from_url,relative_url) 1256 h.http_error_302(req, MockFile(), 302, "That's fine", 1257 MockHeaders({"location": valid_url})) 1258 self.assertEqual(o.req.get_full_url(), valid_url) 1259 1260 def test_cookie_redirect(self): 1261 # cookies shouldn't leak into redirected requests 1262 from http.cookiejar import CookieJar 1263 from test.test_http_cookiejar import interact_netscape 1264 1265 cj = CookieJar() 1266 interact_netscape(cj, "http://www.example.com/", "spam=eggs") 1267 hh = MockHTTPHandler(302, "Location: http://www.cracker.com/\r\n\r\n") 1268 hdeh = urllib.request.HTTPDefaultErrorHandler() 1269 hrh = urllib.request.HTTPRedirectHandler() 1270 cp = urllib.request.HTTPCookieProcessor(cj) 1271 o = build_test_opener(hh, hdeh, hrh, cp) 1272 o.open("http://www.example.com/") 1273 self.assertFalse(hh.req.has_header("Cookie")) 1274 1275 def test_redirect_fragment(self): 1276 redirected_url = 'http://www.example.com/index.html#OK\r\n\r\n' 1277 hh = MockHTTPHandler(302, 'Location: ' + redirected_url) 1278 hdeh = urllib.request.HTTPDefaultErrorHandler() 1279 hrh = urllib.request.HTTPRedirectHandler() 1280 o = build_test_opener(hh, hdeh, hrh) 1281 fp = o.open('http://www.example.com') 1282 self.assertEqual(fp.geturl(), redirected_url.strip()) 1283 1284 def test_redirect_no_path(self): 1285 # Issue 14132: Relative redirect strips original path 1286 real_class = http.client.HTTPConnection 1287 response1 = b"HTTP/1.1 302 Found\r\nLocation: ?query\r\n\r\n" 1288 http.client.HTTPConnection = test_urllib.fakehttp(response1) 1289 self.addCleanup(setattr, http.client, "HTTPConnection", real_class) 1290 urls = iter(("/path", "/path?query")) 1291 def request(conn, method, url, *pos, **kw): 1292 self.assertEqual(url, next(urls)) 1293 real_class.request(conn, method, url, *pos, **kw) 1294 # Change response for subsequent connection 1295 conn.__class__.fakedata = b"HTTP/1.1 200 OK\r\n\r\nHello!" 1296 http.client.HTTPConnection.request = request 1297 fp = urllib.request.urlopen("http://python.org/path") 1298 self.assertEqual(fp.geturl(), "http://python.org/path?query") 1299 1300 def test_redirect_encoding(self): 1301 # Some characters in the redirect target may need special handling, 1302 # but most ASCII characters should be treated as already encoded 1303 class Handler(urllib.request.HTTPHandler): 1304 def http_open(self, req): 1305 result = self.do_open(self.connection, req) 1306 self.last_buf = self.connection.buf 1307 # Set up a normal response for the next request 1308 self.connection = test_urllib.fakehttp( 1309 b'HTTP/1.1 200 OK\r\n' 1310 b'Content-Length: 3\r\n' 1311 b'\r\n' 1312 b'123' 1313 ) 1314 return result 1315 handler = Handler() 1316 opener = urllib.request.build_opener(handler) 1317 tests = ( 1318 (b'/p\xC3\xA5-dansk/', b'/p%C3%A5-dansk/'), 1319 (b'/spaced%20path/', b'/spaced%20path/'), 1320 (b'/spaced path/', b'/spaced%20path/'), 1321 (b'/?p\xC3\xA5-dansk', b'/?p%C3%A5-dansk'), 1322 ) 1323 for [location, result] in tests: 1324 with self.subTest(repr(location)): 1325 handler.connection = test_urllib.fakehttp( 1326 b'HTTP/1.1 302 Redirect\r\n' 1327 b'Location: ' + location + b'\r\n' 1328 b'\r\n' 1329 ) 1330 response = opener.open('http://example.com/') 1331 expected = b'GET ' + result + b' ' 1332 request = handler.last_buf 1333 self.assertTrue(request.startswith(expected), repr(request)) 1334 1335 def test_proxy(self): 1336 o = OpenerDirector() 1337 ph = urllib.request.ProxyHandler(dict(http="proxy.example.com:3128")) 1338 o.add_handler(ph) 1339 meth_spec = [ 1340 [("http_open", "return response")] 1341 ] 1342 handlers = add_ordered_mock_handlers(o, meth_spec) 1343 1344 req = Request("http://acme.example.com/") 1345 self.assertEqual(req.host, "acme.example.com") 1346 o.open(req) 1347 self.assertEqual(req.host, "proxy.example.com:3128") 1348 1349 self.assertEqual([(handlers[0], "http_open")], 1350 [tup[0:2] for tup in o.calls]) 1351 1352 def test_proxy_no_proxy(self): 1353 os.environ['no_proxy'] = 'python.org' 1354 o = OpenerDirector() 1355 ph = urllib.request.ProxyHandler(dict(http="proxy.example.com")) 1356 o.add_handler(ph) 1357 req = Request("http://www.perl.org/") 1358 self.assertEqual(req.host, "www.perl.org") 1359 o.open(req) 1360 self.assertEqual(req.host, "proxy.example.com") 1361 req = Request("http://www.python.org") 1362 self.assertEqual(req.host, "www.python.org") 1363 o.open(req) 1364 self.assertEqual(req.host, "www.python.org") 1365 del os.environ['no_proxy'] 1366 1367 def test_proxy_no_proxy_all(self): 1368 os.environ['no_proxy'] = '*' 1369 o = OpenerDirector() 1370 ph = urllib.request.ProxyHandler(dict(http="proxy.example.com")) 1371 o.add_handler(ph) 1372 req = Request("http://www.python.org") 1373 self.assertEqual(req.host, "www.python.org") 1374 o.open(req) 1375 self.assertEqual(req.host, "www.python.org") 1376 del os.environ['no_proxy'] 1377 1378 def test_proxy_https(self): 1379 o = OpenerDirector() 1380 ph = urllib.request.ProxyHandler(dict(https="proxy.example.com:3128")) 1381 o.add_handler(ph) 1382 meth_spec = [ 1383 [("https_open", "return response")] 1384 ] 1385 handlers = add_ordered_mock_handlers(o, meth_spec) 1386 1387 req = Request("https://www.example.com/") 1388 self.assertEqual(req.host, "www.example.com") 1389 o.open(req) 1390 self.assertEqual(req.host, "proxy.example.com:3128") 1391 self.assertEqual([(handlers[0], "https_open")], 1392 [tup[0:2] for tup in o.calls]) 1393 1394 def test_proxy_https_proxy_authorization(self): 1395 o = OpenerDirector() 1396 ph = urllib.request.ProxyHandler(dict(https='proxy.example.com:3128')) 1397 o.add_handler(ph) 1398 https_handler = MockHTTPSHandler() 1399 o.add_handler(https_handler) 1400 req = Request("https://www.example.com/") 1401 req.add_header("Proxy-Authorization", "FooBar") 1402 req.add_header("User-Agent", "Grail") 1403 self.assertEqual(req.host, "www.example.com") 1404 self.assertIsNone(req._tunnel_host) 1405 o.open(req) 1406 # Verify Proxy-Authorization gets tunneled to request. 1407 # httpsconn req_headers do not have the Proxy-Authorization header but 1408 # the req will have. 1409 self.assertNotIn(("Proxy-Authorization", "FooBar"), 1410 https_handler.httpconn.req_headers) 1411 self.assertIn(("User-Agent", "Grail"), 1412 https_handler.httpconn.req_headers) 1413 self.assertIsNotNone(req._tunnel_host) 1414 self.assertEqual(req.host, "proxy.example.com:3128") 1415 self.assertEqual(req.get_header("Proxy-authorization"), "FooBar") 1416 1417 # TODO: This should be only for OSX 1418 @unittest.skipUnless(sys.platform == 'darwin', "only relevant for OSX") 1419 def test_osx_proxy_bypass(self): 1420 bypass = { 1421 'exclude_simple': False, 1422 'exceptions': ['foo.bar', '*.bar.com', '127.0.0.1', '10.10', 1423 '10.0/16'] 1424 } 1425 # Check hosts that should trigger the proxy bypass 1426 for host in ('foo.bar', 'www.bar.com', '127.0.0.1', '10.10.0.1', 1427 '10.0.0.1'): 1428 self.assertTrue(_proxy_bypass_macosx_sysconf(host, bypass), 1429 'expected bypass of %s to be True' % host) 1430 # Check hosts that should not trigger the proxy bypass 1431 for host in ('abc.foo.bar', 'bar.com', '127.0.0.2', '10.11.0.1', 1432 'notinbypass'): 1433 self.assertFalse(_proxy_bypass_macosx_sysconf(host, bypass), 1434 'expected bypass of %s to be False' % host) 1435 1436 # Check the exclude_simple flag 1437 bypass = {'exclude_simple': True, 'exceptions': []} 1438 self.assertTrue(_proxy_bypass_macosx_sysconf('test', bypass)) 1439 1440 def test_basic_auth(self, quote_char='"'): 1441 opener = OpenerDirector() 1442 password_manager = MockPasswordManager() 1443 auth_handler = urllib.request.HTTPBasicAuthHandler(password_manager) 1444 realm = "ACME Widget Store" 1445 http_handler = MockHTTPHandler( 1446 401, 'WWW-Authenticate: Basic realm=%s%s%s\r\n\r\n' % 1447 (quote_char, realm, quote_char)) 1448 opener.add_handler(auth_handler) 1449 opener.add_handler(http_handler) 1450 self._test_basic_auth(opener, auth_handler, "Authorization", 1451 realm, http_handler, password_manager, 1452 "http://acme.example.com/protected", 1453 "http://acme.example.com/protected", 1454 ) 1455 1456 def test_basic_auth_with_single_quoted_realm(self): 1457 self.test_basic_auth(quote_char="'") 1458 1459 def test_basic_auth_with_unquoted_realm(self): 1460 opener = OpenerDirector() 1461 password_manager = MockPasswordManager() 1462 auth_handler = urllib.request.HTTPBasicAuthHandler(password_manager) 1463 realm = "ACME Widget Store" 1464 http_handler = MockHTTPHandler( 1465 401, 'WWW-Authenticate: Basic realm=%s\r\n\r\n' % realm) 1466 opener.add_handler(auth_handler) 1467 opener.add_handler(http_handler) 1468 with self.assertWarns(UserWarning): 1469 self._test_basic_auth(opener, auth_handler, "Authorization", 1470 realm, http_handler, password_manager, 1471 "http://acme.example.com/protected", 1472 "http://acme.example.com/protected", 1473 ) 1474 1475 def test_proxy_basic_auth(self): 1476 opener = OpenerDirector() 1477 ph = urllib.request.ProxyHandler(dict(http="proxy.example.com:3128")) 1478 opener.add_handler(ph) 1479 password_manager = MockPasswordManager() 1480 auth_handler = urllib.request.ProxyBasicAuthHandler(password_manager) 1481 realm = "ACME Networks" 1482 http_handler = MockHTTPHandler( 1483 407, 'Proxy-Authenticate: Basic realm="%s"\r\n\r\n' % realm) 1484 opener.add_handler(auth_handler) 1485 opener.add_handler(http_handler) 1486 self._test_basic_auth(opener, auth_handler, "Proxy-authorization", 1487 realm, http_handler, password_manager, 1488 "http://acme.example.com:3128/protected", 1489 "proxy.example.com:3128", 1490 ) 1491 1492 def test_basic_and_digest_auth_handlers(self): 1493 # HTTPDigestAuthHandler raised an exception if it couldn't handle a 40* 1494 # response (http://python.org/sf/1479302), where it should instead 1495 # return None to allow another handler (especially 1496 # HTTPBasicAuthHandler) to handle the response. 1497 1498 # Also (http://python.org/sf/14797027, RFC 2617 section 1.2), we must 1499 # try digest first (since it's the strongest auth scheme), so we record 1500 # order of calls here to check digest comes first: 1501 class RecordingOpenerDirector(OpenerDirector): 1502 def __init__(self): 1503 OpenerDirector.__init__(self) 1504 self.recorded = [] 1505 1506 def record(self, info): 1507 self.recorded.append(info) 1508 1509 class TestDigestAuthHandler(urllib.request.HTTPDigestAuthHandler): 1510 def http_error_401(self, *args, **kwds): 1511 self.parent.record("digest") 1512 urllib.request.HTTPDigestAuthHandler.http_error_401(self, 1513 *args, **kwds) 1514 1515 class TestBasicAuthHandler(urllib.request.HTTPBasicAuthHandler): 1516 def http_error_401(self, *args, **kwds): 1517 self.parent.record("basic") 1518 urllib.request.HTTPBasicAuthHandler.http_error_401(self, 1519 *args, **kwds) 1520 1521 opener = RecordingOpenerDirector() 1522 password_manager = MockPasswordManager() 1523 digest_handler = TestDigestAuthHandler(password_manager) 1524 basic_handler = TestBasicAuthHandler(password_manager) 1525 realm = "ACME Networks" 1526 http_handler = MockHTTPHandler( 1527 401, 'WWW-Authenticate: Basic realm="%s"\r\n\r\n' % realm) 1528 opener.add_handler(basic_handler) 1529 opener.add_handler(digest_handler) 1530 opener.add_handler(http_handler) 1531 1532 # check basic auth isn't blocked by digest handler failing 1533 self._test_basic_auth(opener, basic_handler, "Authorization", 1534 realm, http_handler, password_manager, 1535 "http://acme.example.com/protected", 1536 "http://acme.example.com/protected", 1537 ) 1538 # check digest was tried before basic (twice, because 1539 # _test_basic_auth called .open() twice) 1540 self.assertEqual(opener.recorded, ["digest", "basic"]*2) 1541 1542 def test_unsupported_auth_digest_handler(self): 1543 opener = OpenerDirector() 1544 # While using DigestAuthHandler 1545 digest_auth_handler = urllib.request.HTTPDigestAuthHandler(None) 1546 http_handler = MockHTTPHandler( 1547 401, 'WWW-Authenticate: Kerberos\r\n\r\n') 1548 opener.add_handler(digest_auth_handler) 1549 opener.add_handler(http_handler) 1550 self.assertRaises(ValueError, opener.open, "http://www.example.com") 1551 1552 def test_unsupported_auth_basic_handler(self): 1553 # While using BasicAuthHandler 1554 opener = OpenerDirector() 1555 basic_auth_handler = urllib.request.HTTPBasicAuthHandler(None) 1556 http_handler = MockHTTPHandler( 1557 401, 'WWW-Authenticate: NTLM\r\n\r\n') 1558 opener.add_handler(basic_auth_handler) 1559 opener.add_handler(http_handler) 1560 self.assertRaises(ValueError, opener.open, "http://www.example.com") 1561 1562 def _test_basic_auth(self, opener, auth_handler, auth_header, 1563 realm, http_handler, password_manager, 1564 request_url, protected_url): 1565 import base64 1566 user, password = "wile", "coyote" 1567 1568 # .add_password() fed through to password manager 1569 auth_handler.add_password(realm, request_url, user, password) 1570 self.assertEqual(realm, password_manager.realm) 1571 self.assertEqual(request_url, password_manager.url) 1572 self.assertEqual(user, password_manager.user) 1573 self.assertEqual(password, password_manager.password) 1574 1575 opener.open(request_url) 1576 1577 # should have asked the password manager for the username/password 1578 self.assertEqual(password_manager.target_realm, realm) 1579 self.assertEqual(password_manager.target_url, protected_url) 1580 1581 # expect one request without authorization, then one with 1582 self.assertEqual(len(http_handler.requests), 2) 1583 self.assertFalse(http_handler.requests[0].has_header(auth_header)) 1584 userpass = bytes('%s:%s' % (user, password), "ascii") 1585 auth_hdr_value = ('Basic ' + 1586 base64.encodebytes(userpass).strip().decode()) 1587 self.assertEqual(http_handler.requests[1].get_header(auth_header), 1588 auth_hdr_value) 1589 self.assertEqual(http_handler.requests[1].unredirected_hdrs[auth_header], 1590 auth_hdr_value) 1591 # if the password manager can't find a password, the handler won't 1592 # handle the HTTP auth error 1593 password_manager.user = password_manager.password = None 1594 http_handler.reset() 1595 opener.open(request_url) 1596 self.assertEqual(len(http_handler.requests), 1) 1597 self.assertFalse(http_handler.requests[0].has_header(auth_header)) 1598 1599 def test_basic_prior_auth_auto_send(self): 1600 # Assume already authenticated if is_authenticated=True 1601 # for APIs like Github that don't return 401 1602 1603 user, password = "wile", "coyote" 1604 request_url = "http://acme.example.com/protected" 1605 1606 http_handler = MockHTTPHandlerCheckAuth(200) 1607 1608 pwd_manager = HTTPPasswordMgrWithPriorAuth() 1609 auth_prior_handler = HTTPBasicAuthHandler(pwd_manager) 1610 auth_prior_handler.add_password( 1611 None, request_url, user, password, is_authenticated=True) 1612 1613 is_auth = pwd_manager.is_authenticated(request_url) 1614 self.assertTrue(is_auth) 1615 1616 opener = OpenerDirector() 1617 opener.add_handler(auth_prior_handler) 1618 opener.add_handler(http_handler) 1619 1620 opener.open(request_url) 1621 1622 # expect request to be sent with auth header 1623 self.assertTrue(http_handler.has_auth_header) 1624 1625 def test_basic_prior_auth_send_after_first_success(self): 1626 # Auto send auth header after authentication is successful once 1627 1628 user, password = 'wile', 'coyote' 1629 request_url = 'http://acme.example.com/protected' 1630 realm = 'ACME' 1631 1632 pwd_manager = HTTPPasswordMgrWithPriorAuth() 1633 auth_prior_handler = HTTPBasicAuthHandler(pwd_manager) 1634 auth_prior_handler.add_password(realm, request_url, user, password) 1635 1636 is_auth = pwd_manager.is_authenticated(request_url) 1637 self.assertFalse(is_auth) 1638 1639 opener = OpenerDirector() 1640 opener.add_handler(auth_prior_handler) 1641 1642 http_handler = MockHTTPHandler( 1643 401, 'WWW-Authenticate: Basic realm="%s"\r\n\r\n' % None) 1644 opener.add_handler(http_handler) 1645 1646 opener.open(request_url) 1647 1648 is_auth = pwd_manager.is_authenticated(request_url) 1649 self.assertTrue(is_auth) 1650 1651 http_handler = MockHTTPHandlerCheckAuth(200) 1652 self.assertFalse(http_handler.has_auth_header) 1653 1654 opener = OpenerDirector() 1655 opener.add_handler(auth_prior_handler) 1656 opener.add_handler(http_handler) 1657 1658 # After getting 200 from MockHTTPHandler 1659 # Next request sends header in the first request 1660 opener.open(request_url) 1661 1662 # expect request to be sent with auth header 1663 self.assertTrue(http_handler.has_auth_header) 1664 1665 def test_http_closed(self): 1666 """Test the connection is cleaned up when the response is closed""" 1667 for (transfer, data) in ( 1668 ("Connection: close", b"data"), 1669 ("Transfer-Encoding: chunked", b"4\r\ndata\r\n0\r\n\r\n"), 1670 ("Content-Length: 4", b"data"), 1671 ): 1672 header = "HTTP/1.1 200 OK\r\n{}\r\n\r\n".format(transfer) 1673 conn = test_urllib.fakehttp(header.encode() + data) 1674 handler = urllib.request.AbstractHTTPHandler() 1675 req = Request("http://dummy/") 1676 req.timeout = None 1677 with handler.do_open(conn, req) as resp: 1678 resp.read() 1679 self.assertTrue(conn.fakesock.closed, 1680 "Connection not closed with {!r}".format(transfer)) 1681 1682 def test_invalid_closed(self): 1683 """Test the connection is cleaned up after an invalid response""" 1684 conn = test_urllib.fakehttp(b"") 1685 handler = urllib.request.AbstractHTTPHandler() 1686 req = Request("http://dummy/") 1687 req.timeout = None 1688 with self.assertRaises(http.client.BadStatusLine): 1689 handler.do_open(conn, req) 1690 self.assertTrue(conn.fakesock.closed, "Connection not closed") 1691 1692 1693 1694class MiscTests(unittest.TestCase): 1695 1696 def opener_has_handler(self, opener, handler_class): 1697 self.assertTrue(any(h.__class__ == handler_class 1698 for h in opener.handlers)) 1699 1700 def test_build_opener(self): 1701 class MyHTTPHandler(urllib.request.HTTPHandler): 1702 pass 1703 1704 class FooHandler(urllib.request.BaseHandler): 1705 def foo_open(self): 1706 pass 1707 1708 class BarHandler(urllib.request.BaseHandler): 1709 def bar_open(self): 1710 pass 1711 1712 build_opener = urllib.request.build_opener 1713 1714 o = build_opener(FooHandler, BarHandler) 1715 self.opener_has_handler(o, FooHandler) 1716 self.opener_has_handler(o, BarHandler) 1717 1718 # can take a mix of classes and instances 1719 o = build_opener(FooHandler, BarHandler()) 1720 self.opener_has_handler(o, FooHandler) 1721 self.opener_has_handler(o, BarHandler) 1722 1723 # subclasses of default handlers override default handlers 1724 o = build_opener(MyHTTPHandler) 1725 self.opener_has_handler(o, MyHTTPHandler) 1726 1727 # a particular case of overriding: default handlers can be passed 1728 # in explicitly 1729 o = build_opener() 1730 self.opener_has_handler(o, urllib.request.HTTPHandler) 1731 o = build_opener(urllib.request.HTTPHandler) 1732 self.opener_has_handler(o, urllib.request.HTTPHandler) 1733 o = build_opener(urllib.request.HTTPHandler()) 1734 self.opener_has_handler(o, urllib.request.HTTPHandler) 1735 1736 # Issue2670: multiple handlers sharing the same base class 1737 class MyOtherHTTPHandler(urllib.request.HTTPHandler): 1738 pass 1739 1740 o = build_opener(MyHTTPHandler, MyOtherHTTPHandler) 1741 self.opener_has_handler(o, MyHTTPHandler) 1742 self.opener_has_handler(o, MyOtherHTTPHandler) 1743 1744 @unittest.skipUnless(support.is_resource_enabled('network'), 1745 'test requires network access') 1746 def test_issue16464(self): 1747 with support.transient_internet("http://www.example.com/"): 1748 opener = urllib.request.build_opener() 1749 request = urllib.request.Request("http://www.example.com/") 1750 self.assertEqual(None, request.data) 1751 1752 opener.open(request, "1".encode("us-ascii")) 1753 self.assertEqual(b"1", request.data) 1754 self.assertEqual("1", request.get_header("Content-length")) 1755 1756 opener.open(request, "1234567890".encode("us-ascii")) 1757 self.assertEqual(b"1234567890", request.data) 1758 self.assertEqual("10", request.get_header("Content-length")) 1759 1760 def test_HTTPError_interface(self): 1761 """ 1762 Issue 13211 reveals that HTTPError didn't implement the URLError 1763 interface even though HTTPError is a subclass of URLError. 1764 """ 1765 msg = 'something bad happened' 1766 url = code = fp = None 1767 hdrs = 'Content-Length: 42' 1768 err = urllib.error.HTTPError(url, code, msg, hdrs, fp) 1769 self.assertTrue(hasattr(err, 'reason')) 1770 self.assertEqual(err.reason, 'something bad happened') 1771 self.assertTrue(hasattr(err, 'headers')) 1772 self.assertEqual(err.headers, 'Content-Length: 42') 1773 expected_errmsg = 'HTTP Error %s: %s' % (err.code, err.msg) 1774 self.assertEqual(str(err), expected_errmsg) 1775 expected_errmsg = '<HTTPError %s: %r>' % (err.code, err.msg) 1776 self.assertEqual(repr(err), expected_errmsg) 1777 1778 def test_parse_proxy(self): 1779 parse_proxy_test_cases = [ 1780 ('proxy.example.com', 1781 (None, None, None, 'proxy.example.com')), 1782 ('proxy.example.com:3128', 1783 (None, None, None, 'proxy.example.com:3128')), 1784 ('proxy.example.com', (None, None, None, 'proxy.example.com')), 1785 ('proxy.example.com:3128', 1786 (None, None, None, 'proxy.example.com:3128')), 1787 # The authority component may optionally include userinfo 1788 # (assumed to be # username:password): 1789 ('joe:password@proxy.example.com', 1790 (None, 'joe', 'password', 'proxy.example.com')), 1791 ('joe:password@proxy.example.com:3128', 1792 (None, 'joe', 'password', 'proxy.example.com:3128')), 1793 #Examples with URLS 1794 ('http://proxy.example.com/', 1795 ('http', None, None, 'proxy.example.com')), 1796 ('http://proxy.example.com:3128/', 1797 ('http', None, None, 'proxy.example.com:3128')), 1798 ('http://joe:password@proxy.example.com/', 1799 ('http', 'joe', 'password', 'proxy.example.com')), 1800 ('http://joe:password@proxy.example.com:3128', 1801 ('http', 'joe', 'password', 'proxy.example.com:3128')), 1802 # Everything after the authority is ignored 1803 ('ftp://joe:password@proxy.example.com/rubbish:3128', 1804 ('ftp', 'joe', 'password', 'proxy.example.com')), 1805 # Test for no trailing '/' case 1806 ('http://joe:password@proxy.example.com', 1807 ('http', 'joe', 'password', 'proxy.example.com')) 1808 ] 1809 1810 for tc, expected in parse_proxy_test_cases: 1811 self.assertEqual(_parse_proxy(tc), expected) 1812 1813 self.assertRaises(ValueError, _parse_proxy, 'file:/ftp.example.com'), 1814 1815 def test_unsupported_algorithm(self): 1816 handler = AbstractDigestAuthHandler() 1817 with self.assertRaises(ValueError) as exc: 1818 handler.get_algorithm_impls('invalid') 1819 self.assertEqual( 1820 str(exc.exception), 1821 "Unsupported digest authentication algorithm 'invalid'" 1822 ) 1823 1824 1825class RequestTests(unittest.TestCase): 1826 class PutRequest(Request): 1827 method = 'PUT' 1828 1829 def setUp(self): 1830 self.get = Request("http://www.python.org/~jeremy/") 1831 self.post = Request("http://www.python.org/~jeremy/", 1832 "data", 1833 headers={"X-Test": "test"}) 1834 self.head = Request("http://www.python.org/~jeremy/", method='HEAD') 1835 self.put = self.PutRequest("http://www.python.org/~jeremy/") 1836 self.force_post = self.PutRequest("http://www.python.org/~jeremy/", 1837 method="POST") 1838 1839 def test_method(self): 1840 self.assertEqual("POST", self.post.get_method()) 1841 self.assertEqual("GET", self.get.get_method()) 1842 self.assertEqual("HEAD", self.head.get_method()) 1843 self.assertEqual("PUT", self.put.get_method()) 1844 self.assertEqual("POST", self.force_post.get_method()) 1845 1846 def test_data(self): 1847 self.assertFalse(self.get.data) 1848 self.assertEqual("GET", self.get.get_method()) 1849 self.get.data = "spam" 1850 self.assertTrue(self.get.data) 1851 self.assertEqual("POST", self.get.get_method()) 1852 1853 # issue 16464 1854 # if we change data we need to remove content-length header 1855 # (cause it's most probably calculated for previous value) 1856 def test_setting_data_should_remove_content_length(self): 1857 self.assertNotIn("Content-length", self.get.unredirected_hdrs) 1858 self.get.add_unredirected_header("Content-length", 42) 1859 self.assertEqual(42, self.get.unredirected_hdrs["Content-length"]) 1860 self.get.data = "spam" 1861 self.assertNotIn("Content-length", self.get.unredirected_hdrs) 1862 1863 # issue 17485 same for deleting data. 1864 def test_deleting_data_should_remove_content_length(self): 1865 self.assertNotIn("Content-length", self.get.unredirected_hdrs) 1866 self.get.data = 'foo' 1867 self.get.add_unredirected_header("Content-length", 3) 1868 self.assertEqual(3, self.get.unredirected_hdrs["Content-length"]) 1869 del self.get.data 1870 self.assertNotIn("Content-length", self.get.unredirected_hdrs) 1871 1872 def test_get_full_url(self): 1873 self.assertEqual("http://www.python.org/~jeremy/", 1874 self.get.get_full_url()) 1875 1876 def test_selector(self): 1877 self.assertEqual("/~jeremy/", self.get.selector) 1878 req = Request("http://www.python.org/") 1879 self.assertEqual("/", req.selector) 1880 1881 def test_get_type(self): 1882 self.assertEqual("http", self.get.type) 1883 1884 def test_get_host(self): 1885 self.assertEqual("www.python.org", self.get.host) 1886 1887 def test_get_host_unquote(self): 1888 req = Request("http://www.%70ython.org/") 1889 self.assertEqual("www.python.org", req.host) 1890 1891 def test_proxy(self): 1892 self.assertFalse(self.get.has_proxy()) 1893 self.get.set_proxy("www.perl.org", "http") 1894 self.assertTrue(self.get.has_proxy()) 1895 self.assertEqual("www.python.org", self.get.origin_req_host) 1896 self.assertEqual("www.perl.org", self.get.host) 1897 1898 def test_wrapped_url(self): 1899 req = Request("<URL:http://www.python.org>") 1900 self.assertEqual("www.python.org", req.host) 1901 1902 def test_url_fragment(self): 1903 req = Request("http://www.python.org/?qs=query#fragment=true") 1904 self.assertEqual("/?qs=query", req.selector) 1905 req = Request("http://www.python.org/#fun=true") 1906 self.assertEqual("/", req.selector) 1907 1908 # Issue 11703: geturl() omits fragment in the original URL. 1909 url = 'http://docs.python.org/library/urllib2.html#OK' 1910 req = Request(url) 1911 self.assertEqual(req.get_full_url(), url) 1912 1913 def test_url_fullurl_get_full_url(self): 1914 urls = ['http://docs.python.org', 1915 'http://docs.python.org/library/urllib2.html#OK', 1916 'http://www.python.org/?qs=query#fragment=true'] 1917 for url in urls: 1918 req = Request(url) 1919 self.assertEqual(req.get_full_url(), req.full_url) 1920 1921 1922if __name__ == "__main__": 1923 unittest.main() 1924
