15821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)from __future__ import generators 25821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)""" 35821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)httplib2 45821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 55821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)A caching http interface that supports ETags and gzip 65821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)to conserve bandwidth. 75821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 85821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)Requires Python 2.3 or later 95821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)Changelog: 115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)2007-08-18, Rick: Modified so it's able to use a socks proxy if needed. 125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)""" 145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)__author__ = "Joe Gregorio (joe@bitworking.org)" 165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)__copyright__ = "Copyright 2006, Joe Gregorio" 175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)__contributors__ = ["Thomas Broyer (t.broyer@ltgt.net)", 185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "James Antill", 195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "Xavier Verges Farrero", 205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "Jonathan Feinberg", 215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "Blair Zajac", 225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "Sam Ruby", 235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "Louis Nyffenegger"] 245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)__license__ = "MIT" 255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)__version__ = "$Rev$" 265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import re 285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import sys 295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import email 305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import email.Utils 315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import email.Message 325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import email.FeedParser 335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import StringIO 345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import gzip 355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import zlib 365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import httplib 375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import urlparse 385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import base64 395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import os 405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import copy 415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import calendar 425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import time 435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import random 445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# remove depracated warning in python2.6 455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)try: 465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) from hashlib import sha1 as _sha, md5 as _md5 475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)except ImportError: 485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) import sha 495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) import md5 505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) _sha = sha.new 515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) _md5 = md5.new 525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import hmac 535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)from gettext import gettext as _ 545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import socket 555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)try: 575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) import socks 585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)except ImportError: 595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) socks = None 605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# Build the appropriate socket wrapper for ssl 625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)try: 635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) import ssl # python 2.6 645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) _ssl_wrap_socket = ssl.wrap_socket 655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)except ImportError: 665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def _ssl_wrap_socket(sock, key_file, cert_file): 675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ssl_sock = socket.ssl(sock, key_file, cert_file) 685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return httplib.FakeSocket(sock, ssl_sock) 695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)if sys.version_info >= (2,3): 725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) from iri2uri import iri2uri 735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)else: 745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def iri2uri(uri): 755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return uri 765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)def has_timeout(timeout): # python 2.6 785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if hasattr(socket, '_GLOBAL_DEFAULT_TIMEOUT'): 795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return (timeout is not None and timeout is not socket._GLOBAL_DEFAULT_TIMEOUT) 805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return (timeout is not None) 815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)__all__ = ['Http', 'Response', 'ProxyInfo', 'HttpLib2Error', 835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 'RedirectMissingLocation', 'RedirectLimit', 'FailedToDecompressContent', 845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 'UnimplementedDigestAuthOptionError', 'UnimplementedHmacDigestAuthOptionError', 855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 'debuglevel'] 865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# The httplib debug level, set to a non-zero value to get debug output 895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)debuglevel = 0 905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# Python 2.3 support 935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)if sys.version_info < (2,4): 945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def sorted(seq): 955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) seq.sort() 965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return seq 975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# Python 2.3 support 995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)def HTTPResponse__getheaders(self): 1005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Return list of (header, value) tuples.""" 1015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if self.msg is None: 1025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) raise httplib.ResponseNotReady() 1035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return self.msg.items() 1045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)if not hasattr(httplib.HTTPResponse, 'getheaders'): 1065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) httplib.HTTPResponse.getheaders = HTTPResponse__getheaders 1075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# All exceptions raised here derive from HttpLib2Error 1095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)class HttpLib2Error(Exception): pass 1105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# Some exceptions can be caught and optionally 1125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# be turned back into responses. 1135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)class HttpLib2ErrorWithResponse(HttpLib2Error): 1145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def __init__(self, desc, response, content): 1155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.response = response 1165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.content = content 1175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) HttpLib2Error.__init__(self, desc) 1185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)class RedirectMissingLocation(HttpLib2ErrorWithResponse): pass 1205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)class RedirectLimit(HttpLib2ErrorWithResponse): pass 1215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)class FailedToDecompressContent(HttpLib2ErrorWithResponse): pass 1225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)class UnimplementedDigestAuthOptionError(HttpLib2ErrorWithResponse): pass 1235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)class UnimplementedHmacDigestAuthOptionError(HttpLib2ErrorWithResponse): pass 1245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)class RelativeURIError(HttpLib2Error): pass 1265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)class ServerNotFoundError(HttpLib2Error): pass 1275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# Open Items: 1295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# ----------- 1305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# Proxy support 1315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# Are we removing the cached content too soon on PUT (only delete on 200 Maybe?) 1335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# Pluggable cache storage (supports storing the cache in 1355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# flat files by default. We need a plug-in architecture 1365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# that can support Berkeley DB and Squid) 1375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# == Known Issues == 1395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# Does not handle a resource that uses conneg and Last-Modified but no ETag as a cache validator. 1405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# Does not handle Cache-Control: max-stale 1415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# Does not use Age: headers when calculating cache freshness. 1425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# The number of redirections to follow before giving up. 1455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# Note that only GET redirects are automatically followed. 1465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# Will also honor 301 requests by saving that info and never 1475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# requesting that URI again. 1485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)DEFAULT_MAX_REDIRECTS = 5 1495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# Which headers are hop-by-hop headers by default 1515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)HOP_BY_HOP = ['connection', 'keep-alive', 'proxy-authenticate', 'proxy-authorization', 'te', 'trailers', 'transfer-encoding', 'upgrade'] 1525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)def _get_end2end_headers(response): 1545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) hopbyhop = list(HOP_BY_HOP) 1555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) hopbyhop.extend([x.strip() for x in response.get('connection', '').split(',')]) 1565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return [header for header in response.keys() if header not in hopbyhop] 1575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)URI = re.compile(r"^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?") 1595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)def parse_uri(uri): 1615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Parses a URI using the regex given in Appendix B of RFC 3986. 1625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) (scheme, authority, path, query, fragment) = parse_uri(uri) 1645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 1655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) groups = URI.match(uri).groups() 1665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return (groups[1], groups[3], groups[4], groups[6], groups[8]) 1675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)def urlnorm(uri): 1695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) (scheme, authority, path, query, fragment) = parse_uri(uri) 1705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if not scheme or not authority: 1715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) raise RelativeURIError("Only absolute URIs are allowed. uri = %s" % uri) 1725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) authority = authority.lower() 1735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) scheme = scheme.lower() 1745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if not path: 1755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) path = "/" 1765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Could do syntax based normalization of the URI before 1775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # computing the digest. See Section 6.2.2 of Std 66. 1785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) request_uri = query and "?".join([path, query]) or path 1795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) scheme = scheme.lower() 1805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) defrag_uri = scheme + "://" + authority + request_uri 1815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return scheme, authority, request_uri, defrag_uri 1825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# Cache filename construction (original borrowed from Venus http://intertwingly.net/code/venus/) 1855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)re_url_scheme = re.compile(r'^\w+://') 1865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)re_slash = re.compile(r'[?/:|]+') 1875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)def safename(filename): 1895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Return a filename suitable for the cache. 1905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Strips dangerous and common characters to create a filename we 1925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) can use to store the cache in. 1935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 1945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) try: 1965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if re_url_scheme.match(filename): 1975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if isinstance(filename,str): 1985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) filename = filename.decode('utf-8') 1995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) filename = filename.encode('idna') 2005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) else: 2015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) filename = filename.encode('idna') 2025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) except UnicodeError: 2035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pass 2045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if isinstance(filename,unicode): 2055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) filename=filename.encode('utf-8') 2065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) filemd5 = _md5(filename).hexdigest() 2075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) filename = re_url_scheme.sub("", filename) 2085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) filename = re_slash.sub(",", filename) 2095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # limit length of filename 2115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if len(filename)>200: 2125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) filename=filename[:200] 2135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return ",".join((filename, filemd5)) 2145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)NORMALIZE_SPACE = re.compile(r'(?:\r\n)?[ \t]+') 2165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)def _normalize_headers(headers): 2175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return dict([ (key.lower(), NORMALIZE_SPACE.sub(value, ' ').strip()) for (key, value) in headers.iteritems()]) 2185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)def _parse_cache_control(headers): 2205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) retval = {} 2215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if headers.has_key('cache-control'): 2225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) parts = headers['cache-control'].split(',') 2235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) parts_with_args = [tuple([x.strip().lower() for x in part.split("=", 1)]) for part in parts if -1 != part.find("=")] 2245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) parts_wo_args = [(name.strip().lower(), 1) for name in parts if -1 == name.find("=")] 2255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) retval = dict(parts_with_args + parts_wo_args) 2265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return retval 2275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# Whether to use a strict mode to parse WWW-Authenticate headers 2295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# Might lead to bad results in case of ill-formed header value, 2305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# so disabled by default, falling back to relaxed parsing. 2315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# Set to true to turn on, usefull for testing servers. 2325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)USE_WWW_AUTH_STRICT_PARSING = 0 2335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# In regex below: 2355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# [^\0-\x1f\x7f-\xff()<>@,;:\\\"/[\]?={} \t]+ matches a "token" as defined by HTTP 2365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# "(?:[^\0-\x08\x0A-\x1f\x7f-\xff\\\"]|\\[\0-\x7f])*?" matches a "quoted-string" as defined by HTTP, when LWS have already been replaced by a single space 2375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# Actually, as an auth-param value can be either a token or a quoted-string, they are combined in a single pattern which matches both: 2385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# \"?((?<=\")(?:[^\0-\x1f\x7f-\xff\\\"]|\\[\0-\x7f])*?(?=\")|(?<!\")[^\0-\x08\x0A-\x1f\x7f-\xff()<>@,;:\\\"/[\]?={} \t]+(?!\"))\"? 2395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)WWW_AUTH_STRICT = re.compile(r"^(?:\s*(?:,\s*)?([^\0-\x1f\x7f-\xff()<>@,;:\\\"/[\]?={} \t]+)\s*=\s*\"?((?<=\")(?:[^\0-\x08\x0A-\x1f\x7f-\xff\\\"]|\\[\0-\x7f])*?(?=\")|(?<!\")[^\0-\x1f\x7f-\xff()<>@,;:\\\"/[\]?={} \t]+(?!\"))\"?)(.*)$") 2405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)WWW_AUTH_RELAXED = re.compile(r"^(?:\s*(?:,\s*)?([^ \t\r\n=]+)\s*=\s*\"?((?<=\")(?:[^\\\"]|\\.)*?(?=\")|(?<!\")[^ \t\r\n,]+(?!\"))\"?)(.*)$") 2415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)UNQUOTE_PAIRS = re.compile(r'\\(.)') 2425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)def _parse_www_authenticate(headers, headername='www-authenticate'): 2435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Returns a dictionary of dictionaries, one dict 2445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) per auth_scheme.""" 2455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) retval = {} 2465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if headers.has_key(headername): 2475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) authenticate = headers[headername].strip() 2485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) www_auth = USE_WWW_AUTH_STRICT_PARSING and WWW_AUTH_STRICT or WWW_AUTH_RELAXED 2495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) while authenticate: 2505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Break off the scheme at the beginning of the line 2515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if headername == 'authentication-info': 2525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) (auth_scheme, the_rest) = ('digest', authenticate) 2535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) else: 2545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) (auth_scheme, the_rest) = authenticate.split(" ", 1) 2555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Now loop over all the key value pairs that come after the scheme, 2565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # being careful not to roll into the next scheme 2575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) match = www_auth.search(the_rest) 2585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) auth_params = {} 2595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) while match: 2605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if match and len(match.groups()) == 3: 2615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) (key, value, the_rest) = match.groups() 2625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) auth_params[key.lower()] = UNQUOTE_PAIRS.sub(r'\1', value) # '\\'.join([x.replace('\\', '') for x in value.split('\\\\')]) 2635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) match = www_auth.search(the_rest) 2645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) retval[auth_scheme.lower()] = auth_params 2655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) authenticate = the_rest.strip() 2665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return retval 2675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)def _entry_disposition(response_headers, request_headers): 2705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Determine freshness from the Date, Expires and Cache-Control headers. 2715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) We don't handle the following: 2735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1. Cache-Control: max-stale 2755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2. Age: headers are not used in the calculations. 2765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Not that this algorithm is simpler than you might think 2785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) because we are operating as a private (non-shared) cache. 2795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) This lets us ignore 's-maxage'. We can also ignore 2805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 'proxy-invalidate' since we aren't a proxy. 2815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) We will never return a stale document as 2825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) fresh as a design decision, and thus the non-implementation 2835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) of 'max-stale'. This also lets us safely ignore 'must-revalidate' 2845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) since we operate as if every server has sent 'must-revalidate'. 2855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Since we are private we get to ignore both 'public' and 2865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 'private' parameters. We also ignore 'no-transform' since 2875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) we don't do any transformations. 2885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) The 'no-store' parameter is handled at a higher level. 2895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) So the only Cache-Control parameters we look at are: 2905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) no-cache 2925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) only-if-cached 2935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) max-age 2945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) min-fresh 2955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 2965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) retval = "STALE" 2985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) cc = _parse_cache_control(request_headers) 2995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) cc_response = _parse_cache_control(response_headers) 3005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if request_headers.has_key('pragma') and request_headers['pragma'].lower().find('no-cache') != -1: 3025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) retval = "TRANSPARENT" 3035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if 'cache-control' not in request_headers: 3045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) request_headers['cache-control'] = 'no-cache' 3055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) elif cc.has_key('no-cache'): 3065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) retval = "TRANSPARENT" 3075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) elif cc_response.has_key('no-cache'): 3085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) retval = "STALE" 3095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) elif cc.has_key('only-if-cached'): 3105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) retval = "FRESH" 3115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) elif response_headers.has_key('date'): 3125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) date = calendar.timegm(email.Utils.parsedate_tz(response_headers['date'])) 3135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) now = time.time() 3145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) current_age = max(0, now - date) 3155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if cc_response.has_key('max-age'): 3165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) try: 3175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) freshness_lifetime = int(cc_response['max-age']) 3185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) except ValueError: 3195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) freshness_lifetime = 0 3205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) elif response_headers.has_key('expires'): 3215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) expires = email.Utils.parsedate_tz(response_headers['expires']) 3225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if None == expires: 3235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) freshness_lifetime = 0 3245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) else: 3255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) freshness_lifetime = max(0, calendar.timegm(expires) - date) 3265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) else: 3275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) freshness_lifetime = 0 3285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if cc.has_key('max-age'): 3295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) try: 3305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) freshness_lifetime = int(cc['max-age']) 3315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) except ValueError: 3325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) freshness_lifetime = 0 3335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if cc.has_key('min-fresh'): 3345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) try: 3355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) min_fresh = int(cc['min-fresh']) 3365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) except ValueError: 3375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) min_fresh = 0 3385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) current_age += min_fresh 3395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if freshness_lifetime > current_age: 3405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) retval = "FRESH" 3415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return retval 3425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)def _decompressContent(response, new_content): 3445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) content = new_content 3455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) try: 3465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) encoding = response.get('content-encoding', None) 3475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if encoding in ['gzip', 'deflate']: 3485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if encoding == 'gzip': 3495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) content = gzip.GzipFile(fileobj=StringIO.StringIO(new_content)).read() 3505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if encoding == 'deflate': 3515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) content = zlib.decompress(content) 3525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) response['content-length'] = str(len(content)) 3535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Record the historical presence of the encoding in a way the won't interfere. 3545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) response['-content-encoding'] = response['content-encoding'] 3555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) del response['content-encoding'] 3565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) except IOError: 3575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) content = "" 3585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) raise FailedToDecompressContent(_("Content purported to be compressed with %s but failed to decompress.") % response.get('content-encoding'), response, content) 3595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return content 3605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)def _updateCache(request_headers, response_headers, content, cache, cachekey): 3625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if cachekey: 3635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) cc = _parse_cache_control(request_headers) 3645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) cc_response = _parse_cache_control(response_headers) 3655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if cc.has_key('no-store') or cc_response.has_key('no-store'): 3665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) cache.delete(cachekey) 3675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) else: 3685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) info = email.Message.Message() 3695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for key, value in response_headers.iteritems(): 3705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if key not in ['status','content-encoding','transfer-encoding']: 3715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) info[key] = value 3725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Add annotations to the cache to indicate what headers 3745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # are variant for this request. 3755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) vary = response_headers.get('vary', None) 3765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if vary: 3775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) vary_headers = vary.lower().replace(' ', '').split(',') 3785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for header in vary_headers: 3795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) key = '-varied-%s' % header 3805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) try: 3815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) info[key] = request_headers[header] 3825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) except KeyError: 3835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pass 3845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) status = response_headers.status 3865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if status == 304: 3875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) status = 200 3885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) status_header = 'status: %d\r\n' % response_headers.status 3905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) header_str = info.as_string() 3925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) header_str = re.sub("\r(?!\n)|(?<!\r)\n", "\r\n", header_str) 3945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) text = "".join([status_header, header_str, content]) 3955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) cache.set(cachekey, text) 3975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)def _cnonce(): 3995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) dig = _md5("%s:%s" % (time.ctime(), ["0123456789"[random.randrange(0, 9)] for i in range(20)])).hexdigest() 4005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return dig[:16] 4015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 4025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)def _wsse_username_token(cnonce, iso_now, password): 4035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return base64.b64encode(_sha("%s%s%s" % (cnonce, iso_now, password)).digest()).strip() 4045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 4055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 4065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# For credentials we need two things, first 4075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# a pool of credential to try (not necesarily tied to BAsic, Digest, etc.) 4085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# Then we also need a list of URIs that have already demanded authentication 4095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# That list is tricky since sub-URIs can take the same auth, or the 4105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# auth scheme may change as you descend the tree. 4115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# So we also need each Auth instance to be able to tell us 4125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# how close to the 'top' it is. 4135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 4145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)class Authentication(object): 4155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def __init__(self, credentials, host, request_uri, headers, response, content, http): 4165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) (scheme, authority, path, query, fragment) = parse_uri(request_uri) 4175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.path = path 4185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.host = host 4195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.credentials = credentials 4205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.http = http 4215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 4225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def depth(self, request_uri): 4235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) (scheme, authority, path, query, fragment) = parse_uri(request_uri) 4245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return request_uri[len(self.path):].count("/") 4255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 4265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def inscope(self, host, request_uri): 4275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # XXX Should we normalize the request_uri? 4285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) (scheme, authority, path, query, fragment) = parse_uri(request_uri) 4295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return (host == self.host) and path.startswith(self.path) 4305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 4315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def request(self, method, request_uri, headers, content): 4325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Modify the request headers to add the appropriate 4335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Authorization header. Over-rise this in sub-classes.""" 4345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pass 4355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 4365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def response(self, response, content): 4375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Gives us a chance to update with new nonces 4385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) or such returned from the last authorized response. 4395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Over-rise this in sub-classes if necessary. 4405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 4415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Return TRUE is the request is to be retried, for 4425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) example Digest may return stale=true. 4435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 4445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return False 4455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 4465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 4475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 4485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)class BasicAuthentication(Authentication): 4495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def __init__(self, credentials, host, request_uri, headers, response, content, http): 4505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Authentication.__init__(self, credentials, host, request_uri, headers, response, content, http) 4515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 4525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def request(self, method, request_uri, headers, content): 4535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Modify the request headers to add the appropriate 4545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Authorization header.""" 4555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) headers['authorization'] = 'Basic ' + base64.b64encode("%s:%s" % self.credentials).strip() 4565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 4575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 4585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)class DigestAuthentication(Authentication): 4595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Only do qop='auth' and MD5, since that 4605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) is all Apache currently implements""" 4615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def __init__(self, credentials, host, request_uri, headers, response, content, http): 4625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Authentication.__init__(self, credentials, host, request_uri, headers, response, content, http) 4635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) challenge = _parse_www_authenticate(response, 'www-authenticate') 4645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.challenge = challenge['digest'] 4655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) qop = self.challenge.get('qop', 'auth') 4665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.challenge['qop'] = ('auth' in [x.strip() for x in qop.split()]) and 'auth' or None 4675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if self.challenge['qop'] is None: 4685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) raise UnimplementedDigestAuthOptionError( _("Unsupported value for qop: %s." % qop)) 4695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.challenge['algorithm'] = self.challenge.get('algorithm', 'MD5').upper() 4705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if self.challenge['algorithm'] != 'MD5': 4715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) raise UnimplementedDigestAuthOptionError( _("Unsupported value for algorithm: %s." % self.challenge['algorithm'])) 4725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.A1 = "".join([self.credentials[0], ":", self.challenge['realm'], ":", self.credentials[1]]) 4735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.challenge['nc'] = 1 4745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 4755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def request(self, method, request_uri, headers, content, cnonce = None): 4765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Modify the request headers""" 4775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) H = lambda x: _md5(x).hexdigest() 4785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) KD = lambda s, d: H("%s:%s" % (s, d)) 4795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) A2 = "".join([method, ":", request_uri]) 4805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.challenge['cnonce'] = cnonce or _cnonce() 4815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) request_digest = '"%s"' % KD(H(self.A1), "%s:%s:%s:%s:%s" % (self.challenge['nonce'], 4825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) '%08x' % self.challenge['nc'], 4835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.challenge['cnonce'], 4845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.challenge['qop'], H(A2) 4855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) )) 4865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) headers['Authorization'] = 'Digest username="%s", realm="%s", nonce="%s", uri="%s", algorithm=%s, response=%s, qop=%s, nc=%08x, cnonce="%s"' % ( 4875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.credentials[0], 4885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.challenge['realm'], 4895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.challenge['nonce'], 4905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) request_uri, 4915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.challenge['algorithm'], 4925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) request_digest, 4935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.challenge['qop'], 4945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.challenge['nc'], 4955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.challenge['cnonce'], 4965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ) 4975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.challenge['nc'] += 1 4985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 4995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def response(self, response, content): 5005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if not response.has_key('authentication-info'): 5015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) challenge = _parse_www_authenticate(response, 'www-authenticate').get('digest', {}) 5025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if 'true' == challenge.get('stale'): 5035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.challenge['nonce'] = challenge['nonce'] 5045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.challenge['nc'] = 1 5055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return True 5065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) else: 5075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) updated_challenge = _parse_www_authenticate(response, 'authentication-info').get('digest', {}) 5085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 5095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if updated_challenge.has_key('nextnonce'): 5105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.challenge['nonce'] = updated_challenge['nextnonce'] 5115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.challenge['nc'] = 1 5125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return False 5135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 5145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 5155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)class HmacDigestAuthentication(Authentication): 5165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Adapted from Robert Sayre's code and DigestAuthentication above.""" 5175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) __author__ = "Thomas Broyer (t.broyer@ltgt.net)" 5185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 5195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def __init__(self, credentials, host, request_uri, headers, response, content, http): 5205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Authentication.__init__(self, credentials, host, request_uri, headers, response, content, http) 5215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) challenge = _parse_www_authenticate(response, 'www-authenticate') 5225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.challenge = challenge['hmacdigest'] 5235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # TODO: self.challenge['domain'] 5245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.challenge['reason'] = self.challenge.get('reason', 'unauthorized') 5255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if self.challenge['reason'] not in ['unauthorized', 'integrity']: 5265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.challenge['reason'] = 'unauthorized' 5275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.challenge['salt'] = self.challenge.get('salt', '') 5285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if not self.challenge.get('snonce'): 5295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) raise UnimplementedHmacDigestAuthOptionError( _("The challenge doesn't contain a server nonce, or this one is empty.")) 5305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.challenge['algorithm'] = self.challenge.get('algorithm', 'HMAC-SHA-1') 5315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if self.challenge['algorithm'] not in ['HMAC-SHA-1', 'HMAC-MD5']: 5325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) raise UnimplementedHmacDigestAuthOptionError( _("Unsupported value for algorithm: %s." % self.challenge['algorithm'])) 5335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.challenge['pw-algorithm'] = self.challenge.get('pw-algorithm', 'SHA-1') 5345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if self.challenge['pw-algorithm'] not in ['SHA-1', 'MD5']: 5355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) raise UnimplementedHmacDigestAuthOptionError( _("Unsupported value for pw-algorithm: %s." % self.challenge['pw-algorithm'])) 5365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if self.challenge['algorithm'] == 'HMAC-MD5': 5375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.hashmod = _md5 5385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) else: 5395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.hashmod = _sha 5405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if self.challenge['pw-algorithm'] == 'MD5': 5415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.pwhashmod = _md5 5425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) else: 5435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.pwhashmod = _sha 5445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.key = "".join([self.credentials[0], ":", 5455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.pwhashmod.new("".join([self.credentials[1], self.challenge['salt']])).hexdigest().lower(), 5465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ":", self.challenge['realm'] 5475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ]) 5485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.key = self.pwhashmod.new(self.key).hexdigest().lower() 5495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 5505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def request(self, method, request_uri, headers, content): 5515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Modify the request headers""" 5525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) keys = _get_end2end_headers(headers) 5535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) keylist = "".join(["%s " % k for k in keys]) 5545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) headers_val = "".join([headers[k] for k in keys]) 5555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) created = time.strftime('%Y-%m-%dT%H:%M:%SZ',time.gmtime()) 5565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) cnonce = _cnonce() 5575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) request_digest = "%s:%s:%s:%s:%s" % (method, request_uri, cnonce, self.challenge['snonce'], headers_val) 5585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) request_digest = hmac.new(self.key, request_digest, self.hashmod).hexdigest().lower() 5595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) headers['Authorization'] = 'HMACDigest username="%s", realm="%s", snonce="%s", cnonce="%s", uri="%s", created="%s", response="%s", headers="%s"' % ( 5605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.credentials[0], 5615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.challenge['realm'], 5625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.challenge['snonce'], 5635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) cnonce, 5645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) request_uri, 5655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) created, 5665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) request_digest, 5675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) keylist, 5685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ) 5695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 5705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def response(self, response, content): 5715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) challenge = _parse_www_authenticate(response, 'www-authenticate').get('hmacdigest', {}) 5725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if challenge.get('reason') in ['integrity', 'stale']: 5735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return True 5745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return False 5755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 5765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 5775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)class WsseAuthentication(Authentication): 5785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """This is thinly tested and should not be relied upon. 5795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) At this time there isn't any third party server to test against. 5805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Blogger and TypePad implemented this algorithm at one point 5815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) but Blogger has since switched to Basic over HTTPS and 5825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) TypePad has implemented it wrong, by never issuing a 401 5835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) challenge but instead requiring your client to telepathically know that 5845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) their endpoint is expecting WSSE profile="UsernameToken".""" 5855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def __init__(self, credentials, host, request_uri, headers, response, content, http): 5865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Authentication.__init__(self, credentials, host, request_uri, headers, response, content, http) 5875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 5885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def request(self, method, request_uri, headers, content): 5895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Modify the request headers to add the appropriate 5905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Authorization header.""" 5915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) headers['Authorization'] = 'WSSE profile="UsernameToken"' 5925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) iso_now = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) 5935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) cnonce = _cnonce() 5945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) password_digest = _wsse_username_token(cnonce, iso_now, self.credentials[1]) 5955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) headers['X-WSSE'] = 'UsernameToken Username="%s", PasswordDigest="%s", Nonce="%s", Created="%s"' % ( 5965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.credentials[0], 5975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) password_digest, 5985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) cnonce, 5995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) iso_now) 6005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 6015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)class GoogleLoginAuthentication(Authentication): 6025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def __init__(self, credentials, host, request_uri, headers, response, content, http): 6035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) from urllib import urlencode 6045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Authentication.__init__(self, credentials, host, request_uri, headers, response, content, http) 6055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) challenge = _parse_www_authenticate(response, 'www-authenticate') 6065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) service = challenge['googlelogin'].get('service', 'xapi') 6075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Bloggger actually returns the service in the challenge 6085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # For the rest we guess based on the URI 6095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if service == 'xapi' and request_uri.find("calendar") > 0: 6105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) service = "cl" 6115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # No point in guessing Base or Spreadsheet 6125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) #elif request_uri.find("spreadsheets") > 0: 6135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # service = "wise" 6145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 6155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) auth = dict(Email=credentials[0], Passwd=credentials[1], service=service, source=headers['user-agent']) 6165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) resp, content = self.http.request("https://www.google.com/accounts/ClientLogin", method="POST", body=urlencode(auth), headers={'Content-Type': 'application/x-www-form-urlencoded'}) 6175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) lines = content.split('\n') 6185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) d = dict([tuple(line.split("=", 1)) for line in lines if line]) 6195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if resp.status == 403: 6205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.Auth = "" 6215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) else: 6225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.Auth = d['Auth'] 6235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 6245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def request(self, method, request_uri, headers, content): 6255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Modify the request headers to add the appropriate 6265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Authorization header.""" 6275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) headers['authorization'] = 'GoogleLogin Auth=' + self.Auth 6285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 6295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 6305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)AUTH_SCHEME_CLASSES = { 6315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "basic": BasicAuthentication, 6325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "wsse": WsseAuthentication, 6335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "digest": DigestAuthentication, 6345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "hmacdigest": HmacDigestAuthentication, 6355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "googlelogin": GoogleLoginAuthentication 6365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 6375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 6385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)AUTH_SCHEME_ORDER = ["hmacdigest", "googlelogin", "digest", "wsse", "basic"] 6395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 6405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)class FileCache(object): 6415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Uses a local directory as a store for cached files. 6425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Not really safe to use if multiple threads or processes are going to 6435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) be running on the same cache. 6445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 6455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def __init__(self, cache, safe=safename): # use safe=lambda x: md5.new(x).hexdigest() for the old behavior 6465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.cache = cache 6475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.safe = safe 6485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if not os.path.exists(cache): 6495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) os.makedirs(self.cache) 6505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 6515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def get(self, key): 6525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) retval = None 6535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) cacheFullPath = os.path.join(self.cache, self.safe(key)) 6545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) try: 6555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) f = file(cacheFullPath, "rb") 6565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) retval = f.read() 6575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) f.close() 6585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) except IOError: 6595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pass 6605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return retval 6615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 6625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def set(self, key, value): 6635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) cacheFullPath = os.path.join(self.cache, self.safe(key)) 6645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) f = file(cacheFullPath, "wb") 6655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) f.write(value) 6665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) f.close() 6675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 6685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def delete(self, key): 6695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) cacheFullPath = os.path.join(self.cache, self.safe(key)) 6705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if os.path.exists(cacheFullPath): 6715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) os.remove(cacheFullPath) 6725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 6735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)class Credentials(object): 6745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def __init__(self): 6755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.credentials = [] 6765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 6775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def add(self, name, password, domain=""): 6785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.credentials.append((domain.lower(), name, password)) 6795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 6805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def clear(self): 6815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.credentials = [] 6825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 6835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def iter(self, domain): 6845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for (cdomain, name, password) in self.credentials: 6855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if cdomain == "" or domain == cdomain: 6865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) yield (name, password) 6875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 6885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)class KeyCerts(Credentials): 6895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Identical to Credentials except that 6905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) name/password are mapped to key/cert.""" 6915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pass 6925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 6935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 6945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)class ProxyInfo(object): 6955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Collect information required to use a proxy.""" 6965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def __init__(self, proxy_type, proxy_host, proxy_port, proxy_rdns=None, proxy_user=None, proxy_pass=None): 6975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """The parameter proxy_type must be set to one of socks.PROXY_TYPE_XXX 6985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) constants. For example: 6995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 7005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)p = ProxyInfo(proxy_type=socks.PROXY_TYPE_HTTP, proxy_host='localhost', proxy_port=8000) 7015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 7025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.proxy_type, self.proxy_host, self.proxy_port, self.proxy_rdns, self.proxy_user, self.proxy_pass = proxy_type, proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass 7035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 7045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def astuple(self): 7055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return (self.proxy_type, self.proxy_host, self.proxy_port, self.proxy_rdns, 7065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.proxy_user, self.proxy_pass) 7075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 7085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def isgood(self): 7095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return socks and (self.proxy_host != None) and (self.proxy_port != None) 7105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 7115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 7125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)class HTTPConnectionWithTimeout(httplib.HTTPConnection): 7135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """HTTPConnection subclass that supports timeouts""" 7145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 7155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def __init__(self, host, port=None, strict=None, timeout=None, proxy_info=None): 7165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) httplib.HTTPConnection.__init__(self, host, port, strict) 7175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.timeout = timeout 7185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.proxy_info = proxy_info 7195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 7205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def connect(self): 7215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Connect to the host and port specified in __init__.""" 7225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Mostly verbatim from httplib.py. 7235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) msg = "getaddrinfo returns an empty list" 7245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for res in socket.getaddrinfo(self.host, self.port, 0, 7255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) socket.SOCK_STREAM): 7265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) af, socktype, proto, canonname, sa = res 7275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) try: 7285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if self.proxy_info and self.proxy_info.isgood(): 7295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.sock = socks.socksocket(af, socktype, proto) 7305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.sock.setproxy(*self.proxy_info.astuple()) 7315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) else: 7325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.sock = socket.socket(af, socktype, proto) 7335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Different from httplib: support timeouts. 7345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if has_timeout(self.timeout): 7355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.sock.settimeout(self.timeout) 7365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # End of difference from httplib. 7375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if self.debuglevel > 0: 7385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) print "connect: (%s, %s)" % (self.host, self.port) 7395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 7405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.sock.connect(sa) 7415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) except socket.error, msg: 7425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if self.debuglevel > 0: 7435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) print 'connect fail:', (self.host, self.port) 7445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if self.sock: 7455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.sock.close() 7465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.sock = None 7475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) continue 7485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) break 7495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if not self.sock: 7505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) raise socket.error, msg 7515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 7525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)class HTTPSConnectionWithTimeout(httplib.HTTPSConnection): 7535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "This class allows communication via SSL." 7545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 7555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def __init__(self, host, port=None, key_file=None, cert_file=None, 7565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) strict=None, timeout=None, proxy_info=None): 7575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) httplib.HTTPSConnection.__init__(self, host, port=port, key_file=key_file, 7585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) cert_file=cert_file, strict=strict) 7595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.timeout = timeout 7605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.proxy_info = proxy_info 7615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 7625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def connect(self): 7635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "Connect to a host on a given (SSL) port." 7645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 7655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if self.proxy_info and self.proxy_info.isgood(): 7665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sock = socks.socksocket(socket.AF_INET, socket.SOCK_STREAM) 7675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sock.setproxy(*self.proxy_info.astuple()) 7685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) else: 7695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 7705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 7715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if has_timeout(self.timeout): 7725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sock.settimeout(self.timeout) 7735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sock.connect((self.host, self.port)) 7745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.sock =_ssl_wrap_socket(sock, self.key_file, self.cert_file) 7755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 7765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 7775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 7785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)class Http(object): 7795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """An HTTP client that handles: 7805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)- all methods 7815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)- caching 7825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)- ETags 7835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)- compression, 7845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)- HTTPS 7855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)- Basic 7865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)- Digest 7875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)- WSSE 7885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 7895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)and more. 7905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 7915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def __init__(self, cache=None, timeout=None, proxy_info=None): 7925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """The value of proxy_info is a ProxyInfo instance. 7935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 7945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)If 'cache' is a string then it is used as a directory name 7955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)for a disk cache. Otherwise it must be an object that supports 7965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)the same interface as FileCache.""" 7975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.proxy_info = proxy_info 7985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Map domain name to an httplib connection 7995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.connections = {} 8005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # The location of the cache, for now a directory 8015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # where cached responses are held. 8025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if cache and isinstance(cache, str): 8035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.cache = FileCache(cache) 8045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) else: 8055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.cache = cache 8065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 8075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Name/password 8085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.credentials = Credentials() 8095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 8105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Key/cert 8115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.certificates = KeyCerts() 8125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 8135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # authorization objects 8145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.authorizations = [] 8155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 8165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # If set to False then no redirects are followed, even safe ones. 8175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.follow_redirects = True 8185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 8195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Which HTTP methods do we apply optimistic concurrency to, i.e. 8205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # which methods get an "if-match:" etag header added to them. 8215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.optimistic_concurrency_methods = ["PUT"] 8225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 8235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # If 'follow_redirects' is True, and this is set to True then 8245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # all redirecs are followed, including unsafe ones. 8255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.follow_all_redirects = False 8265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 8275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.ignore_etag = False 8285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 8295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.force_exception_to_status_code = False 8305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 8315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.timeout = timeout 8325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 8335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def _auth_from_challenge(self, host, request_uri, headers, response, content): 8345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """A generator that creates Authorization objects 8355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) that can be applied to requests. 8365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 8375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) challenges = _parse_www_authenticate(response, 'www-authenticate') 8385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for cred in self.credentials.iter(host): 8395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for scheme in AUTH_SCHEME_ORDER: 8405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if challenges.has_key(scheme): 8415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) yield AUTH_SCHEME_CLASSES[scheme](cred, host, request_uri, headers, response, content, self) 8425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 8435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def add_credentials(self, name, password, domain=""): 8445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Add a name and password that will be used 8455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) any time a request requires authentication.""" 8465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.credentials.add(name, password, domain) 8475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 8485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def add_certificate(self, key, cert, domain): 8495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Add a key and cert that will be used 8505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) any time a request requires authentication.""" 8515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.certificates.add(key, cert, domain) 8525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 8535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def clear_credentials(self): 8545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Remove all the names and passwords 8555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) that are used for authentication""" 8565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.credentials.clear() 8575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.authorizations = [] 8585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 8595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def _conn_request(self, conn, request_uri, method, body, headers): 8605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for i in range(2): 8615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) try: 8625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) conn.request(method, request_uri, body, headers) 8635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) except socket.gaierror: 8645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) conn.close() 8655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) raise ServerNotFoundError("Unable to find the server at %s" % conn.host) 8665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) except (socket.error, httplib.HTTPException): 8675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Just because the server closed the connection doesn't apparently mean 8685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # that the server didn't send a response. 8695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pass 8705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) try: 8715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) response = conn.getresponse() 8725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) except (socket.error, httplib.HTTPException): 8735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if i == 0: 8745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) conn.close() 8755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) conn.connect() 8765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) continue 8775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) else: 8785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) raise 8795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) else: 8805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) content = "" 8815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if method == "HEAD": 8825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) response.close() 8835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) else: 8845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) content = response.read() 8855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) response = Response(response) 8865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if method != "HEAD": 8875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) content = _decompressContent(response, content) 8885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) break 8895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return (response, content) 8905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 8915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 8925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def _request(self, conn, host, absolute_uri, request_uri, method, body, headers, redirections, cachekey): 8935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Do the actual request using the connection object 8945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) and also follow one level of redirects if necessary""" 8955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 8965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) auths = [(auth.depth(request_uri), auth) for auth in self.authorizations if auth.inscope(host, request_uri)] 8975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) auth = auths and sorted(auths)[0][1] or None 8985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if auth: 8995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) auth.request(method, request_uri, headers, body) 9005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 9015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) (response, content) = self._conn_request(conn, request_uri, method, body, headers) 9025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 9035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if auth: 9045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if auth.response(response, body): 9055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) auth.request(method, request_uri, headers, body) 9065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) (response, content) = self._conn_request(conn, request_uri, method, body, headers ) 9075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) response._stale_digest = 1 9085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 9095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if response.status == 401: 9105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for authorization in self._auth_from_challenge(host, request_uri, headers, response, content): 9115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) authorization.request(method, request_uri, headers, body) 9125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) (response, content) = self._conn_request(conn, request_uri, method, body, headers, ) 9135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if response.status != 401: 9145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.authorizations.append(authorization) 9155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) authorization.response(response, body) 9165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) break 9175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 9185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (self.follow_all_redirects or (method in ["GET", "HEAD"]) or response.status == 303): 9195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if self.follow_redirects and response.status in [300, 301, 302, 303, 307]: 9205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Pick out the location header and basically start from the beginning 9215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # remembering first to strip the ETag header and decrement our 'depth' 9225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if redirections: 9235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if not response.has_key('location') and response.status != 300: 9245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) raise RedirectMissingLocation( _("Redirected but the response is missing a Location: header."), response, content) 9255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Fix-up relative redirects (which violate an RFC 2616 MUST) 9265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if response.has_key('location'): 9275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) location = response['location'] 9285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) (scheme, authority, path, query, fragment) = parse_uri(location) 9295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if authority == None: 9305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) response['location'] = urlparse.urljoin(absolute_uri, location) 9315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if response.status == 301 and method in ["GET", "HEAD"]: 9325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) response['-x-permanent-redirect-url'] = response['location'] 9335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if not response.has_key('content-location'): 9345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) response['content-location'] = absolute_uri 9355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) _updateCache(headers, response, content, self.cache, cachekey) 9365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if headers.has_key('if-none-match'): 9375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) del headers['if-none-match'] 9385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if headers.has_key('if-modified-since'): 9395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) del headers['if-modified-since'] 9405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if response.has_key('location'): 9415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) location = response['location'] 9425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) old_response = copy.deepcopy(response) 9435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if not old_response.has_key('content-location'): 9445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) old_response['content-location'] = absolute_uri 9455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) redirect_method = ((response.status == 303) and (method not in ["GET", "HEAD"])) and "GET" or method 9465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) (response, content) = self.request(location, redirect_method, body=body, headers = headers, redirections = redirections - 1) 9475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) response.previous = old_response 9485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) else: 9495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) raise RedirectLimit( _("Redirected more times than rediection_limit allows."), response, content) 9505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) elif response.status in [200, 203] and method == "GET": 9515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Don't cache 206's since we aren't going to handle byte range requests 9525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if not response.has_key('content-location'): 9535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) response['content-location'] = absolute_uri 9545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) _updateCache(headers, response, content, self.cache, cachekey) 9555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 9565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return (response, content) 9575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 9585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def _normalize_headers(self, headers): 9595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return _normalize_headers(headers) 9605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 9615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# Need to catch and rebrand some exceptions 9625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# Then need to optionally turn all exceptions into status codes 9635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# including all socket.* and httplib.* exceptions. 9645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 9655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 9665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def request(self, uri, method="GET", body=None, headers=None, redirections=DEFAULT_MAX_REDIRECTS, connection_type=None): 9675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ Performs a single HTTP request. 9685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)The 'uri' is the URI of the HTTP resource and can begin 9695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)with either 'http' or 'https'. The value of 'uri' must be an absolute URI. 9705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 9715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)The 'method' is the HTTP method to perform, such as GET, POST, DELETE, etc. 9725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)There is no restriction on the methods allowed. 9735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 9745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)The 'body' is the entity body to be sent with the request. It is a string 9755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)object. 9765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 9775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)Any extra headers that are to be sent with the request should be provided in the 9785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)'headers' dictionary. 9795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 9805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)The maximum number of redirect to follow before raising an 9815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)exception is 'redirections. The default is 5. 9825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 9835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)The return value is a tuple of (response, content), the first 9845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)being and instance of the 'Response' class, the second being 9855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)a string that contains the response entity body. 9865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 9875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) try: 9885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if headers is None: 9895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) headers = {} 9905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) else: 9915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) headers = self._normalize_headers(headers) 9925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 9935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if not headers.has_key('user-agent'): 9945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) headers['user-agent'] = "Python-httplib2/%s" % __version__ 9955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 9965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) uri = iri2uri(uri) 9975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 9985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) (scheme, authority, request_uri, defrag_uri) = urlnorm(uri) 9995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) domain_port = authority.split(":")[0:2] 10005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if len(domain_port) == 2 and domain_port[1] == '443' and scheme == 'http': 10015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) scheme = 'https' 10025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) authority = domain_port[0] 10035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 10045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) conn_key = scheme+":"+authority 10055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if conn_key in self.connections: 10065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) conn = self.connections[conn_key] 10075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) else: 10085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if not connection_type: 10095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) connection_type = (scheme == 'https') and HTTPSConnectionWithTimeout or HTTPConnectionWithTimeout 10105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) certs = list(self.certificates.iter(authority)) 10115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if scheme == 'https' and certs: 10125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) conn = self.connections[conn_key] = connection_type(authority, key_file=certs[0][0], 10135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) cert_file=certs[0][1], timeout=self.timeout, proxy_info=self.proxy_info) 10145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) else: 10155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) conn = self.connections[conn_key] = connection_type(authority, timeout=self.timeout, proxy_info=self.proxy_info) 10165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) conn.set_debuglevel(debuglevel) 10175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 10185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if method in ["GET", "HEAD"] and 'range' not in headers and 'accept-encoding' not in headers: 10195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) headers['accept-encoding'] = 'gzip, deflate' 10205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 10215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) info = email.Message.Message() 10225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) cached_value = None 10235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if self.cache: 10245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) cachekey = defrag_uri 10255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) cached_value = self.cache.get(cachekey) 10265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if cached_value: 10275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # info = email.message_from_string(cached_value) 10285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # 10295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Need to replace the line above with the kludge below 10305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # to fix the non-existent bug not fixed in this 10315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # bug report: http://mail.python.org/pipermail/python-bugs-list/2005-September/030289.html 10325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) try: 10335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) info, content = cached_value.split('\r\n\r\n', 1) 10345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) feedparser = email.FeedParser.FeedParser() 10355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) feedparser.feed(info) 10365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) info = feedparser.close() 10375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) feedparser._parse = None 10385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) except IndexError: 10395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.cache.delete(cachekey) 10405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) cachekey = None 10415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) cached_value = None 10425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) else: 10435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) cachekey = None 10445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 10455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if method in self.optimistic_concurrency_methods and self.cache and info.has_key('etag') and not self.ignore_etag and 'if-match' not in headers: 10465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # http://www.w3.org/1999/04/Editing/ 10475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) headers['if-match'] = info['etag'] 10485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 10495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if method not in ["GET", "HEAD"] and self.cache and cachekey: 10505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # RFC 2616 Section 13.10 10515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.cache.delete(cachekey) 10525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 10535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Check the vary header in the cache to see if this request 10545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # matches what varies in the cache. 10555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if method in ['GET', 'HEAD'] and 'vary' in info: 10565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) vary = info['vary'] 10575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) vary_headers = vary.lower().replace(' ', '').split(',') 10585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for header in vary_headers: 10595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) key = '-varied-%s' % header 10605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) value = info[key] 10615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if headers.get(header, '') != value: 10625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) cached_value = None 10635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) break 10645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 10655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if cached_value and method in ["GET", "HEAD"] and self.cache and 'range' not in headers: 10665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if info.has_key('-x-permanent-redirect-url'): 10675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Should cached permanent redirects be counted in our redirection count? For now, yes. 10685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) (response, new_content) = self.request(info['-x-permanent-redirect-url'], "GET", headers = headers, redirections = redirections - 1) 10695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) response.previous = Response(info) 10705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) response.previous.fromcache = True 10715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) else: 10725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Determine our course of action: 10735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Is the cached entry fresh or stale? 10745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Has the client requested a non-cached response? 10755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # 10765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # There seems to be three possible answers: 10775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # 1. [FRESH] Return the cache entry w/o doing a GET 10785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # 2. [STALE] Do the GET (but add in cache validators if available) 10795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # 3. [TRANSPARENT] Do a GET w/o any cache validators (Cache-Control: no-cache) on the request 10805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) entry_disposition = _entry_disposition(info, headers) 10815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 10825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if entry_disposition == "FRESH": 10835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if not cached_value: 10845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) info['status'] = '504' 10855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) content = "" 10865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) response = Response(info) 10875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if cached_value: 10885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) response.fromcache = True 10895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return (response, content) 10905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 10915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if entry_disposition == "STALE": 10925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if info.has_key('etag') and not self.ignore_etag and not 'if-none-match' in headers: 10935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) headers['if-none-match'] = info['etag'] 10945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if info.has_key('last-modified') and not 'last-modified' in headers: 10955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) headers['if-modified-since'] = info['last-modified'] 10965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) elif entry_disposition == "TRANSPARENT": 10975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pass 10985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 10995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) (response, new_content) = self._request(conn, authority, uri, request_uri, method, body, headers, redirections, cachekey) 11005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 11015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if response.status == 304 and method == "GET": 11025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Rewrite the cache entry with the new end-to-end headers 11035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Take all headers that are in response 11045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # and overwrite their values in info. 11055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # unless they are hop-by-hop, or are listed in the connection header. 11065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 11075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for key in _get_end2end_headers(response): 11085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) info[key] = response[key] 11095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) merged_response = Response(info) 11105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if hasattr(response, "_stale_digest"): 11115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) merged_response._stale_digest = response._stale_digest 11125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) _updateCache(headers, merged_response, content, self.cache, cachekey) 11135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) response = merged_response 11145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) response.status = 200 11155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) response.fromcache = True 11165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 11175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) elif response.status == 200: 11185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) content = new_content 11195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) else: 11205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.cache.delete(cachekey) 11215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) content = new_content 11225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) else: 11235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) cc = _parse_cache_control(headers) 11245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if cc.has_key('only-if-cached'): 11255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) info['status'] = '504' 11265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) response = Response(info) 11275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) content = "" 11285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) else: 11295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) (response, content) = self._request(conn, authority, uri, request_uri, method, body, headers, redirections, cachekey) 11305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) except Exception, e: 11315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if self.force_exception_to_status_code: 11325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if isinstance(e, HttpLib2ErrorWithResponse): 11335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) response = e.response 11345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) content = e.content 11355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) response.status = 500 11365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) response.reason = str(e) 11375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) elif isinstance(e, socket.timeout): 11385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) content = "Request Timeout" 11395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) response = Response( { 11405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "content-type": "text/plain", 11415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "status": "408", 11425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "content-length": len(content) 11435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) }) 11445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) response.reason = "Request Timeout" 11455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) else: 11465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) content = str(e) 11475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) response = Response( { 11485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "content-type": "text/plain", 11495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "status": "400", 11505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "content-length": len(content) 11515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) }) 11525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) response.reason = "Bad Request" 11535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) else: 11545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) raise 11555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 11565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 11575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return (response, content) 11585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 11595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 11605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 11615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)class Response(dict): 11625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """An object more like email.Message than httplib.HTTPResponse.""" 11635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 11645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Is this response from our local cache""" 11655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) fromcache = False 11665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 11675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """HTTP protocol version used by server. 10 for HTTP/1.0, 11 for HTTP/1.1. """ 11685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) version = 11 11695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 11705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "Status code returned by server. " 11715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) status = 200 11725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 11735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Reason phrase returned by server.""" 11745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) reason = "Ok" 11755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 11765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) previous = None 11775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 11785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def __init__(self, info): 11795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # info is either an email.Message or 11805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # an httplib.HTTPResponse object. 11815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if isinstance(info, httplib.HTTPResponse): 11825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for key, value in info.getheaders(): 11835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self[key.lower()] = value 11845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.status = info.status 11855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self['status'] = str(self.status) 11865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.reason = info.reason 11875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.version = info.version 11885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) elif isinstance(info, email.Message.Message): 11895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for key, value in info.items(): 11905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self[key] = value 11915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.status = int(self['status']) 11925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) else: 11935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for key, value in info.iteritems(): 11945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self[key] = value 11955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.status = int(self.get('status', self.status)) 11965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 11975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 11985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def __getattr__(self, name): 11995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if name == 'dict': 12005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return self 12015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) else: 12025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) raise AttributeError, name 1203