10a8c90248264a8b26970b4473770bcc3df8515fJosh Gao"""Create portable serialized representations of Python objects. 20a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 30a8c90248264a8b26970b4473770bcc3df8515fJosh GaoSee module cPickle for a (much) faster implementation. 40a8c90248264a8b26970b4473770bcc3df8515fJosh GaoSee module copy_reg for a mechanism for registering custom picklers. 50a8c90248264a8b26970b4473770bcc3df8515fJosh GaoSee module pickletools source for extensive comments. 60a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 70a8c90248264a8b26970b4473770bcc3df8515fJosh GaoClasses: 80a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 90a8c90248264a8b26970b4473770bcc3df8515fJosh Gao Pickler 100a8c90248264a8b26970b4473770bcc3df8515fJosh Gao Unpickler 110a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 120a8c90248264a8b26970b4473770bcc3df8515fJosh GaoFunctions: 130a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 140a8c90248264a8b26970b4473770bcc3df8515fJosh Gao dump(object, file) 150a8c90248264a8b26970b4473770bcc3df8515fJosh Gao dumps(object) -> string 160a8c90248264a8b26970b4473770bcc3df8515fJosh Gao load(file) -> object 170a8c90248264a8b26970b4473770bcc3df8515fJosh Gao loads(string) -> object 180a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 190a8c90248264a8b26970b4473770bcc3df8515fJosh GaoMisc variables: 200a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 210a8c90248264a8b26970b4473770bcc3df8515fJosh Gao __version__ 220a8c90248264a8b26970b4473770bcc3df8515fJosh Gao format_version 230a8c90248264a8b26970b4473770bcc3df8515fJosh Gao compatible_formats 240a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 250a8c90248264a8b26970b4473770bcc3df8515fJosh Gao""" 260a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 270a8c90248264a8b26970b4473770bcc3df8515fJosh Gao__version__ = "$Revision: 72223 $" # Code version 280a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 290a8c90248264a8b26970b4473770bcc3df8515fJosh Gaofrom types import * 300a8c90248264a8b26970b4473770bcc3df8515fJosh Gaofrom copy_reg import dispatch_table 310a8c90248264a8b26970b4473770bcc3df8515fJosh Gaofrom copy_reg import _extension_registry, _inverted_registry, _extension_cache 320a8c90248264a8b26970b4473770bcc3df8515fJosh Gaoimport marshal 330a8c90248264a8b26970b4473770bcc3df8515fJosh Gaoimport sys 340a8c90248264a8b26970b4473770bcc3df8515fJosh Gaoimport struct 350a8c90248264a8b26970b4473770bcc3df8515fJosh Gaoimport re 360a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 370a8c90248264a8b26970b4473770bcc3df8515fJosh Gao__all__ = ["PickleError", "PicklingError", "UnpicklingError", "Pickler", 380a8c90248264a8b26970b4473770bcc3df8515fJosh Gao "Unpickler", "dump", "dumps", "load", "loads"] 390a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 400a8c90248264a8b26970b4473770bcc3df8515fJosh Gao# These are purely informational; no code uses these. 410a8c90248264a8b26970b4473770bcc3df8515fJosh Gaoformat_version = "2.0" # File format version we write 420a8c90248264a8b26970b4473770bcc3df8515fJosh Gaocompatible_formats = ["1.0", # Original protocol 0 430a8c90248264a8b26970b4473770bcc3df8515fJosh Gao "1.1", # Protocol 0 with INST added 440a8c90248264a8b26970b4473770bcc3df8515fJosh Gao "1.2", # Original protocol 1 450a8c90248264a8b26970b4473770bcc3df8515fJosh Gao "1.3", # Protocol 1 with BINFLOAT added 460a8c90248264a8b26970b4473770bcc3df8515fJosh Gao "2.0", # Protocol 2 470a8c90248264a8b26970b4473770bcc3df8515fJosh Gao ] # Old format versions we can read 480a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 490a8c90248264a8b26970b4473770bcc3df8515fJosh Gao# Keep in synch with cPickle. This is the highest protocol number we 500a8c90248264a8b26970b4473770bcc3df8515fJosh Gao# know how to read. 510a8c90248264a8b26970b4473770bcc3df8515fJosh GaoHIGHEST_PROTOCOL = 2 520a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 530a8c90248264a8b26970b4473770bcc3df8515fJosh Gao# Why use struct.pack() for pickling but marshal.loads() for 540a8c90248264a8b26970b4473770bcc3df8515fJosh Gao# unpickling? struct.pack() is 40% faster than marshal.dumps(), but 550a8c90248264a8b26970b4473770bcc3df8515fJosh Gao# marshal.loads() is twice as fast as struct.unpack()! 560a8c90248264a8b26970b4473770bcc3df8515fJosh Gaomloads = marshal.loads 570a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 580a8c90248264a8b26970b4473770bcc3df8515fJosh Gaoclass PickleError(Exception): 590a8c90248264a8b26970b4473770bcc3df8515fJosh Gao """A common base class for the other pickling exceptions.""" 600a8c90248264a8b26970b4473770bcc3df8515fJosh Gao pass 610a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 620a8c90248264a8b26970b4473770bcc3df8515fJosh Gaoclass PicklingError(PickleError): 630a8c90248264a8b26970b4473770bcc3df8515fJosh Gao """This exception is raised when an unpicklable object is passed to the 640a8c90248264a8b26970b4473770bcc3df8515fJosh Gao dump() method. 650a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 660a8c90248264a8b26970b4473770bcc3df8515fJosh Gao """ 670a8c90248264a8b26970b4473770bcc3df8515fJosh Gao pass 680a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 690a8c90248264a8b26970b4473770bcc3df8515fJosh Gaoclass UnpicklingError(PickleError): 700a8c90248264a8b26970b4473770bcc3df8515fJosh Gao """This exception is raised when there is a problem unpickling an object, 710a8c90248264a8b26970b4473770bcc3df8515fJosh Gao such as a security violation. 720a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 730a8c90248264a8b26970b4473770bcc3df8515fJosh Gao Note that other exceptions may also be raised during unpickling, including 740a8c90248264a8b26970b4473770bcc3df8515fJosh Gao (but not necessarily limited to) AttributeError, EOFError, ImportError, 750a8c90248264a8b26970b4473770bcc3df8515fJosh Gao and IndexError. 760a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 770a8c90248264a8b26970b4473770bcc3df8515fJosh Gao """ 780a8c90248264a8b26970b4473770bcc3df8515fJosh Gao pass 790a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 800a8c90248264a8b26970b4473770bcc3df8515fJosh Gao# An instance of _Stop is raised by Unpickler.load_stop() in response to 810a8c90248264a8b26970b4473770bcc3df8515fJosh Gao# the STOP opcode, passing the object that is the result of unpickling. 820a8c90248264a8b26970b4473770bcc3df8515fJosh Gaoclass _Stop(Exception): 830a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def __init__(self, value): 840a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.value = value 850a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 860a8c90248264a8b26970b4473770bcc3df8515fJosh Gao# Jython has PyStringMap; it's a dict subclass with string keys 870a8c90248264a8b26970b4473770bcc3df8515fJosh Gaotry: 880a8c90248264a8b26970b4473770bcc3df8515fJosh Gao from org.python.core import PyStringMap 890a8c90248264a8b26970b4473770bcc3df8515fJosh Gaoexcept ImportError: 900a8c90248264a8b26970b4473770bcc3df8515fJosh Gao PyStringMap = None 910a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 920a8c90248264a8b26970b4473770bcc3df8515fJosh Gao# UnicodeType may or may not be exported (normally imported from types) 930a8c90248264a8b26970b4473770bcc3df8515fJosh Gaotry: 940a8c90248264a8b26970b4473770bcc3df8515fJosh Gao UnicodeType 950a8c90248264a8b26970b4473770bcc3df8515fJosh Gaoexcept NameError: 960a8c90248264a8b26970b4473770bcc3df8515fJosh Gao UnicodeType = None 970a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 980a8c90248264a8b26970b4473770bcc3df8515fJosh Gao# Pickle opcodes. See pickletools.py for extensive docs. The listing 990a8c90248264a8b26970b4473770bcc3df8515fJosh Gao# here is in kind-of alphabetical order of 1-character pickle code. 1000a8c90248264a8b26970b4473770bcc3df8515fJosh Gao# pickletools groups them by purpose. 1010a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 1020a8c90248264a8b26970b4473770bcc3df8515fJosh GaoMARK = '(' # push special markobject on stack 1030a8c90248264a8b26970b4473770bcc3df8515fJosh GaoSTOP = '.' # every pickle ends with STOP 1040a8c90248264a8b26970b4473770bcc3df8515fJosh GaoPOP = '0' # discard topmost stack item 1050a8c90248264a8b26970b4473770bcc3df8515fJosh GaoPOP_MARK = '1' # discard stack top through topmost markobject 1060a8c90248264a8b26970b4473770bcc3df8515fJosh GaoDUP = '2' # duplicate top stack item 1070a8c90248264a8b26970b4473770bcc3df8515fJosh GaoFLOAT = 'F' # push float object; decimal string argument 1080a8c90248264a8b26970b4473770bcc3df8515fJosh GaoINT = 'I' # push integer or bool; decimal string argument 1090a8c90248264a8b26970b4473770bcc3df8515fJosh GaoBININT = 'J' # push four-byte signed int 1100a8c90248264a8b26970b4473770bcc3df8515fJosh GaoBININT1 = 'K' # push 1-byte unsigned int 1110a8c90248264a8b26970b4473770bcc3df8515fJosh GaoLONG = 'L' # push long; decimal string argument 1120a8c90248264a8b26970b4473770bcc3df8515fJosh GaoBININT2 = 'M' # push 2-byte unsigned int 1130a8c90248264a8b26970b4473770bcc3df8515fJosh GaoNONE = 'N' # push None 1140a8c90248264a8b26970b4473770bcc3df8515fJosh GaoPERSID = 'P' # push persistent object; id is taken from string arg 1150a8c90248264a8b26970b4473770bcc3df8515fJosh GaoBINPERSID = 'Q' # " " " ; " " " " stack 1160a8c90248264a8b26970b4473770bcc3df8515fJosh GaoREDUCE = 'R' # apply callable to argtuple, both on stack 1170a8c90248264a8b26970b4473770bcc3df8515fJosh GaoSTRING = 'S' # push string; NL-terminated string argument 1180a8c90248264a8b26970b4473770bcc3df8515fJosh GaoBINSTRING = 'T' # push string; counted binary string argument 1190a8c90248264a8b26970b4473770bcc3df8515fJosh GaoSHORT_BINSTRING = 'U' # " " ; " " " " < 256 bytes 1200a8c90248264a8b26970b4473770bcc3df8515fJosh GaoUNICODE = 'V' # push Unicode string; raw-unicode-escaped'd argument 1210a8c90248264a8b26970b4473770bcc3df8515fJosh GaoBINUNICODE = 'X' # " " " ; counted UTF-8 string argument 1220a8c90248264a8b26970b4473770bcc3df8515fJosh GaoAPPEND = 'a' # append stack top to list below it 1230a8c90248264a8b26970b4473770bcc3df8515fJosh GaoBUILD = 'b' # call __setstate__ or __dict__.update() 1240a8c90248264a8b26970b4473770bcc3df8515fJosh GaoGLOBAL = 'c' # push self.find_class(modname, name); 2 string args 1250a8c90248264a8b26970b4473770bcc3df8515fJosh GaoDICT = 'd' # build a dict from stack items 1260a8c90248264a8b26970b4473770bcc3df8515fJosh GaoEMPTY_DICT = '}' # push empty dict 1270a8c90248264a8b26970b4473770bcc3df8515fJosh GaoAPPENDS = 'e' # extend list on stack by topmost stack slice 1280a8c90248264a8b26970b4473770bcc3df8515fJosh GaoGET = 'g' # push item from memo on stack; index is string arg 1290a8c90248264a8b26970b4473770bcc3df8515fJosh GaoBINGET = 'h' # " " " " " " ; " " 1-byte arg 1300a8c90248264a8b26970b4473770bcc3df8515fJosh GaoINST = 'i' # build & push class instance 1310a8c90248264a8b26970b4473770bcc3df8515fJosh GaoLONG_BINGET = 'j' # push item from memo on stack; index is 4-byte arg 1320a8c90248264a8b26970b4473770bcc3df8515fJosh GaoLIST = 'l' # build list from topmost stack items 1330a8c90248264a8b26970b4473770bcc3df8515fJosh GaoEMPTY_LIST = ']' # push empty list 1340a8c90248264a8b26970b4473770bcc3df8515fJosh GaoOBJ = 'o' # build & push class instance 1350a8c90248264a8b26970b4473770bcc3df8515fJosh GaoPUT = 'p' # store stack top in memo; index is string arg 1360a8c90248264a8b26970b4473770bcc3df8515fJosh GaoBINPUT = 'q' # " " " " " ; " " 1-byte arg 1370a8c90248264a8b26970b4473770bcc3df8515fJosh GaoLONG_BINPUT = 'r' # " " " " " ; " " 4-byte arg 1380a8c90248264a8b26970b4473770bcc3df8515fJosh GaoSETITEM = 's' # add key+value pair to dict 1390a8c90248264a8b26970b4473770bcc3df8515fJosh GaoTUPLE = 't' # build tuple from topmost stack items 1400a8c90248264a8b26970b4473770bcc3df8515fJosh GaoEMPTY_TUPLE = ')' # push empty tuple 1410a8c90248264a8b26970b4473770bcc3df8515fJosh GaoSETITEMS = 'u' # modify dict by adding topmost key+value pairs 1420a8c90248264a8b26970b4473770bcc3df8515fJosh GaoBINFLOAT = 'G' # push float; arg is 8-byte float encoding 1430a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 1440a8c90248264a8b26970b4473770bcc3df8515fJosh GaoTRUE = 'I01\n' # not an opcode; see INT docs in pickletools.py 1450a8c90248264a8b26970b4473770bcc3df8515fJosh GaoFALSE = 'I00\n' # not an opcode; see INT docs in pickletools.py 1460a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 1470a8c90248264a8b26970b4473770bcc3df8515fJosh Gao# Protocol 2 1480a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 1490a8c90248264a8b26970b4473770bcc3df8515fJosh GaoPROTO = '\x80' # identify pickle protocol 1500a8c90248264a8b26970b4473770bcc3df8515fJosh GaoNEWOBJ = '\x81' # build object by applying cls.__new__ to argtuple 1510a8c90248264a8b26970b4473770bcc3df8515fJosh GaoEXT1 = '\x82' # push object from extension registry; 1-byte index 1520a8c90248264a8b26970b4473770bcc3df8515fJosh GaoEXT2 = '\x83' # ditto, but 2-byte index 1530a8c90248264a8b26970b4473770bcc3df8515fJosh GaoEXT4 = '\x84' # ditto, but 4-byte index 1540a8c90248264a8b26970b4473770bcc3df8515fJosh GaoTUPLE1 = '\x85' # build 1-tuple from stack top 1550a8c90248264a8b26970b4473770bcc3df8515fJosh GaoTUPLE2 = '\x86' # build 2-tuple from two topmost stack items 1560a8c90248264a8b26970b4473770bcc3df8515fJosh GaoTUPLE3 = '\x87' # build 3-tuple from three topmost stack items 1570a8c90248264a8b26970b4473770bcc3df8515fJosh GaoNEWTRUE = '\x88' # push True 1580a8c90248264a8b26970b4473770bcc3df8515fJosh GaoNEWFALSE = '\x89' # push False 1590a8c90248264a8b26970b4473770bcc3df8515fJosh GaoLONG1 = '\x8a' # push long from < 256 bytes 1600a8c90248264a8b26970b4473770bcc3df8515fJosh GaoLONG4 = '\x8b' # push really big long 1610a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 1620a8c90248264a8b26970b4473770bcc3df8515fJosh Gao_tuplesize2code = [EMPTY_TUPLE, TUPLE1, TUPLE2, TUPLE3] 1630a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 1640a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 1650a8c90248264a8b26970b4473770bcc3df8515fJosh Gao__all__.extend([x for x in dir() if re.match("[A-Z][A-Z0-9_]+$",x)]) 1660a8c90248264a8b26970b4473770bcc3df8515fJosh Gaodel x 1670a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 1680a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 1690a8c90248264a8b26970b4473770bcc3df8515fJosh Gao# Pickling machinery 1700a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 1710a8c90248264a8b26970b4473770bcc3df8515fJosh Gaoclass Pickler: 1720a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 1730a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def __init__(self, file, protocol=None): 1740a8c90248264a8b26970b4473770bcc3df8515fJosh Gao """This takes a file-like object for writing a pickle data stream. 1750a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 1760a8c90248264a8b26970b4473770bcc3df8515fJosh Gao The optional protocol argument tells the pickler to use the 1770a8c90248264a8b26970b4473770bcc3df8515fJosh Gao given protocol; supported protocols are 0, 1, 2. The default 1780a8c90248264a8b26970b4473770bcc3df8515fJosh Gao protocol is 0, to be backwards compatible. (Protocol 0 is the 1790a8c90248264a8b26970b4473770bcc3df8515fJosh Gao only protocol that can be written to a file opened in text 1800a8c90248264a8b26970b4473770bcc3df8515fJosh Gao mode and read back successfully. When using a protocol higher 1810a8c90248264a8b26970b4473770bcc3df8515fJosh Gao than 0, make sure the file is opened in binary mode, both when 1820a8c90248264a8b26970b4473770bcc3df8515fJosh Gao pickling and unpickling.) 1830a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 1840a8c90248264a8b26970b4473770bcc3df8515fJosh Gao Protocol 1 is more efficient than protocol 0; protocol 2 is 1850a8c90248264a8b26970b4473770bcc3df8515fJosh Gao more efficient than protocol 1. 1860a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 1870a8c90248264a8b26970b4473770bcc3df8515fJosh Gao Specifying a negative protocol version selects the highest 1880a8c90248264a8b26970b4473770bcc3df8515fJosh Gao protocol version supported. The higher the protocol used, the 1890a8c90248264a8b26970b4473770bcc3df8515fJosh Gao more recent the version of Python needed to read the pickle 1900a8c90248264a8b26970b4473770bcc3df8515fJosh Gao produced. 1910a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 1920a8c90248264a8b26970b4473770bcc3df8515fJosh Gao The file parameter must have a write() method that accepts a single 1930a8c90248264a8b26970b4473770bcc3df8515fJosh Gao string argument. It can thus be an open file object, a StringIO 1940a8c90248264a8b26970b4473770bcc3df8515fJosh Gao object, or any other custom object that meets this interface. 1950a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 1960a8c90248264a8b26970b4473770bcc3df8515fJosh Gao """ 1970a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if protocol is None: 1980a8c90248264a8b26970b4473770bcc3df8515fJosh Gao protocol = 0 1990a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if protocol < 0: 2000a8c90248264a8b26970b4473770bcc3df8515fJosh Gao protocol = HIGHEST_PROTOCOL 2010a8c90248264a8b26970b4473770bcc3df8515fJosh Gao elif not 0 <= protocol <= HIGHEST_PROTOCOL: 2020a8c90248264a8b26970b4473770bcc3df8515fJosh Gao raise ValueError("pickle protocol must be <= %d" % HIGHEST_PROTOCOL) 2030a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.write = file.write 2040a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.memo = {} 2050a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.proto = int(protocol) 2060a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.bin = protocol >= 1 2070a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.fast = 0 2080a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 2090a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def clear_memo(self): 2100a8c90248264a8b26970b4473770bcc3df8515fJosh Gao """Clears the pickler's "memo". 2110a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 2120a8c90248264a8b26970b4473770bcc3df8515fJosh Gao The memo is the data structure that remembers which objects the 2130a8c90248264a8b26970b4473770bcc3df8515fJosh Gao pickler has already seen, so that shared or recursive objects are 2140a8c90248264a8b26970b4473770bcc3df8515fJosh Gao pickled by reference and not by value. This method is useful when 2150a8c90248264a8b26970b4473770bcc3df8515fJosh Gao re-using picklers. 2160a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 2170a8c90248264a8b26970b4473770bcc3df8515fJosh Gao """ 2180a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.memo.clear() 2190a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 2200a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def dump(self, obj): 2210a8c90248264a8b26970b4473770bcc3df8515fJosh Gao """Write a pickled representation of obj to the open file.""" 2220a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if self.proto >= 2: 2230a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.write(PROTO + chr(self.proto)) 2240a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.save(obj) 2250a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.write(STOP) 2260a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 2270a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def memoize(self, obj): 2280a8c90248264a8b26970b4473770bcc3df8515fJosh Gao """Store an object in the memo.""" 2290a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 2300a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # The Pickler memo is a dictionary mapping object ids to 2-tuples 2310a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # that contain the Unpickler memo key and the object being memoized. 2320a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # The memo key is written to the pickle and will become 2330a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # the key in the Unpickler's memo. The object is stored in the 2340a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # Pickler memo so that transient objects are kept alive during 2350a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # pickling. 2360a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 2370a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # The use of the Unpickler memo length as the memo key is just a 2380a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # convention. The only requirement is that the memo values be unique. 2390a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # But there appears no advantage to any other scheme, and this 2400a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # scheme allows the Unpickler memo to be implemented as a plain (but 2410a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # growable) array, indexed by memo key. 2420a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if self.fast: 2430a8c90248264a8b26970b4473770bcc3df8515fJosh Gao return 2440a8c90248264a8b26970b4473770bcc3df8515fJosh Gao assert id(obj) not in self.memo 2450a8c90248264a8b26970b4473770bcc3df8515fJosh Gao memo_len = len(self.memo) 2460a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.write(self.put(memo_len)) 2470a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.memo[id(obj)] = memo_len, obj 2480a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 2490a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # Return a PUT (BINPUT, LONG_BINPUT) opcode string, with argument i. 2500a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def put(self, i, pack=struct.pack): 2510a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if self.bin: 2520a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if i < 256: 2530a8c90248264a8b26970b4473770bcc3df8515fJosh Gao return BINPUT + chr(i) 2540a8c90248264a8b26970b4473770bcc3df8515fJosh Gao else: 2550a8c90248264a8b26970b4473770bcc3df8515fJosh Gao return LONG_BINPUT + pack("<i", i) 2560a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 2570a8c90248264a8b26970b4473770bcc3df8515fJosh Gao return PUT + repr(i) + '\n' 2580a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 2590a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # Return a GET (BINGET, LONG_BINGET) opcode string, with argument i. 2600a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def get(self, i, pack=struct.pack): 2610a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if self.bin: 2620a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if i < 256: 2630a8c90248264a8b26970b4473770bcc3df8515fJosh Gao return BINGET + chr(i) 2640a8c90248264a8b26970b4473770bcc3df8515fJosh Gao else: 2650a8c90248264a8b26970b4473770bcc3df8515fJosh Gao return LONG_BINGET + pack("<i", i) 2660a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 2670a8c90248264a8b26970b4473770bcc3df8515fJosh Gao return GET + repr(i) + '\n' 2680a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 2690a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def save(self, obj): 2700a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # Check for persistent id (defined by a subclass) 2710a8c90248264a8b26970b4473770bcc3df8515fJosh Gao pid = self.persistent_id(obj) 2720a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if pid: 2730a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.save_pers(pid) 2740a8c90248264a8b26970b4473770bcc3df8515fJosh Gao return 2750a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 2760a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # Check the memo 2770a8c90248264a8b26970b4473770bcc3df8515fJosh Gao x = self.memo.get(id(obj)) 2780a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if x: 2790a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.write(self.get(x[0])) 2800a8c90248264a8b26970b4473770bcc3df8515fJosh Gao return 2810a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 2820a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # Check the type dispatch table 2830a8c90248264a8b26970b4473770bcc3df8515fJosh Gao t = type(obj) 2840a8c90248264a8b26970b4473770bcc3df8515fJosh Gao f = self.dispatch.get(t) 2850a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if f: 2860a8c90248264a8b26970b4473770bcc3df8515fJosh Gao f(self, obj) # Call unbound method with explicit self 2870a8c90248264a8b26970b4473770bcc3df8515fJosh Gao return 2880a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 2890a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # Check copy_reg.dispatch_table 2900a8c90248264a8b26970b4473770bcc3df8515fJosh Gao reduce = dispatch_table.get(t) 2910a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if reduce: 2920a8c90248264a8b26970b4473770bcc3df8515fJosh Gao rv = reduce(obj) 2930a8c90248264a8b26970b4473770bcc3df8515fJosh Gao else: 2940a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # Check for a class with a custom metaclass; treat as regular class 2950a8c90248264a8b26970b4473770bcc3df8515fJosh Gao try: 2960a8c90248264a8b26970b4473770bcc3df8515fJosh Gao issc = issubclass(t, TypeType) 2970a8c90248264a8b26970b4473770bcc3df8515fJosh Gao except TypeError: # t is not a class (old Boost; see SF #502085) 2980a8c90248264a8b26970b4473770bcc3df8515fJosh Gao issc = 0 2990a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if issc: 3000a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.save_global(obj) 3010a8c90248264a8b26970b4473770bcc3df8515fJosh Gao return 3020a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 3030a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # Check for a __reduce_ex__ method, fall back to __reduce__ 3040a8c90248264a8b26970b4473770bcc3df8515fJosh Gao reduce = getattr(obj, "__reduce_ex__", None) 3050a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if reduce: 3060a8c90248264a8b26970b4473770bcc3df8515fJosh Gao rv = reduce(self.proto) 3070a8c90248264a8b26970b4473770bcc3df8515fJosh Gao else: 3080a8c90248264a8b26970b4473770bcc3df8515fJosh Gao reduce = getattr(obj, "__reduce__", None) 3090a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if reduce: 3100a8c90248264a8b26970b4473770bcc3df8515fJosh Gao rv = reduce() 3110a8c90248264a8b26970b4473770bcc3df8515fJosh Gao else: 3120a8c90248264a8b26970b4473770bcc3df8515fJosh Gao raise PicklingError("Can't pickle %r object: %r" % 3130a8c90248264a8b26970b4473770bcc3df8515fJosh Gao (t.__name__, obj)) 3140a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 3150a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # Check for string returned by reduce(), meaning "save as global" 3160a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if type(rv) is StringType: 3170a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.save_global(obj, rv) 3180a8c90248264a8b26970b4473770bcc3df8515fJosh Gao return 3190a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 3200a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # Assert that reduce() returned a tuple 3210a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if type(rv) is not TupleType: 3220a8c90248264a8b26970b4473770bcc3df8515fJosh Gao raise PicklingError("%s must return string or tuple" % reduce) 3230a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 3240a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # Assert that it returned an appropriately sized tuple 3250a8c90248264a8b26970b4473770bcc3df8515fJosh Gao l = len(rv) 3260a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if not (2 <= l <= 5): 3270a8c90248264a8b26970b4473770bcc3df8515fJosh Gao raise PicklingError("Tuple returned by %s must have " 3280a8c90248264a8b26970b4473770bcc3df8515fJosh Gao "two to five elements" % reduce) 3290a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 3300a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # Save the reduce() output and finally memoize the object 3310a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.save_reduce(obj=obj, *rv) 3320a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 3330a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def persistent_id(self, obj): 3340a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # This exists so a subclass can override it 3350a8c90248264a8b26970b4473770bcc3df8515fJosh Gao return None 3360a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 3370a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def save_pers(self, pid): 3380a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # Save a persistent id reference 3390a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if self.bin: 3400a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.save(pid) 3410a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.write(BINPERSID) 3420a8c90248264a8b26970b4473770bcc3df8515fJosh Gao else: 3430a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.write(PERSID + str(pid) + '\n') 3440a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 3450a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def save_reduce(self, func, args, state=None, 3460a8c90248264a8b26970b4473770bcc3df8515fJosh Gao listitems=None, dictitems=None, obj=None): 3470a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # This API is called by some subclasses 3480a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 3490a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # Assert that args is a tuple or None 3500a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if not isinstance(args, TupleType): 3510a8c90248264a8b26970b4473770bcc3df8515fJosh Gao raise PicklingError("args from reduce() should be a tuple") 3520a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 3530a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # Assert that func is callable 3540a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if not hasattr(func, '__call__'): 3550a8c90248264a8b26970b4473770bcc3df8515fJosh Gao raise PicklingError("func from reduce should be callable") 3560a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 3570a8c90248264a8b26970b4473770bcc3df8515fJosh Gao save = self.save 3580a8c90248264a8b26970b4473770bcc3df8515fJosh Gao write = self.write 3590a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 3600a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # Protocol 2 special case: if func's name is __newobj__, use NEWOBJ 3610a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if self.proto >= 2 and getattr(func, "__name__", "") == "__newobj__": 3620a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # A __reduce__ implementation can direct protocol 2 to 3630a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # use the more efficient NEWOBJ opcode, while still 3640a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # allowing protocol 0 and 1 to work normally. For this to 3650a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # work, the function returned by __reduce__ should be 3660a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # called __newobj__, and its first argument should be a 3670a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # new-style class. The implementation for __newobj__ 3680a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # should be as follows, although pickle has no way to 3690a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # verify this: 3700a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # 3710a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # def __newobj__(cls, *args): 3720a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # return cls.__new__(cls, *args) 3730a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # 3740a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # Protocols 0 and 1 will pickle a reference to __newobj__, 3750a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # while protocol 2 (and above) will pickle a reference to 3760a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # cls, the remaining args tuple, and the NEWOBJ code, 3770a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # which calls cls.__new__(cls, *args) at unpickling time 3780a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # (see load_newobj below). If __reduce__ returns a 3790a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # three-tuple, the state from the third tuple item will be 3800a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # pickled regardless of the protocol, calling __setstate__ 3810a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # at unpickling time (see load_build below). 3820a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # 3830a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # Note that no standard __newobj__ implementation exists; 3840a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # you have to provide your own. This is to enforce 3850a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # compatibility with Python 2.2 (pickles written using 3860a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # protocol 0 or 1 in Python 2.3 should be unpicklable by 3870a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # Python 2.2). 3880a8c90248264a8b26970b4473770bcc3df8515fJosh Gao cls = args[0] 3890a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if not hasattr(cls, "__new__"): 3900a8c90248264a8b26970b4473770bcc3df8515fJosh Gao raise PicklingError( 3910a8c90248264a8b26970b4473770bcc3df8515fJosh Gao "args[0] from __newobj__ args has no __new__") 3920a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if obj is not None and cls is not obj.__class__: 3930a8c90248264a8b26970b4473770bcc3df8515fJosh Gao raise PicklingError( 3940a8c90248264a8b26970b4473770bcc3df8515fJosh Gao "args[0] from __newobj__ args has the wrong class") 3950a8c90248264a8b26970b4473770bcc3df8515fJosh Gao args = args[1:] 3960a8c90248264a8b26970b4473770bcc3df8515fJosh Gao save(cls) 3970a8c90248264a8b26970b4473770bcc3df8515fJosh Gao save(args) 3980a8c90248264a8b26970b4473770bcc3df8515fJosh Gao write(NEWOBJ) 3990a8c90248264a8b26970b4473770bcc3df8515fJosh Gao else: 4000a8c90248264a8b26970b4473770bcc3df8515fJosh Gao save(func) 4010a8c90248264a8b26970b4473770bcc3df8515fJosh Gao save(args) 4020a8c90248264a8b26970b4473770bcc3df8515fJosh Gao write(REDUCE) 4030a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 4040a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if obj is not None: 4050a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.memoize(obj) 4060a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 4070a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # More new special cases (that work with older protocols as 4080a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # well): when __reduce__ returns a tuple with 4 or 5 items, 4090a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # the 4th and 5th item should be iterators that provide list 4100a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # items and dict items (as (key, value) tuples), or None. 4110a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 4120a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if listitems is not None: 4130a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self._batch_appends(listitems) 4140a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 4150a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if dictitems is not None: 4160a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self._batch_setitems(dictitems) 4170a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 4180a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if state is not None: 4190a8c90248264a8b26970b4473770bcc3df8515fJosh Gao save(state) 4200a8c90248264a8b26970b4473770bcc3df8515fJosh Gao write(BUILD) 4210a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 4220a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # Methods below this point are dispatched through the dispatch table 4230a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 4240a8c90248264a8b26970b4473770bcc3df8515fJosh Gao dispatch = {} 4250a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 4260a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def save_none(self, obj): 4270a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.write(NONE) 4280a8c90248264a8b26970b4473770bcc3df8515fJosh Gao dispatch[NoneType] = save_none 4290a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 4300a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def save_bool(self, obj): 4310a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if self.proto >= 2: 4320a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.write(obj and NEWTRUE or NEWFALSE) 4330a8c90248264a8b26970b4473770bcc3df8515fJosh Gao else: 4340a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.write(obj and TRUE or FALSE) 4350a8c90248264a8b26970b4473770bcc3df8515fJosh Gao dispatch[bool] = save_bool 4360a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 4370a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def save_int(self, obj, pack=struct.pack): 4380a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if self.bin: 4390a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # If the int is small enough to fit in a signed 4-byte 2's-comp 4400a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # format, we can store it more efficiently than the general 4410a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # case. 4420a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # First one- and two-byte unsigned ints: 4430a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if obj >= 0: 4440a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if obj <= 0xff: 4450a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.write(BININT1 + chr(obj)) 4460a8c90248264a8b26970b4473770bcc3df8515fJosh Gao return 4470a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if obj <= 0xffff: 4480a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.write("%c%c%c" % (BININT2, obj&0xff, obj>>8)) 4490a8c90248264a8b26970b4473770bcc3df8515fJosh Gao return 4500a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # Next check for 4-byte signed ints: 4510a8c90248264a8b26970b4473770bcc3df8515fJosh Gao high_bits = obj >> 31 # note that Python shift sign-extends 4520a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if high_bits == 0 or high_bits == -1: 4530a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # All high bits are copies of bit 2**31, so the value 4540a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # fits in a 4-byte signed int. 4550a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.write(BININT + pack("<i", obj)) 4560a8c90248264a8b26970b4473770bcc3df8515fJosh Gao return 4570a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # Text pickle, or int too big to fit in signed 4-byte format. 4580a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.write(INT + repr(obj) + '\n') 4590a8c90248264a8b26970b4473770bcc3df8515fJosh Gao dispatch[IntType] = save_int 4600a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 4610a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def save_long(self, obj, pack=struct.pack): 4620a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if self.proto >= 2: 4630a8c90248264a8b26970b4473770bcc3df8515fJosh Gao bytes = encode_long(obj) 4640a8c90248264a8b26970b4473770bcc3df8515fJosh Gao n = len(bytes) 4650a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if n < 256: 4660a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.write(LONG1 + chr(n) + bytes) 4670a8c90248264a8b26970b4473770bcc3df8515fJosh Gao else: 4680a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.write(LONG4 + pack("<i", n) + bytes) 4690a8c90248264a8b26970b4473770bcc3df8515fJosh Gao return 4700a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.write(LONG + repr(obj) + '\n') 4710a8c90248264a8b26970b4473770bcc3df8515fJosh Gao dispatch[LongType] = save_long 4720a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 4730a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def save_float(self, obj, pack=struct.pack): 4740a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if self.bin: 4750a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.write(BINFLOAT + pack('>d', obj)) 4760a8c90248264a8b26970b4473770bcc3df8515fJosh Gao else: 4770a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.write(FLOAT + repr(obj) + '\n') 4780a8c90248264a8b26970b4473770bcc3df8515fJosh Gao dispatch[FloatType] = save_float 4790a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 4800a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def save_string(self, obj, pack=struct.pack): 4810a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if self.bin: 4820a8c90248264a8b26970b4473770bcc3df8515fJosh Gao n = len(obj) 4830a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if n < 256: 4840a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.write(SHORT_BINSTRING + chr(n) + obj) 4850a8c90248264a8b26970b4473770bcc3df8515fJosh Gao else: 4860a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.write(BINSTRING + pack("<i", n) + obj) 4870a8c90248264a8b26970b4473770bcc3df8515fJosh Gao else: 4880a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.write(STRING + repr(obj) + '\n') 4890a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.memoize(obj) 4900a8c90248264a8b26970b4473770bcc3df8515fJosh Gao dispatch[StringType] = save_string 4910a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 4920a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def save_unicode(self, obj, pack=struct.pack): 4930a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if self.bin: 4940a8c90248264a8b26970b4473770bcc3df8515fJosh Gao encoding = obj.encode('utf-8') 4950a8c90248264a8b26970b4473770bcc3df8515fJosh Gao n = len(encoding) 4960a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.write(BINUNICODE + pack("<i", n) + encoding) 4970a8c90248264a8b26970b4473770bcc3df8515fJosh Gao else: 4980a8c90248264a8b26970b4473770bcc3df8515fJosh Gao obj = obj.replace("\\", "\\u005c") 4990a8c90248264a8b26970b4473770bcc3df8515fJosh Gao obj = obj.replace("\n", "\\u000a") 5000a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.write(UNICODE + obj.encode('raw-unicode-escape') + '\n') 5010a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.memoize(obj) 5020a8c90248264a8b26970b4473770bcc3df8515fJosh Gao dispatch[UnicodeType] = save_unicode 5030a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 5040a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if StringType is UnicodeType: 5050a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # This is true for Jython 5060a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def save_string(self, obj, pack=struct.pack): 5070a8c90248264a8b26970b4473770bcc3df8515fJosh Gao unicode = obj.isunicode() 5080a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 5090a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if self.bin: 5100a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if unicode: 5110a8c90248264a8b26970b4473770bcc3df8515fJosh Gao obj = obj.encode("utf-8") 5120a8c90248264a8b26970b4473770bcc3df8515fJosh Gao l = len(obj) 5130a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if l < 256 and not unicode: 5140a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.write(SHORT_BINSTRING + chr(l) + obj) 5150a8c90248264a8b26970b4473770bcc3df8515fJosh Gao else: 5160a8c90248264a8b26970b4473770bcc3df8515fJosh Gao s = pack("<i", l) 5170a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if unicode: 5180a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.write(BINUNICODE + s + obj) 5190a8c90248264a8b26970b4473770bcc3df8515fJosh Gao else: 5200a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.write(BINSTRING + s + obj) 5210a8c90248264a8b26970b4473770bcc3df8515fJosh Gao else: 5220a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if unicode: 5230a8c90248264a8b26970b4473770bcc3df8515fJosh Gao obj = obj.replace("\\", "\\u005c") 5240a8c90248264a8b26970b4473770bcc3df8515fJosh Gao obj = obj.replace("\n", "\\u000a") 5250a8c90248264a8b26970b4473770bcc3df8515fJosh Gao obj = obj.encode('raw-unicode-escape') 5260a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.write(UNICODE + obj + '\n') 5270a8c90248264a8b26970b4473770bcc3df8515fJosh Gao else: 5280a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.write(STRING + repr(obj) + '\n') 5290a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.memoize(obj) 5300a8c90248264a8b26970b4473770bcc3df8515fJosh Gao dispatch[StringType] = save_string 5310a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 5320a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def save_tuple(self, obj): 5330a8c90248264a8b26970b4473770bcc3df8515fJosh Gao write = self.write 5340a8c90248264a8b26970b4473770bcc3df8515fJosh Gao proto = self.proto 5350a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 5360a8c90248264a8b26970b4473770bcc3df8515fJosh Gao n = len(obj) 5370a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if n == 0: 5380a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if proto: 5390a8c90248264a8b26970b4473770bcc3df8515fJosh Gao write(EMPTY_TUPLE) 5400a8c90248264a8b26970b4473770bcc3df8515fJosh Gao else: 5410a8c90248264a8b26970b4473770bcc3df8515fJosh Gao write(MARK + TUPLE) 5420a8c90248264a8b26970b4473770bcc3df8515fJosh Gao return 5430a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 5440a8c90248264a8b26970b4473770bcc3df8515fJosh Gao save = self.save 5450a8c90248264a8b26970b4473770bcc3df8515fJosh Gao memo = self.memo 5460a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if n <= 3 and proto >= 2: 5470a8c90248264a8b26970b4473770bcc3df8515fJosh Gao for element in obj: 5480a8c90248264a8b26970b4473770bcc3df8515fJosh Gao save(element) 5490a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # Subtle. Same as in the big comment below. 5500a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if id(obj) in memo: 5510a8c90248264a8b26970b4473770bcc3df8515fJosh Gao get = self.get(memo[id(obj)][0]) 5520a8c90248264a8b26970b4473770bcc3df8515fJosh Gao write(POP * n + get) 5530a8c90248264a8b26970b4473770bcc3df8515fJosh Gao else: 5540a8c90248264a8b26970b4473770bcc3df8515fJosh Gao write(_tuplesize2code[n]) 5550a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.memoize(obj) 5560a8c90248264a8b26970b4473770bcc3df8515fJosh Gao return 5570a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 5580a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # proto 0 or proto 1 and tuple isn't empty, or proto > 1 and tuple 5590a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # has more than 3 elements. 5600a8c90248264a8b26970b4473770bcc3df8515fJosh Gao write(MARK) 5610a8c90248264a8b26970b4473770bcc3df8515fJosh Gao for element in obj: 5620a8c90248264a8b26970b4473770bcc3df8515fJosh Gao save(element) 5630a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 5640a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if id(obj) in memo: 5650a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # Subtle. d was not in memo when we entered save_tuple(), so 5660a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # the process of saving the tuple's elements must have saved 5670a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # the tuple itself: the tuple is recursive. The proper action 5680a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # now is to throw away everything we put on the stack, and 5690a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # simply GET the tuple (it's already constructed). This check 5700a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # could have been done in the "for element" loop instead, but 5710a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # recursive tuples are a rare thing. 5720a8c90248264a8b26970b4473770bcc3df8515fJosh Gao get = self.get(memo[id(obj)][0]) 5730a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if proto: 5740a8c90248264a8b26970b4473770bcc3df8515fJosh Gao write(POP_MARK + get) 5750a8c90248264a8b26970b4473770bcc3df8515fJosh Gao else: # proto 0 -- POP_MARK not available 5760a8c90248264a8b26970b4473770bcc3df8515fJosh Gao write(POP * (n+1) + get) 5770a8c90248264a8b26970b4473770bcc3df8515fJosh Gao return 5780a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 5790a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # No recursion. 5800a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.write(TUPLE) 5810a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.memoize(obj) 5820a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 5830a8c90248264a8b26970b4473770bcc3df8515fJosh Gao dispatch[TupleType] = save_tuple 5840a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 5850a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # save_empty_tuple() isn't used by anything in Python 2.3. However, I 5860a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # found a Pickler subclass in Zope3 that calls it, so it's not harmless 5870a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # to remove it. 5880a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def save_empty_tuple(self, obj): 5890a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.write(EMPTY_TUPLE) 5900a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 5910a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def save_list(self, obj): 5920a8c90248264a8b26970b4473770bcc3df8515fJosh Gao write = self.write 5930a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 5940a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if self.bin: 5950a8c90248264a8b26970b4473770bcc3df8515fJosh Gao write(EMPTY_LIST) 5960a8c90248264a8b26970b4473770bcc3df8515fJosh Gao else: # proto 0 -- can't use EMPTY_LIST 5970a8c90248264a8b26970b4473770bcc3df8515fJosh Gao write(MARK + LIST) 5980a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 5990a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.memoize(obj) 6000a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self._batch_appends(iter(obj)) 6010a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 6020a8c90248264a8b26970b4473770bcc3df8515fJosh Gao dispatch[ListType] = save_list 6030a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 6040a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # Keep in synch with cPickle's BATCHSIZE. Nothing will break if it gets 6050a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # out of synch, though. 6060a8c90248264a8b26970b4473770bcc3df8515fJosh Gao _BATCHSIZE = 1000 6070a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 6080a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def _batch_appends(self, items): 6090a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # Helper to batch up APPENDS sequences 6100a8c90248264a8b26970b4473770bcc3df8515fJosh Gao save = self.save 6110a8c90248264a8b26970b4473770bcc3df8515fJosh Gao write = self.write 6120a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 6130a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if not self.bin: 6140a8c90248264a8b26970b4473770bcc3df8515fJosh Gao for x in items: 6150a8c90248264a8b26970b4473770bcc3df8515fJosh Gao save(x) 6160a8c90248264a8b26970b4473770bcc3df8515fJosh Gao write(APPEND) 6170a8c90248264a8b26970b4473770bcc3df8515fJosh Gao return 6180a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 6190a8c90248264a8b26970b4473770bcc3df8515fJosh Gao r = xrange(self._BATCHSIZE) 6200a8c90248264a8b26970b4473770bcc3df8515fJosh Gao while items is not None: 6210a8c90248264a8b26970b4473770bcc3df8515fJosh Gao tmp = [] 6220a8c90248264a8b26970b4473770bcc3df8515fJosh Gao for i in r: 6230a8c90248264a8b26970b4473770bcc3df8515fJosh Gao try: 6240a8c90248264a8b26970b4473770bcc3df8515fJosh Gao x = items.next() 6250a8c90248264a8b26970b4473770bcc3df8515fJosh Gao tmp.append(x) 6260a8c90248264a8b26970b4473770bcc3df8515fJosh Gao except StopIteration: 6270a8c90248264a8b26970b4473770bcc3df8515fJosh Gao items = None 6280a8c90248264a8b26970b4473770bcc3df8515fJosh Gao break 6290a8c90248264a8b26970b4473770bcc3df8515fJosh Gao n = len(tmp) 6300a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if n > 1: 6310a8c90248264a8b26970b4473770bcc3df8515fJosh Gao write(MARK) 6320a8c90248264a8b26970b4473770bcc3df8515fJosh Gao for x in tmp: 6330a8c90248264a8b26970b4473770bcc3df8515fJosh Gao save(x) 6340a8c90248264a8b26970b4473770bcc3df8515fJosh Gao write(APPENDS) 6350a8c90248264a8b26970b4473770bcc3df8515fJosh Gao elif n: 6360a8c90248264a8b26970b4473770bcc3df8515fJosh Gao save(tmp[0]) 6370a8c90248264a8b26970b4473770bcc3df8515fJosh Gao write(APPEND) 6380a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # else tmp is empty, and we're done 6390a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 6400a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def save_dict(self, obj): 6410a8c90248264a8b26970b4473770bcc3df8515fJosh Gao write = self.write 6420a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 6430a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if self.bin: 6440a8c90248264a8b26970b4473770bcc3df8515fJosh Gao write(EMPTY_DICT) 6450a8c90248264a8b26970b4473770bcc3df8515fJosh Gao else: # proto 0 -- can't use EMPTY_DICT 6460a8c90248264a8b26970b4473770bcc3df8515fJosh Gao write(MARK + DICT) 6470a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 6480a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.memoize(obj) 6490a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self._batch_setitems(obj.iteritems()) 6500a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 6510a8c90248264a8b26970b4473770bcc3df8515fJosh Gao dispatch[DictionaryType] = save_dict 6520a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if not PyStringMap is None: 6530a8c90248264a8b26970b4473770bcc3df8515fJosh Gao dispatch[PyStringMap] = save_dict 6540a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 6550a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def _batch_setitems(self, items): 6560a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # Helper to batch up SETITEMS sequences; proto >= 1 only 6570a8c90248264a8b26970b4473770bcc3df8515fJosh Gao save = self.save 6580a8c90248264a8b26970b4473770bcc3df8515fJosh Gao write = self.write 6590a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 6600a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if not self.bin: 6610a8c90248264a8b26970b4473770bcc3df8515fJosh Gao for k, v in items: 6620a8c90248264a8b26970b4473770bcc3df8515fJosh Gao save(k) 6630a8c90248264a8b26970b4473770bcc3df8515fJosh Gao save(v) 6640a8c90248264a8b26970b4473770bcc3df8515fJosh Gao write(SETITEM) 6650a8c90248264a8b26970b4473770bcc3df8515fJosh Gao return 6660a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 6670a8c90248264a8b26970b4473770bcc3df8515fJosh Gao r = xrange(self._BATCHSIZE) 6680a8c90248264a8b26970b4473770bcc3df8515fJosh Gao while items is not None: 6690a8c90248264a8b26970b4473770bcc3df8515fJosh Gao tmp = [] 6700a8c90248264a8b26970b4473770bcc3df8515fJosh Gao for i in r: 6710a8c90248264a8b26970b4473770bcc3df8515fJosh Gao try: 6720a8c90248264a8b26970b4473770bcc3df8515fJosh Gao tmp.append(items.next()) 6730a8c90248264a8b26970b4473770bcc3df8515fJosh Gao except StopIteration: 6740a8c90248264a8b26970b4473770bcc3df8515fJosh Gao items = None 6750a8c90248264a8b26970b4473770bcc3df8515fJosh Gao break 6760a8c90248264a8b26970b4473770bcc3df8515fJosh Gao n = len(tmp) 6770a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if n > 1: 6780a8c90248264a8b26970b4473770bcc3df8515fJosh Gao write(MARK) 6790a8c90248264a8b26970b4473770bcc3df8515fJosh Gao for k, v in tmp: 6800a8c90248264a8b26970b4473770bcc3df8515fJosh Gao save(k) 6810a8c90248264a8b26970b4473770bcc3df8515fJosh Gao save(v) 6820a8c90248264a8b26970b4473770bcc3df8515fJosh Gao write(SETITEMS) 6830a8c90248264a8b26970b4473770bcc3df8515fJosh Gao elif n: 6840a8c90248264a8b26970b4473770bcc3df8515fJosh Gao k, v = tmp[0] 6850a8c90248264a8b26970b4473770bcc3df8515fJosh Gao save(k) 6860a8c90248264a8b26970b4473770bcc3df8515fJosh Gao save(v) 6870a8c90248264a8b26970b4473770bcc3df8515fJosh Gao write(SETITEM) 6880a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # else tmp is empty, and we're done 6890a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 6900a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def save_inst(self, obj): 6910a8c90248264a8b26970b4473770bcc3df8515fJosh Gao cls = obj.__class__ 6920a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 6930a8c90248264a8b26970b4473770bcc3df8515fJosh Gao memo = self.memo 6940a8c90248264a8b26970b4473770bcc3df8515fJosh Gao write = self.write 6950a8c90248264a8b26970b4473770bcc3df8515fJosh Gao save = self.save 6960a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 6970a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if hasattr(obj, '__getinitargs__'): 6980a8c90248264a8b26970b4473770bcc3df8515fJosh Gao args = obj.__getinitargs__() 6990a8c90248264a8b26970b4473770bcc3df8515fJosh Gao len(args) # XXX Assert it's a sequence 7000a8c90248264a8b26970b4473770bcc3df8515fJosh Gao _keep_alive(args, memo) 7010a8c90248264a8b26970b4473770bcc3df8515fJosh Gao else: 7020a8c90248264a8b26970b4473770bcc3df8515fJosh Gao args = () 7030a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 7040a8c90248264a8b26970b4473770bcc3df8515fJosh Gao write(MARK) 7050a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 7060a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if self.bin: 7070a8c90248264a8b26970b4473770bcc3df8515fJosh Gao save(cls) 7080a8c90248264a8b26970b4473770bcc3df8515fJosh Gao for arg in args: 7090a8c90248264a8b26970b4473770bcc3df8515fJosh Gao save(arg) 7100a8c90248264a8b26970b4473770bcc3df8515fJosh Gao write(OBJ) 7110a8c90248264a8b26970b4473770bcc3df8515fJosh Gao else: 7120a8c90248264a8b26970b4473770bcc3df8515fJosh Gao for arg in args: 7130a8c90248264a8b26970b4473770bcc3df8515fJosh Gao save(arg) 7140a8c90248264a8b26970b4473770bcc3df8515fJosh Gao write(INST + cls.__module__ + '\n' + cls.__name__ + '\n') 7150a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 7160a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.memoize(obj) 7170a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 7180a8c90248264a8b26970b4473770bcc3df8515fJosh Gao try: 7190a8c90248264a8b26970b4473770bcc3df8515fJosh Gao getstate = obj.__getstate__ 7200a8c90248264a8b26970b4473770bcc3df8515fJosh Gao except AttributeError: 7210a8c90248264a8b26970b4473770bcc3df8515fJosh Gao stuff = obj.__dict__ 7220a8c90248264a8b26970b4473770bcc3df8515fJosh Gao else: 7230a8c90248264a8b26970b4473770bcc3df8515fJosh Gao stuff = getstate() 7240a8c90248264a8b26970b4473770bcc3df8515fJosh Gao _keep_alive(stuff, memo) 7250a8c90248264a8b26970b4473770bcc3df8515fJosh Gao save(stuff) 7260a8c90248264a8b26970b4473770bcc3df8515fJosh Gao write(BUILD) 7270a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 7280a8c90248264a8b26970b4473770bcc3df8515fJosh Gao dispatch[InstanceType] = save_inst 7290a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 7300a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def save_global(self, obj, name=None, pack=struct.pack): 7310a8c90248264a8b26970b4473770bcc3df8515fJosh Gao write = self.write 7320a8c90248264a8b26970b4473770bcc3df8515fJosh Gao memo = self.memo 7330a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 7340a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if name is None: 7350a8c90248264a8b26970b4473770bcc3df8515fJosh Gao name = obj.__name__ 7360a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 7370a8c90248264a8b26970b4473770bcc3df8515fJosh Gao module = getattr(obj, "__module__", None) 7380a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if module is None: 7390a8c90248264a8b26970b4473770bcc3df8515fJosh Gao module = whichmodule(obj, name) 7400a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 7410a8c90248264a8b26970b4473770bcc3df8515fJosh Gao try: 7420a8c90248264a8b26970b4473770bcc3df8515fJosh Gao __import__(module) 7430a8c90248264a8b26970b4473770bcc3df8515fJosh Gao mod = sys.modules[module] 7440a8c90248264a8b26970b4473770bcc3df8515fJosh Gao klass = getattr(mod, name) 7450a8c90248264a8b26970b4473770bcc3df8515fJosh Gao except (ImportError, KeyError, AttributeError): 7460a8c90248264a8b26970b4473770bcc3df8515fJosh Gao raise PicklingError( 7470a8c90248264a8b26970b4473770bcc3df8515fJosh Gao "Can't pickle %r: it's not found as %s.%s" % 7480a8c90248264a8b26970b4473770bcc3df8515fJosh Gao (obj, module, name)) 7490a8c90248264a8b26970b4473770bcc3df8515fJosh Gao else: 7500a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if klass is not obj: 7510a8c90248264a8b26970b4473770bcc3df8515fJosh Gao raise PicklingError( 7520a8c90248264a8b26970b4473770bcc3df8515fJosh Gao "Can't pickle %r: it's not the same object as %s.%s" % 7530a8c90248264a8b26970b4473770bcc3df8515fJosh Gao (obj, module, name)) 7540a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 7550a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if self.proto >= 2: 7560a8c90248264a8b26970b4473770bcc3df8515fJosh Gao code = _extension_registry.get((module, name)) 7570a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if code: 7580a8c90248264a8b26970b4473770bcc3df8515fJosh Gao assert code > 0 7590a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if code <= 0xff: 7600a8c90248264a8b26970b4473770bcc3df8515fJosh Gao write(EXT1 + chr(code)) 7610a8c90248264a8b26970b4473770bcc3df8515fJosh Gao elif code <= 0xffff: 7620a8c90248264a8b26970b4473770bcc3df8515fJosh Gao write("%c%c%c" % (EXT2, code&0xff, code>>8)) 7630a8c90248264a8b26970b4473770bcc3df8515fJosh Gao else: 7640a8c90248264a8b26970b4473770bcc3df8515fJosh Gao write(EXT4 + pack("<i", code)) 7650a8c90248264a8b26970b4473770bcc3df8515fJosh Gao return 7660a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 7670a8c90248264a8b26970b4473770bcc3df8515fJosh Gao write(GLOBAL + module + '\n' + name + '\n') 7680a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.memoize(obj) 7690a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 7700a8c90248264a8b26970b4473770bcc3df8515fJosh Gao dispatch[ClassType] = save_global 7710a8c90248264a8b26970b4473770bcc3df8515fJosh Gao dispatch[FunctionType] = save_global 7720a8c90248264a8b26970b4473770bcc3df8515fJosh Gao dispatch[BuiltinFunctionType] = save_global 7730a8c90248264a8b26970b4473770bcc3df8515fJosh Gao dispatch[TypeType] = save_global 7740a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 7750a8c90248264a8b26970b4473770bcc3df8515fJosh Gao# Pickling helpers 7760a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 7770a8c90248264a8b26970b4473770bcc3df8515fJosh Gaodef _keep_alive(x, memo): 7780a8c90248264a8b26970b4473770bcc3df8515fJosh Gao """Keeps a reference to the object x in the memo. 7790a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 7800a8c90248264a8b26970b4473770bcc3df8515fJosh Gao Because we remember objects by their id, we have 7810a8c90248264a8b26970b4473770bcc3df8515fJosh Gao to assure that possibly temporary objects are kept 7820a8c90248264a8b26970b4473770bcc3df8515fJosh Gao alive by referencing them. 7830a8c90248264a8b26970b4473770bcc3df8515fJosh Gao We store a reference at the id of the memo, which should 7840a8c90248264a8b26970b4473770bcc3df8515fJosh Gao normally not be used unless someone tries to deepcopy 7850a8c90248264a8b26970b4473770bcc3df8515fJosh Gao the memo itself... 7860a8c90248264a8b26970b4473770bcc3df8515fJosh Gao """ 7870a8c90248264a8b26970b4473770bcc3df8515fJosh Gao try: 7880a8c90248264a8b26970b4473770bcc3df8515fJosh Gao memo[id(memo)].append(x) 7890a8c90248264a8b26970b4473770bcc3df8515fJosh Gao except KeyError: 7900a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # aha, this is the first one :-) 7910a8c90248264a8b26970b4473770bcc3df8515fJosh Gao memo[id(memo)]=[x] 7920a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 7930a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 7940a8c90248264a8b26970b4473770bcc3df8515fJosh Gao# A cache for whichmodule(), mapping a function object to the name of 7950a8c90248264a8b26970b4473770bcc3df8515fJosh Gao# the module in which the function was found. 7960a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 7970a8c90248264a8b26970b4473770bcc3df8515fJosh Gaoclassmap = {} # called classmap for backwards compatibility 7980a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 7990a8c90248264a8b26970b4473770bcc3df8515fJosh Gaodef whichmodule(func, funcname): 8000a8c90248264a8b26970b4473770bcc3df8515fJosh Gao """Figure out the module in which a function occurs. 8010a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 8020a8c90248264a8b26970b4473770bcc3df8515fJosh Gao Search sys.modules for the module. 8030a8c90248264a8b26970b4473770bcc3df8515fJosh Gao Cache in classmap. 8040a8c90248264a8b26970b4473770bcc3df8515fJosh Gao Return a module name. 8050a8c90248264a8b26970b4473770bcc3df8515fJosh Gao If the function cannot be found, return "__main__". 8060a8c90248264a8b26970b4473770bcc3df8515fJosh Gao """ 8070a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # Python functions should always get an __module__ from their globals. 8080a8c90248264a8b26970b4473770bcc3df8515fJosh Gao mod = getattr(func, "__module__", None) 8090a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if mod is not None: 8100a8c90248264a8b26970b4473770bcc3df8515fJosh Gao return mod 8110a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if func in classmap: 8120a8c90248264a8b26970b4473770bcc3df8515fJosh Gao return classmap[func] 8130a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 8140a8c90248264a8b26970b4473770bcc3df8515fJosh Gao for name, module in sys.modules.items(): 8150a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if module is None: 8160a8c90248264a8b26970b4473770bcc3df8515fJosh Gao continue # skip dummy package entries 8170a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if name != '__main__' and getattr(module, funcname, None) is func: 8180a8c90248264a8b26970b4473770bcc3df8515fJosh Gao break 8190a8c90248264a8b26970b4473770bcc3df8515fJosh Gao else: 8200a8c90248264a8b26970b4473770bcc3df8515fJosh Gao name = '__main__' 8210a8c90248264a8b26970b4473770bcc3df8515fJosh Gao classmap[func] = name 8220a8c90248264a8b26970b4473770bcc3df8515fJosh Gao return name 8230a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 8240a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 8250a8c90248264a8b26970b4473770bcc3df8515fJosh Gao# Unpickling machinery 8260a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 8270a8c90248264a8b26970b4473770bcc3df8515fJosh Gaoclass Unpickler: 8280a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 8290a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def __init__(self, file): 8300a8c90248264a8b26970b4473770bcc3df8515fJosh Gao """This takes a file-like object for reading a pickle data stream. 8310a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 8320a8c90248264a8b26970b4473770bcc3df8515fJosh Gao The protocol version of the pickle is detected automatically, so no 8330a8c90248264a8b26970b4473770bcc3df8515fJosh Gao proto argument is needed. 8340a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 8350a8c90248264a8b26970b4473770bcc3df8515fJosh Gao The file-like object must have two methods, a read() method that 8360a8c90248264a8b26970b4473770bcc3df8515fJosh Gao takes an integer argument, and a readline() method that requires no 8370a8c90248264a8b26970b4473770bcc3df8515fJosh Gao arguments. Both methods should return a string. Thus file-like 8380a8c90248264a8b26970b4473770bcc3df8515fJosh Gao object can be a file object opened for reading, a StringIO object, 8390a8c90248264a8b26970b4473770bcc3df8515fJosh Gao or any other custom object that meets this interface. 8400a8c90248264a8b26970b4473770bcc3df8515fJosh Gao """ 8410a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.readline = file.readline 8420a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.read = file.read 8430a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.memo = {} 8440a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 8450a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def load(self): 8460a8c90248264a8b26970b4473770bcc3df8515fJosh Gao """Read a pickled object representation from the open file. 8470a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 8480a8c90248264a8b26970b4473770bcc3df8515fJosh Gao Return the reconstituted object hierarchy specified in the file. 8490a8c90248264a8b26970b4473770bcc3df8515fJosh Gao """ 8500a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.mark = object() # any new unique object 8510a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.stack = [] 8520a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.append = self.stack.append 8530a8c90248264a8b26970b4473770bcc3df8515fJosh Gao read = self.read 8540a8c90248264a8b26970b4473770bcc3df8515fJosh Gao dispatch = self.dispatch 8550a8c90248264a8b26970b4473770bcc3df8515fJosh Gao try: 8560a8c90248264a8b26970b4473770bcc3df8515fJosh Gao while 1: 8570a8c90248264a8b26970b4473770bcc3df8515fJosh Gao key = read(1) 8580a8c90248264a8b26970b4473770bcc3df8515fJosh Gao dispatch[key](self) 8590a8c90248264a8b26970b4473770bcc3df8515fJosh Gao except _Stop, stopinst: 8600a8c90248264a8b26970b4473770bcc3df8515fJosh Gao return stopinst.value 8610a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 8620a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # Return largest index k such that self.stack[k] is self.mark. 8630a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # If the stack doesn't contain a mark, eventually raises IndexError. 8640a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # This could be sped by maintaining another stack, of indices at which 8650a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # the mark appears. For that matter, the latter stack would suffice, 8660a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # and we wouldn't need to push mark objects on self.stack at all. 8670a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # Doing so is probably a good thing, though, since if the pickle is 8680a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # corrupt (or hostile) we may get a clue from finding self.mark embedded 8690a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # in unpickled objects. 8700a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def marker(self): 8710a8c90248264a8b26970b4473770bcc3df8515fJosh Gao stack = self.stack 8720a8c90248264a8b26970b4473770bcc3df8515fJosh Gao mark = self.mark 8730a8c90248264a8b26970b4473770bcc3df8515fJosh Gao k = len(stack)-1 8740a8c90248264a8b26970b4473770bcc3df8515fJosh Gao while stack[k] is not mark: k = k-1 8750a8c90248264a8b26970b4473770bcc3df8515fJosh Gao return k 8760a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 8770a8c90248264a8b26970b4473770bcc3df8515fJosh Gao dispatch = {} 8780a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 8790a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def load_eof(self): 8800a8c90248264a8b26970b4473770bcc3df8515fJosh Gao raise EOFError 8810a8c90248264a8b26970b4473770bcc3df8515fJosh Gao dispatch[''] = load_eof 8820a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 8830a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def load_proto(self): 8840a8c90248264a8b26970b4473770bcc3df8515fJosh Gao proto = ord(self.read(1)) 8850a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if not 0 <= proto <= 2: 8860a8c90248264a8b26970b4473770bcc3df8515fJosh Gao raise ValueError, "unsupported pickle protocol: %d" % proto 8870a8c90248264a8b26970b4473770bcc3df8515fJosh Gao dispatch[PROTO] = load_proto 8880a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 8890a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def load_persid(self): 8900a8c90248264a8b26970b4473770bcc3df8515fJosh Gao pid = self.readline()[:-1] 8910a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.append(self.persistent_load(pid)) 8920a8c90248264a8b26970b4473770bcc3df8515fJosh Gao dispatch[PERSID] = load_persid 8930a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 8940a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def load_binpersid(self): 8950a8c90248264a8b26970b4473770bcc3df8515fJosh Gao pid = self.stack.pop() 8960a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.append(self.persistent_load(pid)) 8970a8c90248264a8b26970b4473770bcc3df8515fJosh Gao dispatch[BINPERSID] = load_binpersid 8980a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 8990a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def load_none(self): 9000a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.append(None) 9010a8c90248264a8b26970b4473770bcc3df8515fJosh Gao dispatch[NONE] = load_none 9020a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 9030a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def load_false(self): 9040a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.append(False) 9050a8c90248264a8b26970b4473770bcc3df8515fJosh Gao dispatch[NEWFALSE] = load_false 9060a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 9070a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def load_true(self): 9080a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.append(True) 9090a8c90248264a8b26970b4473770bcc3df8515fJosh Gao dispatch[NEWTRUE] = load_true 9100a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 9110a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def load_int(self): 9120a8c90248264a8b26970b4473770bcc3df8515fJosh Gao data = self.readline() 9130a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if data == FALSE[1:]: 9140a8c90248264a8b26970b4473770bcc3df8515fJosh Gao val = False 9150a8c90248264a8b26970b4473770bcc3df8515fJosh Gao elif data == TRUE[1:]: 9160a8c90248264a8b26970b4473770bcc3df8515fJosh Gao val = True 9170a8c90248264a8b26970b4473770bcc3df8515fJosh Gao else: 9180a8c90248264a8b26970b4473770bcc3df8515fJosh Gao try: 9190a8c90248264a8b26970b4473770bcc3df8515fJosh Gao val = int(data) 9200a8c90248264a8b26970b4473770bcc3df8515fJosh Gao except ValueError: 9210a8c90248264a8b26970b4473770bcc3df8515fJosh Gao val = long(data) 9220a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.append(val) 9230a8c90248264a8b26970b4473770bcc3df8515fJosh Gao dispatch[INT] = load_int 9240a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 9250a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def load_binint(self): 9260a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.append(mloads('i' + self.read(4))) 9270a8c90248264a8b26970b4473770bcc3df8515fJosh Gao dispatch[BININT] = load_binint 9280a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 9290a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def load_binint1(self): 9300a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.append(ord(self.read(1))) 9310a8c90248264a8b26970b4473770bcc3df8515fJosh Gao dispatch[BININT1] = load_binint1 9320a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 9330a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def load_binint2(self): 9340a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.append(mloads('i' + self.read(2) + '\000\000')) 9350a8c90248264a8b26970b4473770bcc3df8515fJosh Gao dispatch[BININT2] = load_binint2 9360a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 9370a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def load_long(self): 9380a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.append(long(self.readline()[:-1], 0)) 9390a8c90248264a8b26970b4473770bcc3df8515fJosh Gao dispatch[LONG] = load_long 9400a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 9410a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def load_long1(self): 9420a8c90248264a8b26970b4473770bcc3df8515fJosh Gao n = ord(self.read(1)) 9430a8c90248264a8b26970b4473770bcc3df8515fJosh Gao bytes = self.read(n) 9440a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.append(decode_long(bytes)) 9450a8c90248264a8b26970b4473770bcc3df8515fJosh Gao dispatch[LONG1] = load_long1 9460a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 9470a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def load_long4(self): 9480a8c90248264a8b26970b4473770bcc3df8515fJosh Gao n = mloads('i' + self.read(4)) 9490a8c90248264a8b26970b4473770bcc3df8515fJosh Gao bytes = self.read(n) 9500a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.append(decode_long(bytes)) 9510a8c90248264a8b26970b4473770bcc3df8515fJosh Gao dispatch[LONG4] = load_long4 9520a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 9530a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def load_float(self): 9540a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.append(float(self.readline()[:-1])) 9550a8c90248264a8b26970b4473770bcc3df8515fJosh Gao dispatch[FLOAT] = load_float 9560a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 9570a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def load_binfloat(self, unpack=struct.unpack): 9580a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.append(unpack('>d', self.read(8))[0]) 9590a8c90248264a8b26970b4473770bcc3df8515fJosh Gao dispatch[BINFLOAT] = load_binfloat 9600a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 9610a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def load_string(self): 9620a8c90248264a8b26970b4473770bcc3df8515fJosh Gao rep = self.readline()[:-1] 9630a8c90248264a8b26970b4473770bcc3df8515fJosh Gao for q in "\"'": # double or single quote 9640a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if rep.startswith(q): 9650a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if len(rep) < 2 or not rep.endswith(q): 9660a8c90248264a8b26970b4473770bcc3df8515fJosh Gao raise ValueError, "insecure string pickle" 9670a8c90248264a8b26970b4473770bcc3df8515fJosh Gao rep = rep[len(q):-len(q)] 9680a8c90248264a8b26970b4473770bcc3df8515fJosh Gao break 9690a8c90248264a8b26970b4473770bcc3df8515fJosh Gao else: 9700a8c90248264a8b26970b4473770bcc3df8515fJosh Gao raise ValueError, "insecure string pickle" 9710a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.append(rep.decode("string-escape")) 9720a8c90248264a8b26970b4473770bcc3df8515fJosh Gao dispatch[STRING] = load_string 9730a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 9740a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def load_binstring(self): 9750a8c90248264a8b26970b4473770bcc3df8515fJosh Gao len = mloads('i' + self.read(4)) 9760a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.append(self.read(len)) 9770a8c90248264a8b26970b4473770bcc3df8515fJosh Gao dispatch[BINSTRING] = load_binstring 9780a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 9790a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def load_unicode(self): 9800a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.append(unicode(self.readline()[:-1],'raw-unicode-escape')) 9810a8c90248264a8b26970b4473770bcc3df8515fJosh Gao dispatch[UNICODE] = load_unicode 9820a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 9830a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def load_binunicode(self): 9840a8c90248264a8b26970b4473770bcc3df8515fJosh Gao len = mloads('i' + self.read(4)) 9850a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.append(unicode(self.read(len),'utf-8')) 9860a8c90248264a8b26970b4473770bcc3df8515fJosh Gao dispatch[BINUNICODE] = load_binunicode 9870a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 9880a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def load_short_binstring(self): 9890a8c90248264a8b26970b4473770bcc3df8515fJosh Gao len = ord(self.read(1)) 9900a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.append(self.read(len)) 9910a8c90248264a8b26970b4473770bcc3df8515fJosh Gao dispatch[SHORT_BINSTRING] = load_short_binstring 9920a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 9930a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def load_tuple(self): 9940a8c90248264a8b26970b4473770bcc3df8515fJosh Gao k = self.marker() 9950a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.stack[k:] = [tuple(self.stack[k+1:])] 9960a8c90248264a8b26970b4473770bcc3df8515fJosh Gao dispatch[TUPLE] = load_tuple 9970a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 9980a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def load_empty_tuple(self): 9990a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.stack.append(()) 10000a8c90248264a8b26970b4473770bcc3df8515fJosh Gao dispatch[EMPTY_TUPLE] = load_empty_tuple 10010a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 10020a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def load_tuple1(self): 10030a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.stack[-1] = (self.stack[-1],) 10040a8c90248264a8b26970b4473770bcc3df8515fJosh Gao dispatch[TUPLE1] = load_tuple1 10050a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 10060a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def load_tuple2(self): 10070a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.stack[-2:] = [(self.stack[-2], self.stack[-1])] 10080a8c90248264a8b26970b4473770bcc3df8515fJosh Gao dispatch[TUPLE2] = load_tuple2 10090a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 10100a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def load_tuple3(self): 10110a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.stack[-3:] = [(self.stack[-3], self.stack[-2], self.stack[-1])] 10120a8c90248264a8b26970b4473770bcc3df8515fJosh Gao dispatch[TUPLE3] = load_tuple3 10130a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 10140a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def load_empty_list(self): 10150a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.stack.append([]) 10160a8c90248264a8b26970b4473770bcc3df8515fJosh Gao dispatch[EMPTY_LIST] = load_empty_list 10170a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 10180a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def load_empty_dictionary(self): 10190a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.stack.append({}) 10200a8c90248264a8b26970b4473770bcc3df8515fJosh Gao dispatch[EMPTY_DICT] = load_empty_dictionary 10210a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 10220a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def load_list(self): 10230a8c90248264a8b26970b4473770bcc3df8515fJosh Gao k = self.marker() 10240a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.stack[k:] = [self.stack[k+1:]] 10250a8c90248264a8b26970b4473770bcc3df8515fJosh Gao dispatch[LIST] = load_list 10260a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 10270a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def load_dict(self): 10280a8c90248264a8b26970b4473770bcc3df8515fJosh Gao k = self.marker() 10290a8c90248264a8b26970b4473770bcc3df8515fJosh Gao d = {} 10300a8c90248264a8b26970b4473770bcc3df8515fJosh Gao items = self.stack[k+1:] 10310a8c90248264a8b26970b4473770bcc3df8515fJosh Gao for i in range(0, len(items), 2): 10320a8c90248264a8b26970b4473770bcc3df8515fJosh Gao key = items[i] 10330a8c90248264a8b26970b4473770bcc3df8515fJosh Gao value = items[i+1] 10340a8c90248264a8b26970b4473770bcc3df8515fJosh Gao d[key] = value 10350a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.stack[k:] = [d] 10360a8c90248264a8b26970b4473770bcc3df8515fJosh Gao dispatch[DICT] = load_dict 10370a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 10380a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # INST and OBJ differ only in how they get a class object. It's not 10390a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # only sensible to do the rest in a common routine, the two routines 10400a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # previously diverged and grew different bugs. 10410a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # klass is the class to instantiate, and k points to the topmost mark 10420a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # object, following which are the arguments for klass.__init__. 10430a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def _instantiate(self, klass, k): 10440a8c90248264a8b26970b4473770bcc3df8515fJosh Gao args = tuple(self.stack[k+1:]) 10450a8c90248264a8b26970b4473770bcc3df8515fJosh Gao del self.stack[k:] 10460a8c90248264a8b26970b4473770bcc3df8515fJosh Gao instantiated = 0 10470a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if (not args and 10480a8c90248264a8b26970b4473770bcc3df8515fJosh Gao type(klass) is ClassType and 10490a8c90248264a8b26970b4473770bcc3df8515fJosh Gao not hasattr(klass, "__getinitargs__")): 10500a8c90248264a8b26970b4473770bcc3df8515fJosh Gao try: 10510a8c90248264a8b26970b4473770bcc3df8515fJosh Gao value = _EmptyClass() 10520a8c90248264a8b26970b4473770bcc3df8515fJosh Gao value.__class__ = klass 10530a8c90248264a8b26970b4473770bcc3df8515fJosh Gao instantiated = 1 10540a8c90248264a8b26970b4473770bcc3df8515fJosh Gao except RuntimeError: 10550a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # In restricted execution, assignment to inst.__class__ is 10560a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # prohibited 10570a8c90248264a8b26970b4473770bcc3df8515fJosh Gao pass 10580a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if not instantiated: 10590a8c90248264a8b26970b4473770bcc3df8515fJosh Gao try: 10600a8c90248264a8b26970b4473770bcc3df8515fJosh Gao value = klass(*args) 10610a8c90248264a8b26970b4473770bcc3df8515fJosh Gao except TypeError, err: 10620a8c90248264a8b26970b4473770bcc3df8515fJosh Gao raise TypeError, "in constructor for %s: %s" % ( 10630a8c90248264a8b26970b4473770bcc3df8515fJosh Gao klass.__name__, str(err)), sys.exc_info()[2] 10640a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.append(value) 10650a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 10660a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def load_inst(self): 10670a8c90248264a8b26970b4473770bcc3df8515fJosh Gao module = self.readline()[:-1] 10680a8c90248264a8b26970b4473770bcc3df8515fJosh Gao name = self.readline()[:-1] 10690a8c90248264a8b26970b4473770bcc3df8515fJosh Gao klass = self.find_class(module, name) 10700a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self._instantiate(klass, self.marker()) 10710a8c90248264a8b26970b4473770bcc3df8515fJosh Gao dispatch[INST] = load_inst 10720a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 10730a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def load_obj(self): 10740a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # Stack is ... markobject classobject arg1 arg2 ... 10750a8c90248264a8b26970b4473770bcc3df8515fJosh Gao k = self.marker() 10760a8c90248264a8b26970b4473770bcc3df8515fJosh Gao klass = self.stack.pop(k+1) 10770a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self._instantiate(klass, k) 10780a8c90248264a8b26970b4473770bcc3df8515fJosh Gao dispatch[OBJ] = load_obj 10790a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 10800a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def load_newobj(self): 10810a8c90248264a8b26970b4473770bcc3df8515fJosh Gao args = self.stack.pop() 10820a8c90248264a8b26970b4473770bcc3df8515fJosh Gao cls = self.stack[-1] 10830a8c90248264a8b26970b4473770bcc3df8515fJosh Gao obj = cls.__new__(cls, *args) 10840a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.stack[-1] = obj 10850a8c90248264a8b26970b4473770bcc3df8515fJosh Gao dispatch[NEWOBJ] = load_newobj 10860a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 10870a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def load_global(self): 10880a8c90248264a8b26970b4473770bcc3df8515fJosh Gao module = self.readline()[:-1] 10890a8c90248264a8b26970b4473770bcc3df8515fJosh Gao name = self.readline()[:-1] 10900a8c90248264a8b26970b4473770bcc3df8515fJosh Gao klass = self.find_class(module, name) 10910a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.append(klass) 10920a8c90248264a8b26970b4473770bcc3df8515fJosh Gao dispatch[GLOBAL] = load_global 10930a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 10940a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def load_ext1(self): 10950a8c90248264a8b26970b4473770bcc3df8515fJosh Gao code = ord(self.read(1)) 10960a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.get_extension(code) 10970a8c90248264a8b26970b4473770bcc3df8515fJosh Gao dispatch[EXT1] = load_ext1 10980a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 10990a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def load_ext2(self): 11000a8c90248264a8b26970b4473770bcc3df8515fJosh Gao code = mloads('i' + self.read(2) + '\000\000') 11010a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.get_extension(code) 11020a8c90248264a8b26970b4473770bcc3df8515fJosh Gao dispatch[EXT2] = load_ext2 11030a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 11040a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def load_ext4(self): 11050a8c90248264a8b26970b4473770bcc3df8515fJosh Gao code = mloads('i' + self.read(4)) 11060a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.get_extension(code) 11070a8c90248264a8b26970b4473770bcc3df8515fJosh Gao dispatch[EXT4] = load_ext4 11080a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 11090a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def get_extension(self, code): 11100a8c90248264a8b26970b4473770bcc3df8515fJosh Gao nil = [] 11110a8c90248264a8b26970b4473770bcc3df8515fJosh Gao obj = _extension_cache.get(code, nil) 11120a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if obj is not nil: 11130a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.append(obj) 11140a8c90248264a8b26970b4473770bcc3df8515fJosh Gao return 11150a8c90248264a8b26970b4473770bcc3df8515fJosh Gao key = _inverted_registry.get(code) 11160a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if not key: 11170a8c90248264a8b26970b4473770bcc3df8515fJosh Gao raise ValueError("unregistered extension code %d" % code) 11180a8c90248264a8b26970b4473770bcc3df8515fJosh Gao obj = self.find_class(*key) 11190a8c90248264a8b26970b4473770bcc3df8515fJosh Gao _extension_cache[code] = obj 11200a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.append(obj) 11210a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 11220a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def find_class(self, module, name): 11230a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # Subclasses may override this 11240a8c90248264a8b26970b4473770bcc3df8515fJosh Gao __import__(module) 11250a8c90248264a8b26970b4473770bcc3df8515fJosh Gao mod = sys.modules[module] 11260a8c90248264a8b26970b4473770bcc3df8515fJosh Gao klass = getattr(mod, name) 11270a8c90248264a8b26970b4473770bcc3df8515fJosh Gao return klass 11280a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 11290a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def load_reduce(self): 11300a8c90248264a8b26970b4473770bcc3df8515fJosh Gao stack = self.stack 11310a8c90248264a8b26970b4473770bcc3df8515fJosh Gao args = stack.pop() 11320a8c90248264a8b26970b4473770bcc3df8515fJosh Gao func = stack[-1] 11330a8c90248264a8b26970b4473770bcc3df8515fJosh Gao value = func(*args) 11340a8c90248264a8b26970b4473770bcc3df8515fJosh Gao stack[-1] = value 11350a8c90248264a8b26970b4473770bcc3df8515fJosh Gao dispatch[REDUCE] = load_reduce 11360a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 11370a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def load_pop(self): 11380a8c90248264a8b26970b4473770bcc3df8515fJosh Gao del self.stack[-1] 11390a8c90248264a8b26970b4473770bcc3df8515fJosh Gao dispatch[POP] = load_pop 11400a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 11410a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def load_pop_mark(self): 11420a8c90248264a8b26970b4473770bcc3df8515fJosh Gao k = self.marker() 11430a8c90248264a8b26970b4473770bcc3df8515fJosh Gao del self.stack[k:] 11440a8c90248264a8b26970b4473770bcc3df8515fJosh Gao dispatch[POP_MARK] = load_pop_mark 11450a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 11460a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def load_dup(self): 11470a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.append(self.stack[-1]) 11480a8c90248264a8b26970b4473770bcc3df8515fJosh Gao dispatch[DUP] = load_dup 11490a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 11500a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def load_get(self): 11510a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.append(self.memo[self.readline()[:-1]]) 11520a8c90248264a8b26970b4473770bcc3df8515fJosh Gao dispatch[GET] = load_get 11530a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 11540a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def load_binget(self): 11550a8c90248264a8b26970b4473770bcc3df8515fJosh Gao i = ord(self.read(1)) 11560a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.append(self.memo[repr(i)]) 11570a8c90248264a8b26970b4473770bcc3df8515fJosh Gao dispatch[BINGET] = load_binget 11580a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 11590a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def load_long_binget(self): 11600a8c90248264a8b26970b4473770bcc3df8515fJosh Gao i = mloads('i' + self.read(4)) 11610a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.append(self.memo[repr(i)]) 11620a8c90248264a8b26970b4473770bcc3df8515fJosh Gao dispatch[LONG_BINGET] = load_long_binget 11630a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 11640a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def load_put(self): 11650a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.memo[self.readline()[:-1]] = self.stack[-1] 11660a8c90248264a8b26970b4473770bcc3df8515fJosh Gao dispatch[PUT] = load_put 11670a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 11680a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def load_binput(self): 11690a8c90248264a8b26970b4473770bcc3df8515fJosh Gao i = ord(self.read(1)) 11700a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.memo[repr(i)] = self.stack[-1] 11710a8c90248264a8b26970b4473770bcc3df8515fJosh Gao dispatch[BINPUT] = load_binput 11720a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 11730a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def load_long_binput(self): 11740a8c90248264a8b26970b4473770bcc3df8515fJosh Gao i = mloads('i' + self.read(4)) 11750a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.memo[repr(i)] = self.stack[-1] 11760a8c90248264a8b26970b4473770bcc3df8515fJosh Gao dispatch[LONG_BINPUT] = load_long_binput 11770a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 11780a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def load_append(self): 11790a8c90248264a8b26970b4473770bcc3df8515fJosh Gao stack = self.stack 11800a8c90248264a8b26970b4473770bcc3df8515fJosh Gao value = stack.pop() 11810a8c90248264a8b26970b4473770bcc3df8515fJosh Gao list = stack[-1] 11820a8c90248264a8b26970b4473770bcc3df8515fJosh Gao list.append(value) 11830a8c90248264a8b26970b4473770bcc3df8515fJosh Gao dispatch[APPEND] = load_append 11840a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 11850a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def load_appends(self): 11860a8c90248264a8b26970b4473770bcc3df8515fJosh Gao stack = self.stack 11870a8c90248264a8b26970b4473770bcc3df8515fJosh Gao mark = self.marker() 11880a8c90248264a8b26970b4473770bcc3df8515fJosh Gao list = stack[mark - 1] 11890a8c90248264a8b26970b4473770bcc3df8515fJosh Gao list.extend(stack[mark + 1:]) 11900a8c90248264a8b26970b4473770bcc3df8515fJosh Gao del stack[mark:] 11910a8c90248264a8b26970b4473770bcc3df8515fJosh Gao dispatch[APPENDS] = load_appends 11920a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 11930a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def load_setitem(self): 11940a8c90248264a8b26970b4473770bcc3df8515fJosh Gao stack = self.stack 11950a8c90248264a8b26970b4473770bcc3df8515fJosh Gao value = stack.pop() 11960a8c90248264a8b26970b4473770bcc3df8515fJosh Gao key = stack.pop() 11970a8c90248264a8b26970b4473770bcc3df8515fJosh Gao dict = stack[-1] 11980a8c90248264a8b26970b4473770bcc3df8515fJosh Gao dict[key] = value 11990a8c90248264a8b26970b4473770bcc3df8515fJosh Gao dispatch[SETITEM] = load_setitem 12000a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 12010a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def load_setitems(self): 12020a8c90248264a8b26970b4473770bcc3df8515fJosh Gao stack = self.stack 12030a8c90248264a8b26970b4473770bcc3df8515fJosh Gao mark = self.marker() 12040a8c90248264a8b26970b4473770bcc3df8515fJosh Gao dict = stack[mark - 1] 12050a8c90248264a8b26970b4473770bcc3df8515fJosh Gao for i in range(mark + 1, len(stack), 2): 12060a8c90248264a8b26970b4473770bcc3df8515fJosh Gao dict[stack[i]] = stack[i + 1] 12070a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 12080a8c90248264a8b26970b4473770bcc3df8515fJosh Gao del stack[mark:] 12090a8c90248264a8b26970b4473770bcc3df8515fJosh Gao dispatch[SETITEMS] = load_setitems 12100a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 12110a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def load_build(self): 12120a8c90248264a8b26970b4473770bcc3df8515fJosh Gao stack = self.stack 12130a8c90248264a8b26970b4473770bcc3df8515fJosh Gao state = stack.pop() 12140a8c90248264a8b26970b4473770bcc3df8515fJosh Gao inst = stack[-1] 12150a8c90248264a8b26970b4473770bcc3df8515fJosh Gao setstate = getattr(inst, "__setstate__", None) 12160a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if setstate: 12170a8c90248264a8b26970b4473770bcc3df8515fJosh Gao setstate(state) 12180a8c90248264a8b26970b4473770bcc3df8515fJosh Gao return 12190a8c90248264a8b26970b4473770bcc3df8515fJosh Gao slotstate = None 12200a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if isinstance(state, tuple) and len(state) == 2: 12210a8c90248264a8b26970b4473770bcc3df8515fJosh Gao state, slotstate = state 12220a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if state: 12230a8c90248264a8b26970b4473770bcc3df8515fJosh Gao try: 12240a8c90248264a8b26970b4473770bcc3df8515fJosh Gao d = inst.__dict__ 12250a8c90248264a8b26970b4473770bcc3df8515fJosh Gao try: 12260a8c90248264a8b26970b4473770bcc3df8515fJosh Gao for k, v in state.iteritems(): 12270a8c90248264a8b26970b4473770bcc3df8515fJosh Gao d[intern(k)] = v 12280a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # keys in state don't have to be strings 12290a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # don't blow up, but don't go out of our way 12300a8c90248264a8b26970b4473770bcc3df8515fJosh Gao except TypeError: 12310a8c90248264a8b26970b4473770bcc3df8515fJosh Gao d.update(state) 12320a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 12330a8c90248264a8b26970b4473770bcc3df8515fJosh Gao except RuntimeError: 12340a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # XXX In restricted execution, the instance's __dict__ 12350a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # is not accessible. Use the old way of unpickling 12360a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # the instance variables. This is a semantic 12370a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # difference when unpickling in restricted 12380a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # vs. unrestricted modes. 12390a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # Note, however, that cPickle has never tried to do the 12400a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # .update() business, and always uses 12410a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # PyObject_SetItem(inst.__dict__, key, value) in a 12420a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # loop over state.items(). 12430a8c90248264a8b26970b4473770bcc3df8515fJosh Gao for k, v in state.items(): 12440a8c90248264a8b26970b4473770bcc3df8515fJosh Gao setattr(inst, k, v) 12450a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if slotstate: 12460a8c90248264a8b26970b4473770bcc3df8515fJosh Gao for k, v in slotstate.items(): 12470a8c90248264a8b26970b4473770bcc3df8515fJosh Gao setattr(inst, k, v) 12480a8c90248264a8b26970b4473770bcc3df8515fJosh Gao dispatch[BUILD] = load_build 12490a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 12500a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def load_mark(self): 12510a8c90248264a8b26970b4473770bcc3df8515fJosh Gao self.append(self.mark) 12520a8c90248264a8b26970b4473770bcc3df8515fJosh Gao dispatch[MARK] = load_mark 12530a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 12540a8c90248264a8b26970b4473770bcc3df8515fJosh Gao def load_stop(self): 12550a8c90248264a8b26970b4473770bcc3df8515fJosh Gao value = self.stack.pop() 12560a8c90248264a8b26970b4473770bcc3df8515fJosh Gao raise _Stop(value) 12570a8c90248264a8b26970b4473770bcc3df8515fJosh Gao dispatch[STOP] = load_stop 12580a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 12590a8c90248264a8b26970b4473770bcc3df8515fJosh Gao# Helper class for load_inst/load_obj 12600a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 12610a8c90248264a8b26970b4473770bcc3df8515fJosh Gaoclass _EmptyClass: 12620a8c90248264a8b26970b4473770bcc3df8515fJosh Gao pass 12630a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 12640a8c90248264a8b26970b4473770bcc3df8515fJosh Gao# Encode/decode longs in linear time. 12650a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 12660a8c90248264a8b26970b4473770bcc3df8515fJosh Gaoimport binascii as _binascii 12670a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 12680a8c90248264a8b26970b4473770bcc3df8515fJosh Gaodef encode_long(x): 12690a8c90248264a8b26970b4473770bcc3df8515fJosh Gao r"""Encode a long to a two's complement little-endian binary string. 12700a8c90248264a8b26970b4473770bcc3df8515fJosh Gao Note that 0L is a special case, returning an empty string, to save a 12710a8c90248264a8b26970b4473770bcc3df8515fJosh Gao byte in the LONG1 pickling context. 12720a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 12730a8c90248264a8b26970b4473770bcc3df8515fJosh Gao >>> encode_long(0L) 12740a8c90248264a8b26970b4473770bcc3df8515fJosh Gao '' 12750a8c90248264a8b26970b4473770bcc3df8515fJosh Gao >>> encode_long(255L) 12760a8c90248264a8b26970b4473770bcc3df8515fJosh Gao '\xff\x00' 12770a8c90248264a8b26970b4473770bcc3df8515fJosh Gao >>> encode_long(32767L) 12780a8c90248264a8b26970b4473770bcc3df8515fJosh Gao '\xff\x7f' 12790a8c90248264a8b26970b4473770bcc3df8515fJosh Gao >>> encode_long(-256L) 12800a8c90248264a8b26970b4473770bcc3df8515fJosh Gao '\x00\xff' 12810a8c90248264a8b26970b4473770bcc3df8515fJosh Gao >>> encode_long(-32768L) 12820a8c90248264a8b26970b4473770bcc3df8515fJosh Gao '\x00\x80' 12830a8c90248264a8b26970b4473770bcc3df8515fJosh Gao >>> encode_long(-128L) 12840a8c90248264a8b26970b4473770bcc3df8515fJosh Gao '\x80' 12850a8c90248264a8b26970b4473770bcc3df8515fJosh Gao >>> encode_long(127L) 12860a8c90248264a8b26970b4473770bcc3df8515fJosh Gao '\x7f' 12870a8c90248264a8b26970b4473770bcc3df8515fJosh Gao >>> 12880a8c90248264a8b26970b4473770bcc3df8515fJosh Gao """ 12890a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 12900a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if x == 0: 12910a8c90248264a8b26970b4473770bcc3df8515fJosh Gao return '' 12920a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if x > 0: 12930a8c90248264a8b26970b4473770bcc3df8515fJosh Gao ashex = hex(x) 12940a8c90248264a8b26970b4473770bcc3df8515fJosh Gao assert ashex.startswith("0x") 12950a8c90248264a8b26970b4473770bcc3df8515fJosh Gao njunkchars = 2 + ashex.endswith('L') 12960a8c90248264a8b26970b4473770bcc3df8515fJosh Gao nibbles = len(ashex) - njunkchars 12970a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if nibbles & 1: 12980a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # need an even # of nibbles for unhexlify 12990a8c90248264a8b26970b4473770bcc3df8515fJosh Gao ashex = "0x0" + ashex[2:] 13000a8c90248264a8b26970b4473770bcc3df8515fJosh Gao elif int(ashex[2], 16) >= 8: 13010a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # "looks negative", so need a byte of sign bits 13020a8c90248264a8b26970b4473770bcc3df8515fJosh Gao ashex = "0x00" + ashex[2:] 13030a8c90248264a8b26970b4473770bcc3df8515fJosh Gao else: 13040a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # Build the 256's-complement: (1L << nbytes) + x. The trick is 13050a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # to find the number of bytes in linear time (although that should 13060a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # really be a constant-time task). 13070a8c90248264a8b26970b4473770bcc3df8515fJosh Gao ashex = hex(-x) 13080a8c90248264a8b26970b4473770bcc3df8515fJosh Gao assert ashex.startswith("0x") 13090a8c90248264a8b26970b4473770bcc3df8515fJosh Gao njunkchars = 2 + ashex.endswith('L') 13100a8c90248264a8b26970b4473770bcc3df8515fJosh Gao nibbles = len(ashex) - njunkchars 13110a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if nibbles & 1: 13120a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # Extend to a full byte. 13130a8c90248264a8b26970b4473770bcc3df8515fJosh Gao nibbles += 1 13140a8c90248264a8b26970b4473770bcc3df8515fJosh Gao nbits = nibbles * 4 13150a8c90248264a8b26970b4473770bcc3df8515fJosh Gao x += 1L << nbits 13160a8c90248264a8b26970b4473770bcc3df8515fJosh Gao assert x > 0 13170a8c90248264a8b26970b4473770bcc3df8515fJosh Gao ashex = hex(x) 13180a8c90248264a8b26970b4473770bcc3df8515fJosh Gao njunkchars = 2 + ashex.endswith('L') 13190a8c90248264a8b26970b4473770bcc3df8515fJosh Gao newnibbles = len(ashex) - njunkchars 13200a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if newnibbles < nibbles: 13210a8c90248264a8b26970b4473770bcc3df8515fJosh Gao ashex = "0x" + "0" * (nibbles - newnibbles) + ashex[2:] 13220a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if int(ashex[2], 16) < 8: 13230a8c90248264a8b26970b4473770bcc3df8515fJosh Gao # "looks positive", so need a byte of sign bits 13240a8c90248264a8b26970b4473770bcc3df8515fJosh Gao ashex = "0xff" + ashex[2:] 13250a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 13260a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if ashex.endswith('L'): 13270a8c90248264a8b26970b4473770bcc3df8515fJosh Gao ashex = ashex[2:-1] 13280a8c90248264a8b26970b4473770bcc3df8515fJosh Gao else: 13290a8c90248264a8b26970b4473770bcc3df8515fJosh Gao ashex = ashex[2:] 13300a8c90248264a8b26970b4473770bcc3df8515fJosh Gao assert len(ashex) & 1 == 0, (x, ashex) 13310a8c90248264a8b26970b4473770bcc3df8515fJosh Gao binary = _binascii.unhexlify(ashex) 13320a8c90248264a8b26970b4473770bcc3df8515fJosh Gao return binary[::-1] 13330a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 13340a8c90248264a8b26970b4473770bcc3df8515fJosh Gaodef decode_long(data): 13350a8c90248264a8b26970b4473770bcc3df8515fJosh Gao r"""Decode a long from a two's complement little-endian binary string. 13360a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 13370a8c90248264a8b26970b4473770bcc3df8515fJosh Gao >>> decode_long('') 13380a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 0L 13390a8c90248264a8b26970b4473770bcc3df8515fJosh Gao >>> decode_long("\xff\x00") 13400a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 255L 13410a8c90248264a8b26970b4473770bcc3df8515fJosh Gao >>> decode_long("\xff\x7f") 13420a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 32767L 13430a8c90248264a8b26970b4473770bcc3df8515fJosh Gao >>> decode_long("\x00\xff") 13440a8c90248264a8b26970b4473770bcc3df8515fJosh Gao -256L 13450a8c90248264a8b26970b4473770bcc3df8515fJosh Gao >>> decode_long("\x00\x80") 13460a8c90248264a8b26970b4473770bcc3df8515fJosh Gao -32768L 13470a8c90248264a8b26970b4473770bcc3df8515fJosh Gao >>> decode_long("\x80") 13480a8c90248264a8b26970b4473770bcc3df8515fJosh Gao -128L 13490a8c90248264a8b26970b4473770bcc3df8515fJosh Gao >>> decode_long("\x7f") 13500a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 127L 13510a8c90248264a8b26970b4473770bcc3df8515fJosh Gao """ 13520a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 13530a8c90248264a8b26970b4473770bcc3df8515fJosh Gao nbytes = len(data) 13540a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if nbytes == 0: 13550a8c90248264a8b26970b4473770bcc3df8515fJosh Gao return 0L 13560a8c90248264a8b26970b4473770bcc3df8515fJosh Gao ashex = _binascii.hexlify(data[::-1]) 13570a8c90248264a8b26970b4473770bcc3df8515fJosh Gao n = long(ashex, 16) # quadratic time before Python 2.3; linear now 13580a8c90248264a8b26970b4473770bcc3df8515fJosh Gao if data[-1] >= '\x80': 13590a8c90248264a8b26970b4473770bcc3df8515fJosh Gao n -= 1L << (nbytes * 8) 13600a8c90248264a8b26970b4473770bcc3df8515fJosh Gao return n 13610a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 13620a8c90248264a8b26970b4473770bcc3df8515fJosh Gao# Shorthands 13630a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 13640a8c90248264a8b26970b4473770bcc3df8515fJosh Gaotry: 13650a8c90248264a8b26970b4473770bcc3df8515fJosh Gao from cStringIO import StringIO 13660a8c90248264a8b26970b4473770bcc3df8515fJosh Gaoexcept ImportError: 13670a8c90248264a8b26970b4473770bcc3df8515fJosh Gao from StringIO import StringIO 13680a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 13690a8c90248264a8b26970b4473770bcc3df8515fJosh Gaodef dump(obj, file, protocol=None): 13700a8c90248264a8b26970b4473770bcc3df8515fJosh Gao Pickler(file, protocol).dump(obj) 13710a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 13720a8c90248264a8b26970b4473770bcc3df8515fJosh Gaodef dumps(obj, protocol=None): 13730a8c90248264a8b26970b4473770bcc3df8515fJosh Gao file = StringIO() 13740a8c90248264a8b26970b4473770bcc3df8515fJosh Gao Pickler(file, protocol).dump(obj) 13750a8c90248264a8b26970b4473770bcc3df8515fJosh Gao return file.getvalue() 13760a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 13770a8c90248264a8b26970b4473770bcc3df8515fJosh Gaodef load(file): 13780a8c90248264a8b26970b4473770bcc3df8515fJosh Gao return Unpickler(file).load() 13790a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 13800a8c90248264a8b26970b4473770bcc3df8515fJosh Gaodef loads(str): 13810a8c90248264a8b26970b4473770bcc3df8515fJosh Gao file = StringIO(str) 13820a8c90248264a8b26970b4473770bcc3df8515fJosh Gao return Unpickler(file).load() 13830a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 13840a8c90248264a8b26970b4473770bcc3df8515fJosh Gao# Doctest 13850a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 13860a8c90248264a8b26970b4473770bcc3df8515fJosh Gaodef _test(): 13870a8c90248264a8b26970b4473770bcc3df8515fJosh Gao import doctest 13880a8c90248264a8b26970b4473770bcc3df8515fJosh Gao return doctest.testmod() 13890a8c90248264a8b26970b4473770bcc3df8515fJosh Gao 13900a8c90248264a8b26970b4473770bcc3df8515fJosh Gaoif __name__ == "__main__": 13910a8c90248264a8b26970b4473770bcc3df8515fJosh Gao _test() 1392