1import collections 2import copyreg 3import dbm 4import io 5import functools 6import pickle 7import pickletools 8import struct 9import sys 10import unittest 11import weakref 12from http.cookies import SimpleCookie 13 14from test import support 15from test.support import ( 16 TestFailed, TESTFN, run_with_locale, no_tracing, 17 _2G, _4G, bigmemtest, 18 ) 19 20from pickle import bytes_types 21 22requires_32b = unittest.skipUnless(sys.maxsize < 2**32, 23 "test is only meaningful on 32-bit builds") 24 25# Tests that try a number of pickle protocols should have a 26# for proto in protocols: 27# kind of outer loop. 28protocols = range(pickle.HIGHEST_PROTOCOL + 1) 29 30 31# Return True if opcode code appears in the pickle, else False. 32def opcode_in_pickle(code, pickle): 33 for op, dummy, dummy in pickletools.genops(pickle): 34 if op.code == code.decode("latin-1"): 35 return True 36 return False 37 38# Return the number of times opcode code appears in pickle. 39def count_opcode(code, pickle): 40 n = 0 41 for op, dummy, dummy in pickletools.genops(pickle): 42 if op.code == code.decode("latin-1"): 43 n += 1 44 return n 45 46 47class UnseekableIO(io.BytesIO): 48 def peek(self, *args): 49 raise NotImplementedError 50 51 def seekable(self): 52 return False 53 54 def seek(self, *args): 55 raise io.UnsupportedOperation 56 57 def tell(self): 58 raise io.UnsupportedOperation 59 60 61# We can't very well test the extension registry without putting known stuff 62# in it, but we have to be careful to restore its original state. Code 63# should do this: 64# 65# e = ExtensionSaver(extension_code) 66# try: 67# fiddle w/ the extension registry's stuff for extension_code 68# finally: 69# e.restore() 70 71class ExtensionSaver: 72 # Remember current registration for code (if any), and remove it (if 73 # there is one). 74 def __init__(self, code): 75 self.code = code 76 if code in copyreg._inverted_registry: 77 self.pair = copyreg._inverted_registry[code] 78 copyreg.remove_extension(self.pair[0], self.pair[1], code) 79 else: 80 self.pair = None 81 82 # Restore previous registration for code. 83 def restore(self): 84 code = self.code 85 curpair = copyreg._inverted_registry.get(code) 86 if curpair is not None: 87 copyreg.remove_extension(curpair[0], curpair[1], code) 88 pair = self.pair 89 if pair is not None: 90 copyreg.add_extension(pair[0], pair[1], code) 91 92class C: 93 def __eq__(self, other): 94 return self.__dict__ == other.__dict__ 95 96class D(C): 97 def __init__(self, arg): 98 pass 99 100class E(C): 101 def __getinitargs__(self): 102 return () 103 104class H(object): 105 pass 106 107# Hashable mutable key 108class K(object): 109 def __init__(self, value): 110 self.value = value 111 112 def __reduce__(self): 113 # Shouldn't support the recursion itself 114 return K, (self.value,) 115 116import __main__ 117__main__.C = C 118C.__module__ = "__main__" 119__main__.D = D 120D.__module__ = "__main__" 121__main__.E = E 122E.__module__ = "__main__" 123__main__.H = H 124H.__module__ = "__main__" 125__main__.K = K 126K.__module__ = "__main__" 127 128class myint(int): 129 def __init__(self, x): 130 self.str = str(x) 131 132class initarg(C): 133 134 def __init__(self, a, b): 135 self.a = a 136 self.b = b 137 138 def __getinitargs__(self): 139 return self.a, self.b 140 141class metaclass(type): 142 pass 143 144class use_metaclass(object, metaclass=metaclass): 145 pass 146 147class pickling_metaclass(type): 148 def __eq__(self, other): 149 return (type(self) == type(other) and 150 self.reduce_args == other.reduce_args) 151 152 def __reduce__(self): 153 return (create_dynamic_class, self.reduce_args) 154 155def create_dynamic_class(name, bases): 156 result = pickling_metaclass(name, bases, dict()) 157 result.reduce_args = (name, bases) 158 return result 159 160# DATA0 .. DATA4 are the pickles we expect under the various protocols, for 161# the object returned by create_data(). 162 163DATA0 = ( 164 b'(lp0\nL0L\naL1L\naF2.0\n' 165 b'ac__builtin__\ncomple' 166 b'x\np1\n(F3.0\nF0.0\ntp2\n' 167 b'Rp3\naL1L\naL-1L\naL255' 168 b'L\naL-255L\naL-256L\naL' 169 b'65535L\naL-65535L\naL-' 170 b'65536L\naL2147483647L' 171 b'\naL-2147483647L\naL-2' 172 b'147483648L\na(Vabc\np4' 173 b'\ng4\nccopy_reg\n_recon' 174 b'structor\np5\n(c__main' 175 b'__\nC\np6\nc__builtin__' 176 b'\nobject\np7\nNtp8\nRp9\n' 177 b'(dp10\nVfoo\np11\nL1L\ns' 178 b'Vbar\np12\nL2L\nsbg9\ntp' 179 b'13\nag13\naL5L\na.' 180) 181 182# Disassembly of DATA0 183DATA0_DIS = """\ 184 0: ( MARK 185 1: l LIST (MARK at 0) 186 2: p PUT 0 187 5: L LONG 0 188 9: a APPEND 189 10: L LONG 1 190 14: a APPEND 191 15: F FLOAT 2.0 192 20: a APPEND 193 21: c GLOBAL '__builtin__ complex' 194 42: p PUT 1 195 45: ( MARK 196 46: F FLOAT 3.0 197 51: F FLOAT 0.0 198 56: t TUPLE (MARK at 45) 199 57: p PUT 2 200 60: R REDUCE 201 61: p PUT 3 202 64: a APPEND 203 65: L LONG 1 204 69: a APPEND 205 70: L LONG -1 206 75: a APPEND 207 76: L LONG 255 208 82: a APPEND 209 83: L LONG -255 210 90: a APPEND 211 91: L LONG -256 212 98: a APPEND 213 99: L LONG 65535 214 107: a APPEND 215 108: L LONG -65535 216 117: a APPEND 217 118: L LONG -65536 218 127: a APPEND 219 128: L LONG 2147483647 220 141: a APPEND 221 142: L LONG -2147483647 222 156: a APPEND 223 157: L LONG -2147483648 224 171: a APPEND 225 172: ( MARK 226 173: V UNICODE 'abc' 227 178: p PUT 4 228 181: g GET 4 229 184: c GLOBAL 'copy_reg _reconstructor' 230 209: p PUT 5 231 212: ( MARK 232 213: c GLOBAL '__main__ C' 233 225: p PUT 6 234 228: c GLOBAL '__builtin__ object' 235 248: p PUT 7 236 251: N NONE 237 252: t TUPLE (MARK at 212) 238 253: p PUT 8 239 256: R REDUCE 240 257: p PUT 9 241 260: ( MARK 242 261: d DICT (MARK at 260) 243 262: p PUT 10 244 266: V UNICODE 'foo' 245 271: p PUT 11 246 275: L LONG 1 247 279: s SETITEM 248 280: V UNICODE 'bar' 249 285: p PUT 12 250 289: L LONG 2 251 293: s SETITEM 252 294: b BUILD 253 295: g GET 9 254 298: t TUPLE (MARK at 172) 255 299: p PUT 13 256 303: a APPEND 257 304: g GET 13 258 308: a APPEND 259 309: L LONG 5 260 313: a APPEND 261 314: . STOP 262highest protocol among opcodes = 0 263""" 264 265DATA1 = ( 266 b']q\x00(K\x00K\x01G@\x00\x00\x00\x00\x00\x00\x00c__' 267 b'builtin__\ncomplex\nq\x01' 268 b'(G@\x08\x00\x00\x00\x00\x00\x00G\x00\x00\x00\x00\x00\x00\x00\x00t' 269 b'q\x02Rq\x03K\x01J\xff\xff\xff\xffK\xffJ\x01\xff\xff\xffJ' 270 b'\x00\xff\xff\xffM\xff\xffJ\x01\x00\xff\xffJ\x00\x00\xff\xffJ\xff\xff' 271 b'\xff\x7fJ\x01\x00\x00\x80J\x00\x00\x00\x80(X\x03\x00\x00\x00ab' 272 b'cq\x04h\x04ccopy_reg\n_reco' 273 b'nstructor\nq\x05(c__main' 274 b'__\nC\nq\x06c__builtin__\n' 275 b'object\nq\x07Ntq\x08Rq\t}q\n(' 276 b'X\x03\x00\x00\x00fooq\x0bK\x01X\x03\x00\x00\x00bar' 277 b'q\x0cK\x02ubh\ttq\rh\rK\x05e.' 278) 279 280# Disassembly of DATA1 281DATA1_DIS = """\ 282 0: ] EMPTY_LIST 283 1: q BINPUT 0 284 3: ( MARK 285 4: K BININT1 0 286 6: K BININT1 1 287 8: G BINFLOAT 2.0 288 17: c GLOBAL '__builtin__ complex' 289 38: q BINPUT 1 290 40: ( MARK 291 41: G BINFLOAT 3.0 292 50: G BINFLOAT 0.0 293 59: t TUPLE (MARK at 40) 294 60: q BINPUT 2 295 62: R REDUCE 296 63: q BINPUT 3 297 65: K BININT1 1 298 67: J BININT -1 299 72: K BININT1 255 300 74: J BININT -255 301 79: J BININT -256 302 84: M BININT2 65535 303 87: J BININT -65535 304 92: J BININT -65536 305 97: J BININT 2147483647 306 102: J BININT -2147483647 307 107: J BININT -2147483648 308 112: ( MARK 309 113: X BINUNICODE 'abc' 310 121: q BINPUT 4 311 123: h BINGET 4 312 125: c GLOBAL 'copy_reg _reconstructor' 313 150: q BINPUT 5 314 152: ( MARK 315 153: c GLOBAL '__main__ C' 316 165: q BINPUT 6 317 167: c GLOBAL '__builtin__ object' 318 187: q BINPUT 7 319 189: N NONE 320 190: t TUPLE (MARK at 152) 321 191: q BINPUT 8 322 193: R REDUCE 323 194: q BINPUT 9 324 196: } EMPTY_DICT 325 197: q BINPUT 10 326 199: ( MARK 327 200: X BINUNICODE 'foo' 328 208: q BINPUT 11 329 210: K BININT1 1 330 212: X BINUNICODE 'bar' 331 220: q BINPUT 12 332 222: K BININT1 2 333 224: u SETITEMS (MARK at 199) 334 225: b BUILD 335 226: h BINGET 9 336 228: t TUPLE (MARK at 112) 337 229: q BINPUT 13 338 231: h BINGET 13 339 233: K BININT1 5 340 235: e APPENDS (MARK at 3) 341 236: . STOP 342highest protocol among opcodes = 1 343""" 344 345DATA2 = ( 346 b'\x80\x02]q\x00(K\x00K\x01G@\x00\x00\x00\x00\x00\x00\x00c' 347 b'__builtin__\ncomplex\n' 348 b'q\x01G@\x08\x00\x00\x00\x00\x00\x00G\x00\x00\x00\x00\x00\x00\x00\x00' 349 b'\x86q\x02Rq\x03K\x01J\xff\xff\xff\xffK\xffJ\x01\xff\xff\xff' 350 b'J\x00\xff\xff\xffM\xff\xffJ\x01\x00\xff\xffJ\x00\x00\xff\xffJ\xff' 351 b'\xff\xff\x7fJ\x01\x00\x00\x80J\x00\x00\x00\x80(X\x03\x00\x00\x00a' 352 b'bcq\x04h\x04c__main__\nC\nq\x05' 353 b')\x81q\x06}q\x07(X\x03\x00\x00\x00fooq\x08K\x01' 354 b'X\x03\x00\x00\x00barq\tK\x02ubh\x06tq\nh' 355 b'\nK\x05e.' 356) 357 358# Disassembly of DATA2 359DATA2_DIS = """\ 360 0: \x80 PROTO 2 361 2: ] EMPTY_LIST 362 3: q BINPUT 0 363 5: ( MARK 364 6: K BININT1 0 365 8: K BININT1 1 366 10: G BINFLOAT 2.0 367 19: c GLOBAL '__builtin__ complex' 368 40: q BINPUT 1 369 42: G BINFLOAT 3.0 370 51: G BINFLOAT 0.0 371 60: \x86 TUPLE2 372 61: q BINPUT 2 373 63: R REDUCE 374 64: q BINPUT 3 375 66: K BININT1 1 376 68: J BININT -1 377 73: K BININT1 255 378 75: J BININT -255 379 80: J BININT -256 380 85: M BININT2 65535 381 88: J BININT -65535 382 93: J BININT -65536 383 98: J BININT 2147483647 384 103: J BININT -2147483647 385 108: J BININT -2147483648 386 113: ( MARK 387 114: X BINUNICODE 'abc' 388 122: q BINPUT 4 389 124: h BINGET 4 390 126: c GLOBAL '__main__ C' 391 138: q BINPUT 5 392 140: ) EMPTY_TUPLE 393 141: \x81 NEWOBJ 394 142: q BINPUT 6 395 144: } EMPTY_DICT 396 145: q BINPUT 7 397 147: ( MARK 398 148: X BINUNICODE 'foo' 399 156: q BINPUT 8 400 158: K BININT1 1 401 160: X BINUNICODE 'bar' 402 168: q BINPUT 9 403 170: K BININT1 2 404 172: u SETITEMS (MARK at 147) 405 173: b BUILD 406 174: h BINGET 6 407 176: t TUPLE (MARK at 113) 408 177: q BINPUT 10 409 179: h BINGET 10 410 181: K BININT1 5 411 183: e APPENDS (MARK at 5) 412 184: . STOP 413highest protocol among opcodes = 2 414""" 415 416DATA3 = ( 417 b'\x80\x03]q\x00(K\x00K\x01G@\x00\x00\x00\x00\x00\x00\x00c' 418 b'builtins\ncomplex\nq\x01G' 419 b'@\x08\x00\x00\x00\x00\x00\x00G\x00\x00\x00\x00\x00\x00\x00\x00\x86q\x02' 420 b'Rq\x03K\x01J\xff\xff\xff\xffK\xffJ\x01\xff\xff\xffJ\x00\xff' 421 b'\xff\xffM\xff\xffJ\x01\x00\xff\xffJ\x00\x00\xff\xffJ\xff\xff\xff\x7f' 422 b'J\x01\x00\x00\x80J\x00\x00\x00\x80(X\x03\x00\x00\x00abcq' 423 b'\x04h\x04c__main__\nC\nq\x05)\x81q' 424 b'\x06}q\x07(X\x03\x00\x00\x00barq\x08K\x02X\x03\x00' 425 b'\x00\x00fooq\tK\x01ubh\x06tq\nh\nK\x05' 426 b'e.' 427) 428 429# Disassembly of DATA3 430DATA3_DIS = """\ 431 0: \x80 PROTO 3 432 2: ] EMPTY_LIST 433 3: q BINPUT 0 434 5: ( MARK 435 6: K BININT1 0 436 8: K BININT1 1 437 10: G BINFLOAT 2.0 438 19: c GLOBAL 'builtins complex' 439 37: q BINPUT 1 440 39: G BINFLOAT 3.0 441 48: G BINFLOAT 0.0 442 57: \x86 TUPLE2 443 58: q BINPUT 2 444 60: R REDUCE 445 61: q BINPUT 3 446 63: K BININT1 1 447 65: J BININT -1 448 70: K BININT1 255 449 72: J BININT -255 450 77: J BININT -256 451 82: M BININT2 65535 452 85: J BININT -65535 453 90: J BININT -65536 454 95: J BININT 2147483647 455 100: J BININT -2147483647 456 105: J BININT -2147483648 457 110: ( MARK 458 111: X BINUNICODE 'abc' 459 119: q BINPUT 4 460 121: h BINGET 4 461 123: c GLOBAL '__main__ C' 462 135: q BINPUT 5 463 137: ) EMPTY_TUPLE 464 138: \x81 NEWOBJ 465 139: q BINPUT 6 466 141: } EMPTY_DICT 467 142: q BINPUT 7 468 144: ( MARK 469 145: X BINUNICODE 'bar' 470 153: q BINPUT 8 471 155: K BININT1 2 472 157: X BINUNICODE 'foo' 473 165: q BINPUT 9 474 167: K BININT1 1 475 169: u SETITEMS (MARK at 144) 476 170: b BUILD 477 171: h BINGET 6 478 173: t TUPLE (MARK at 110) 479 174: q BINPUT 10 480 176: h BINGET 10 481 178: K BININT1 5 482 180: e APPENDS (MARK at 5) 483 181: . STOP 484highest protocol among opcodes = 2 485""" 486 487DATA4 = ( 488 b'\x80\x04\x95\xa8\x00\x00\x00\x00\x00\x00\x00]\x94(K\x00K\x01G@' 489 b'\x00\x00\x00\x00\x00\x00\x00\x8c\x08builtins\x94\x8c\x07' 490 b'complex\x94\x93\x94G@\x08\x00\x00\x00\x00\x00\x00G' 491 b'\x00\x00\x00\x00\x00\x00\x00\x00\x86\x94R\x94K\x01J\xff\xff\xff\xffK' 492 b'\xffJ\x01\xff\xff\xffJ\x00\xff\xff\xffM\xff\xffJ\x01\x00\xff\xffJ' 493 b'\x00\x00\xff\xffJ\xff\xff\xff\x7fJ\x01\x00\x00\x80J\x00\x00\x00\x80(' 494 b'\x8c\x03abc\x94h\x06\x8c\x08__main__\x94\x8c' 495 b'\x01C\x94\x93\x94)\x81\x94}\x94(\x8c\x03bar\x94K\x02\x8c' 496 b'\x03foo\x94K\x01ubh\nt\x94h\x0eK\x05e.' 497) 498 499# Disassembly of DATA4 500DATA4_DIS = """\ 501 0: \x80 PROTO 4 502 2: \x95 FRAME 168 503 11: ] EMPTY_LIST 504 12: \x94 MEMOIZE 505 13: ( MARK 506 14: K BININT1 0 507 16: K BININT1 1 508 18: G BINFLOAT 2.0 509 27: \x8c SHORT_BINUNICODE 'builtins' 510 37: \x94 MEMOIZE 511 38: \x8c SHORT_BINUNICODE 'complex' 512 47: \x94 MEMOIZE 513 48: \x93 STACK_GLOBAL 514 49: \x94 MEMOIZE 515 50: G BINFLOAT 3.0 516 59: G BINFLOAT 0.0 517 68: \x86 TUPLE2 518 69: \x94 MEMOIZE 519 70: R REDUCE 520 71: \x94 MEMOIZE 521 72: K BININT1 1 522 74: J BININT -1 523 79: K BININT1 255 524 81: J BININT -255 525 86: J BININT -256 526 91: M BININT2 65535 527 94: J BININT -65535 528 99: J BININT -65536 529 104: J BININT 2147483647 530 109: J BININT -2147483647 531 114: J BININT -2147483648 532 119: ( MARK 533 120: \x8c SHORT_BINUNICODE 'abc' 534 125: \x94 MEMOIZE 535 126: h BINGET 6 536 128: \x8c SHORT_BINUNICODE '__main__' 537 138: \x94 MEMOIZE 538 139: \x8c SHORT_BINUNICODE 'C' 539 142: \x94 MEMOIZE 540 143: \x93 STACK_GLOBAL 541 144: \x94 MEMOIZE 542 145: ) EMPTY_TUPLE 543 146: \x81 NEWOBJ 544 147: \x94 MEMOIZE 545 148: } EMPTY_DICT 546 149: \x94 MEMOIZE 547 150: ( MARK 548 151: \x8c SHORT_BINUNICODE 'bar' 549 156: \x94 MEMOIZE 550 157: K BININT1 2 551 159: \x8c SHORT_BINUNICODE 'foo' 552 164: \x94 MEMOIZE 553 165: K BININT1 1 554 167: u SETITEMS (MARK at 150) 555 168: b BUILD 556 169: h BINGET 10 557 171: t TUPLE (MARK at 119) 558 172: \x94 MEMOIZE 559 173: h BINGET 14 560 175: K BININT1 5 561 177: e APPENDS (MARK at 13) 562 178: . STOP 563highest protocol among opcodes = 4 564""" 565 566# set([1,2]) pickled from 2.x with protocol 2 567DATA_SET = b'\x80\x02c__builtin__\nset\nq\x00]q\x01(K\x01K\x02e\x85q\x02Rq\x03.' 568 569# xrange(5) pickled from 2.x with protocol 2 570DATA_XRANGE = b'\x80\x02c__builtin__\nxrange\nq\x00K\x00K\x05K\x01\x87q\x01Rq\x02.' 571 572# a SimpleCookie() object pickled from 2.x with protocol 2 573DATA_COOKIE = (b'\x80\x02cCookie\nSimpleCookie\nq\x00)\x81q\x01U\x03key' 574 b'q\x02cCookie\nMorsel\nq\x03)\x81q\x04(U\x07commentq\x05U' 575 b'\x00q\x06U\x06domainq\x07h\x06U\x06secureq\x08h\x06U\x07' 576 b'expiresq\th\x06U\x07max-ageq\nh\x06U\x07versionq\x0bh\x06U' 577 b'\x04pathq\x0ch\x06U\x08httponlyq\rh\x06u}q\x0e(U\x0b' 578 b'coded_valueq\x0fU\x05valueq\x10h\x10h\x10h\x02h\x02ubs}q\x11b.') 579 580# set([3]) pickled from 2.x with protocol 2 581DATA_SET2 = b'\x80\x02c__builtin__\nset\nq\x00]q\x01K\x03a\x85q\x02Rq\x03.' 582 583python2_exceptions_without_args = ( 584 ArithmeticError, 585 AssertionError, 586 AttributeError, 587 BaseException, 588 BufferError, 589 BytesWarning, 590 DeprecationWarning, 591 EOFError, 592 EnvironmentError, 593 Exception, 594 FloatingPointError, 595 FutureWarning, 596 GeneratorExit, 597 IOError, 598 ImportError, 599 ImportWarning, 600 IndentationError, 601 IndexError, 602 KeyError, 603 KeyboardInterrupt, 604 LookupError, 605 MemoryError, 606 NameError, 607 NotImplementedError, 608 OSError, 609 OverflowError, 610 PendingDeprecationWarning, 611 ReferenceError, 612 RuntimeError, 613 RuntimeWarning, 614 # StandardError is gone in Python 3, we map it to Exception 615 StopIteration, 616 SyntaxError, 617 SyntaxWarning, 618 SystemError, 619 SystemExit, 620 TabError, 621 TypeError, 622 UnboundLocalError, 623 UnicodeError, 624 UnicodeWarning, 625 UserWarning, 626 ValueError, 627 Warning, 628 ZeroDivisionError, 629) 630 631exception_pickle = b'\x80\x02cexceptions\n?\nq\x00)Rq\x01.' 632 633# UnicodeEncodeError object pickled from 2.x with protocol 2 634DATA_UEERR = (b'\x80\x02cexceptions\nUnicodeEncodeError\n' 635 b'q\x00(U\x05asciiq\x01X\x03\x00\x00\x00fooq\x02K\x00K\x01' 636 b'U\x03badq\x03tq\x04Rq\x05.') 637 638 639def create_data(): 640 c = C() 641 c.foo = 1 642 c.bar = 2 643 x = [0, 1, 2.0, 3.0+0j] 644 # Append some integer test cases at cPickle.c's internal size 645 # cutoffs. 646 uint1max = 0xff 647 uint2max = 0xffff 648 int4max = 0x7fffffff 649 x.extend([1, -1, 650 uint1max, -uint1max, -uint1max-1, 651 uint2max, -uint2max, -uint2max-1, 652 int4max, -int4max, -int4max-1]) 653 y = ('abc', 'abc', c, c) 654 x.append(y) 655 x.append(y) 656 x.append(5) 657 return x 658 659 660class AbstractUnpickleTests(unittest.TestCase): 661 # Subclass must define self.loads. 662 663 _testdata = create_data() 664 665 def assert_is_copy(self, obj, objcopy, msg=None): 666 """Utility method to verify if two objects are copies of each others. 667 """ 668 if msg is None: 669 msg = "{!r} is not a copy of {!r}".format(obj, objcopy) 670 self.assertEqual(obj, objcopy, msg=msg) 671 self.assertIs(type(obj), type(objcopy), msg=msg) 672 if hasattr(obj, '__dict__'): 673 self.assertDictEqual(obj.__dict__, objcopy.__dict__, msg=msg) 674 self.assertIsNot(obj.__dict__, objcopy.__dict__, msg=msg) 675 if hasattr(obj, '__slots__'): 676 self.assertListEqual(obj.__slots__, objcopy.__slots__, msg=msg) 677 for slot in obj.__slots__: 678 self.assertEqual( 679 hasattr(obj, slot), hasattr(objcopy, slot), msg=msg) 680 self.assertEqual(getattr(obj, slot, None), 681 getattr(objcopy, slot, None), msg=msg) 682 683 def check_unpickling_error(self, errors, data): 684 with self.subTest(data=data), \ 685 self.assertRaises(errors): 686 try: 687 self.loads(data) 688 except BaseException as exc: 689 if support.verbose > 1: 690 print('%-32r - %s: %s' % 691 (data, exc.__class__.__name__, exc)) 692 raise 693 694 def test_load_from_data0(self): 695 self.assert_is_copy(self._testdata, self.loads(DATA0)) 696 697 def test_load_from_data1(self): 698 self.assert_is_copy(self._testdata, self.loads(DATA1)) 699 700 def test_load_from_data2(self): 701 self.assert_is_copy(self._testdata, self.loads(DATA2)) 702 703 def test_load_from_data3(self): 704 self.assert_is_copy(self._testdata, self.loads(DATA3)) 705 706 def test_load_from_data4(self): 707 self.assert_is_copy(self._testdata, self.loads(DATA4)) 708 709 def test_load_classic_instance(self): 710 # See issue5180. Test loading 2.x pickles that 711 # contain an instance of old style class. 712 for X, args in [(C, ()), (D, ('x',)), (E, ())]: 713 xname = X.__name__.encode('ascii') 714 # Protocol 0 (text mode pickle): 715 """ 716 0: ( MARK 717 1: i INST '__main__ X' (MARK at 0) 718 13: p PUT 0 719 16: ( MARK 720 17: d DICT (MARK at 16) 721 18: p PUT 1 722 21: b BUILD 723 22: . STOP 724 """ 725 pickle0 = (b"(i__main__\n" 726 b"X\n" 727 b"p0\n" 728 b"(dp1\nb.").replace(b'X', xname) 729 self.assert_is_copy(X(*args), self.loads(pickle0)) 730 731 # Protocol 1 (binary mode pickle) 732 """ 733 0: ( MARK 734 1: c GLOBAL '__main__ X' 735 13: q BINPUT 0 736 15: o OBJ (MARK at 0) 737 16: q BINPUT 1 738 18: } EMPTY_DICT 739 19: q BINPUT 2 740 21: b BUILD 741 22: . STOP 742 """ 743 pickle1 = (b'(c__main__\n' 744 b'X\n' 745 b'q\x00oq\x01}q\x02b.').replace(b'X', xname) 746 self.assert_is_copy(X(*args), self.loads(pickle1)) 747 748 # Protocol 2 (pickle2 = b'\x80\x02' + pickle1) 749 """ 750 0: \x80 PROTO 2 751 2: ( MARK 752 3: c GLOBAL '__main__ X' 753 15: q BINPUT 0 754 17: o OBJ (MARK at 2) 755 18: q BINPUT 1 756 20: } EMPTY_DICT 757 21: q BINPUT 2 758 23: b BUILD 759 24: . STOP 760 """ 761 pickle2 = (b'\x80\x02(c__main__\n' 762 b'X\n' 763 b'q\x00oq\x01}q\x02b.').replace(b'X', xname) 764 self.assert_is_copy(X(*args), self.loads(pickle2)) 765 766 def test_maxint64(self): 767 maxint64 = (1 << 63) - 1 768 data = b'I' + str(maxint64).encode("ascii") + b'\n.' 769 got = self.loads(data) 770 self.assert_is_copy(maxint64, got) 771 772 # Try too with a bogus literal. 773 data = b'I' + str(maxint64).encode("ascii") + b'JUNK\n.' 774 self.check_unpickling_error(ValueError, data) 775 776 def test_unpickle_from_2x(self): 777 # Unpickle non-trivial data from Python 2.x. 778 loaded = self.loads(DATA_SET) 779 self.assertEqual(loaded, set([1, 2])) 780 loaded = self.loads(DATA_XRANGE) 781 self.assertEqual(type(loaded), type(range(0))) 782 self.assertEqual(list(loaded), list(range(5))) 783 loaded = self.loads(DATA_COOKIE) 784 self.assertEqual(type(loaded), SimpleCookie) 785 self.assertEqual(list(loaded.keys()), ["key"]) 786 self.assertEqual(loaded["key"].value, "value") 787 788 # Exception objects without arguments pickled from 2.x with protocol 2 789 for exc in python2_exceptions_without_args: 790 data = exception_pickle.replace(b'?', exc.__name__.encode("ascii")) 791 loaded = self.loads(data) 792 self.assertIs(type(loaded), exc) 793 794 # StandardError is mapped to Exception, test that separately 795 loaded = self.loads(exception_pickle.replace(b'?', b'StandardError')) 796 self.assertIs(type(loaded), Exception) 797 798 loaded = self.loads(DATA_UEERR) 799 self.assertIs(type(loaded), UnicodeEncodeError) 800 self.assertEqual(loaded.object, "foo") 801 self.assertEqual(loaded.encoding, "ascii") 802 self.assertEqual(loaded.start, 0) 803 self.assertEqual(loaded.end, 1) 804 self.assertEqual(loaded.reason, "bad") 805 806 def test_load_python2_str_as_bytes(self): 807 # From Python 2: pickle.dumps('a\x00\xa0', protocol=0) 808 self.assertEqual(self.loads(b"S'a\\x00\\xa0'\n.", 809 encoding="bytes"), b'a\x00\xa0') 810 # From Python 2: pickle.dumps('a\x00\xa0', protocol=1) 811 self.assertEqual(self.loads(b'U\x03a\x00\xa0.', 812 encoding="bytes"), b'a\x00\xa0') 813 # From Python 2: pickle.dumps('a\x00\xa0', protocol=2) 814 self.assertEqual(self.loads(b'\x80\x02U\x03a\x00\xa0.', 815 encoding="bytes"), b'a\x00\xa0') 816 817 def test_load_python2_unicode_as_str(self): 818 # From Python 2: pickle.dumps(u'π', protocol=0) 819 self.assertEqual(self.loads(b'V\\u03c0\n.', 820 encoding='bytes'), 'π') 821 # From Python 2: pickle.dumps(u'π', protocol=1) 822 self.assertEqual(self.loads(b'X\x02\x00\x00\x00\xcf\x80.', 823 encoding="bytes"), 'π') 824 # From Python 2: pickle.dumps(u'π', protocol=2) 825 self.assertEqual(self.loads(b'\x80\x02X\x02\x00\x00\x00\xcf\x80.', 826 encoding="bytes"), 'π') 827 828 def test_load_long_python2_str_as_bytes(self): 829 # From Python 2: pickle.dumps('x' * 300, protocol=1) 830 self.assertEqual(self.loads(pickle.BINSTRING + 831 struct.pack("<I", 300) + 832 b'x' * 300 + pickle.STOP, 833 encoding='bytes'), b'x' * 300) 834 835 def test_constants(self): 836 self.assertIsNone(self.loads(b'N.')) 837 self.assertIs(self.loads(b'\x88.'), True) 838 self.assertIs(self.loads(b'\x89.'), False) 839 self.assertIs(self.loads(b'I01\n.'), True) 840 self.assertIs(self.loads(b'I00\n.'), False) 841 842 def test_empty_bytestring(self): 843 # issue 11286 844 empty = self.loads(b'\x80\x03U\x00q\x00.', encoding='koi8-r') 845 self.assertEqual(empty, '') 846 847 def test_short_binbytes(self): 848 dumped = b'\x80\x03C\x04\xe2\x82\xac\x00.' 849 self.assertEqual(self.loads(dumped), b'\xe2\x82\xac\x00') 850 851 def test_binbytes(self): 852 dumped = b'\x80\x03B\x04\x00\x00\x00\xe2\x82\xac\x00.' 853 self.assertEqual(self.loads(dumped), b'\xe2\x82\xac\x00') 854 855 @requires_32b 856 def test_negative_32b_binbytes(self): 857 # On 32-bit builds, a BINBYTES of 2**31 or more is refused 858 dumped = b'\x80\x03B\xff\xff\xff\xffxyzq\x00.' 859 self.check_unpickling_error((pickle.UnpicklingError, OverflowError), 860 dumped) 861 862 @requires_32b 863 def test_negative_32b_binunicode(self): 864 # On 32-bit builds, a BINUNICODE of 2**31 or more is refused 865 dumped = b'\x80\x03X\xff\xff\xff\xffxyzq\x00.' 866 self.check_unpickling_error((pickle.UnpicklingError, OverflowError), 867 dumped) 868 869 def test_short_binunicode(self): 870 dumped = b'\x80\x04\x8c\x04\xe2\x82\xac\x00.' 871 self.assertEqual(self.loads(dumped), '\u20ac\x00') 872 873 def test_misc_get(self): 874 self.check_unpickling_error(KeyError, b'g0\np0') 875 self.assert_is_copy([(100,), (100,)], 876 self.loads(b'((Kdtp0\nh\x00l.))')) 877 878 def test_binbytes8(self): 879 dumped = b'\x80\x04\x8e\4\0\0\0\0\0\0\0\xe2\x82\xac\x00.' 880 self.assertEqual(self.loads(dumped), b'\xe2\x82\xac\x00') 881 882 def test_binunicode8(self): 883 dumped = b'\x80\x04\x8d\4\0\0\0\0\0\0\0\xe2\x82\xac\x00.' 884 self.assertEqual(self.loads(dumped), '\u20ac\x00') 885 886 @requires_32b 887 def test_large_32b_binbytes8(self): 888 dumped = b'\x80\x04\x8e\4\0\0\0\1\0\0\0\xe2\x82\xac\x00.' 889 self.check_unpickling_error((pickle.UnpicklingError, OverflowError), 890 dumped) 891 892 @requires_32b 893 def test_large_32b_binunicode8(self): 894 dumped = b'\x80\x04\x8d\4\0\0\0\1\0\0\0\xe2\x82\xac\x00.' 895 self.check_unpickling_error((pickle.UnpicklingError, OverflowError), 896 dumped) 897 898 def test_get(self): 899 pickled = b'((lp100000\ng100000\nt.' 900 unpickled = self.loads(pickled) 901 self.assertEqual(unpickled, ([],)*2) 902 self.assertIs(unpickled[0], unpickled[1]) 903 904 def test_binget(self): 905 pickled = b'(]q\xffh\xfft.' 906 unpickled = self.loads(pickled) 907 self.assertEqual(unpickled, ([],)*2) 908 self.assertIs(unpickled[0], unpickled[1]) 909 910 def test_long_binget(self): 911 pickled = b'(]r\x00\x00\x01\x00j\x00\x00\x01\x00t.' 912 unpickled = self.loads(pickled) 913 self.assertEqual(unpickled, ([],)*2) 914 self.assertIs(unpickled[0], unpickled[1]) 915 916 def test_dup(self): 917 pickled = b'((l2t.' 918 unpickled = self.loads(pickled) 919 self.assertEqual(unpickled, ([],)*2) 920 self.assertIs(unpickled[0], unpickled[1]) 921 922 def test_negative_put(self): 923 # Issue #12847 924 dumped = b'Va\np-1\n.' 925 self.check_unpickling_error(ValueError, dumped) 926 927 @requires_32b 928 def test_negative_32b_binput(self): 929 # Issue #12847 930 dumped = b'\x80\x03X\x01\x00\x00\x00ar\xff\xff\xff\xff.' 931 self.check_unpickling_error(ValueError, dumped) 932 933 def test_badly_escaped_string(self): 934 self.check_unpickling_error(ValueError, b"S'\\'\n.") 935 936 def test_badly_quoted_string(self): 937 # Issue #17710 938 badpickles = [b"S'\n.", 939 b'S"\n.', 940 b'S\' \n.', 941 b'S" \n.', 942 b'S\'"\n.', 943 b'S"\'\n.', 944 b"S' ' \n.", 945 b'S" " \n.', 946 b"S ''\n.", 947 b'S ""\n.', 948 b'S \n.', 949 b'S\n.', 950 b'S.'] 951 for p in badpickles: 952 self.check_unpickling_error(pickle.UnpicklingError, p) 953 954 def test_correctly_quoted_string(self): 955 goodpickles = [(b"S''\n.", ''), 956 (b'S""\n.', ''), 957 (b'S"\\n"\n.', '\n'), 958 (b"S'\\n'\n.", '\n')] 959 for p, expected in goodpickles: 960 self.assertEqual(self.loads(p), expected) 961 962 def test_frame_readline(self): 963 pickled = b'\x80\x04\x95\x05\x00\x00\x00\x00\x00\x00\x00I42\n.' 964 # 0: \x80 PROTO 4 965 # 2: \x95 FRAME 5 966 # 11: I INT 42 967 # 15: . STOP 968 self.assertEqual(self.loads(pickled), 42) 969 970 def test_compat_unpickle(self): 971 # xrange(1, 7) 972 pickled = b'\x80\x02c__builtin__\nxrange\nK\x01K\x07K\x01\x87R.' 973 unpickled = self.loads(pickled) 974 self.assertIs(type(unpickled), range) 975 self.assertEqual(unpickled, range(1, 7)) 976 self.assertEqual(list(unpickled), [1, 2, 3, 4, 5, 6]) 977 # reduce 978 pickled = b'\x80\x02c__builtin__\nreduce\n.' 979 self.assertIs(self.loads(pickled), functools.reduce) 980 # whichdb.whichdb 981 pickled = b'\x80\x02cwhichdb\nwhichdb\n.' 982 self.assertIs(self.loads(pickled), dbm.whichdb) 983 # Exception(), StandardError() 984 for name in (b'Exception', b'StandardError'): 985 pickled = (b'\x80\x02cexceptions\n' + name + b'\nU\x03ugh\x85R.') 986 unpickled = self.loads(pickled) 987 self.assertIs(type(unpickled), Exception) 988 self.assertEqual(str(unpickled), 'ugh') 989 # UserDict.UserDict({1: 2}), UserDict.IterableUserDict({1: 2}) 990 for name in (b'UserDict', b'IterableUserDict'): 991 pickled = (b'\x80\x02(cUserDict\n' + name + 992 b'\no}U\x04data}K\x01K\x02ssb.') 993 unpickled = self.loads(pickled) 994 self.assertIs(type(unpickled), collections.UserDict) 995 self.assertEqual(unpickled, collections.UserDict({1: 2})) 996 997 def test_bad_stack(self): 998 badpickles = [ 999 b'.', # STOP 1000 b'0', # POP 1001 b'1', # POP_MARK 1002 b'2', # DUP 1003 b'(2', 1004 b'R', # REDUCE 1005 b')R', 1006 b'a', # APPEND 1007 b'Na', 1008 b'b', # BUILD 1009 b'Nb', 1010 b'd', # DICT 1011 b'e', # APPENDS 1012 b'(e', 1013 b'ibuiltins\nlist\n', # INST 1014 b'l', # LIST 1015 b'o', # OBJ 1016 b'(o', 1017 b'p1\n', # PUT 1018 b'q\x00', # BINPUT 1019 b'r\x00\x00\x00\x00', # LONG_BINPUT 1020 b's', # SETITEM 1021 b'Ns', 1022 b'NNs', 1023 b't', # TUPLE 1024 b'u', # SETITEMS 1025 b'(u', 1026 b'}(Nu', 1027 b'\x81', # NEWOBJ 1028 b')\x81', 1029 b'\x85', # TUPLE1 1030 b'\x86', # TUPLE2 1031 b'N\x86', 1032 b'\x87', # TUPLE3 1033 b'N\x87', 1034 b'NN\x87', 1035 b'\x90', # ADDITEMS 1036 b'(\x90', 1037 b'\x91', # FROZENSET 1038 b'\x92', # NEWOBJ_EX 1039 b')}\x92', 1040 b'\x93', # STACK_GLOBAL 1041 b'Vlist\n\x93', 1042 b'\x94', # MEMOIZE 1043 ] 1044 for p in badpickles: 1045 self.check_unpickling_error(self.bad_stack_errors, p) 1046 1047 def test_bad_mark(self): 1048 badpickles = [ 1049 b'N(.', # STOP 1050 b'N(2', # DUP 1051 b'cbuiltins\nlist\n)(R', # REDUCE 1052 b'cbuiltins\nlist\n()R', 1053 b']N(a', # APPEND 1054 # BUILD 1055 b'cbuiltins\nValueError\n)R}(b', 1056 b'cbuiltins\nValueError\n)R(}b', 1057 b'(Nd', # DICT 1058 b'N(p1\n', # PUT 1059 b'N(q\x00', # BINPUT 1060 b'N(r\x00\x00\x00\x00', # LONG_BINPUT 1061 b'}NN(s', # SETITEM 1062 b'}N(Ns', 1063 b'}(NNs', 1064 b'}((u', # SETITEMS 1065 b'cbuiltins\nlist\n)(\x81', # NEWOBJ 1066 b'cbuiltins\nlist\n()\x81', 1067 b'N(\x85', # TUPLE1 1068 b'NN(\x86', # TUPLE2 1069 b'N(N\x86', 1070 b'NNN(\x87', # TUPLE3 1071 b'NN(N\x87', 1072 b'N(NN\x87', 1073 b']((\x90', # ADDITEMS 1074 # NEWOBJ_EX 1075 b'cbuiltins\nlist\n)}(\x92', 1076 b'cbuiltins\nlist\n)(}\x92', 1077 b'cbuiltins\nlist\n()}\x92', 1078 # STACK_GLOBAL 1079 b'Vbuiltins\n(Vlist\n\x93', 1080 b'Vbuiltins\nVlist\n(\x93', 1081 b'N(\x94', # MEMOIZE 1082 ] 1083 for p in badpickles: 1084 self.check_unpickling_error(self.bad_stack_errors, p) 1085 1086 def test_truncated_data(self): 1087 self.check_unpickling_error(EOFError, b'') 1088 self.check_unpickling_error(EOFError, b'N') 1089 badpickles = [ 1090 b'B', # BINBYTES 1091 b'B\x03\x00\x00', 1092 b'B\x03\x00\x00\x00', 1093 b'B\x03\x00\x00\x00ab', 1094 b'C', # SHORT_BINBYTES 1095 b'C\x03', 1096 b'C\x03ab', 1097 b'F', # FLOAT 1098 b'F0.0', 1099 b'F0.00', 1100 b'G', # BINFLOAT 1101 b'G\x00\x00\x00\x00\x00\x00\x00', 1102 b'I', # INT 1103 b'I0', 1104 b'J', # BININT 1105 b'J\x00\x00\x00', 1106 b'K', # BININT1 1107 b'L', # LONG 1108 b'L0', 1109 b'L10', 1110 b'L0L', 1111 b'L10L', 1112 b'M', # BININT2 1113 b'M\x00', 1114 # b'P', # PERSID 1115 # b'Pabc', 1116 b'S', # STRING 1117 b"S'abc'", 1118 b'T', # BINSTRING 1119 b'T\x03\x00\x00', 1120 b'T\x03\x00\x00\x00', 1121 b'T\x03\x00\x00\x00ab', 1122 b'U', # SHORT_BINSTRING 1123 b'U\x03', 1124 b'U\x03ab', 1125 b'V', # UNICODE 1126 b'Vabc', 1127 b'X', # BINUNICODE 1128 b'X\x03\x00\x00', 1129 b'X\x03\x00\x00\x00', 1130 b'X\x03\x00\x00\x00ab', 1131 b'(c', # GLOBAL 1132 b'(cbuiltins', 1133 b'(cbuiltins\n', 1134 b'(cbuiltins\nlist', 1135 b'Ng', # GET 1136 b'Ng0', 1137 b'(i', # INST 1138 b'(ibuiltins', 1139 b'(ibuiltins\n', 1140 b'(ibuiltins\nlist', 1141 b'Nh', # BINGET 1142 b'Nj', # LONG_BINGET 1143 b'Nj\x00\x00\x00', 1144 b'Np', # PUT 1145 b'Np0', 1146 b'Nq', # BINPUT 1147 b'Nr', # LONG_BINPUT 1148 b'Nr\x00\x00\x00', 1149 b'\x80', # PROTO 1150 b'\x82', # EXT1 1151 b'\x83', # EXT2 1152 b'\x84\x01', 1153 b'\x84', # EXT4 1154 b'\x84\x01\x00\x00', 1155 b'\x8a', # LONG1 1156 b'\x8b', # LONG4 1157 b'\x8b\x00\x00\x00', 1158 b'\x8c', # SHORT_BINUNICODE 1159 b'\x8c\x03', 1160 b'\x8c\x03ab', 1161 b'\x8d', # BINUNICODE8 1162 b'\x8d\x03\x00\x00\x00\x00\x00\x00', 1163 b'\x8d\x03\x00\x00\x00\x00\x00\x00\x00', 1164 b'\x8d\x03\x00\x00\x00\x00\x00\x00\x00ab', 1165 b'\x8e', # BINBYTES8 1166 b'\x8e\x03\x00\x00\x00\x00\x00\x00', 1167 b'\x8e\x03\x00\x00\x00\x00\x00\x00\x00', 1168 b'\x8e\x03\x00\x00\x00\x00\x00\x00\x00ab', 1169 b'\x95', # FRAME 1170 b'\x95\x02\x00\x00\x00\x00\x00\x00', 1171 b'\x95\x02\x00\x00\x00\x00\x00\x00\x00', 1172 b'\x95\x02\x00\x00\x00\x00\x00\x00\x00N', 1173 ] 1174 for p in badpickles: 1175 self.check_unpickling_error(self.truncated_errors, p) 1176 1177 1178class AbstractPickleTests(unittest.TestCase): 1179 # Subclass must define self.dumps, self.loads. 1180 1181 optimized = False 1182 1183 _testdata = AbstractUnpickleTests._testdata 1184 1185 def setUp(self): 1186 pass 1187 1188 assert_is_copy = AbstractUnpickleTests.assert_is_copy 1189 1190 def test_misc(self): 1191 # test various datatypes not tested by testdata 1192 for proto in protocols: 1193 x = myint(4) 1194 s = self.dumps(x, proto) 1195 y = self.loads(s) 1196 self.assert_is_copy(x, y) 1197 1198 x = (1, ()) 1199 s = self.dumps(x, proto) 1200 y = self.loads(s) 1201 self.assert_is_copy(x, y) 1202 1203 x = initarg(1, x) 1204 s = self.dumps(x, proto) 1205 y = self.loads(s) 1206 self.assert_is_copy(x, y) 1207 1208 # XXX test __reduce__ protocol? 1209 1210 def test_roundtrip_equality(self): 1211 expected = self._testdata 1212 for proto in protocols: 1213 s = self.dumps(expected, proto) 1214 got = self.loads(s) 1215 self.assert_is_copy(expected, got) 1216 1217 # There are gratuitous differences between pickles produced by 1218 # pickle and cPickle, largely because cPickle starts PUT indices at 1219 # 1 and pickle starts them at 0. See XXX comment in cPickle's put2() -- 1220 # there's a comment with an exclamation point there whose meaning 1221 # is a mystery. cPickle also suppresses PUT for objects with a refcount 1222 # of 1. 1223 def dont_test_disassembly(self): 1224 from io import StringIO 1225 from pickletools import dis 1226 1227 for proto, expected in (0, DATA0_DIS), (1, DATA1_DIS): 1228 s = self.dumps(self._testdata, proto) 1229 filelike = StringIO() 1230 dis(s, out=filelike) 1231 got = filelike.getvalue() 1232 self.assertEqual(expected, got) 1233 1234 def test_recursive_list(self): 1235 l = [] 1236 l.append(l) 1237 for proto in protocols: 1238 s = self.dumps(l, proto) 1239 x = self.loads(s) 1240 self.assertIsInstance(x, list) 1241 self.assertEqual(len(x), 1) 1242 self.assertIs(x[0], x) 1243 1244 def test_recursive_tuple_and_list(self): 1245 t = ([],) 1246 t[0].append(t) 1247 for proto in protocols: 1248 s = self.dumps(t, proto) 1249 x = self.loads(s) 1250 self.assertIsInstance(x, tuple) 1251 self.assertEqual(len(x), 1) 1252 self.assertIsInstance(x[0], list) 1253 self.assertEqual(len(x[0]), 1) 1254 self.assertIs(x[0][0], x) 1255 1256 def test_recursive_dict(self): 1257 d = {} 1258 d[1] = d 1259 for proto in protocols: 1260 s = self.dumps(d, proto) 1261 x = self.loads(s) 1262 self.assertIsInstance(x, dict) 1263 self.assertEqual(list(x.keys()), [1]) 1264 self.assertIs(x[1], x) 1265 1266 def test_recursive_dict_key(self): 1267 d = {} 1268 k = K(d) 1269 d[k] = 1 1270 for proto in protocols: 1271 s = self.dumps(d, proto) 1272 x = self.loads(s) 1273 self.assertIsInstance(x, dict) 1274 self.assertEqual(len(x.keys()), 1) 1275 self.assertIsInstance(list(x.keys())[0], K) 1276 self.assertIs(list(x.keys())[0].value, x) 1277 1278 def test_recursive_set(self): 1279 y = set() 1280 k = K(y) 1281 y.add(k) 1282 for proto in range(4, pickle.HIGHEST_PROTOCOL + 1): 1283 s = self.dumps(y, proto) 1284 x = self.loads(s) 1285 self.assertIsInstance(x, set) 1286 self.assertEqual(len(x), 1) 1287 self.assertIsInstance(list(x)[0], K) 1288 self.assertIs(list(x)[0].value, x) 1289 1290 def test_recursive_list_subclass(self): 1291 y = MyList() 1292 y.append(y) 1293 for proto in range(2, pickle.HIGHEST_PROTOCOL + 1): 1294 s = self.dumps(y, proto) 1295 x = self.loads(s) 1296 self.assertIsInstance(x, MyList) 1297 self.assertEqual(len(x), 1) 1298 self.assertIs(x[0], x) 1299 1300 def test_recursive_dict_subclass(self): 1301 d = MyDict() 1302 d[1] = d 1303 for proto in range(2, pickle.HIGHEST_PROTOCOL + 1): 1304 s = self.dumps(d, proto) 1305 x = self.loads(s) 1306 self.assertIsInstance(x, MyDict) 1307 self.assertEqual(list(x.keys()), [1]) 1308 self.assertIs(x[1], x) 1309 1310 def test_recursive_dict_subclass_key(self): 1311 d = MyDict() 1312 k = K(d) 1313 d[k] = 1 1314 for proto in range(2, pickle.HIGHEST_PROTOCOL + 1): 1315 s = self.dumps(d, proto) 1316 x = self.loads(s) 1317 self.assertIsInstance(x, MyDict) 1318 self.assertEqual(len(list(x.keys())), 1) 1319 self.assertIsInstance(list(x.keys())[0], K) 1320 self.assertIs(list(x.keys())[0].value, x) 1321 1322 def test_recursive_inst(self): 1323 i = C() 1324 i.attr = i 1325 for proto in protocols: 1326 s = self.dumps(i, proto) 1327 x = self.loads(s) 1328 self.assertIsInstance(x, C) 1329 self.assertEqual(dir(x), dir(i)) 1330 self.assertIs(x.attr, x) 1331 1332 def test_recursive_multi(self): 1333 l = [] 1334 d = {1:l} 1335 i = C() 1336 i.attr = d 1337 l.append(i) 1338 for proto in protocols: 1339 s = self.dumps(l, proto) 1340 x = self.loads(s) 1341 self.assertIsInstance(x, list) 1342 self.assertEqual(len(x), 1) 1343 self.assertEqual(dir(x[0]), dir(i)) 1344 self.assertEqual(list(x[0].attr.keys()), [1]) 1345 self.assertTrue(x[0].attr[1] is x) 1346 1347 def check_recursive_collection_and_inst(self, factory): 1348 h = H() 1349 y = factory([h]) 1350 h.attr = y 1351 for proto in protocols: 1352 s = self.dumps(y, proto) 1353 x = self.loads(s) 1354 self.assertIsInstance(x, type(y)) 1355 self.assertEqual(len(x), 1) 1356 self.assertIsInstance(list(x)[0], H) 1357 self.assertIs(list(x)[0].attr, x) 1358 1359 def test_recursive_list_and_inst(self): 1360 self.check_recursive_collection_and_inst(list) 1361 1362 def test_recursive_tuple_and_inst(self): 1363 self.check_recursive_collection_and_inst(tuple) 1364 1365 def test_recursive_dict_and_inst(self): 1366 self.check_recursive_collection_and_inst(dict.fromkeys) 1367 1368 def test_recursive_set_and_inst(self): 1369 self.check_recursive_collection_and_inst(set) 1370 1371 def test_recursive_frozenset_and_inst(self): 1372 self.check_recursive_collection_and_inst(frozenset) 1373 1374 def test_recursive_list_subclass_and_inst(self): 1375 self.check_recursive_collection_and_inst(MyList) 1376 1377 def test_recursive_tuple_subclass_and_inst(self): 1378 self.check_recursive_collection_and_inst(MyTuple) 1379 1380 def test_recursive_dict_subclass_and_inst(self): 1381 self.check_recursive_collection_and_inst(MyDict.fromkeys) 1382 1383 def test_recursive_set_subclass_and_inst(self): 1384 self.check_recursive_collection_and_inst(MySet) 1385 1386 def test_recursive_frozenset_subclass_and_inst(self): 1387 self.check_recursive_collection_and_inst(MyFrozenSet) 1388 1389 def test_unicode(self): 1390 endcases = ['', '<\\u>', '<\\\u1234>', '<\n>', 1391 '<\\>', '<\\\U00012345>', 1392 # surrogates 1393 '<\udc80>'] 1394 for proto in protocols: 1395 for u in endcases: 1396 p = self.dumps(u, proto) 1397 u2 = self.loads(p) 1398 self.assert_is_copy(u, u2) 1399 1400 def test_unicode_high_plane(self): 1401 t = '\U00012345' 1402 for proto in protocols: 1403 p = self.dumps(t, proto) 1404 t2 = self.loads(p) 1405 self.assert_is_copy(t, t2) 1406 1407 def test_bytes(self): 1408 for proto in protocols: 1409 for s in b'', b'xyz', b'xyz'*100: 1410 p = self.dumps(s, proto) 1411 self.assert_is_copy(s, self.loads(p)) 1412 for s in [bytes([i]) for i in range(256)]: 1413 p = self.dumps(s, proto) 1414 self.assert_is_copy(s, self.loads(p)) 1415 for s in [bytes([i, i]) for i in range(256)]: 1416 p = self.dumps(s, proto) 1417 self.assert_is_copy(s, self.loads(p)) 1418 1419 def test_ints(self): 1420 for proto in protocols: 1421 n = sys.maxsize 1422 while n: 1423 for expected in (-n, n): 1424 s = self.dumps(expected, proto) 1425 n2 = self.loads(s) 1426 self.assert_is_copy(expected, n2) 1427 n = n >> 1 1428 1429 def test_long(self): 1430 for proto in protocols: 1431 # 256 bytes is where LONG4 begins. 1432 for nbits in 1, 8, 8*254, 8*255, 8*256, 8*257: 1433 nbase = 1 << nbits 1434 for npos in nbase-1, nbase, nbase+1: 1435 for n in npos, -npos: 1436 pickle = self.dumps(n, proto) 1437 got = self.loads(pickle) 1438 self.assert_is_copy(n, got) 1439 # Try a monster. This is quadratic-time in protos 0 & 1, so don't 1440 # bother with those. 1441 nbase = int("deadbeeffeedface", 16) 1442 nbase += nbase << 1000000 1443 for n in nbase, -nbase: 1444 p = self.dumps(n, 2) 1445 got = self.loads(p) 1446 # assert_is_copy is very expensive here as it precomputes 1447 # a failure message by computing the repr() of n and got, 1448 # we just do the check ourselves. 1449 self.assertIs(type(got), int) 1450 self.assertEqual(n, got) 1451 1452 def test_float(self): 1453 test_values = [0.0, 4.94e-324, 1e-310, 7e-308, 6.626e-34, 0.1, 0.5, 1454 3.14, 263.44582062374053, 6.022e23, 1e30] 1455 test_values = test_values + [-x for x in test_values] 1456 for proto in protocols: 1457 for value in test_values: 1458 pickle = self.dumps(value, proto) 1459 got = self.loads(pickle) 1460 self.assert_is_copy(value, got) 1461 1462 @run_with_locale('LC_ALL', 'de_DE', 'fr_FR') 1463 def test_float_format(self): 1464 # make sure that floats are formatted locale independent with proto 0 1465 self.assertEqual(self.dumps(1.2, 0)[0:3], b'F1.') 1466 1467 def test_reduce(self): 1468 for proto in protocols: 1469 inst = AAA() 1470 dumped = self.dumps(inst, proto) 1471 loaded = self.loads(dumped) 1472 self.assertEqual(loaded, REDUCE_A) 1473 1474 def test_getinitargs(self): 1475 for proto in protocols: 1476 inst = initarg(1, 2) 1477 dumped = self.dumps(inst, proto) 1478 loaded = self.loads(dumped) 1479 self.assert_is_copy(inst, loaded) 1480 1481 def test_metaclass(self): 1482 a = use_metaclass() 1483 for proto in protocols: 1484 s = self.dumps(a, proto) 1485 b = self.loads(s) 1486 self.assertEqual(a.__class__, b.__class__) 1487 1488 def test_dynamic_class(self): 1489 a = create_dynamic_class("my_dynamic_class", (object,)) 1490 copyreg.pickle(pickling_metaclass, pickling_metaclass.__reduce__) 1491 for proto in protocols: 1492 s = self.dumps(a, proto) 1493 b = self.loads(s) 1494 self.assertEqual(a, b) 1495 self.assertIs(type(a), type(b)) 1496 1497 def test_structseq(self): 1498 import time 1499 import os 1500 1501 t = time.localtime() 1502 for proto in protocols: 1503 s = self.dumps(t, proto) 1504 u = self.loads(s) 1505 self.assert_is_copy(t, u) 1506 if hasattr(os, "stat"): 1507 t = os.stat(os.curdir) 1508 s = self.dumps(t, proto) 1509 u = self.loads(s) 1510 self.assert_is_copy(t, u) 1511 if hasattr(os, "statvfs"): 1512 t = os.statvfs(os.curdir) 1513 s = self.dumps(t, proto) 1514 u = self.loads(s) 1515 self.assert_is_copy(t, u) 1516 1517 def test_ellipsis(self): 1518 for proto in protocols: 1519 s = self.dumps(..., proto) 1520 u = self.loads(s) 1521 self.assertIs(..., u) 1522 1523 def test_notimplemented(self): 1524 for proto in protocols: 1525 s = self.dumps(NotImplemented, proto) 1526 u = self.loads(s) 1527 self.assertIs(NotImplemented, u) 1528 1529 def test_singleton_types(self): 1530 # Issue #6477: Test that types of built-in singletons can be pickled. 1531 singletons = [None, ..., NotImplemented] 1532 for singleton in singletons: 1533 for proto in protocols: 1534 s = self.dumps(type(singleton), proto) 1535 u = self.loads(s) 1536 self.assertIs(type(singleton), u) 1537 1538 # Tests for protocol 2 1539 1540 def test_proto(self): 1541 for proto in protocols: 1542 pickled = self.dumps(None, proto) 1543 if proto >= 2: 1544 proto_header = pickle.PROTO + bytes([proto]) 1545 self.assertTrue(pickled.startswith(proto_header)) 1546 else: 1547 self.assertEqual(count_opcode(pickle.PROTO, pickled), 0) 1548 1549 oob = protocols[-1] + 1 # a future protocol 1550 build_none = pickle.NONE + pickle.STOP 1551 badpickle = pickle.PROTO + bytes([oob]) + build_none 1552 try: 1553 self.loads(badpickle) 1554 except ValueError as err: 1555 self.assertIn("unsupported pickle protocol", str(err)) 1556 else: 1557 self.fail("expected bad protocol number to raise ValueError") 1558 1559 def test_long1(self): 1560 x = 12345678910111213141516178920 1561 for proto in protocols: 1562 s = self.dumps(x, proto) 1563 y = self.loads(s) 1564 self.assert_is_copy(x, y) 1565 self.assertEqual(opcode_in_pickle(pickle.LONG1, s), proto >= 2) 1566 1567 def test_long4(self): 1568 x = 12345678910111213141516178920 << (256*8) 1569 for proto in protocols: 1570 s = self.dumps(x, proto) 1571 y = self.loads(s) 1572 self.assert_is_copy(x, y) 1573 self.assertEqual(opcode_in_pickle(pickle.LONG4, s), proto >= 2) 1574 1575 def test_short_tuples(self): 1576 # Map (proto, len(tuple)) to expected opcode. 1577 expected_opcode = {(0, 0): pickle.TUPLE, 1578 (0, 1): pickle.TUPLE, 1579 (0, 2): pickle.TUPLE, 1580 (0, 3): pickle.TUPLE, 1581 (0, 4): pickle.TUPLE, 1582 1583 (1, 0): pickle.EMPTY_TUPLE, 1584 (1, 1): pickle.TUPLE, 1585 (1, 2): pickle.TUPLE, 1586 (1, 3): pickle.TUPLE, 1587 (1, 4): pickle.TUPLE, 1588 1589 (2, 0): pickle.EMPTY_TUPLE, 1590 (2, 1): pickle.TUPLE1, 1591 (2, 2): pickle.TUPLE2, 1592 (2, 3): pickle.TUPLE3, 1593 (2, 4): pickle.TUPLE, 1594 1595 (3, 0): pickle.EMPTY_TUPLE, 1596 (3, 1): pickle.TUPLE1, 1597 (3, 2): pickle.TUPLE2, 1598 (3, 3): pickle.TUPLE3, 1599 (3, 4): pickle.TUPLE, 1600 } 1601 a = () 1602 b = (1,) 1603 c = (1, 2) 1604 d = (1, 2, 3) 1605 e = (1, 2, 3, 4) 1606 for proto in protocols: 1607 for x in a, b, c, d, e: 1608 s = self.dumps(x, proto) 1609 y = self.loads(s) 1610 self.assert_is_copy(x, y) 1611 expected = expected_opcode[min(proto, 3), len(x)] 1612 self.assertTrue(opcode_in_pickle(expected, s)) 1613 1614 def test_singletons(self): 1615 # Map (proto, singleton) to expected opcode. 1616 expected_opcode = {(0, None): pickle.NONE, 1617 (1, None): pickle.NONE, 1618 (2, None): pickle.NONE, 1619 (3, None): pickle.NONE, 1620 1621 (0, True): pickle.INT, 1622 (1, True): pickle.INT, 1623 (2, True): pickle.NEWTRUE, 1624 (3, True): pickle.NEWTRUE, 1625 1626 (0, False): pickle.INT, 1627 (1, False): pickle.INT, 1628 (2, False): pickle.NEWFALSE, 1629 (3, False): pickle.NEWFALSE, 1630 } 1631 for proto in protocols: 1632 for x in None, False, True: 1633 s = self.dumps(x, proto) 1634 y = self.loads(s) 1635 self.assertTrue(x is y, (proto, x, s, y)) 1636 expected = expected_opcode[min(proto, 3), x] 1637 self.assertTrue(opcode_in_pickle(expected, s)) 1638 1639 def test_newobj_tuple(self): 1640 x = MyTuple([1, 2, 3]) 1641 x.foo = 42 1642 x.bar = "hello" 1643 for proto in protocols: 1644 s = self.dumps(x, proto) 1645 y = self.loads(s) 1646 self.assert_is_copy(x, y) 1647 1648 def test_newobj_list(self): 1649 x = MyList([1, 2, 3]) 1650 x.foo = 42 1651 x.bar = "hello" 1652 for proto in protocols: 1653 s = self.dumps(x, proto) 1654 y = self.loads(s) 1655 self.assert_is_copy(x, y) 1656 1657 def test_newobj_generic(self): 1658 for proto in protocols: 1659 for C in myclasses: 1660 B = C.__base__ 1661 x = C(C.sample) 1662 x.foo = 42 1663 s = self.dumps(x, proto) 1664 y = self.loads(s) 1665 detail = (proto, C, B, x, y, type(y)) 1666 self.assert_is_copy(x, y) # XXX revisit 1667 self.assertEqual(B(x), B(y), detail) 1668 self.assertEqual(x.__dict__, y.__dict__, detail) 1669 1670 def test_newobj_proxies(self): 1671 # NEWOBJ should use the __class__ rather than the raw type 1672 classes = myclasses[:] 1673 # Cannot create weakproxies to these classes 1674 for c in (MyInt, MyTuple): 1675 classes.remove(c) 1676 for proto in protocols: 1677 for C in classes: 1678 B = C.__base__ 1679 x = C(C.sample) 1680 x.foo = 42 1681 p = weakref.proxy(x) 1682 s = self.dumps(p, proto) 1683 y = self.loads(s) 1684 self.assertEqual(type(y), type(x)) # rather than type(p) 1685 detail = (proto, C, B, x, y, type(y)) 1686 self.assertEqual(B(x), B(y), detail) 1687 self.assertEqual(x.__dict__, y.__dict__, detail) 1688 1689 def test_newobj_not_class(self): 1690 # Issue 24552 1691 global SimpleNewObj 1692 save = SimpleNewObj 1693 o = SimpleNewObj.__new__(SimpleNewObj) 1694 b = self.dumps(o, 4) 1695 try: 1696 SimpleNewObj = 42 1697 self.assertRaises((TypeError, pickle.UnpicklingError), self.loads, b) 1698 finally: 1699 SimpleNewObj = save 1700 1701 # Register a type with copyreg, with extension code extcode. Pickle 1702 # an object of that type. Check that the resulting pickle uses opcode 1703 # (EXT[124]) under proto 2, and not in proto 1. 1704 1705 def produce_global_ext(self, extcode, opcode): 1706 e = ExtensionSaver(extcode) 1707 try: 1708 copyreg.add_extension(__name__, "MyList", extcode) 1709 x = MyList([1, 2, 3]) 1710 x.foo = 42 1711 x.bar = "hello" 1712 1713 # Dump using protocol 1 for comparison. 1714 s1 = self.dumps(x, 1) 1715 self.assertIn(__name__.encode("utf-8"), s1) 1716 self.assertIn(b"MyList", s1) 1717 self.assertFalse(opcode_in_pickle(opcode, s1)) 1718 1719 y = self.loads(s1) 1720 self.assert_is_copy(x, y) 1721 1722 # Dump using protocol 2 for test. 1723 s2 = self.dumps(x, 2) 1724 self.assertNotIn(__name__.encode("utf-8"), s2) 1725 self.assertNotIn(b"MyList", s2) 1726 self.assertEqual(opcode_in_pickle(opcode, s2), True, repr(s2)) 1727 1728 y = self.loads(s2) 1729 self.assert_is_copy(x, y) 1730 finally: 1731 e.restore() 1732 1733 def test_global_ext1(self): 1734 self.produce_global_ext(0x00000001, pickle.EXT1) # smallest EXT1 code 1735 self.produce_global_ext(0x000000ff, pickle.EXT1) # largest EXT1 code 1736 1737 def test_global_ext2(self): 1738 self.produce_global_ext(0x00000100, pickle.EXT2) # smallest EXT2 code 1739 self.produce_global_ext(0x0000ffff, pickle.EXT2) # largest EXT2 code 1740 self.produce_global_ext(0x0000abcd, pickle.EXT2) # check endianness 1741 1742 def test_global_ext4(self): 1743 self.produce_global_ext(0x00010000, pickle.EXT4) # smallest EXT4 code 1744 self.produce_global_ext(0x7fffffff, pickle.EXT4) # largest EXT4 code 1745 self.produce_global_ext(0x12abcdef, pickle.EXT4) # check endianness 1746 1747 def test_list_chunking(self): 1748 n = 10 # too small to chunk 1749 x = list(range(n)) 1750 for proto in protocols: 1751 s = self.dumps(x, proto) 1752 y = self.loads(s) 1753 self.assert_is_copy(x, y) 1754 num_appends = count_opcode(pickle.APPENDS, s) 1755 self.assertEqual(num_appends, proto > 0) 1756 1757 n = 2500 # expect at least two chunks when proto > 0 1758 x = list(range(n)) 1759 for proto in protocols: 1760 s = self.dumps(x, proto) 1761 y = self.loads(s) 1762 self.assert_is_copy(x, y) 1763 num_appends = count_opcode(pickle.APPENDS, s) 1764 if proto == 0: 1765 self.assertEqual(num_appends, 0) 1766 else: 1767 self.assertTrue(num_appends >= 2) 1768 1769 def test_dict_chunking(self): 1770 n = 10 # too small to chunk 1771 x = dict.fromkeys(range(n)) 1772 for proto in protocols: 1773 s = self.dumps(x, proto) 1774 self.assertIsInstance(s, bytes_types) 1775 y = self.loads(s) 1776 self.assert_is_copy(x, y) 1777 num_setitems = count_opcode(pickle.SETITEMS, s) 1778 self.assertEqual(num_setitems, proto > 0) 1779 1780 n = 2500 # expect at least two chunks when proto > 0 1781 x = dict.fromkeys(range(n)) 1782 for proto in protocols: 1783 s = self.dumps(x, proto) 1784 y = self.loads(s) 1785 self.assert_is_copy(x, y) 1786 num_setitems = count_opcode(pickle.SETITEMS, s) 1787 if proto == 0: 1788 self.assertEqual(num_setitems, 0) 1789 else: 1790 self.assertTrue(num_setitems >= 2) 1791 1792 def test_set_chunking(self): 1793 n = 10 # too small to chunk 1794 x = set(range(n)) 1795 for proto in protocols: 1796 s = self.dumps(x, proto) 1797 y = self.loads(s) 1798 self.assert_is_copy(x, y) 1799 num_additems = count_opcode(pickle.ADDITEMS, s) 1800 if proto < 4: 1801 self.assertEqual(num_additems, 0) 1802 else: 1803 self.assertEqual(num_additems, 1) 1804 1805 n = 2500 # expect at least two chunks when proto >= 4 1806 x = set(range(n)) 1807 for proto in protocols: 1808 s = self.dumps(x, proto) 1809 y = self.loads(s) 1810 self.assert_is_copy(x, y) 1811 num_additems = count_opcode(pickle.ADDITEMS, s) 1812 if proto < 4: 1813 self.assertEqual(num_additems, 0) 1814 else: 1815 self.assertGreaterEqual(num_additems, 2) 1816 1817 def test_simple_newobj(self): 1818 x = SimpleNewObj.__new__(SimpleNewObj, 0xface) # avoid __init__ 1819 x.abc = 666 1820 for proto in protocols: 1821 with self.subTest(proto=proto): 1822 s = self.dumps(x, proto) 1823 if proto < 1: 1824 self.assertIn(b'\nL64206', s) # LONG 1825 else: 1826 self.assertIn(b'M\xce\xfa', s) # BININT2 1827 self.assertEqual(opcode_in_pickle(pickle.NEWOBJ, s), 1828 2 <= proto) 1829 self.assertFalse(opcode_in_pickle(pickle.NEWOBJ_EX, s)) 1830 y = self.loads(s) # will raise TypeError if __init__ called 1831 self.assert_is_copy(x, y) 1832 1833 def test_complex_newobj(self): 1834 x = ComplexNewObj.__new__(ComplexNewObj, 0xface) # avoid __init__ 1835 x.abc = 666 1836 for proto in protocols: 1837 with self.subTest(proto=proto): 1838 s = self.dumps(x, proto) 1839 if proto < 1: 1840 self.assertIn(b'\nL64206', s) # LONG 1841 elif proto < 2: 1842 self.assertIn(b'M\xce\xfa', s) # BININT2 1843 elif proto < 4: 1844 self.assertIn(b'X\x04\x00\x00\x00FACE', s) # BINUNICODE 1845 else: 1846 self.assertIn(b'\x8c\x04FACE', s) # SHORT_BINUNICODE 1847 self.assertEqual(opcode_in_pickle(pickle.NEWOBJ, s), 1848 2 <= proto) 1849 self.assertFalse(opcode_in_pickle(pickle.NEWOBJ_EX, s)) 1850 y = self.loads(s) # will raise TypeError if __init__ called 1851 self.assert_is_copy(x, y) 1852 1853 def test_complex_newobj_ex(self): 1854 x = ComplexNewObjEx.__new__(ComplexNewObjEx, 0xface) # avoid __init__ 1855 x.abc = 666 1856 for proto in protocols: 1857 with self.subTest(proto=proto): 1858 s = self.dumps(x, proto) 1859 if proto < 1: 1860 self.assertIn(b'\nL64206', s) # LONG 1861 elif proto < 2: 1862 self.assertIn(b'M\xce\xfa', s) # BININT2 1863 elif proto < 4: 1864 self.assertIn(b'X\x04\x00\x00\x00FACE', s) # BINUNICODE 1865 else: 1866 self.assertIn(b'\x8c\x04FACE', s) # SHORT_BINUNICODE 1867 self.assertFalse(opcode_in_pickle(pickle.NEWOBJ, s)) 1868 self.assertEqual(opcode_in_pickle(pickle.NEWOBJ_EX, s), 1869 4 <= proto) 1870 y = self.loads(s) # will raise TypeError if __init__ called 1871 self.assert_is_copy(x, y) 1872 1873 def test_newobj_list_slots(self): 1874 x = SlotList([1, 2, 3]) 1875 x.foo = 42 1876 x.bar = "hello" 1877 s = self.dumps(x, 2) 1878 y = self.loads(s) 1879 self.assert_is_copy(x, y) 1880 1881 def test_reduce_overrides_default_reduce_ex(self): 1882 for proto in protocols: 1883 x = REX_one() 1884 self.assertEqual(x._reduce_called, 0) 1885 s = self.dumps(x, proto) 1886 self.assertEqual(x._reduce_called, 1) 1887 y = self.loads(s) 1888 self.assertEqual(y._reduce_called, 0) 1889 1890 def test_reduce_ex_called(self): 1891 for proto in protocols: 1892 x = REX_two() 1893 self.assertEqual(x._proto, None) 1894 s = self.dumps(x, proto) 1895 self.assertEqual(x._proto, proto) 1896 y = self.loads(s) 1897 self.assertEqual(y._proto, None) 1898 1899 def test_reduce_ex_overrides_reduce(self): 1900 for proto in protocols: 1901 x = REX_three() 1902 self.assertEqual(x._proto, None) 1903 s = self.dumps(x, proto) 1904 self.assertEqual(x._proto, proto) 1905 y = self.loads(s) 1906 self.assertEqual(y._proto, None) 1907 1908 def test_reduce_ex_calls_base(self): 1909 for proto in protocols: 1910 x = REX_four() 1911 self.assertEqual(x._proto, None) 1912 s = self.dumps(x, proto) 1913 self.assertEqual(x._proto, proto) 1914 y = self.loads(s) 1915 self.assertEqual(y._proto, proto) 1916 1917 def test_reduce_calls_base(self): 1918 for proto in protocols: 1919 x = REX_five() 1920 self.assertEqual(x._reduce_called, 0) 1921 s = self.dumps(x, proto) 1922 self.assertEqual(x._reduce_called, 1) 1923 y = self.loads(s) 1924 self.assertEqual(y._reduce_called, 1) 1925 1926 @no_tracing 1927 def test_bad_getattr(self): 1928 # Issue #3514: crash when there is an infinite loop in __getattr__ 1929 x = BadGetattr() 1930 for proto in protocols: 1931 self.assertRaises(RuntimeError, self.dumps, x, proto) 1932 1933 def test_reduce_bad_iterator(self): 1934 # Issue4176: crash when 4th and 5th items of __reduce__() 1935 # are not iterators 1936 class C(object): 1937 def __reduce__(self): 1938 # 4th item is not an iterator 1939 return list, (), None, [], None 1940 class D(object): 1941 def __reduce__(self): 1942 # 5th item is not an iterator 1943 return dict, (), None, None, [] 1944 1945 # Python implementation is less strict and also accepts iterables. 1946 for proto in protocols: 1947 try: 1948 self.dumps(C(), proto) 1949 except pickle.PicklingError: 1950 pass 1951 try: 1952 self.dumps(D(), proto) 1953 except pickle.PicklingError: 1954 pass 1955 1956 def test_many_puts_and_gets(self): 1957 # Test that internal data structures correctly deal with lots of 1958 # puts/gets. 1959 keys = ("aaa" + str(i) for i in range(100)) 1960 large_dict = dict((k, [4, 5, 6]) for k in keys) 1961 obj = [dict(large_dict), dict(large_dict), dict(large_dict)] 1962 1963 for proto in protocols: 1964 with self.subTest(proto=proto): 1965 dumped = self.dumps(obj, proto) 1966 loaded = self.loads(dumped) 1967 self.assert_is_copy(obj, loaded) 1968 1969 def test_attribute_name_interning(self): 1970 # Test that attribute names of pickled objects are interned when 1971 # unpickling. 1972 for proto in protocols: 1973 x = C() 1974 x.foo = 42 1975 x.bar = "hello" 1976 s = self.dumps(x, proto) 1977 y = self.loads(s) 1978 x_keys = sorted(x.__dict__) 1979 y_keys = sorted(y.__dict__) 1980 for x_key, y_key in zip(x_keys, y_keys): 1981 self.assertIs(x_key, y_key) 1982 1983 def test_pickle_to_2x(self): 1984 # Pickle non-trivial data with protocol 2, expecting that it yields 1985 # the same result as Python 2.x did. 1986 # NOTE: this test is a bit too strong since we can produce different 1987 # bytecode that 2.x will still understand. 1988 dumped = self.dumps(range(5), 2) 1989 self.assertEqual(dumped, DATA_XRANGE) 1990 dumped = self.dumps(set([3]), 2) 1991 self.assertEqual(dumped, DATA_SET2) 1992 1993 def test_large_pickles(self): 1994 # Test the correctness of internal buffering routines when handling 1995 # large data. 1996 for proto in protocols: 1997 data = (1, min, b'xy' * (30 * 1024), len) 1998 dumped = self.dumps(data, proto) 1999 loaded = self.loads(dumped) 2000 self.assertEqual(len(loaded), len(data)) 2001 self.assertEqual(loaded, data) 2002 2003 def test_int_pickling_efficiency(self): 2004 # Test compacity of int representation (see issue #12744) 2005 for proto in protocols: 2006 with self.subTest(proto=proto): 2007 pickles = [self.dumps(2**n, proto) for n in range(70)] 2008 sizes = list(map(len, pickles)) 2009 # the size function is monotonic 2010 self.assertEqual(sorted(sizes), sizes) 2011 if proto >= 2: 2012 for p in pickles: 2013 self.assertFalse(opcode_in_pickle(pickle.LONG, p)) 2014 2015 def _check_pickling_with_opcode(self, obj, opcode, proto): 2016 pickled = self.dumps(obj, proto) 2017 self.assertTrue(opcode_in_pickle(opcode, pickled)) 2018 unpickled = self.loads(pickled) 2019 self.assertEqual(obj, unpickled) 2020 2021 def test_appends_on_non_lists(self): 2022 # Issue #17720 2023 obj = REX_six([1, 2, 3]) 2024 for proto in protocols: 2025 if proto == 0: 2026 self._check_pickling_with_opcode(obj, pickle.APPEND, proto) 2027 else: 2028 self._check_pickling_with_opcode(obj, pickle.APPENDS, proto) 2029 2030 def test_setitems_on_non_dicts(self): 2031 obj = REX_seven({1: -1, 2: -2, 3: -3}) 2032 for proto in protocols: 2033 if proto == 0: 2034 self._check_pickling_with_opcode(obj, pickle.SETITEM, proto) 2035 else: 2036 self._check_pickling_with_opcode(obj, pickle.SETITEMS, proto) 2037 2038 # Exercise framing (proto >= 4) for significant workloads 2039 2040 FRAME_SIZE_TARGET = 64 * 1024 2041 2042 def check_frame_opcodes(self, pickled): 2043 """ 2044 Check the arguments of FRAME opcodes in a protocol 4+ pickle. 2045 """ 2046 frame_opcode_size = 9 2047 last_arg = last_pos = None 2048 for op, arg, pos in pickletools.genops(pickled): 2049 if op.name != 'FRAME': 2050 continue 2051 if last_pos is not None: 2052 # The previous frame's size should be equal to the number 2053 # of bytes up to the current frame. 2054 frame_size = pos - last_pos - frame_opcode_size 2055 self.assertEqual(frame_size, last_arg) 2056 last_arg, last_pos = arg, pos 2057 # The last frame's size should be equal to the number of bytes up 2058 # to the pickle's end. 2059 frame_size = len(pickled) - last_pos - frame_opcode_size 2060 self.assertEqual(frame_size, last_arg) 2061 2062 def test_framing_many_objects(self): 2063 obj = list(range(10**5)) 2064 for proto in range(4, pickle.HIGHEST_PROTOCOL + 1): 2065 with self.subTest(proto=proto): 2066 pickled = self.dumps(obj, proto) 2067 unpickled = self.loads(pickled) 2068 self.assertEqual(obj, unpickled) 2069 bytes_per_frame = (len(pickled) / 2070 count_opcode(pickle.FRAME, pickled)) 2071 self.assertGreater(bytes_per_frame, 2072 self.FRAME_SIZE_TARGET / 2) 2073 self.assertLessEqual(bytes_per_frame, 2074 self.FRAME_SIZE_TARGET * 1) 2075 self.check_frame_opcodes(pickled) 2076 2077 def test_framing_large_objects(self): 2078 N = 1024 * 1024 2079 obj = [b'x' * N, b'y' * N, b'z' * N] 2080 for proto in range(4, pickle.HIGHEST_PROTOCOL + 1): 2081 with self.subTest(proto=proto): 2082 pickled = self.dumps(obj, proto) 2083 unpickled = self.loads(pickled) 2084 self.assertEqual(obj, unpickled) 2085 n_frames = count_opcode(pickle.FRAME, pickled) 2086 self.assertGreaterEqual(n_frames, len(obj)) 2087 self.check_frame_opcodes(pickled) 2088 2089 def test_optional_frames(self): 2090 if pickle.HIGHEST_PROTOCOL < 4: 2091 return 2092 2093 def remove_frames(pickled, keep_frame=None): 2094 """Remove frame opcodes from the given pickle.""" 2095 frame_starts = [] 2096 # 1 byte for the opcode and 8 for the argument 2097 frame_opcode_size = 9 2098 for opcode, _, pos in pickletools.genops(pickled): 2099 if opcode.name == 'FRAME': 2100 frame_starts.append(pos) 2101 2102 newpickle = bytearray() 2103 last_frame_end = 0 2104 for i, pos in enumerate(frame_starts): 2105 if keep_frame and keep_frame(i): 2106 continue 2107 newpickle += pickled[last_frame_end:pos] 2108 last_frame_end = pos + frame_opcode_size 2109 newpickle += pickled[last_frame_end:] 2110 return newpickle 2111 2112 frame_size = self.FRAME_SIZE_TARGET 2113 num_frames = 20 2114 obj = [bytes([i]) * frame_size for i in range(num_frames)] 2115 2116 for proto in range(4, pickle.HIGHEST_PROTOCOL + 1): 2117 pickled = self.dumps(obj, proto) 2118 2119 frameless_pickle = remove_frames(pickled) 2120 self.assertEqual(count_opcode(pickle.FRAME, frameless_pickle), 0) 2121 self.assertEqual(obj, self.loads(frameless_pickle)) 2122 2123 some_frames_pickle = remove_frames(pickled, lambda i: i % 2) 2124 self.assertLess(count_opcode(pickle.FRAME, some_frames_pickle), 2125 count_opcode(pickle.FRAME, pickled)) 2126 self.assertEqual(obj, self.loads(some_frames_pickle)) 2127 2128 def test_nested_names(self): 2129 global Nested 2130 class Nested: 2131 class A: 2132 class B: 2133 class C: 2134 pass 2135 for proto in range(pickle.HIGHEST_PROTOCOL + 1): 2136 for obj in [Nested.A, Nested.A.B, Nested.A.B.C]: 2137 with self.subTest(proto=proto, obj=obj): 2138 unpickled = self.loads(self.dumps(obj, proto)) 2139 self.assertIs(obj, unpickled) 2140 2141 def test_recursive_nested_names(self): 2142 global Recursive 2143 class Recursive: 2144 pass 2145 Recursive.mod = sys.modules[Recursive.__module__] 2146 Recursive.__qualname__ = 'Recursive.mod.Recursive' 2147 for proto in range(pickle.HIGHEST_PROTOCOL + 1): 2148 with self.subTest(proto=proto): 2149 unpickled = self.loads(self.dumps(Recursive, proto)) 2150 self.assertIs(unpickled, Recursive) 2151 del Recursive.mod # break reference loop 2152 2153 def test_py_methods(self): 2154 global PyMethodsTest 2155 class PyMethodsTest: 2156 @staticmethod 2157 def cheese(): 2158 return "cheese" 2159 @classmethod 2160 def wine(cls): 2161 assert cls is PyMethodsTest 2162 return "wine" 2163 def biscuits(self): 2164 assert isinstance(self, PyMethodsTest) 2165 return "biscuits" 2166 class Nested: 2167 "Nested class" 2168 @staticmethod 2169 def ketchup(): 2170 return "ketchup" 2171 @classmethod 2172 def maple(cls): 2173 assert cls is PyMethodsTest.Nested 2174 return "maple" 2175 def pie(self): 2176 assert isinstance(self, PyMethodsTest.Nested) 2177 return "pie" 2178 2179 py_methods = ( 2180 PyMethodsTest.cheese, 2181 PyMethodsTest.wine, 2182 PyMethodsTest().biscuits, 2183 PyMethodsTest.Nested.ketchup, 2184 PyMethodsTest.Nested.maple, 2185 PyMethodsTest.Nested().pie 2186 ) 2187 py_unbound_methods = ( 2188 (PyMethodsTest.biscuits, PyMethodsTest), 2189 (PyMethodsTest.Nested.pie, PyMethodsTest.Nested) 2190 ) 2191 for proto in range(pickle.HIGHEST_PROTOCOL + 1): 2192 for method in py_methods: 2193 with self.subTest(proto=proto, method=method): 2194 unpickled = self.loads(self.dumps(method, proto)) 2195 self.assertEqual(method(), unpickled()) 2196 for method, cls in py_unbound_methods: 2197 obj = cls() 2198 with self.subTest(proto=proto, method=method): 2199 unpickled = self.loads(self.dumps(method, proto)) 2200 self.assertEqual(method(obj), unpickled(obj)) 2201 2202 def test_c_methods(self): 2203 global Subclass 2204 class Subclass(tuple): 2205 class Nested(str): 2206 pass 2207 2208 c_methods = ( 2209 # bound built-in method 2210 ("abcd".index, ("c",)), 2211 # unbound built-in method 2212 (str.index, ("abcd", "c")), 2213 # bound "slot" method 2214 ([1, 2, 3].__len__, ()), 2215 # unbound "slot" method 2216 (list.__len__, ([1, 2, 3],)), 2217 # bound "coexist" method 2218 ({1, 2}.__contains__, (2,)), 2219 # unbound "coexist" method 2220 (set.__contains__, ({1, 2}, 2)), 2221 # built-in class method 2222 (dict.fromkeys, (("a", 1), ("b", 2))), 2223 # built-in static method 2224 (bytearray.maketrans, (b"abc", b"xyz")), 2225 # subclass methods 2226 (Subclass([1,2,2]).count, (2,)), 2227 (Subclass.count, (Subclass([1,2,2]), 2)), 2228 (Subclass.Nested("sweet").count, ("e",)), 2229 (Subclass.Nested.count, (Subclass.Nested("sweet"), "e")), 2230 ) 2231 for proto in range(pickle.HIGHEST_PROTOCOL + 1): 2232 for method, args in c_methods: 2233 with self.subTest(proto=proto, method=method): 2234 unpickled = self.loads(self.dumps(method, proto)) 2235 self.assertEqual(method(*args), unpickled(*args)) 2236 2237 def test_compat_pickle(self): 2238 tests = [ 2239 (range(1, 7), '__builtin__', 'xrange'), 2240 (map(int, '123'), 'itertools', 'imap'), 2241 (functools.reduce, '__builtin__', 'reduce'), 2242 (dbm.whichdb, 'whichdb', 'whichdb'), 2243 (Exception(), 'exceptions', 'Exception'), 2244 (collections.UserDict(), 'UserDict', 'IterableUserDict'), 2245 (collections.UserList(), 'UserList', 'UserList'), 2246 (collections.defaultdict(), 'collections', 'defaultdict'), 2247 ] 2248 for val, mod, name in tests: 2249 for proto in range(3): 2250 with self.subTest(type=type(val), proto=proto): 2251 pickled = self.dumps(val, proto) 2252 self.assertIn(('c%s\n%s' % (mod, name)).encode(), pickled) 2253 self.assertIs(type(self.loads(pickled)), type(val)) 2254 2255 def test_local_lookup_error(self): 2256 # Test that whichmodule() errors out cleanly when looking up 2257 # an assumed globally-reachable object fails. 2258 def f(): 2259 pass 2260 # Since the function is local, lookup will fail 2261 for proto in range(0, pickle.HIGHEST_PROTOCOL + 1): 2262 with self.assertRaises((AttributeError, pickle.PicklingError)): 2263 pickletools.dis(self.dumps(f, proto)) 2264 # Same without a __module__ attribute (exercises a different path 2265 # in _pickle.c). 2266 del f.__module__ 2267 for proto in range(0, pickle.HIGHEST_PROTOCOL + 1): 2268 with self.assertRaises((AttributeError, pickle.PicklingError)): 2269 pickletools.dis(self.dumps(f, proto)) 2270 # Yet a different path. 2271 f.__name__ = f.__qualname__ 2272 for proto in range(0, pickle.HIGHEST_PROTOCOL + 1): 2273 with self.assertRaises((AttributeError, pickle.PicklingError)): 2274 pickletools.dis(self.dumps(f, proto)) 2275 2276 2277class BigmemPickleTests(unittest.TestCase): 2278 2279 # Binary protocols can serialize longs of up to 2GB-1 2280 2281 @bigmemtest(size=_2G, memuse=3.6, dry_run=False) 2282 def test_huge_long_32b(self, size): 2283 data = 1 << (8 * size) 2284 try: 2285 for proto in protocols: 2286 if proto < 2: 2287 continue 2288 with self.subTest(proto=proto): 2289 with self.assertRaises((ValueError, OverflowError)): 2290 self.dumps(data, protocol=proto) 2291 finally: 2292 data = None 2293 2294 # Protocol 3 can serialize up to 4GB-1 as a bytes object 2295 # (older protocols don't have a dedicated opcode for bytes and are 2296 # too inefficient) 2297 2298 @bigmemtest(size=_2G, memuse=2.5, dry_run=False) 2299 def test_huge_bytes_32b(self, size): 2300 data = b"abcd" * (size // 4) 2301 try: 2302 for proto in protocols: 2303 if proto < 3: 2304 continue 2305 with self.subTest(proto=proto): 2306 try: 2307 pickled = self.dumps(data, protocol=proto) 2308 header = (pickle.BINBYTES + 2309 struct.pack("<I", len(data))) 2310 data_start = pickled.index(data) 2311 self.assertEqual( 2312 header, 2313 pickled[data_start-len(header):data_start]) 2314 finally: 2315 pickled = None 2316 finally: 2317 data = None 2318 2319 @bigmemtest(size=_4G, memuse=2.5, dry_run=False) 2320 def test_huge_bytes_64b(self, size): 2321 data = b"acbd" * (size // 4) 2322 try: 2323 for proto in protocols: 2324 if proto < 3: 2325 continue 2326 with self.subTest(proto=proto): 2327 if proto == 3: 2328 # Protocol 3 does not support large bytes objects. 2329 # Verify that we do not crash when processing one. 2330 with self.assertRaises((ValueError, OverflowError)): 2331 self.dumps(data, protocol=proto) 2332 continue 2333 try: 2334 pickled = self.dumps(data, protocol=proto) 2335 header = (pickle.BINBYTES8 + 2336 struct.pack("<Q", len(data))) 2337 data_start = pickled.index(data) 2338 self.assertEqual( 2339 header, 2340 pickled[data_start-len(header):data_start]) 2341 finally: 2342 pickled = None 2343 finally: 2344 data = None 2345 2346 # All protocols use 1-byte per printable ASCII character; we add another 2347 # byte because the encoded form has to be copied into the internal buffer. 2348 2349 @bigmemtest(size=_2G, memuse=8, dry_run=False) 2350 def test_huge_str_32b(self, size): 2351 data = "abcd" * (size // 4) 2352 try: 2353 for proto in protocols: 2354 if proto == 0: 2355 continue 2356 with self.subTest(proto=proto): 2357 try: 2358 pickled = self.dumps(data, protocol=proto) 2359 header = (pickle.BINUNICODE + 2360 struct.pack("<I", len(data))) 2361 data_start = pickled.index(b'abcd') 2362 self.assertEqual( 2363 header, 2364 pickled[data_start-len(header):data_start]) 2365 self.assertEqual((pickled.rindex(b"abcd") + len(b"abcd") - 2366 pickled.index(b"abcd")), len(data)) 2367 finally: 2368 pickled = None 2369 finally: 2370 data = None 2371 2372 # BINUNICODE (protocols 1, 2 and 3) cannot carry more than 2**32 - 1 bytes 2373 # of utf-8 encoded unicode. BINUNICODE8 (protocol 4) supports these huge 2374 # unicode strings however. 2375 2376 @bigmemtest(size=_4G, memuse=8, dry_run=False) 2377 def test_huge_str_64b(self, size): 2378 data = "abcd" * (size // 4) 2379 try: 2380 for proto in protocols: 2381 if proto == 0: 2382 continue 2383 with self.subTest(proto=proto): 2384 if proto < 4: 2385 with self.assertRaises((ValueError, OverflowError)): 2386 self.dumps(data, protocol=proto) 2387 continue 2388 try: 2389 pickled = self.dumps(data, protocol=proto) 2390 header = (pickle.BINUNICODE8 + 2391 struct.pack("<Q", len(data))) 2392 data_start = pickled.index(b'abcd') 2393 self.assertEqual( 2394 header, 2395 pickled[data_start-len(header):data_start]) 2396 self.assertEqual((pickled.rindex(b"abcd") + len(b"abcd") - 2397 pickled.index(b"abcd")), len(data)) 2398 finally: 2399 pickled = None 2400 finally: 2401 data = None 2402 2403 2404# Test classes for reduce_ex 2405 2406class REX_one(object): 2407 """No __reduce_ex__ here, but inheriting it from object""" 2408 _reduce_called = 0 2409 def __reduce__(self): 2410 self._reduce_called = 1 2411 return REX_one, () 2412 2413class REX_two(object): 2414 """No __reduce__ here, but inheriting it from object""" 2415 _proto = None 2416 def __reduce_ex__(self, proto): 2417 self._proto = proto 2418 return REX_two, () 2419 2420class REX_three(object): 2421 _proto = None 2422 def __reduce_ex__(self, proto): 2423 self._proto = proto 2424 return REX_two, () 2425 def __reduce__(self): 2426 raise TestFailed("This __reduce__ shouldn't be called") 2427 2428class REX_four(object): 2429 """Calling base class method should succeed""" 2430 _proto = None 2431 def __reduce_ex__(self, proto): 2432 self._proto = proto 2433 return object.__reduce_ex__(self, proto) 2434 2435class REX_five(object): 2436 """This one used to fail with infinite recursion""" 2437 _reduce_called = 0 2438 def __reduce__(self): 2439 self._reduce_called = 1 2440 return object.__reduce__(self) 2441 2442class REX_six(object): 2443 """This class is used to check the 4th argument (list iterator) of 2444 the reduce protocol. 2445 """ 2446 def __init__(self, items=None): 2447 self.items = items if items is not None else [] 2448 def __eq__(self, other): 2449 return type(self) is type(other) and self.items == other.items 2450 def append(self, item): 2451 self.items.append(item) 2452 def __reduce__(self): 2453 return type(self), (), None, iter(self.items), None 2454 2455class REX_seven(object): 2456 """This class is used to check the 5th argument (dict iterator) of 2457 the reduce protocol. 2458 """ 2459 def __init__(self, table=None): 2460 self.table = table if table is not None else {} 2461 def __eq__(self, other): 2462 return type(self) is type(other) and self.table == other.table 2463 def __setitem__(self, key, value): 2464 self.table[key] = value 2465 def __reduce__(self): 2466 return type(self), (), None, None, iter(self.table.items()) 2467 2468 2469# Test classes for newobj 2470 2471class MyInt(int): 2472 sample = 1 2473 2474class MyFloat(float): 2475 sample = 1.0 2476 2477class MyComplex(complex): 2478 sample = 1.0 + 0.0j 2479 2480class MyStr(str): 2481 sample = "hello" 2482 2483class MyUnicode(str): 2484 sample = "hello \u1234" 2485 2486class MyTuple(tuple): 2487 sample = (1, 2, 3) 2488 2489class MyList(list): 2490 sample = [1, 2, 3] 2491 2492class MyDict(dict): 2493 sample = {"a": 1, "b": 2} 2494 2495class MySet(set): 2496 sample = {"a", "b"} 2497 2498class MyFrozenSet(frozenset): 2499 sample = frozenset({"a", "b"}) 2500 2501myclasses = [MyInt, MyFloat, 2502 MyComplex, 2503 MyStr, MyUnicode, 2504 MyTuple, MyList, MyDict, MySet, MyFrozenSet] 2505 2506 2507class SlotList(MyList): 2508 __slots__ = ["foo"] 2509 2510class SimpleNewObj(int): 2511 def __init__(self, *args, **kwargs): 2512 # raise an error, to make sure this isn't called 2513 raise TypeError("SimpleNewObj.__init__() didn't expect to get called") 2514 def __eq__(self, other): 2515 return int(self) == int(other) and self.__dict__ == other.__dict__ 2516 2517class ComplexNewObj(SimpleNewObj): 2518 def __getnewargs__(self): 2519 return ('%X' % self, 16) 2520 2521class ComplexNewObjEx(SimpleNewObj): 2522 def __getnewargs_ex__(self): 2523 return ('%X' % self,), {'base': 16} 2524 2525class BadGetattr: 2526 def __getattr__(self, key): 2527 self.foo 2528 2529 2530class AbstractPickleModuleTests(unittest.TestCase): 2531 2532 def test_dump_closed_file(self): 2533 import os 2534 f = open(TESTFN, "wb") 2535 try: 2536 f.close() 2537 self.assertRaises(ValueError, pickle.dump, 123, f) 2538 finally: 2539 os.remove(TESTFN) 2540 2541 def test_load_closed_file(self): 2542 import os 2543 f = open(TESTFN, "wb") 2544 try: 2545 f.close() 2546 self.assertRaises(ValueError, pickle.dump, 123, f) 2547 finally: 2548 os.remove(TESTFN) 2549 2550 def test_load_from_and_dump_to_file(self): 2551 stream = io.BytesIO() 2552 data = [123, {}, 124] 2553 pickle.dump(data, stream) 2554 stream.seek(0) 2555 unpickled = pickle.load(stream) 2556 self.assertEqual(unpickled, data) 2557 2558 def test_highest_protocol(self): 2559 # Of course this needs to be changed when HIGHEST_PROTOCOL changes. 2560 self.assertEqual(pickle.HIGHEST_PROTOCOL, 4) 2561 2562 def test_callapi(self): 2563 f = io.BytesIO() 2564 # With and without keyword arguments 2565 pickle.dump(123, f, -1) 2566 pickle.dump(123, file=f, protocol=-1) 2567 pickle.dumps(123, -1) 2568 pickle.dumps(123, protocol=-1) 2569 pickle.Pickler(f, -1) 2570 pickle.Pickler(f, protocol=-1) 2571 2572 def test_bad_init(self): 2573 # Test issue3664 (pickle can segfault from a badly initialized Pickler). 2574 # Override initialization without calling __init__() of the superclass. 2575 class BadPickler(pickle.Pickler): 2576 def __init__(self): pass 2577 2578 class BadUnpickler(pickle.Unpickler): 2579 def __init__(self): pass 2580 2581 self.assertRaises(pickle.PicklingError, BadPickler().dump, 0) 2582 self.assertRaises(pickle.UnpicklingError, BadUnpickler().load) 2583 2584 2585class AbstractPersistentPicklerTests(unittest.TestCase): 2586 2587 # This class defines persistent_id() and persistent_load() 2588 # functions that should be used by the pickler. All even integers 2589 # are pickled using persistent ids. 2590 2591 def persistent_id(self, object): 2592 if isinstance(object, int) and object % 2 == 0: 2593 self.id_count += 1 2594 return str(object) 2595 elif object == "test_false_value": 2596 self.false_count += 1 2597 return "" 2598 else: 2599 return None 2600 2601 def persistent_load(self, oid): 2602 if not oid: 2603 self.load_false_count += 1 2604 return "test_false_value" 2605 else: 2606 self.load_count += 1 2607 object = int(oid) 2608 assert object % 2 == 0 2609 return object 2610 2611 def test_persistence(self): 2612 L = list(range(10)) + ["test_false_value"] 2613 for proto in protocols: 2614 self.id_count = 0 2615 self.false_count = 0 2616 self.load_false_count = 0 2617 self.load_count = 0 2618 self.assertEqual(self.loads(self.dumps(L, proto)), L) 2619 self.assertEqual(self.id_count, 5) 2620 self.assertEqual(self.false_count, 1) 2621 self.assertEqual(self.load_count, 5) 2622 self.assertEqual(self.load_false_count, 1) 2623 2624 2625class AbstractIdentityPersistentPicklerTests(unittest.TestCase): 2626 2627 def persistent_id(self, obj): 2628 return obj 2629 2630 def persistent_load(self, pid): 2631 return pid 2632 2633 def _check_return_correct_type(self, obj, proto): 2634 unpickled = self.loads(self.dumps(obj, proto)) 2635 self.assertIsInstance(unpickled, type(obj)) 2636 self.assertEqual(unpickled, obj) 2637 2638 def test_return_correct_type(self): 2639 for proto in protocols: 2640 # Protocol 0 supports only ASCII strings. 2641 if proto == 0: 2642 self._check_return_correct_type("abc", 0) 2643 else: 2644 for obj in [b"abc\n", "abc\n", -1, -1.1 * 0.1, str]: 2645 self._check_return_correct_type(obj, proto) 2646 2647 def test_protocol0_is_ascii_only(self): 2648 non_ascii_str = "\N{EMPTY SET}" 2649 self.assertRaises(pickle.PicklingError, self.dumps, non_ascii_str, 0) 2650 pickled = pickle.PERSID + non_ascii_str.encode('utf-8') + b'\n.' 2651 self.assertRaises(pickle.UnpicklingError, self.loads, pickled) 2652 2653 2654class AbstractPicklerUnpicklerObjectTests(unittest.TestCase): 2655 2656 pickler_class = None 2657 unpickler_class = None 2658 2659 def setUp(self): 2660 assert self.pickler_class 2661 assert self.unpickler_class 2662 2663 def test_clear_pickler_memo(self): 2664 # To test whether clear_memo() has any effect, we pickle an object, 2665 # then pickle it again without clearing the memo; the two serialized 2666 # forms should be different. If we clear_memo() and then pickle the 2667 # object again, the third serialized form should be identical to the 2668 # first one we obtained. 2669 data = ["abcdefg", "abcdefg", 44] 2670 f = io.BytesIO() 2671 pickler = self.pickler_class(f) 2672 2673 pickler.dump(data) 2674 first_pickled = f.getvalue() 2675 2676 # Reset BytesIO object. 2677 f.seek(0) 2678 f.truncate() 2679 2680 pickler.dump(data) 2681 second_pickled = f.getvalue() 2682 2683 # Reset the Pickler and BytesIO objects. 2684 pickler.clear_memo() 2685 f.seek(0) 2686 f.truncate() 2687 2688 pickler.dump(data) 2689 third_pickled = f.getvalue() 2690 2691 self.assertNotEqual(first_pickled, second_pickled) 2692 self.assertEqual(first_pickled, third_pickled) 2693 2694 def test_priming_pickler_memo(self): 2695 # Verify that we can set the Pickler's memo attribute. 2696 data = ["abcdefg", "abcdefg", 44] 2697 f = io.BytesIO() 2698 pickler = self.pickler_class(f) 2699 2700 pickler.dump(data) 2701 first_pickled = f.getvalue() 2702 2703 f = io.BytesIO() 2704 primed = self.pickler_class(f) 2705 primed.memo = pickler.memo 2706 2707 primed.dump(data) 2708 primed_pickled = f.getvalue() 2709 2710 self.assertNotEqual(first_pickled, primed_pickled) 2711 2712 def test_priming_unpickler_memo(self): 2713 # Verify that we can set the Unpickler's memo attribute. 2714 data = ["abcdefg", "abcdefg", 44] 2715 f = io.BytesIO() 2716 pickler = self.pickler_class(f) 2717 2718 pickler.dump(data) 2719 first_pickled = f.getvalue() 2720 2721 f = io.BytesIO() 2722 primed = self.pickler_class(f) 2723 primed.memo = pickler.memo 2724 2725 primed.dump(data) 2726 primed_pickled = f.getvalue() 2727 2728 unpickler = self.unpickler_class(io.BytesIO(first_pickled)) 2729 unpickled_data1 = unpickler.load() 2730 2731 self.assertEqual(unpickled_data1, data) 2732 2733 primed = self.unpickler_class(io.BytesIO(primed_pickled)) 2734 primed.memo = unpickler.memo 2735 unpickled_data2 = primed.load() 2736 2737 primed.memo.clear() 2738 2739 self.assertEqual(unpickled_data2, data) 2740 self.assertTrue(unpickled_data2 is unpickled_data1) 2741 2742 def test_reusing_unpickler_objects(self): 2743 data1 = ["abcdefg", "abcdefg", 44] 2744 f = io.BytesIO() 2745 pickler = self.pickler_class(f) 2746 pickler.dump(data1) 2747 pickled1 = f.getvalue() 2748 2749 data2 = ["abcdefg", 44, 44] 2750 f = io.BytesIO() 2751 pickler = self.pickler_class(f) 2752 pickler.dump(data2) 2753 pickled2 = f.getvalue() 2754 2755 f = io.BytesIO() 2756 f.write(pickled1) 2757 f.seek(0) 2758 unpickler = self.unpickler_class(f) 2759 self.assertEqual(unpickler.load(), data1) 2760 2761 f.seek(0) 2762 f.truncate() 2763 f.write(pickled2) 2764 f.seek(0) 2765 self.assertEqual(unpickler.load(), data2) 2766 2767 def _check_multiple_unpicklings(self, ioclass): 2768 for proto in protocols: 2769 with self.subTest(proto=proto): 2770 data1 = [(x, str(x)) for x in range(2000)] + [b"abcde", len] 2771 f = ioclass() 2772 pickler = self.pickler_class(f, protocol=proto) 2773 pickler.dump(data1) 2774 pickled = f.getvalue() 2775 2776 N = 5 2777 f = ioclass(pickled * N) 2778 unpickler = self.unpickler_class(f) 2779 for i in range(N): 2780 if f.seekable(): 2781 pos = f.tell() 2782 self.assertEqual(unpickler.load(), data1) 2783 if f.seekable(): 2784 self.assertEqual(f.tell(), pos + len(pickled)) 2785 self.assertRaises(EOFError, unpickler.load) 2786 2787 def test_multiple_unpicklings_seekable(self): 2788 self._check_multiple_unpicklings(io.BytesIO) 2789 2790 def test_multiple_unpicklings_unseekable(self): 2791 self._check_multiple_unpicklings(UnseekableIO) 2792 2793 def test_unpickling_buffering_readline(self): 2794 # Issue #12687: the unpickler's buffering logic could fail with 2795 # text mode opcodes. 2796 data = list(range(10)) 2797 for proto in protocols: 2798 for buf_size in range(1, 11): 2799 f = io.BufferedRandom(io.BytesIO(), buffer_size=buf_size) 2800 pickler = self.pickler_class(f, protocol=proto) 2801 pickler.dump(data) 2802 f.seek(0) 2803 unpickler = self.unpickler_class(f) 2804 self.assertEqual(unpickler.load(), data) 2805 2806 2807# Tests for dispatch_table attribute 2808 2809REDUCE_A = 'reduce_A' 2810 2811class AAA(object): 2812 def __reduce__(self): 2813 return str, (REDUCE_A,) 2814 2815class BBB(object): 2816 pass 2817 2818class AbstractDispatchTableTests(unittest.TestCase): 2819 2820 def test_default_dispatch_table(self): 2821 # No dispatch_table attribute by default 2822 f = io.BytesIO() 2823 p = self.pickler_class(f, 0) 2824 with self.assertRaises(AttributeError): 2825 p.dispatch_table 2826 self.assertFalse(hasattr(p, 'dispatch_table')) 2827 2828 def test_class_dispatch_table(self): 2829 # A dispatch_table attribute can be specified class-wide 2830 dt = self.get_dispatch_table() 2831 2832 class MyPickler(self.pickler_class): 2833 dispatch_table = dt 2834 2835 def dumps(obj, protocol=None): 2836 f = io.BytesIO() 2837 p = MyPickler(f, protocol) 2838 self.assertEqual(p.dispatch_table, dt) 2839 p.dump(obj) 2840 return f.getvalue() 2841 2842 self._test_dispatch_table(dumps, dt) 2843 2844 def test_instance_dispatch_table(self): 2845 # A dispatch_table attribute can also be specified instance-wide 2846 dt = self.get_dispatch_table() 2847 2848 def dumps(obj, protocol=None): 2849 f = io.BytesIO() 2850 p = self.pickler_class(f, protocol) 2851 p.dispatch_table = dt 2852 self.assertEqual(p.dispatch_table, dt) 2853 p.dump(obj) 2854 return f.getvalue() 2855 2856 self._test_dispatch_table(dumps, dt) 2857 2858 def _test_dispatch_table(self, dumps, dispatch_table): 2859 def custom_load_dump(obj): 2860 return pickle.loads(dumps(obj, 0)) 2861 2862 def default_load_dump(obj): 2863 return pickle.loads(pickle.dumps(obj, 0)) 2864 2865 # pickling complex numbers using protocol 0 relies on copyreg 2866 # so check pickling a complex number still works 2867 z = 1 + 2j 2868 self.assertEqual(custom_load_dump(z), z) 2869 self.assertEqual(default_load_dump(z), z) 2870 2871 # modify pickling of complex 2872 REDUCE_1 = 'reduce_1' 2873 def reduce_1(obj): 2874 return str, (REDUCE_1,) 2875 dispatch_table[complex] = reduce_1 2876 self.assertEqual(custom_load_dump(z), REDUCE_1) 2877 self.assertEqual(default_load_dump(z), z) 2878 2879 # check picklability of AAA and BBB 2880 a = AAA() 2881 b = BBB() 2882 self.assertEqual(custom_load_dump(a), REDUCE_A) 2883 self.assertIsInstance(custom_load_dump(b), BBB) 2884 self.assertEqual(default_load_dump(a), REDUCE_A) 2885 self.assertIsInstance(default_load_dump(b), BBB) 2886 2887 # modify pickling of BBB 2888 dispatch_table[BBB] = reduce_1 2889 self.assertEqual(custom_load_dump(a), REDUCE_A) 2890 self.assertEqual(custom_load_dump(b), REDUCE_1) 2891 self.assertEqual(default_load_dump(a), REDUCE_A) 2892 self.assertIsInstance(default_load_dump(b), BBB) 2893 2894 # revert pickling of BBB and modify pickling of AAA 2895 REDUCE_2 = 'reduce_2' 2896 def reduce_2(obj): 2897 return str, (REDUCE_2,) 2898 dispatch_table[AAA] = reduce_2 2899 del dispatch_table[BBB] 2900 self.assertEqual(custom_load_dump(a), REDUCE_2) 2901 self.assertIsInstance(custom_load_dump(b), BBB) 2902 self.assertEqual(default_load_dump(a), REDUCE_A) 2903 self.assertIsInstance(default_load_dump(b), BBB) 2904 2905 2906if __name__ == "__main__": 2907 # Print some stuff that can be used to rewrite DATA{0,1,2} 2908 from pickletools import dis 2909 x = create_data() 2910 for i in range(pickle.HIGHEST_PROTOCOL+1): 2911 p = pickle.dumps(x, i) 2912 print("DATA{0} = (".format(i)) 2913 for j in range(0, len(p), 20): 2914 b = bytes(p[j:j+20]) 2915 print(" {0!r}".format(b)) 2916 print(")") 2917 print() 2918 print("# Disassembly of DATA{0}".format(i)) 2919 print("DATA{0}_DIS = \"\"\"\\".format(i)) 2920 dis(p) 2921 print("\"\"\"") 2922 print() 2923