marshal.c revision 0a7697b718898f4ebc17ba261b786502a03ed738 
1
2/* Write Python objects to files and read them back.
3   This is primarily intended for writing and reading compiled Python code,
4   even though dicts, lists, sets and frozensets, not commonly seen in
5   code objects, are supported.
6   Version 3 of this protocol properly supports circular links
7   and sharing. */
8
9#define PY_SSIZE_T_CLEAN
10
11#include "Python.h"
12#include "longintrepr.h"
13#include "code.h"
14#include "marshal.h"
15
16#define ABS(x) ((x) < 0 ? -(x) : (x))
17
18/* High water mark to determine when the marshalled object is dangerously deep
19 * and risks coring the interpreter.  When the object stack gets this deep,
20 * raise an exception instead of continuing.
21 * On Windows debug builds, reduce this value.
22 */
23#if defined(MS_WINDOWS) && defined(_DEBUG)
24#define MAX_MARSHAL_STACK_DEPTH 1500
25#else
26#define MAX_MARSHAL_STACK_DEPTH 2000
27#endif
28
29#define TYPE_NULL               '0'
30#define TYPE_NONE               'N'
31#define TYPE_FALSE              'F'
32#define TYPE_TRUE               'T'
33#define TYPE_STOPITER           'S'
34#define TYPE_ELLIPSIS           '.'
35#define TYPE_INT                'i'
36#define TYPE_FLOAT              'f'
37#define TYPE_BINARY_FLOAT       'g'
38#define TYPE_COMPLEX            'x'
39#define TYPE_BINARY_COMPLEX     'y'
40#define TYPE_LONG               'l'
41#define TYPE_STRING             's'
42#define TYPE_INTERNED           't'
43#define TYPE_REF                'r'
44#define TYPE_TUPLE              '('
45#define TYPE_LIST               '['
46#define TYPE_DICT               '{'
47#define TYPE_CODE               'c'
48#define TYPE_UNICODE            'u'
49#define TYPE_UNKNOWN            '?'
50#define TYPE_SET                '<'
51#define TYPE_FROZENSET          '>'
52#define FLAG_REF                '\x80' /* with a type, add obj to index */
53
54#define TYPE_ASCII              'a'
55#define TYPE_ASCII_INTERNED     'A'
56#define TYPE_SMALL_TUPLE        ')'
57#define TYPE_SHORT_ASCII        'z'
58#define TYPE_SHORT_ASCII_INTERNED 'Z'
59
60#define WFERR_OK 0
61#define WFERR_UNMARSHALLABLE 1
62#define WFERR_NESTEDTOODEEP 2
63#define WFERR_NOMEMORY 3
64
65typedef struct {
66    FILE *fp;
67    int error;  /* see WFERR_* values */
68    int depth;
69    /* If fp == NULL, the following are valid: */
70    PyObject *readable;    /* Stream-like object being read from */
71    PyObject *str;
72    PyObject *current_filename;
73    char *ptr;
74    char *end;
75    char *buf;
76    Py_ssize_t buf_size;
77    PyObject *refs; /* dict on marshal, list on unmarshal */
78    int version;
79} WFILE;
80
81#define w_byte(c, p) if (((p)->fp)) putc((c), (p)->fp); \
82                      else if ((p)->ptr != (p)->end) *(p)->ptr++ = (c); \
83                           else w_more(c, p)
84
85static void
86w_more(char c, WFILE *p)
87{
88    Py_ssize_t size, newsize;
89    if (p->str == NULL)
90        return; /* An error already occurred */
91    size = PyBytes_Size(p->str);
92    newsize = size + size + 1024;
93    if (newsize > 32*1024*1024) {
94        newsize = size + (size >> 3);           /* 12.5% overallocation */
95    }
96    if (_PyBytes_Resize(&p->str, newsize) != 0) {
97        p->ptr = p->end = NULL;
98    }
99    else {
100        p->ptr = PyBytes_AS_STRING((PyBytesObject *)p->str) + size;
101        p->end =
102            PyBytes_AS_STRING((PyBytesObject *)p->str) + newsize;
103        *p->ptr++ = c;
104    }
105}
106
107static void
108w_string(const char *s, Py_ssize_t n, WFILE *p)
109{
110    if (p->fp != NULL) {
111        fwrite(s, 1, n, p->fp);
112    }
113    else {
114        while (--n >= 0) {
115            w_byte(*s, p);
116            s++;
117        }
118    }
119}
120
121static void
122w_short(int x, WFILE *p)
123{
124    w_byte((char)( x      & 0xff), p);
125    w_byte((char)((x>> 8) & 0xff), p);
126}
127
128static void
129w_long(long x, WFILE *p)
130{
131    w_byte((char)( x      & 0xff), p);
132    w_byte((char)((x>> 8) & 0xff), p);
133    w_byte((char)((x>>16) & 0xff), p);
134    w_byte((char)((x>>24) & 0xff), p);
135}
136
137#define SIZE32_MAX  0x7FFFFFFF
138
139#if SIZEOF_SIZE_T > 4
140# define W_SIZE(n, p)  do {                     \
141        if ((n) > SIZE32_MAX) {                 \
142            (p)->depth--;                       \
143            (p)->error = WFERR_UNMARSHALLABLE;  \
144            return;                             \
145        }                                       \
146        w_long((long)(n), p);                   \
147    } while(0)
148#else
149# define W_SIZE  w_long
150#endif
151
152static void
153w_pstring(const char *s, Py_ssize_t n, WFILE *p)
154{
155        W_SIZE(n, p);
156        w_string(s, n, p);
157}
158
159static void
160w_short_pstring(const char *s, Py_ssize_t n, WFILE *p)
161{
162    w_byte(Py_SAFE_DOWNCAST(n, Py_ssize_t, unsigned char), p);
163    w_string(s, n, p);
164}
165
166/* We assume that Python ints are stored internally in base some power of
167   2**15; for the sake of portability we'll always read and write them in base
168   exactly 2**15. */
169
170#define PyLong_MARSHAL_SHIFT 15
171#define PyLong_MARSHAL_BASE ((short)1 << PyLong_MARSHAL_SHIFT)
172#define PyLong_MARSHAL_MASK (PyLong_MARSHAL_BASE - 1)
173#if PyLong_SHIFT % PyLong_MARSHAL_SHIFT != 0
174#error "PyLong_SHIFT must be a multiple of PyLong_MARSHAL_SHIFT"
175#endif
176#define PyLong_MARSHAL_RATIO (PyLong_SHIFT / PyLong_MARSHAL_SHIFT)
177
178#define W_TYPE(t, p) do { \
179    w_byte((t) | flag, (p)); \
180} while(0)
181
182static void
183w_PyLong(const PyLongObject *ob, char flag, WFILE *p)
184{
185    Py_ssize_t i, j, n, l;
186    digit d;
187
188    W_TYPE(TYPE_LONG, p);
189    if (Py_SIZE(ob) == 0) {
190        w_long((long)0, p);
191        return;
192    }
193
194    /* set l to number of base PyLong_MARSHAL_BASE digits */
195    n = ABS(Py_SIZE(ob));
196    l = (n-1) * PyLong_MARSHAL_RATIO;
197    d = ob->ob_digit[n-1];
198    assert(d != 0); /* a PyLong is always normalized */
199    do {
200        d >>= PyLong_MARSHAL_SHIFT;
201        l++;
202    } while (d != 0);
203    if (l > SIZE32_MAX) {
204        p->depth--;
205        p->error = WFERR_UNMARSHALLABLE;
206        return;
207    }
208    w_long((long)(Py_SIZE(ob) > 0 ? l : -l), p);
209
210    for (i=0; i < n-1; i++) {
211        d = ob->ob_digit[i];
212        for (j=0; j < PyLong_MARSHAL_RATIO; j++) {
213            w_short(d & PyLong_MARSHAL_MASK, p);
214            d >>= PyLong_MARSHAL_SHIFT;
215        }
216        assert (d == 0);
217    }
218    d = ob->ob_digit[n-1];
219    do {
220        w_short(d & PyLong_MARSHAL_MASK, p);
221        d >>= PyLong_MARSHAL_SHIFT;
222    } while (d != 0);
223}
224
225static int
226w_ref(PyObject *v, char *flag, WFILE *p)
227{
228    PyObject *id;
229    PyObject *idx;
230
231    if (p->version < 3 || p->refs == NULL)
232        return 0; /* not writing object references */
233
234    /* if it has only one reference, it definitely isn't shared */
235    if (Py_REFCNT(v) == 1)
236        return 0;
237
238    id = PyLong_FromVoidPtr((void*)v);
239    if (id == NULL)
240        goto err;
241    idx = PyDict_GetItem(p->refs, id);
242    if (idx != NULL) {
243        /* write the reference index to the stream */
244        long w = PyLong_AsLong(idx);
245        Py_DECREF(id);
246        if (w == -1 && PyErr_Occurred()) {
247            goto err;
248        }
249        /* we don't store "long" indices in the dict */
250        assert(0 <= w && w <= 0x7fffffff);
251        w_byte(TYPE_REF, p);
252        w_long(w, p);
253        return 1;
254    } else {
255        int ok;
256        Py_ssize_t s = PyDict_Size(p->refs);
257        /* we don't support long indices */
258        if (s >= 0x7fffffff) {
259            PyErr_SetString(PyExc_ValueError, "too many objects");
260            goto err;
261        }
262        idx = PyLong_FromSsize_t(s);
263        ok = idx && PyDict_SetItem(p->refs, id, idx) == 0;
264        Py_DECREF(id);
265        Py_XDECREF(idx);
266        if (!ok)
267            goto err;
268        *flag |= FLAG_REF;
269        return 0;
270    }
271err:
272    p->error = WFERR_UNMARSHALLABLE;
273    return 1;
274}
275
276static void
277w_complex_object(PyObject *v, char flag, WFILE *p);
278
279static void
280w_object(PyObject *v, WFILE *p)
281{
282    char flag = '\0';
283
284    p->depth++;
285
286    if (p->depth > MAX_MARSHAL_STACK_DEPTH) {
287        p->error = WFERR_NESTEDTOODEEP;
288    }
289    else if (v == NULL) {
290        w_byte(TYPE_NULL, p);
291    }
292    else if (v == Py_None) {
293        w_byte(TYPE_NONE, p);
294    }
295    else if (v == PyExc_StopIteration) {
296        w_byte(TYPE_STOPITER, p);
297    }
298    else if (v == Py_Ellipsis) {
299        w_byte(TYPE_ELLIPSIS, p);
300    }
301    else if (v == Py_False) {
302        w_byte(TYPE_FALSE, p);
303    }
304    else if (v == Py_True) {
305        w_byte(TYPE_TRUE, p);
306    }
307    else if (!w_ref(v, &flag, p))
308        w_complex_object(v, flag, p);
309
310    p->depth--;
311}
312
313static void
314w_complex_object(PyObject *v, char flag, WFILE *p)
315{
316    Py_ssize_t i, n;
317
318    if (PyLong_CheckExact(v)) {
319        long x = PyLong_AsLong(v);
320        if ((x == -1)  && PyErr_Occurred()) {
321            PyLongObject *ob = (PyLongObject *)v;
322            PyErr_Clear();
323            w_PyLong(ob, flag, p);
324        }
325        else {
326#if SIZEOF_LONG > 4
327            long y = Py_ARITHMETIC_RIGHT_SHIFT(long, x, 31);
328            if (y && y != -1) {
329                /* Too large for TYPE_INT */
330                w_PyLong((PyLongObject*)v, flag, p);
331            }
332            else
333#endif
334            {
335                W_TYPE(TYPE_INT, p);
336                w_long(x, p);
337            }
338        }
339    }
340    else if (PyFloat_CheckExact(v)) {
341        if (p->version > 1) {
342            unsigned char buf[8];
343            if (_PyFloat_Pack8(PyFloat_AsDouble(v),
344                               buf, 1) < 0) {
345                p->error = WFERR_UNMARSHALLABLE;
346                return;
347            }
348            W_TYPE(TYPE_BINARY_FLOAT, p);
349            w_string((char*)buf, 8, p);
350        }
351        else {
352            char *buf = PyOS_double_to_string(PyFloat_AS_DOUBLE(v),
353                                              'g', 17, 0, NULL);
354            if (!buf) {
355                p->error = WFERR_NOMEMORY;
356                return;
357            }
358            n = strlen(buf);
359            W_TYPE(TYPE_FLOAT, p);
360            w_byte((int)n, p);
361            w_string(buf, n, p);
362            PyMem_Free(buf);
363        }
364    }
365    else if (PyComplex_CheckExact(v)) {
366        if (p->version > 1) {
367            unsigned char buf[8];
368            if (_PyFloat_Pack8(PyComplex_RealAsDouble(v),
369                               buf, 1) < 0) {
370                p->error = WFERR_UNMARSHALLABLE;
371                return;
372            }
373            W_TYPE(TYPE_BINARY_COMPLEX, p);
374            w_string((char*)buf, 8, p);
375            if (_PyFloat_Pack8(PyComplex_ImagAsDouble(v),
376                               buf, 1) < 0) {
377                p->error = WFERR_UNMARSHALLABLE;
378                return;
379            }
380            w_string((char*)buf, 8, p);
381        }
382        else {
383            char *buf;
384            W_TYPE(TYPE_COMPLEX, p);
385            buf = PyOS_double_to_string(PyComplex_RealAsDouble(v),
386                                        'g', 17, 0, NULL);
387            if (!buf) {
388                p->error = WFERR_NOMEMORY;
389                return;
390            }
391            n = strlen(buf);
392            w_byte((int)n, p);
393            w_string(buf, n, p);
394            PyMem_Free(buf);
395            buf = PyOS_double_to_string(PyComplex_ImagAsDouble(v),
396                                        'g', 17, 0, NULL);
397            if (!buf) {
398                p->error = WFERR_NOMEMORY;
399                return;
400            }
401            n = strlen(buf);
402            w_byte((int)n, p);
403            w_string(buf, n, p);
404            PyMem_Free(buf);
405        }
406    }
407    else if (PyBytes_CheckExact(v)) {
408        W_TYPE(TYPE_STRING, p);
409        w_pstring(PyBytes_AS_STRING(v), PyBytes_GET_SIZE(v), p);
410    }
411    else if (PyUnicode_CheckExact(v)) {
412        if (p->version >= 4 && PyUnicode_IS_ASCII(v)) {
413            int is_short = PyUnicode_GET_LENGTH(v) < 256;
414            if (is_short) {
415                if (PyUnicode_CHECK_INTERNED(v))
416                    W_TYPE(TYPE_SHORT_ASCII_INTERNED, p);
417                else
418                    W_TYPE(TYPE_SHORT_ASCII, p);
419                w_short_pstring((char *) PyUnicode_1BYTE_DATA(v),
420                                PyUnicode_GET_LENGTH(v), p);
421            }
422            else {
423                if (PyUnicode_CHECK_INTERNED(v))
424                    W_TYPE(TYPE_ASCII_INTERNED, p);
425                else
426                    W_TYPE(TYPE_ASCII, p);
427                w_pstring((char *) PyUnicode_1BYTE_DATA(v),
428                          PyUnicode_GET_LENGTH(v), p);
429            }
430        }
431        else {
432            PyObject *utf8;
433            utf8 = PyUnicode_AsEncodedString(v, "utf8", "surrogatepass");
434            if (utf8 == NULL) {
435                p->depth--;
436                p->error = WFERR_UNMARSHALLABLE;
437                return;
438            }
439            if (p->version >= 3 &&  PyUnicode_CHECK_INTERNED(v))
440                W_TYPE(TYPE_INTERNED, p);
441            else
442                W_TYPE(TYPE_UNICODE, p);
443            w_pstring(PyBytes_AS_STRING(utf8), PyBytes_GET_SIZE(utf8), p);
444            Py_DECREF(utf8);
445        }
446    }
447    else if (PyTuple_CheckExact(v)) {
448        n = PyTuple_Size(v);
449        if (p->version >= 4 && n < 256) {
450            W_TYPE(TYPE_SMALL_TUPLE, p);
451            w_byte(n, p);
452        }
453        else {
454            W_TYPE(TYPE_TUPLE, p);
455            W_SIZE(n, p);
456        }
457        for (i = 0; i < n; i++) {
458            w_object(PyTuple_GET_ITEM(v, i), p);
459        }
460    }
461    else if (PyList_CheckExact(v)) {
462        W_TYPE(TYPE_LIST, p);
463        n = PyList_GET_SIZE(v);
464        W_SIZE(n, p);
465        for (i = 0; i < n; i++) {
466            w_object(PyList_GET_ITEM(v, i), p);
467        }
468    }
469    else if (PyDict_CheckExact(v)) {
470        Py_ssize_t pos;
471        PyObject *key, *value;
472        W_TYPE(TYPE_DICT, p);
473        /* This one is NULL object terminated! */
474        pos = 0;
475        while (PyDict_Next(v, &pos, &key, &value)) {
476            w_object(key, p);
477            w_object(value, p);
478        }
479        w_object((PyObject *)NULL, p);
480    }
481    else if (PyAnySet_CheckExact(v)) {
482        PyObject *value, *it;
483
484        if (PyObject_TypeCheck(v, &PySet_Type))
485            W_TYPE(TYPE_SET, p);
486        else
487            W_TYPE(TYPE_FROZENSET, p);
488        n = PyObject_Size(v);
489        if (n == -1) {
490            p->depth--;
491            p->error = WFERR_UNMARSHALLABLE;
492            return;
493        }
494        W_SIZE(n, p);
495        it = PyObject_GetIter(v);
496        if (it == NULL) {
497            p->depth--;
498            p->error = WFERR_UNMARSHALLABLE;
499            return;
500        }
501        while ((value = PyIter_Next(it)) != NULL) {
502            w_object(value, p);
503            Py_DECREF(value);
504        }
505        Py_DECREF(it);
506        if (PyErr_Occurred()) {
507            p->depth--;
508            p->error = WFERR_UNMARSHALLABLE;
509            return;
510        }
511    }
512    else if (PyCode_Check(v)) {
513        PyCodeObject *co = (PyCodeObject *)v;
514        W_TYPE(TYPE_CODE, p);
515        w_long(co->co_argcount, p);
516        w_long(co->co_kwonlyargcount, p);
517        w_long(co->co_nlocals, p);
518        w_long(co->co_stacksize, p);
519        w_long(co->co_flags, p);
520        w_object(co->co_code, p);
521        w_object(co->co_consts, p);
522        w_object(co->co_names, p);
523        w_object(co->co_varnames, p);
524        w_object(co->co_freevars, p);
525        w_object(co->co_cellvars, p);
526        w_object(co->co_filename, p);
527        w_object(co->co_name, p);
528        w_long(co->co_firstlineno, p);
529        w_object(co->co_lnotab, p);
530    }
531    else if (PyObject_CheckBuffer(v)) {
532        /* Write unknown buffer-style objects as a string */
533        Py_buffer view;
534        if (PyObject_GetBuffer(v, &view, PyBUF_SIMPLE) != 0) {
535            w_byte(TYPE_UNKNOWN, p);
536            p->depth--;
537            p->error = WFERR_UNMARSHALLABLE;
538            return;
539        }
540        W_TYPE(TYPE_STRING, p);
541        w_pstring(view.buf, view.len, p);
542        PyBuffer_Release(&view);
543    }
544    else {
545        W_TYPE(TYPE_UNKNOWN, p);
546        p->error = WFERR_UNMARSHALLABLE;
547    }
548}
549
550/* version currently has no effect for writing ints. */
551void
552PyMarshal_WriteLongToFile(long x, FILE *fp, int version)
553{
554    WFILE wf;
555    wf.fp = fp;
556    wf.error = WFERR_OK;
557    wf.depth = 0;
558    wf.refs = NULL;
559    wf.version = version;
560    w_long(x, &wf);
561}
562
563void
564PyMarshal_WriteObjectToFile(PyObject *x, FILE *fp, int version)
565{
566    WFILE wf;
567    wf.fp = fp;
568    wf.error = WFERR_OK;
569    wf.depth = 0;
570    if (version >= 3) {
571        if ((wf.refs = PyDict_New()) == NULL)
572            return; /* caller mush check PyErr_Occurred() */
573    } else
574        wf.refs = NULL;
575    wf.version = version;
576    w_object(x, &wf);
577    Py_XDECREF(wf.refs);
578}
579
580typedef WFILE RFILE; /* Same struct with different invariants */
581
582static char *
583r_string(Py_ssize_t n, RFILE *p)
584{
585    Py_ssize_t read = -1;
586
587    if (p->ptr != NULL) {
588        /* Fast path for loads() */
589        char *res = p->ptr;
590        Py_ssize_t left = p->end - p->ptr;
591        if (left < n) {
592            PyErr_SetString(PyExc_EOFError,
593                            "marshal data too short");
594            return NULL;
595        }
596        p->ptr += n;
597        return res;
598    }
599    if (p->buf == NULL) {
600        p->buf = PyMem_MALLOC(n);
601        if (p->buf == NULL) {
602            PyErr_NoMemory();
603            return NULL;
604        }
605        p->buf_size = n;
606    }
607    else if (p->buf_size < n) {
608        p->buf = PyMem_REALLOC(p->buf, n);
609        if (p->buf == NULL) {
610            PyErr_NoMemory();
611            return NULL;
612        }
613        p->buf_size = n;
614    }
615    if (!p->readable) {
616        assert(p->fp != NULL);
617        /* The result fits into int because it must be <=n. */
618        read = fread(p->buf, 1, n, p->fp);
619    }
620    else {
621        _Py_IDENTIFIER(readinto);
622        PyObject *res, *mview;
623        Py_buffer buf;
624
625        if (PyBuffer_FillInfo(&buf, NULL, p->buf, n, 0, PyBUF_CONTIG) == -1)
626            return NULL;
627        mview = PyMemoryView_FromBuffer(&buf);
628        if (mview == NULL)
629            return NULL;
630
631        res = _PyObject_CallMethodId(p->readable, &PyId_readinto, "N", mview);
632        if (res != NULL) {
633            read = PyNumber_AsSsize_t(res, PyExc_ValueError);
634            Py_DECREF(res);
635        }
636    }
637    if (read != n) {
638        if (!PyErr_Occurred()) {
639            if (read > n)
640                PyErr_Format(PyExc_ValueError,
641                             "read() returned too much data: "
642                             "%zd bytes requested, %zd returned",
643                             n, read);
644            else
645                PyErr_SetString(PyExc_EOFError,
646                                "EOF read where not expected");
647        }
648        return NULL;
649    }
650    return p->buf;
651}
652
653
654static int
655r_byte(RFILE *p)
656{
657    int c = EOF;
658
659    if (p->ptr != NULL) {
660        if (p->ptr < p->end)
661            c = (unsigned char) *p->ptr++;
662        return c;
663    }
664    if (!p->readable) {
665        assert(p->fp);
666        c = getc(p->fp);
667    }
668    else {
669        char *ptr = r_string(1, p);
670        if (ptr != NULL)
671            c = *(unsigned char *) ptr;
672    }
673    return c;
674}
675
676static int
677r_short(RFILE *p)
678{
679    short x = -1;
680    unsigned char *buffer;
681
682    buffer = (unsigned char *) r_string(2, p);
683    if (buffer != NULL) {
684        x = buffer[0];
685        x |= buffer[1] << 8;
686        /* Sign-extension, in case short greater than 16 bits */
687        x |= -(x & 0x8000);
688    }
689    return x;
690}
691
692static long
693r_long(RFILE *p)
694{
695    long x = -1;
696    unsigned char *buffer;
697
698    buffer = (unsigned char *) r_string(4, p);
699    if (buffer != NULL) {
700        x = buffer[0];
701        x |= (long)buffer[1] << 8;
702        x |= (long)buffer[2] << 16;
703        x |= (long)buffer[3] << 24;
704#if SIZEOF_LONG > 4
705        /* Sign extension for 64-bit machines */
706        x |= -(x & 0x80000000L);
707#endif
708    }
709    return x;
710}
711
712static PyObject *
713r_PyLong(RFILE *p)
714{
715    PyLongObject *ob;
716    long n, size, i;
717    int j, md, shorts_in_top_digit;
718    digit d;
719
720    n = r_long(p);
721    if (PyErr_Occurred())
722        return NULL;
723    if (n == 0)
724        return (PyObject *)_PyLong_New(0);
725    if (n < -SIZE32_MAX || n > SIZE32_MAX) {
726        PyErr_SetString(PyExc_ValueError,
727                       "bad marshal data (long size out of range)");
728        return NULL;
729    }
730
731    size = 1 + (ABS(n) - 1) / PyLong_MARSHAL_RATIO;
732    shorts_in_top_digit = 1 + (ABS(n) - 1) % PyLong_MARSHAL_RATIO;
733    ob = _PyLong_New(size);
734    if (ob == NULL)
735        return NULL;
736    Py_SIZE(ob) = n > 0 ? size : -size;
737
738    for (i = 0; i < size-1; i++) {
739        d = 0;
740        for (j=0; j < PyLong_MARSHAL_RATIO; j++) {
741            md = r_short(p);
742            if (PyErr_Occurred())
743                break;
744            if (md < 0 || md > PyLong_MARSHAL_BASE)
745                goto bad_digit;
746            d += (digit)md << j*PyLong_MARSHAL_SHIFT;
747        }
748        ob->ob_digit[i] = d;
749    }
750    d = 0;
751    for (j=0; j < shorts_in_top_digit; j++) {
752        md = r_short(p);
753        if (PyErr_Occurred())
754            break;
755        if (md < 0 || md > PyLong_MARSHAL_BASE)
756            goto bad_digit;
757        /* topmost marshal digit should be nonzero */
758        if (md == 0 && j == shorts_in_top_digit - 1) {
759            Py_DECREF(ob);
760            PyErr_SetString(PyExc_ValueError,
761                "bad marshal data (unnormalized long data)");
762            return NULL;
763        }
764        d += (digit)md << j*PyLong_MARSHAL_SHIFT;
765    }
766    if (PyErr_Occurred()) {
767        Py_DECREF(ob);
768        return NULL;
769    }
770    /* top digit should be nonzero, else the resulting PyLong won't be
771       normalized */
772    ob->ob_digit[size-1] = d;
773    return (PyObject *)ob;
774  bad_digit:
775    Py_DECREF(ob);
776    PyErr_SetString(PyExc_ValueError,
777                    "bad marshal data (digit out of range in long)");
778    return NULL;
779}
780
781/* allocate the reflist index for a new object. Return -1 on failure */
782static Py_ssize_t
783r_ref_reserve(int flag, RFILE *p)
784{
785    if (flag) { /* currently only FLAG_REF is defined */
786        Py_ssize_t idx = PyList_GET_SIZE(p->refs);
787        if (idx >= 0x7ffffffe) {
788            PyErr_SetString(PyExc_ValueError, "bad marshal data (index list too large)");
789            return -1;
790        }
791        if (PyList_Append(p->refs, Py_None) < 0)
792            return -1;
793        return idx;
794    } else
795        return 0;
796}
797
798/* insert the new object 'o' to the reflist at previously
799 * allocated index 'idx'.
800 * 'o' can be NULL, in which case nothing is done.
801 * if 'o' was non-NULL, and the function succeeds, 'o' is returned.
802 * if 'o' was non-NULL, and the function fails, 'o' is released and
803 * NULL returned. This simplifies error checking at the call site since
804 * a single test for NULL for the function result is enough.
805 */
806static PyObject *
807r_ref_insert(PyObject *o, Py_ssize_t idx, int flag, RFILE *p)
808{
809    if (o != NULL && flag) { /* currently only FLAG_REF is defined */
810        PyObject *tmp = PyList_GET_ITEM(p->refs, idx);
811        Py_INCREF(o);
812        PyList_SET_ITEM(p->refs, idx, o);
813        Py_DECREF(tmp);
814    }
815    return o;
816}
817
818/* combination of both above, used when an object can be
819 * created whenever it is seen in the file, as opposed to
820 * after having loaded its sub-objects.
821 */
822static PyObject *
823r_ref(PyObject *o, int flag, RFILE *p)
824{
825    if (o != NULL && flag) { /* currently only FLAG_REF is defined */
826        if (PyList_Append(p->refs, o) < 0) {
827            Py_DECREF(o); /* release the new object */
828            return NULL;
829        }
830    }
831    return o;
832}
833
834static PyObject *
835r_object(RFILE *p)
836{
837    /* NULL is a valid return value, it does not necessarily means that
838       an exception is set. */
839    PyObject *v, *v2;
840    Py_ssize_t idx = 0;
841    long i, n;
842    int type, code = r_byte(p);
843    int flag, is_interned = 0;
844    PyObject *retval = NULL;
845
846    if (code == EOF) {
847        PyErr_SetString(PyExc_EOFError,
848                        "EOF read where object expected");
849        return NULL;
850    }
851
852    p->depth++;
853
854    if (p->depth > MAX_MARSHAL_STACK_DEPTH) {
855        p->depth--;
856        PyErr_SetString(PyExc_ValueError, "recursion limit exceeded");
857        return NULL;
858    }
859
860    flag = code & FLAG_REF;
861    type = code & ~FLAG_REF;
862
863#define R_REF(O) do{\
864    if (flag) \
865        O = r_ref(O, flag, p);\
866} while (0)
867
868    switch (type) {
869
870    case TYPE_NULL:
871        break;
872
873    case TYPE_NONE:
874        Py_INCREF(Py_None);
875        retval = Py_None;
876        break;
877
878    case TYPE_STOPITER:
879        Py_INCREF(PyExc_StopIteration);
880        retval = PyExc_StopIteration;
881        break;
882
883    case TYPE_ELLIPSIS:
884        Py_INCREF(Py_Ellipsis);
885        retval = Py_Ellipsis;
886        break;
887
888    case TYPE_FALSE:
889        Py_INCREF(Py_False);
890        retval = Py_False;
891        break;
892
893    case TYPE_TRUE:
894        Py_INCREF(Py_True);
895        retval = Py_True;
896        break;
897
898    case TYPE_INT:
899        n = r_long(p);
900        retval = PyErr_Occurred() ? NULL : PyLong_FromLong(n);
901        R_REF(retval);
902        break;
903
904    case TYPE_LONG:
905        retval = r_PyLong(p);
906        R_REF(retval);
907        break;
908
909    case TYPE_FLOAT:
910        {
911            char buf[256], *ptr;
912            double dx;
913            n = r_byte(p);
914            if (n == EOF) {
915                PyErr_SetString(PyExc_EOFError,
916                    "EOF read where object expected");
917                break;
918            }
919            ptr = r_string(n, p);
920            if (ptr == NULL)
921                break;
922            memcpy(buf, ptr, n);
923            buf[n] = '\0';
924            dx = PyOS_string_to_double(buf, NULL, NULL);
925            if (dx == -1.0 && PyErr_Occurred())
926                break;
927            retval = PyFloat_FromDouble(dx);
928            R_REF(retval);
929            break;
930        }
931
932    case TYPE_BINARY_FLOAT:
933        {
934            unsigned char *buf;
935            double x;
936            buf = (unsigned char *) r_string(8, p);
937            if (buf == NULL)
938                break;
939            x = _PyFloat_Unpack8(buf, 1);
940            if (x == -1.0 && PyErr_Occurred())
941                break;
942            retval = PyFloat_FromDouble(x);
943            R_REF(retval);
944            break;
945        }
946
947    case TYPE_COMPLEX:
948        {
949            char buf[256], *ptr;
950            Py_complex c;
951            n = r_byte(p);
952            if (n == EOF) {
953                PyErr_SetString(PyExc_EOFError,
954                    "EOF read where object expected");
955                break;
956            }
957            ptr = r_string(n, p);
958            if (ptr == NULL)
959                break;
960            memcpy(buf, ptr, n);
961            buf[n] = '\0';
962            c.real = PyOS_string_to_double(buf, NULL, NULL);
963            if (c.real == -1.0 && PyErr_Occurred())
964                break;
965            n = r_byte(p);
966            if (n == EOF) {
967                PyErr_SetString(PyExc_EOFError,
968                    "EOF read where object expected");
969                break;
970            }
971            ptr = r_string(n, p);
972            if (ptr == NULL)
973                break;
974            memcpy(buf, ptr, n);
975            buf[n] = '\0';
976            c.imag = PyOS_string_to_double(buf, NULL, NULL);
977            if (c.imag == -1.0 && PyErr_Occurred())
978                break;
979            retval = PyComplex_FromCComplex(c);
980            R_REF(retval);
981            break;
982        }
983
984    case TYPE_BINARY_COMPLEX:
985        {
986            unsigned char *buf;
987            Py_complex c;
988            buf = (unsigned char *) r_string(8, p);
989            if (buf == NULL)
990                break;
991            c.real = _PyFloat_Unpack8(buf, 1);
992            if (c.real == -1.0 && PyErr_Occurred())
993                break;
994            buf = (unsigned char *) r_string(8, p);
995            if (buf == NULL)
996                break;
997            c.imag = _PyFloat_Unpack8(buf, 1);
998            if (c.imag == -1.0 && PyErr_Occurred())
999                break;
1000            retval = PyComplex_FromCComplex(c);
1001            R_REF(retval);
1002            break;
1003        }
1004
1005    case TYPE_STRING:
1006        {
1007            char *ptr;
1008            n = r_long(p);
1009            if (PyErr_Occurred())
1010                break;
1011            if (n < 0 || n > SIZE32_MAX) {
1012                PyErr_SetString(PyExc_ValueError, "bad marshal data (string size out of range)");
1013                break;
1014            }
1015            v = PyBytes_FromStringAndSize((char *)NULL, n);
1016            if (v == NULL)
1017                break;
1018            ptr = r_string(n, p);
1019            if (ptr == NULL) {
1020                Py_DECREF(v);
1021                break;
1022            }
1023            memcpy(PyBytes_AS_STRING(v), ptr, n);
1024            retval = v;
1025            R_REF(retval);
1026            break;
1027        }
1028
1029    case TYPE_ASCII_INTERNED:
1030        is_interned = 1;
1031    case TYPE_ASCII:
1032        n = r_long(p);
1033        if (PyErr_Occurred())
1034            break;
1035        if (n < 0 || n > SIZE32_MAX) {
1036            PyErr_SetString(PyExc_ValueError, "bad marshal data (unicode size out of range)");
1037            break;
1038        }
1039        goto _read_ascii;
1040
1041    case TYPE_SHORT_ASCII_INTERNED:
1042        is_interned = 1;
1043    case TYPE_SHORT_ASCII:
1044        n = r_byte(p);
1045        if (n == EOF) {
1046            PyErr_SetString(PyExc_EOFError,
1047                "EOF read where object expected");
1048            break;
1049        }
1050    _read_ascii:
1051        {
1052            char *ptr;
1053            ptr = r_string(n, p);
1054            if (ptr == NULL)
1055                break;
1056            v = PyUnicode_FromKindAndData(PyUnicode_1BYTE_KIND, ptr, n);
1057            if (v == NULL)
1058                break;
1059            if (is_interned)
1060                PyUnicode_InternInPlace(&v);
1061            retval = v;
1062            R_REF(retval);
1063            break;
1064        }
1065
1066    case TYPE_INTERNED:
1067        is_interned = 1;
1068    case TYPE_UNICODE:
1069        {
1070        char *buffer;
1071
1072        n = r_long(p);
1073        if (PyErr_Occurred())
1074            break;
1075        if (n < 0 || n > SIZE32_MAX) {
1076            PyErr_SetString(PyExc_ValueError, "bad marshal data (unicode size out of range)");
1077            break;
1078        }
1079        if (n != 0) {
1080            buffer = r_string(n, p);
1081            if (buffer == NULL)
1082                break;
1083            v = PyUnicode_DecodeUTF8(buffer, n, "surrogatepass");
1084        }
1085        else {
1086            v = PyUnicode_New(0, 0);
1087        }
1088        if (v == NULL)
1089            break;
1090        if (is_interned)
1091            PyUnicode_InternInPlace(&v);
1092        retval = v;
1093        R_REF(retval);
1094        break;
1095        }
1096
1097    case TYPE_SMALL_TUPLE:
1098        n = (unsigned char) r_byte(p);
1099        goto _read_tuple;
1100    case TYPE_TUPLE:
1101        n = r_long(p);
1102        if (PyErr_Occurred())
1103            break;
1104        if (n < 0 || n > SIZE32_MAX) {
1105            PyErr_SetString(PyExc_ValueError, "bad marshal data (tuple size out of range)");
1106            break;
1107        }
1108    _read_tuple:
1109        v = PyTuple_New(n);
1110        R_REF(v);
1111        if (v == NULL)
1112            break;
1113        for (i = 0; i < n; i++) {
1114            v2 = r_object(p);
1115            if ( v2 == NULL ) {
1116                if (!PyErr_Occurred())
1117                    PyErr_SetString(PyExc_TypeError,
1118                        "NULL object in marshal data for tuple");
1119                Py_DECREF(v);
1120                v = NULL;
1121                break;
1122            }
1123            PyTuple_SET_ITEM(v, i, v2);
1124        }
1125        retval = v;
1126        break;
1127
1128    case TYPE_LIST:
1129        n = r_long(p);
1130        if (PyErr_Occurred())
1131            break;
1132        if (n < 0 || n > SIZE32_MAX) {
1133            PyErr_SetString(PyExc_ValueError, "bad marshal data (list size out of range)");
1134            break;
1135        }
1136        v = PyList_New(n);
1137        R_REF(v);
1138        if (v == NULL)
1139            break;
1140        for (i = 0; i < n; i++) {
1141            v2 = r_object(p);
1142            if ( v2 == NULL ) {
1143                if (!PyErr_Occurred())
1144                    PyErr_SetString(PyExc_TypeError,
1145                        "NULL object in marshal data for list");
1146                Py_DECREF(v);
1147                v = NULL;
1148                break;
1149            }
1150            PyList_SET_ITEM(v, i, v2);
1151        }
1152        retval = v;
1153        break;
1154
1155    case TYPE_DICT:
1156        v = PyDict_New();
1157        R_REF(v);
1158        if (v == NULL)
1159            break;
1160        for (;;) {
1161            PyObject *key, *val;
1162            key = r_object(p);
1163            if (key == NULL)
1164                break;
1165            val = r_object(p);
1166            if (val != NULL)
1167                PyDict_SetItem(v, key, val);
1168            Py_DECREF(key);
1169            Py_XDECREF(val);
1170        }
1171        if (PyErr_Occurred()) {
1172            Py_DECREF(v);
1173            v = NULL;
1174        }
1175        retval = v;
1176        break;
1177
1178    case TYPE_SET:
1179    case TYPE_FROZENSET:
1180        n = r_long(p);
1181        if (PyErr_Occurred())
1182            break;
1183        if (n < 0 || n > SIZE32_MAX) {
1184            PyErr_SetString(PyExc_ValueError, "bad marshal data (set size out of range)");
1185            break;
1186        }
1187        v = (type == TYPE_SET) ? PySet_New(NULL) : PyFrozenSet_New(NULL);
1188        if (type == TYPE_SET) {
1189            R_REF(v);
1190        } else {
1191            /* must use delayed registration of frozensets because they must
1192             * be init with a refcount of 1
1193             */
1194            idx = r_ref_reserve(flag, p);
1195            if (idx < 0)
1196                Py_CLEAR(v); /* signal error */
1197        }
1198        if (v == NULL)
1199            break;
1200        for (i = 0; i < n; i++) {
1201            v2 = r_object(p);
1202            if ( v2 == NULL ) {
1203                if (!PyErr_Occurred())
1204                    PyErr_SetString(PyExc_TypeError,
1205                        "NULL object in marshal data for set");
1206                Py_DECREF(v);
1207                v = NULL;
1208                break;
1209            }
1210            if (PySet_Add(v, v2) == -1) {
1211                Py_DECREF(v);
1212                Py_DECREF(v2);
1213                v = NULL;
1214                break;
1215            }
1216            Py_DECREF(v2);
1217        }
1218        if (type != TYPE_SET)
1219            v = r_ref_insert(v, idx, flag, p);
1220        retval = v;
1221        break;
1222
1223    case TYPE_CODE:
1224        {
1225            int argcount;
1226            int kwonlyargcount;
1227            int nlocals;
1228            int stacksize;
1229            int flags;
1230            PyObject *code = NULL;
1231            PyObject *consts = NULL;
1232            PyObject *names = NULL;
1233            PyObject *varnames = NULL;
1234            PyObject *freevars = NULL;
1235            PyObject *cellvars = NULL;
1236            PyObject *filename = NULL;
1237            PyObject *name = NULL;
1238            int firstlineno;
1239            PyObject *lnotab = NULL;
1240
1241            idx = r_ref_reserve(flag, p);
1242            if (idx < 0)
1243                break;
1244
1245            v = NULL;
1246
1247            /* XXX ignore long->int overflows for now */
1248            argcount = (int)r_long(p);
1249            if (PyErr_Occurred())
1250                goto code_error;
1251            kwonlyargcount = (int)r_long(p);
1252            if (PyErr_Occurred())
1253                goto code_error;
1254            nlocals = (int)r_long(p);
1255            if (PyErr_Occurred())
1256                goto code_error;
1257            stacksize = (int)r_long(p);
1258            if (PyErr_Occurred())
1259                goto code_error;
1260            flags = (int)r_long(p);
1261            if (PyErr_Occurred())
1262                goto code_error;
1263            code = r_object(p);
1264            if (code == NULL)
1265                goto code_error;
1266            consts = r_object(p);
1267            if (consts == NULL)
1268                goto code_error;
1269            names = r_object(p);
1270            if (names == NULL)
1271                goto code_error;
1272            varnames = r_object(p);
1273            if (varnames == NULL)
1274                goto code_error;
1275            freevars = r_object(p);
1276            if (freevars == NULL)
1277                goto code_error;
1278            cellvars = r_object(p);
1279            if (cellvars == NULL)
1280                goto code_error;
1281            filename = r_object(p);
1282            if (filename == NULL)
1283                goto code_error;
1284            if (PyUnicode_CheckExact(filename)) {
1285                if (p->current_filename != NULL) {
1286                    if (!PyUnicode_Compare(filename, p->current_filename)) {
1287                        Py_DECREF(filename);
1288                        Py_INCREF(p->current_filename);
1289                        filename = p->current_filename;
1290                    }
1291                }
1292                else {
1293                    p->current_filename = filename;
1294                }
1295            }
1296            name = r_object(p);
1297            if (name == NULL)
1298                goto code_error;
1299            firstlineno = (int)r_long(p);
1300            if (firstlineno == -1 && PyErr_Occurred())
1301                break;
1302            lnotab = r_object(p);
1303            if (lnotab == NULL)
1304                goto code_error;
1305
1306            v = (PyObject *) PyCode_New(
1307                            argcount, kwonlyargcount,
1308                            nlocals, stacksize, flags,
1309                            code, consts, names, varnames,
1310                            freevars, cellvars, filename, name,
1311                            firstlineno, lnotab);
1312            v = r_ref_insert(v, idx, flag, p);
1313
1314          code_error:
1315            Py_XDECREF(code);
1316            Py_XDECREF(consts);
1317            Py_XDECREF(names);
1318            Py_XDECREF(varnames);
1319            Py_XDECREF(freevars);
1320            Py_XDECREF(cellvars);
1321            Py_XDECREF(filename);
1322            Py_XDECREF(name);
1323            Py_XDECREF(lnotab);
1324        }
1325        retval = v;
1326        break;
1327
1328    case TYPE_REF:
1329        n = r_long(p);
1330        if (n < 0 || n >= PyList_GET_SIZE(p->refs)) {
1331            if (n == -1 && PyErr_Occurred())
1332                break;
1333            PyErr_SetString(PyExc_ValueError, "bad marshal data (invalid reference)");
1334            break;
1335        }
1336        v = PyList_GET_ITEM(p->refs, n);
1337        if (v == Py_None) {
1338            PyErr_SetString(PyExc_ValueError, "bad marshal data (invalid reference)");
1339            break;
1340        }
1341        Py_INCREF(v);
1342        retval = v;
1343        break;
1344
1345    default:
1346        /* Bogus data got written, which isn't ideal.
1347           This will let you keep working and recover. */
1348        PyErr_SetString(PyExc_ValueError, "bad marshal data (unknown type code)");
1349        break;
1350
1351    }
1352    p->depth--;
1353    return retval;
1354}
1355
1356static PyObject *
1357read_object(RFILE *p)
1358{
1359    PyObject *v;
1360    if (PyErr_Occurred()) {
1361        fprintf(stderr, "XXX readobject called with exception set\n");
1362        return NULL;
1363    }
1364    v = r_object(p);
1365    if (v == NULL && !PyErr_Occurred())
1366        PyErr_SetString(PyExc_TypeError, "NULL object in marshal data for object");
1367    return v;
1368}
1369
1370int
1371PyMarshal_ReadShortFromFile(FILE *fp)
1372{
1373    RFILE rf;
1374    int res;
1375    assert(fp);
1376    rf.readable = NULL;
1377    rf.fp = fp;
1378    rf.current_filename = NULL;
1379    rf.end = rf.ptr = NULL;
1380    rf.buf = NULL;
1381    res = r_short(&rf);
1382    if (rf.buf != NULL)
1383        PyMem_FREE(rf.buf);
1384    return res;
1385}
1386
1387long
1388PyMarshal_ReadLongFromFile(FILE *fp)
1389{
1390    RFILE rf;
1391    long res;
1392    rf.fp = fp;
1393    rf.readable = NULL;
1394    rf.current_filename = NULL;
1395    rf.ptr = rf.end = NULL;
1396    rf.buf = NULL;
1397    res = r_long(&rf);
1398    if (rf.buf != NULL)
1399        PyMem_FREE(rf.buf);
1400    return res;
1401}
1402
1403#ifdef HAVE_FSTAT
1404/* Return size of file in bytes; < 0 if unknown. */
1405static off_t
1406getfilesize(FILE *fp)
1407{
1408    struct stat st;
1409    if (fstat(fileno(fp), &st) != 0)
1410        return -1;
1411    else
1412        return st.st_size;
1413}
1414#endif
1415
1416/* If we can get the size of the file up-front, and it's reasonably small,
1417 * read it in one gulp and delegate to ...FromString() instead.  Much quicker
1418 * than reading a byte at a time from file; speeds .pyc imports.
1419 * CAUTION:  since this may read the entire remainder of the file, don't
1420 * call it unless you know you're done with the file.
1421 */
1422PyObject *
1423PyMarshal_ReadLastObjectFromFile(FILE *fp)
1424{
1425/* REASONABLE_FILE_LIMIT is by defn something big enough for Tkinter.pyc. */
1426#define REASONABLE_FILE_LIMIT (1L << 18)
1427#ifdef HAVE_FSTAT
1428    off_t filesize;
1429    filesize = getfilesize(fp);
1430    if (filesize > 0 && filesize <= REASONABLE_FILE_LIMIT) {
1431        char* pBuf = (char *)PyMem_MALLOC(filesize);
1432        if (pBuf != NULL) {
1433            size_t n = fread(pBuf, 1, (size_t)filesize, fp);
1434            PyObject* v = PyMarshal_ReadObjectFromString(pBuf, n);
1435            PyMem_FREE(pBuf);
1436            return v;
1437        }
1438
1439    }
1440#endif
1441    /* We don't have fstat, or we do but the file is larger than
1442     * REASONABLE_FILE_LIMIT or malloc failed -- read a byte at a time.
1443     */
1444    return PyMarshal_ReadObjectFromFile(fp);
1445
1446#undef REASONABLE_FILE_LIMIT
1447}
1448
1449PyObject *
1450PyMarshal_ReadObjectFromFile(FILE *fp)
1451{
1452    RFILE rf;
1453    PyObject *result;
1454    rf.fp = fp;
1455    rf.readable = NULL;
1456    rf.current_filename = NULL;
1457    rf.depth = 0;
1458    rf.ptr = rf.end = NULL;
1459    rf.buf = NULL;
1460    rf.refs = PyList_New(0);
1461    if (rf.refs == NULL)
1462        return NULL;
1463    result = r_object(&rf);
1464    Py_DECREF(rf.refs);
1465    if (rf.buf != NULL)
1466        PyMem_FREE(rf.buf);
1467    return result;
1468}
1469
1470PyObject *
1471PyMarshal_ReadObjectFromString(char *str, Py_ssize_t len)
1472{
1473    RFILE rf;
1474    PyObject *result;
1475    rf.fp = NULL;
1476    rf.readable = NULL;
1477    rf.current_filename = NULL;
1478    rf.ptr = str;
1479    rf.end = str + len;
1480    rf.buf = NULL;
1481    rf.depth = 0;
1482    rf.refs = PyList_New(0);
1483    if (rf.refs == NULL)
1484        return NULL;
1485    result = r_object(&rf);
1486    Py_DECREF(rf.refs);
1487    if (rf.buf != NULL)
1488        PyMem_FREE(rf.buf);
1489    return result;
1490}
1491
1492PyObject *
1493PyMarshal_WriteObjectToString(PyObject *x, int version)
1494{
1495    WFILE wf;
1496
1497    wf.fp = NULL;
1498    wf.readable = NULL;
1499    wf.str = PyBytes_FromStringAndSize((char *)NULL, 50);
1500    if (wf.str == NULL)
1501        return NULL;
1502    wf.ptr = PyBytes_AS_STRING((PyBytesObject *)wf.str);
1503    wf.end = wf.ptr + PyBytes_Size(wf.str);
1504    wf.error = WFERR_OK;
1505    wf.depth = 0;
1506    wf.version = version;
1507    if (version >= 3) {
1508        if ((wf.refs = PyDict_New()) == NULL)
1509            return NULL;
1510    } else
1511        wf.refs = NULL;
1512    w_object(x, &wf);
1513    Py_XDECREF(wf.refs);
1514    if (wf.str != NULL) {
1515        char *base = PyBytes_AS_STRING((PyBytesObject *)wf.str);
1516        if (wf.ptr - base > PY_SSIZE_T_MAX) {
1517            Py_DECREF(wf.str);
1518            PyErr_SetString(PyExc_OverflowError,
1519                            "too much marshal data for a string");
1520            return NULL;
1521        }
1522        if (_PyBytes_Resize(&wf.str, (Py_ssize_t)(wf.ptr - base)) < 0)
1523            return NULL;
1524    }
1525    if (wf.error != WFERR_OK) {
1526        Py_XDECREF(wf.str);
1527        if (wf.error == WFERR_NOMEMORY)
1528            PyErr_NoMemory();
1529        else
1530            PyErr_SetString(PyExc_ValueError,
1531              (wf.error==WFERR_UNMARSHALLABLE)?"unmarshallable object"
1532               :"object too deeply nested to marshal");
1533        return NULL;
1534    }
1535    return wf.str;
1536}
1537
1538/* And an interface for Python programs... */
1539
1540static PyObject *
1541marshal_dump(PyObject *self, PyObject *args)
1542{
1543    /* XXX Quick hack -- need to do this differently */
1544    PyObject *x;
1545    PyObject *f;
1546    int version = Py_MARSHAL_VERSION;
1547    PyObject *s;
1548    PyObject *res;
1549    _Py_IDENTIFIER(write);
1550
1551    if (!PyArg_ParseTuple(args, "OO|i:dump", &x, &f, &version))
1552        return NULL;
1553    s = PyMarshal_WriteObjectToString(x, version);
1554    if (s == NULL)
1555        return NULL;
1556    res = _PyObject_CallMethodId(f, &PyId_write, "O", s);
1557    Py_DECREF(s);
1558    return res;
1559}
1560
1561PyDoc_STRVAR(dump_doc,
1562"dump(value, file[, version])\n\
1563\n\
1564Write the value on the open file. The value must be a supported type.\n\
1565The file must be an open file object such as sys.stdout or returned by\n\
1566open() or os.popen(). It must be opened in binary mode ('wb' or 'w+b').\n\
1567\n\
1568If the value has (or contains an object that has) an unsupported type, a\n\
1569ValueError exception is raised — but garbage data will also be written\n\
1570to the file. The object will not be properly read back by load()\n\
1571\n\
1572The version argument indicates the data format that dump should use.");
1573
1574static PyObject *
1575marshal_load(PyObject *self, PyObject *f)
1576{
1577    PyObject *data, *result;
1578    _Py_IDENTIFIER(read);
1579    RFILE rf;
1580
1581    /*
1582     * Make a call to the read method, but read zero bytes.
1583     * This is to ensure that the object passed in at least
1584     * has a read method which returns bytes.
1585     * This can be removed if we guarantee good error handling
1586     * for r_string()
1587     */
1588    data = _PyObject_CallMethodId(f, &PyId_read, "i", 0);
1589    if (data == NULL)
1590        return NULL;
1591    if (!PyBytes_Check(data)) {
1592        PyErr_Format(PyExc_TypeError,
1593                     "f.read() returned not bytes but %.100s",
1594                     data->ob_type->tp_name);
1595        result = NULL;
1596    }
1597    else {
1598        rf.depth = 0;
1599        rf.fp = NULL;
1600        rf.readable = f;
1601        rf.current_filename = NULL;
1602        rf.ptr = rf.end = NULL;
1603        rf.buf = NULL;
1604        if ((rf.refs = PyList_New(0)) != NULL) {
1605            result = read_object(&rf);
1606            Py_DECREF(rf.refs);
1607            if (rf.buf != NULL)
1608                PyMem_FREE(rf.buf);
1609        } else
1610            result = NULL;
1611    }
1612    Py_DECREF(data);
1613    return result;
1614}
1615
1616PyDoc_STRVAR(load_doc,
1617"load(file)\n\
1618\n\
1619Read one value from the open file and return it. If no valid value is\n\
1620read (e.g. because the data has a different Python version’s\n\
1621incompatible marshal format), raise EOFError, ValueError or TypeError.\n\
1622The file must be an open file object opened in binary mode ('rb' or\n\
1623'r+b').\n\
1624\n\
1625Note: If an object containing an unsupported type was marshalled with\n\
1626dump(), load() will substitute None for the unmarshallable type.");
1627
1628
1629static PyObject *
1630marshal_dumps(PyObject *self, PyObject *args)
1631{
1632    PyObject *x;
1633    int version = Py_MARSHAL_VERSION;
1634    if (!PyArg_ParseTuple(args, "O|i:dumps", &x, &version))
1635        return NULL;
1636    return PyMarshal_WriteObjectToString(x, version);
1637}
1638
1639PyDoc_STRVAR(dumps_doc,
1640"dumps(value[, version])\n\
1641\n\
1642Return the string that would be written to a file by dump(value, file).\n\
1643The value must be a supported type. Raise a ValueError exception if\n\
1644value has (or contains an object that has) an unsupported type.\n\
1645\n\
1646The version argument indicates the data format that dumps should use.");
1647
1648
1649static PyObject *
1650marshal_loads(PyObject *self, PyObject *args)
1651{
1652    RFILE rf;
1653    Py_buffer p;
1654    char *s;
1655    Py_ssize_t n;
1656    PyObject* result;
1657    if (!PyArg_ParseTuple(args, "y*:loads", &p))
1658        return NULL;
1659    s = p.buf;
1660    n = p.len;
1661    rf.fp = NULL;
1662    rf.readable = NULL;
1663    rf.current_filename = NULL;
1664    rf.ptr = s;
1665    rf.end = s + n;
1666    rf.depth = 0;
1667    if ((rf.refs = PyList_New(0)) == NULL)
1668        return NULL;
1669    result = read_object(&rf);
1670    PyBuffer_Release(&p);
1671    Py_DECREF(rf.refs);
1672    return result;
1673}
1674
1675PyDoc_STRVAR(loads_doc,
1676"loads(bytes)\n\
1677\n\
1678Convert the bytes object to a value. If no valid value is found, raise\n\
1679EOFError, ValueError or TypeError. Extra characters in the input are\n\
1680ignored.");
1681
1682static PyMethodDef marshal_methods[] = {
1683    {"dump",            marshal_dump,   METH_VARARGS,   dump_doc},
1684    {"load",            marshal_load,   METH_O,         load_doc},
1685    {"dumps",           marshal_dumps,  METH_VARARGS,   dumps_doc},
1686    {"loads",           marshal_loads,  METH_VARARGS,   loads_doc},
1687    {NULL,              NULL}           /* sentinel */
1688};
1689
1690
1691PyDoc_STRVAR(module_doc,
1692"This module contains functions that can read and write Python values in\n\
1693a binary format. The format is specific to Python, but independent of\n\
1694machine architecture issues.\n\
1695\n\
1696Not all Python object types are supported; in general, only objects\n\
1697whose value is independent from a particular invocation of Python can be\n\
1698written and read by this module. The following types are supported:\n\
1699None, integers, floating point numbers, strings, bytes, bytearrays,\n\
1700tuples, lists, sets, dictionaries, and code objects, where it\n\
1701should be understood that tuples, lists and dictionaries are only\n\
1702supported as long as the values contained therein are themselves\n\
1703supported; and recursive lists and dictionaries should not be written\n\
1704(they will cause infinite loops).\n\
1705\n\
1706Variables:\n\
1707\n\
1708version -- indicates the format that the module uses. Version 0 is the\n\
1709    historical format, version 1 shares interned strings and version 2\n\
1710    uses a binary format for floating point numbers.\n\
1711    Version 3 shares common object references (New in version 3.4).\n\
1712\n\
1713Functions:\n\
1714\n\
1715dump() -- write value to a file\n\
1716load() -- read value from a file\n\
1717dumps() -- write value to a string\n\
1718loads() -- read value from a string");
1719
1720
1721
1722static struct PyModuleDef marshalmodule = {
1723    PyModuleDef_HEAD_INIT,
1724    "marshal",
1725    module_doc,
1726    0,
1727    marshal_methods,
1728    NULL,
1729    NULL,
1730    NULL,
1731    NULL
1732};
1733
1734PyMODINIT_FUNC
1735PyMarshal_Init(void)
1736{
1737    PyObject *mod = PyModule_Create(&marshalmodule);
1738    if (mod == NULL)
1739        return NULL;
1740    PyModule_AddIntConstant(mod, "version", Py_MARSHAL_VERSION);
1741    return mod;
1742}
1743