marshal.c revision 74c71f5a624cef653c7da470607ca063b749ef75 
1
2/* Write Python objects to files and read them back.
3   This is intended for writing and reading compiled Python code only;
4   a true persistent storage facility would be much harder, since
5   it would have to take circular links and sharing into account. */
6
7#define PY_SSIZE_T_CLEAN
8
9#include "Python.h"
10#include "longintrepr.h"
11#include "code.h"
12#include "marshal.h"
13
14/* High water mark to determine when the marshalled object is dangerously deep
15 * and risks coring the interpreter.  When the object stack gets this deep,
16 * raise an exception instead of continuing.
17 * On Windows debug builds, reduce this value.
18 */
19#if defined(MS_WINDOWS) && defined(_DEBUG)
20#define MAX_MARSHAL_STACK_DEPTH 1500
21#else
22#define MAX_MARSHAL_STACK_DEPTH 2000
23#endif
24
25#define TYPE_NULL		'0'
26#define TYPE_NONE		'N'
27#define TYPE_FALSE		'F'
28#define TYPE_TRUE		'T'
29#define TYPE_STOPITER		'S'
30#define TYPE_ELLIPSIS   	'.'
31#define TYPE_INT		'i'
32#define TYPE_INT64		'I'
33#define TYPE_FLOAT		'f'
34#define TYPE_BINARY_FLOAT	'g'
35#define TYPE_COMPLEX		'x'
36#define TYPE_BINARY_COMPLEX	'y'
37#define TYPE_LONG		'l'
38#define TYPE_STRING		's'
39#define TYPE_TUPLE		'('
40#define TYPE_LIST		'['
41#define TYPE_DICT		'{'
42#define TYPE_CODE		'c'
43#define TYPE_UNICODE		'u'
44#define TYPE_UNKNOWN		'?'
45#define TYPE_SET		'<'
46#define TYPE_FROZENSET  	'>'
47
48typedef struct {
49	FILE *fp;
50	int error;
51	int depth;
52	/* If fp == NULL, the following are valid: */
53	PyObject *str;
54	char *ptr;
55	char *end;
56	PyObject *strings; /* dict on marshal, list on unmarshal */
57	int version;
58} WFILE;
59
60#define w_byte(c, p) if (((p)->fp)) putc((c), (p)->fp); \
61		      else if ((p)->ptr != (p)->end) *(p)->ptr++ = (c); \
62			   else w_more(c, p)
63
64static void
65w_more(int c, WFILE *p)
66{
67	Py_ssize_t size, newsize;
68	if (p->str == NULL)
69		return; /* An error already occurred */
70	size = PyBytes_Size(p->str);
71	newsize = size + size + 1024;
72	if (newsize > 32*1024*1024) {
73		newsize = size + 1024*1024;
74	}
75	if (_PyBytes_Resize(&p->str, newsize) != 0) {
76		p->ptr = p->end = NULL;
77	}
78	else {
79		p->ptr = PyBytes_AS_STRING((PyBytesObject *)p->str) + size;
80		p->end =
81			PyBytes_AS_STRING((PyBytesObject *)p->str) + newsize;
82		*p->ptr++ = Py_SAFE_DOWNCAST(c, int, char);
83	}
84}
85
86static void
87w_string(char *s, int n, WFILE *p)
88{
89	if (p->fp != NULL) {
90		fwrite(s, 1, n, p->fp);
91	}
92	else {
93		while (--n >= 0) {
94			w_byte(*s, p);
95			s++;
96		}
97	}
98}
99
100static void
101w_short(int x, WFILE *p)
102{
103	w_byte((char)( x      & 0xff), p);
104	w_byte((char)((x>> 8) & 0xff), p);
105}
106
107static void
108w_long(long x, WFILE *p)
109{
110	w_byte((char)( x      & 0xff), p);
111	w_byte((char)((x>> 8) & 0xff), p);
112	w_byte((char)((x>>16) & 0xff), p);
113	w_byte((char)((x>>24) & 0xff), p);
114}
115
116#if SIZEOF_LONG > 4
117static void
118w_long64(long x, WFILE *p)
119{
120	w_long(x, p);
121	w_long(x>>32, p);
122}
123#endif
124
125static void
126w_object(PyObject *v, WFILE *p)
127{
128	Py_ssize_t i, n;
129
130	p->depth++;
131
132	if (p->depth > MAX_MARSHAL_STACK_DEPTH) {
133		p->error = 2;
134	}
135	else if (v == NULL) {
136		w_byte(TYPE_NULL, p);
137	}
138	else if (v == Py_None) {
139		w_byte(TYPE_NONE, p);
140	}
141	else if (v == PyExc_StopIteration) {
142		w_byte(TYPE_STOPITER, p);
143	}
144	else if (v == Py_Ellipsis) {
145	        w_byte(TYPE_ELLIPSIS, p);
146	}
147	else if (v == Py_False) {
148	        w_byte(TYPE_FALSE, p);
149	}
150	else if (v == Py_True) {
151	        w_byte(TYPE_TRUE, p);
152	}
153	else if (PyLong_CheckExact(v)) {
154		long x = PyLong_AsLong(v);
155		if ((x == -1)  && PyErr_Occurred()) {
156			PyLongObject *ob = (PyLongObject *)v;
157			PyErr_Clear();
158			w_byte(TYPE_LONG, p);
159			n = Py_SIZE(ob);
160			w_long((long)n, p);
161			if (n < 0)
162				n = -n;
163			for (i = 0; i < n; i++)
164				w_short(ob->ob_digit[i], p);
165		}
166		else {
167#if SIZEOF_LONG > 4
168			long y = Py_ARITHMETIC_RIGHT_SHIFT(long, x, 31);
169			if (y && y != -1) {
170				w_byte(TYPE_INT64, p);
171				w_long64(x, p);
172			}
173			else
174#endif
175			{
176				w_byte(TYPE_INT, p);
177				w_long(x, p);
178			}
179		}
180	}
181	else if (PyFloat_CheckExact(v)) {
182		if (p->version > 1) {
183			unsigned char buf[8];
184			if (_PyFloat_Pack8(PyFloat_AsDouble(v),
185					   buf, 1) < 0) {
186				p->error = 1;
187				return;
188			}
189			w_byte(TYPE_BINARY_FLOAT, p);
190			w_string((char*)buf, 8, p);
191		}
192		else {
193			char buf[256]; /* Plenty to format any double */
194			n = _PyFloat_Repr(PyFloat_AS_DOUBLE(v),
195					  buf, sizeof(buf));
196			w_byte(TYPE_FLOAT, p);
197			w_byte((int)n, p);
198			w_string(buf, (int)n, p);
199		}
200	}
201#ifndef WITHOUT_COMPLEX
202	else if (PyComplex_CheckExact(v)) {
203		if (p->version > 1) {
204			unsigned char buf[8];
205			if (_PyFloat_Pack8(PyComplex_RealAsDouble(v),
206					   buf, 1) < 0) {
207				p->error = 1;
208				return;
209			}
210			w_byte(TYPE_BINARY_COMPLEX, p);
211			w_string((char*)buf, 8, p);
212			if (_PyFloat_Pack8(PyComplex_ImagAsDouble(v),
213					   buf, 1) < 0) {
214				p->error = 1;
215				return;
216			}
217			w_string((char*)buf, 8, p);
218		}
219		else {
220			char buf[256]; /* Plenty to format any double */
221			w_byte(TYPE_COMPLEX, p);
222			n = _PyFloat_Repr(PyComplex_RealAsDouble(v),
223					  buf, sizeof(buf));
224			n = strlen(buf);
225			w_byte((int)n, p);
226			w_string(buf, (int)n, p);
227			n = _PyFloat_Repr(PyComplex_ImagAsDouble(v),
228					  buf, sizeof(buf));
229			w_byte((int)n, p);
230			w_string(buf, (int)n, p);
231		}
232	}
233#endif
234	else if (PyBytes_CheckExact(v)) {
235		w_byte(TYPE_STRING, p);
236		n = PyBytes_GET_SIZE(v);
237		if (n > INT_MAX) {
238			/* huge strings are not supported */
239			p->depth--;
240			p->error = 1;
241			return;
242		}
243		w_long((long)n, p);
244		w_string(PyBytes_AS_STRING(v), (int)n, p);
245	}
246	else if (PyUnicode_CheckExact(v)) {
247	        PyObject *utf8;
248		utf8 = PyUnicode_AsUTF8String(v);
249		if (utf8 == NULL) {
250			p->depth--;
251			p->error = 1;
252			return;
253		}
254		w_byte(TYPE_UNICODE, p);
255		n = PyBytes_GET_SIZE(utf8);
256		if (n > INT_MAX) {
257			p->depth--;
258			p->error = 1;
259			return;
260		}
261		w_long((long)n, p);
262		w_string(PyBytes_AS_STRING(utf8), (int)n, p);
263		Py_DECREF(utf8);
264	}
265	else if (PyTuple_CheckExact(v)) {
266		w_byte(TYPE_TUPLE, p);
267		n = PyTuple_Size(v);
268		w_long((long)n, p);
269		for (i = 0; i < n; i++) {
270			w_object(PyTuple_GET_ITEM(v, i), p);
271		}
272	}
273	else if (PyList_CheckExact(v)) {
274		w_byte(TYPE_LIST, p);
275		n = PyList_GET_SIZE(v);
276		w_long((long)n, p);
277		for (i = 0; i < n; i++) {
278			w_object(PyList_GET_ITEM(v, i), p);
279		}
280	}
281	else if (PyDict_CheckExact(v)) {
282		Py_ssize_t pos;
283		PyObject *key, *value;
284		w_byte(TYPE_DICT, p);
285		/* This one is NULL object terminated! */
286		pos = 0;
287		while (PyDict_Next(v, &pos, &key, &value)) {
288			w_object(key, p);
289			w_object(value, p);
290		}
291		w_object((PyObject *)NULL, p);
292	}
293	else if (PyAnySet_CheckExact(v)) {
294		PyObject *value, *it;
295
296		if (PyObject_TypeCheck(v, &PySet_Type))
297			w_byte(TYPE_SET, p);
298		else
299			w_byte(TYPE_FROZENSET, p);
300		n = PyObject_Size(v);
301		if (n == -1) {
302			p->depth--;
303			p->error = 1;
304			return;
305		}
306		w_long((long)n, p);
307		it = PyObject_GetIter(v);
308		if (it == NULL) {
309			p->depth--;
310			p->error = 1;
311			return;
312		}
313		while ((value = PyIter_Next(it)) != NULL) {
314			w_object(value, p);
315			Py_DECREF(value);
316		}
317		Py_DECREF(it);
318		if (PyErr_Occurred()) {
319			p->depth--;
320			p->error = 1;
321			return;
322		}
323	}
324	else if (PyCode_Check(v)) {
325		PyCodeObject *co = (PyCodeObject *)v;
326		w_byte(TYPE_CODE, p);
327		w_long(co->co_argcount, p);
328		w_long(co->co_kwonlyargcount, p);
329		w_long(co->co_nlocals, p);
330		w_long(co->co_stacksize, p);
331		w_long(co->co_flags, p);
332		w_object(co->co_code, p);
333		w_object(co->co_consts, p);
334		w_object(co->co_names, p);
335		w_object(co->co_varnames, p);
336		w_object(co->co_freevars, p);
337		w_object(co->co_cellvars, p);
338		w_object(co->co_filename, p);
339		w_object(co->co_name, p);
340		w_long(co->co_firstlineno, p);
341		w_object(co->co_lnotab, p);
342	}
343	else if (PyObject_CheckBuffer(v)) {
344		/* Write unknown buffer-style objects as a string */
345		char *s;
346		PyBufferProcs *pb = v->ob_type->tp_as_buffer;
347                Py_buffer view;
348		if ((*pb->bf_getbuffer)(v, &view, PyBUF_SIMPLE) != 0) {
349                        w_byte(TYPE_UNKNOWN, p);
350                        p->error = 1;
351                }
352		w_byte(TYPE_STRING, p);
353                n = view.len;
354                s = view.buf;
355		if (n > INT_MAX) {
356			p->depth--;
357			p->error = 1;
358			return;
359		}
360		w_long((long)n, p);
361		w_string(s, (int)n, p);
362                if (pb->bf_releasebuffer != NULL)
363                        (*pb->bf_releasebuffer)(v, &view);
364	}
365	else {
366		w_byte(TYPE_UNKNOWN, p);
367		p->error = 1;
368	}
369	p->depth--;
370}
371
372/* version currently has no effect for writing longs. */
373void
374PyMarshal_WriteLongToFile(long x, FILE *fp, int version)
375{
376	WFILE wf;
377	wf.fp = fp;
378	wf.error = 0;
379	wf.depth = 0;
380	wf.strings = NULL;
381	wf.version = version;
382	w_long(x, &wf);
383}
384
385void
386PyMarshal_WriteObjectToFile(PyObject *x, FILE *fp, int version)
387{
388	WFILE wf;
389	wf.fp = fp;
390	wf.error = 0;
391	wf.depth = 0;
392	wf.strings = (version > 0) ? PyDict_New() : NULL;
393	wf.version = version;
394	w_object(x, &wf);
395	Py_XDECREF(wf.strings);
396}
397
398typedef WFILE RFILE; /* Same struct with different invariants */
399
400#define rs_byte(p) (((p)->ptr < (p)->end) ? (unsigned char)*(p)->ptr++ : EOF)
401
402#define r_byte(p) ((p)->fp ? getc((p)->fp) : rs_byte(p))
403
404static int
405r_string(char *s, int n, RFILE *p)
406{
407	if (p->fp != NULL)
408		/* The result fits into int because it must be <=n. */
409		return (int)fread(s, 1, n, p->fp);
410	if (p->end - p->ptr < n)
411		n = (int)(p->end - p->ptr);
412	memcpy(s, p->ptr, n);
413	p->ptr += n;
414	return n;
415}
416
417static int
418r_short(RFILE *p)
419{
420	register short x;
421	x = r_byte(p);
422	x |= r_byte(p) << 8;
423	/* Sign-extension, in case short greater than 16 bits */
424	x |= -(x & 0x8000);
425	return x;
426}
427
428static long
429r_long(RFILE *p)
430{
431	register long x;
432	register FILE *fp = p->fp;
433	if (fp) {
434		x = getc(fp);
435		x |= (long)getc(fp) << 8;
436		x |= (long)getc(fp) << 16;
437		x |= (long)getc(fp) << 24;
438	}
439	else {
440		x = rs_byte(p);
441		x |= (long)rs_byte(p) << 8;
442		x |= (long)rs_byte(p) << 16;
443		x |= (long)rs_byte(p) << 24;
444	}
445#if SIZEOF_LONG > 4
446	/* Sign extension for 64-bit machines */
447	x |= -(x & 0x80000000L);
448#endif
449	return x;
450}
451
452/* r_long64 deals with the TYPE_INT64 code.  On a machine with
453   sizeof(long) > 4, it returns a Python int object, else a Python long
454   object.  Note that w_long64 writes out TYPE_INT if 32 bits is enough,
455   so there's no inefficiency here in returning a PyLong on 32-bit boxes
456   for everything written via TYPE_INT64 (i.e., if an int is written via
457   TYPE_INT64, it *needs* more than 32 bits).
458*/
459static PyObject *
460r_long64(RFILE *p)
461{
462	long lo4 = r_long(p);
463	long hi4 = r_long(p);
464#if SIZEOF_LONG > 4
465	long x = (hi4 << 32) | (lo4 & 0xFFFFFFFFL);
466	return PyLong_FromLong(x);
467#else
468	unsigned char buf[8];
469	int one = 1;
470	int is_little_endian = (int)*(char*)&one;
471	if (is_little_endian) {
472		memcpy(buf, &lo4, 4);
473		memcpy(buf+4, &hi4, 4);
474	}
475	else {
476		memcpy(buf, &hi4, 4);
477		memcpy(buf+4, &lo4, 4);
478	}
479	return _PyLong_FromByteArray(buf, 8, is_little_endian, 1);
480#endif
481}
482
483static PyObject *
484r_object(RFILE *p)
485{
486	/* NULL is a valid return value, it does not necessarily means that
487	   an exception is set. */
488	PyObject *v, *v2;
489	long i, n;
490	int type = r_byte(p);
491	PyObject *retval;
492
493	p->depth++;
494
495	if (p->depth > MAX_MARSHAL_STACK_DEPTH) {
496		p->depth--;
497		PyErr_SetString(PyExc_ValueError, "recursion limit exceeded");
498		return NULL;
499	}
500
501	switch (type) {
502
503	case EOF:
504		PyErr_SetString(PyExc_EOFError,
505				"EOF read where object expected");
506		retval = NULL;
507		break;
508
509	case TYPE_NULL:
510		retval = NULL;
511		break;
512
513	case TYPE_NONE:
514		Py_INCREF(Py_None);
515		retval = Py_None;
516		break;
517
518	case TYPE_STOPITER:
519		Py_INCREF(PyExc_StopIteration);
520		retval = PyExc_StopIteration;
521		break;
522
523	case TYPE_ELLIPSIS:
524		Py_INCREF(Py_Ellipsis);
525		retval = Py_Ellipsis;
526		break;
527
528	case TYPE_FALSE:
529		Py_INCREF(Py_False);
530		retval = Py_False;
531		break;
532
533	case TYPE_TRUE:
534		Py_INCREF(Py_True);
535		retval = Py_True;
536		break;
537
538	case TYPE_INT:
539		retval = PyLong_FromLong(r_long(p));
540		break;
541
542	case TYPE_INT64:
543		retval = r_long64(p);
544		break;
545
546	case TYPE_LONG:
547		{
548			int size;
549			PyLongObject *ob;
550			n = r_long(p);
551			if (n < -INT_MAX || n > INT_MAX) {
552				PyErr_SetString(PyExc_ValueError,
553						"bad marshal data");
554				retval = NULL;
555				break;
556			}
557			size = n<0 ? -n : n;
558			ob = _PyLong_New(size);
559			if (ob == NULL) {
560				retval = NULL;
561				break;
562			}
563			Py_SIZE(ob) = n;
564			for (i = 0; i < size; i++) {
565				int digit = r_short(p);
566				if (digit < 0) {
567					Py_DECREF(ob);
568					PyErr_SetString(PyExc_ValueError,
569							"bad marshal data");
570					ob = NULL;
571					break;
572				}
573				if (ob != NULL)
574					ob->ob_digit[i] = digit;
575			}
576			retval = (PyObject *)ob;
577			break;
578		}
579
580	case TYPE_FLOAT:
581		{
582			char buf[256];
583			double dx;
584			n = r_byte(p);
585			if (n == EOF || r_string(buf, (int)n, p) != n) {
586				PyErr_SetString(PyExc_EOFError,
587					"EOF read where object expected");
588				retval = NULL;
589				break;
590			}
591			buf[n] = '\0';
592			retval = NULL;
593			PyFPE_START_PROTECT("atof", break)
594			dx = PyOS_ascii_atof(buf);
595			PyFPE_END_PROTECT(dx)
596			retval = PyFloat_FromDouble(dx);
597			break;
598		}
599
600	case TYPE_BINARY_FLOAT:
601		{
602			unsigned char buf[8];
603			double x;
604			if (r_string((char*)buf, 8, p) != 8) {
605				PyErr_SetString(PyExc_EOFError,
606					"EOF read where object expected");
607				retval = NULL;
608				break;
609			}
610			x = _PyFloat_Unpack8(buf, 1);
611			if (x == -1.0 && PyErr_Occurred()) {
612				retval = NULL;
613				break;
614			}
615			retval = PyFloat_FromDouble(x);
616			break;
617		}
618
619#ifndef WITHOUT_COMPLEX
620	case TYPE_COMPLEX:
621		{
622			char buf[256];
623			Py_complex c;
624			n = r_byte(p);
625			if (n == EOF || r_string(buf, (int)n, p) != n) {
626				PyErr_SetString(PyExc_EOFError,
627					"EOF read where object expected");
628				retval = NULL;
629				break;
630			}
631			buf[n] = '\0';
632			retval = NULL;
633			PyFPE_START_PROTECT("atof", break;)
634			c.real = PyOS_ascii_atof(buf);
635			PyFPE_END_PROTECT(c)
636			n = r_byte(p);
637			if (n == EOF || r_string(buf, (int)n, p) != n) {
638				PyErr_SetString(PyExc_EOFError,
639					"EOF read where object expected");
640				retval = NULL;
641				break;
642			}
643			buf[n] = '\0';
644			PyFPE_START_PROTECT("atof", break)
645			c.imag = PyOS_ascii_atof(buf);
646			PyFPE_END_PROTECT(c)
647			retval = PyComplex_FromCComplex(c);
648			break;
649		}
650
651	case TYPE_BINARY_COMPLEX:
652		{
653			unsigned char buf[8];
654			Py_complex c;
655			if (r_string((char*)buf, 8, p) != 8) {
656				PyErr_SetString(PyExc_EOFError,
657					"EOF read where object expected");
658				retval = NULL;
659				break;
660			}
661			c.real = _PyFloat_Unpack8(buf, 1);
662			if (c.real == -1.0 && PyErr_Occurred()) {
663				retval = NULL;
664				break;
665			}
666			if (r_string((char*)buf, 8, p) != 8) {
667				PyErr_SetString(PyExc_EOFError,
668					"EOF read where object expected");
669				retval = NULL;
670				break;
671			}
672			c.imag = _PyFloat_Unpack8(buf, 1);
673			if (c.imag == -1.0 && PyErr_Occurred()) {
674				retval = NULL;
675				break;
676			}
677			retval = PyComplex_FromCComplex(c);
678			break;
679		}
680#endif
681
682	case TYPE_STRING:
683		n = r_long(p);
684		if (n < 0 || n > INT_MAX) {
685			PyErr_SetString(PyExc_ValueError, "bad marshal data");
686			retval = NULL;
687			break;
688		}
689		v = PyBytes_FromStringAndSize((char *)NULL, n);
690		if (v == NULL) {
691			retval = NULL;
692			break;
693		}
694		if (r_string(PyBytes_AS_STRING(v), (int)n, p) != n) {
695			Py_DECREF(v);
696			PyErr_SetString(PyExc_EOFError,
697					"EOF read where object expected");
698			retval = NULL;
699			break;
700		}
701		retval = v;
702		break;
703
704	case TYPE_UNICODE:
705	    {
706		char *buffer;
707
708		n = r_long(p);
709		if (n < 0 || n > INT_MAX) {
710			PyErr_SetString(PyExc_ValueError, "bad marshal data");
711			retval = NULL;
712			break;
713		}
714		buffer = PyMem_NEW(char, n);
715		if (buffer == NULL) {
716			retval = PyErr_NoMemory();
717			break;
718		}
719		if (r_string(buffer, (int)n, p) != n) {
720			PyMem_DEL(buffer);
721			PyErr_SetString(PyExc_EOFError,
722				"EOF read where object expected");
723			retval = NULL;
724			break;
725		}
726		v = PyUnicode_DecodeUTF8(buffer, n, NULL);
727		PyMem_DEL(buffer);
728		retval = v;
729		break;
730	    }
731
732	case TYPE_TUPLE:
733		n = r_long(p);
734		if (n < 0 || n > INT_MAX) {
735			PyErr_SetString(PyExc_ValueError, "bad marshal data");
736			retval = NULL;
737			break;
738		}
739		v = PyTuple_New((int)n);
740		if (v == NULL) {
741			retval = NULL;
742			break;
743		}
744		for (i = 0; i < n; i++) {
745			v2 = r_object(p);
746			if ( v2 == NULL ) {
747				if (!PyErr_Occurred())
748					PyErr_SetString(PyExc_TypeError,
749						"NULL object in marshal data");
750				Py_DECREF(v);
751				v = NULL;
752				break;
753			}
754			PyTuple_SET_ITEM(v, (int)i, v2);
755		}
756		retval = v;
757		break;
758
759	case TYPE_LIST:
760		n = r_long(p);
761		if (n < 0 || n > INT_MAX) {
762			PyErr_SetString(PyExc_ValueError, "bad marshal data");
763			retval = NULL;
764			break;
765		}
766		v = PyList_New((int)n);
767		if (v == NULL) {
768			retval = NULL;
769			break;
770		}
771		for (i = 0; i < n; i++) {
772			v2 = r_object(p);
773			if ( v2 == NULL ) {
774				if (!PyErr_Occurred())
775					PyErr_SetString(PyExc_TypeError,
776						"NULL object in marshal data");
777				Py_DECREF(v);
778				v = NULL;
779				break;
780			}
781			PyList_SET_ITEM(v, (int)i, v2);
782		}
783		retval = v;
784		break;
785
786	case TYPE_DICT:
787		v = PyDict_New();
788		if (v == NULL) {
789			retval = NULL;
790			break;
791		}
792		for (;;) {
793			PyObject *key, *val;
794			key = r_object(p);
795			if (key == NULL)
796				break;
797			val = r_object(p);
798			if (val != NULL)
799				PyDict_SetItem(v, key, val);
800			Py_DECREF(key);
801			Py_XDECREF(val);
802		}
803		if (PyErr_Occurred()) {
804			Py_DECREF(v);
805			v = NULL;
806		}
807		retval = v;
808		break;
809
810	case TYPE_SET:
811	case TYPE_FROZENSET:
812		n = r_long(p);
813		if (n < 0 || n > INT_MAX) {
814			PyErr_SetString(PyExc_ValueError, "bad marshal data");
815			retval = NULL;
816			break;
817		}
818                v = (type == TYPE_SET) ? PySet_New(NULL) : PyFrozenSet_New(NULL);
819		if (v == NULL) {
820			retval = NULL;
821			break;
822		}
823		for (i = 0; i < n; i++) {
824			v2 = r_object(p);
825			if ( v2 == NULL ) {
826				if (!PyErr_Occurred())
827					PyErr_SetString(PyExc_TypeError,
828						"NULL object in marshal data");
829				Py_DECREF(v);
830				v = NULL;
831				break;
832			}
833			if (PySet_Add(v, v2) == -1) {
834                                Py_DECREF(v);
835                                Py_DECREF(v2);
836                                v = NULL;
837                                break;
838                        }
839                        Py_DECREF(v2);
840		}
841		retval = v;
842		break;
843
844	case TYPE_CODE:
845		{
846			int argcount;
847			int kwonlyargcount;
848			int nlocals;
849			int stacksize;
850			int flags;
851			PyObject *code = NULL;
852			PyObject *consts = NULL;
853			PyObject *names = NULL;
854			PyObject *varnames = NULL;
855			PyObject *freevars = NULL;
856			PyObject *cellvars = NULL;
857			PyObject *filename = NULL;
858			PyObject *name = NULL;
859			int firstlineno;
860			PyObject *lnotab = NULL;
861
862			v = NULL;
863
864			/* XXX ignore long->int overflows for now */
865			argcount = (int)r_long(p);
866			kwonlyargcount = (int)r_long(p);
867			nlocals = (int)r_long(p);
868			stacksize = (int)r_long(p);
869			flags = (int)r_long(p);
870			code = r_object(p);
871			if (code == NULL)
872				goto code_error;
873			consts = r_object(p);
874			if (consts == NULL)
875				goto code_error;
876			names = r_object(p);
877			if (names == NULL)
878				goto code_error;
879			varnames = r_object(p);
880			if (varnames == NULL)
881				goto code_error;
882			freevars = r_object(p);
883			if (freevars == NULL)
884				goto code_error;
885			cellvars = r_object(p);
886			if (cellvars == NULL)
887				goto code_error;
888			filename = r_object(p);
889			if (filename == NULL)
890				goto code_error;
891			name = r_object(p);
892			if (name == NULL)
893				goto code_error;
894			firstlineno = (int)r_long(p);
895			lnotab = r_object(p);
896			if (lnotab == NULL)
897				goto code_error;
898
899			v = (PyObject *) PyCode_New(
900					argcount, kwonlyargcount,
901					nlocals, stacksize, flags,
902					code, consts, names, varnames,
903					freevars, cellvars, filename, name,
904					firstlineno, lnotab);
905
906		  code_error:
907			Py_XDECREF(code);
908			Py_XDECREF(consts);
909			Py_XDECREF(names);
910			Py_XDECREF(varnames);
911			Py_XDECREF(freevars);
912			Py_XDECREF(cellvars);
913			Py_XDECREF(filename);
914			Py_XDECREF(name);
915			Py_XDECREF(lnotab);
916		}
917		retval = v;
918		break;
919
920	default:
921		/* Bogus data got written, which isn't ideal.
922		   This will let you keep working and recover. */
923		PyErr_SetString(PyExc_ValueError, "bad marshal data");
924		retval = NULL;
925		break;
926
927	}
928	p->depth--;
929	return retval;
930}
931
932static PyObject *
933read_object(RFILE *p)
934{
935	PyObject *v;
936	if (PyErr_Occurred()) {
937		fprintf(stderr, "XXX readobject called with exception set\n");
938		return NULL;
939	}
940	v = r_object(p);
941	if (v == NULL && !PyErr_Occurred())
942		PyErr_SetString(PyExc_TypeError, "NULL object in marshal data");
943	return v;
944}
945
946int
947PyMarshal_ReadShortFromFile(FILE *fp)
948{
949	RFILE rf;
950	assert(fp);
951	rf.fp = fp;
952	rf.strings = NULL;
953	rf.end = rf.ptr = NULL;
954	return r_short(&rf);
955}
956
957long
958PyMarshal_ReadLongFromFile(FILE *fp)
959{
960	RFILE rf;
961	rf.fp = fp;
962	rf.strings = NULL;
963	rf.ptr = rf.end = NULL;
964	return r_long(&rf);
965}
966
967#ifdef HAVE_FSTAT
968/* Return size of file in bytes; < 0 if unknown. */
969static off_t
970getfilesize(FILE *fp)
971{
972	struct stat st;
973	if (fstat(fileno(fp), &st) != 0)
974		return -1;
975	else
976		return st.st_size;
977}
978#endif
979
980/* If we can get the size of the file up-front, and it's reasonably small,
981 * read it in one gulp and delegate to ...FromString() instead.  Much quicker
982 * than reading a byte at a time from file; speeds .pyc imports.
983 * CAUTION:  since this may read the entire remainder of the file, don't
984 * call it unless you know you're done with the file.
985 */
986PyObject *
987PyMarshal_ReadLastObjectFromFile(FILE *fp)
988{
989/* 75% of 2.1's .pyc files can exploit SMALL_FILE_LIMIT.
990 * REASONABLE_FILE_LIMIT is by defn something big enough for Tkinter.pyc.
991 */
992#define SMALL_FILE_LIMIT (1L << 14)
993#define REASONABLE_FILE_LIMIT (1L << 18)
994#ifdef HAVE_FSTAT
995	off_t filesize;
996#endif
997#ifdef HAVE_FSTAT
998	filesize = getfilesize(fp);
999	if (filesize > 0) {
1000		char buf[SMALL_FILE_LIMIT];
1001		char* pBuf = NULL;
1002		if (filesize <= SMALL_FILE_LIMIT)
1003			pBuf = buf;
1004		else if (filesize <= REASONABLE_FILE_LIMIT)
1005			pBuf = (char *)PyMem_MALLOC(filesize);
1006		if (pBuf != NULL) {
1007			PyObject* v;
1008			size_t n;
1009			/* filesize must fit into an int, because it
1010			   is smaller than REASONABLE_FILE_LIMIT */
1011			n = fread(pBuf, 1, (int)filesize, fp);
1012			v = PyMarshal_ReadObjectFromString(pBuf, n);
1013			if (pBuf != buf)
1014				PyMem_FREE(pBuf);
1015			return v;
1016		}
1017
1018	}
1019#endif
1020	/* We don't have fstat, or we do but the file is larger than
1021	 * REASONABLE_FILE_LIMIT or malloc failed -- read a byte at a time.
1022	 */
1023	return PyMarshal_ReadObjectFromFile(fp);
1024
1025#undef SMALL_FILE_LIMIT
1026#undef REASONABLE_FILE_LIMIT
1027}
1028
1029PyObject *
1030PyMarshal_ReadObjectFromFile(FILE *fp)
1031{
1032	RFILE rf;
1033	PyObject *result;
1034	rf.fp = fp;
1035	rf.strings = PyList_New(0);
1036	rf.depth = 0;
1037	rf.ptr = rf.end = NULL;
1038	result = r_object(&rf);
1039	Py_DECREF(rf.strings);
1040	return result;
1041}
1042
1043PyObject *
1044PyMarshal_ReadObjectFromString(char *str, Py_ssize_t len)
1045{
1046	RFILE rf;
1047	PyObject *result;
1048	rf.fp = NULL;
1049	rf.ptr = str;
1050	rf.end = str + len;
1051	rf.strings = PyList_New(0);
1052	rf.depth = 0;
1053	result = r_object(&rf);
1054	Py_DECREF(rf.strings);
1055	return result;
1056}
1057
1058PyObject *
1059PyMarshal_WriteObjectToString(PyObject *x, int version)
1060{
1061	WFILE wf;
1062	PyObject *res = NULL;
1063
1064	wf.fp = NULL;
1065	wf.str = PyBytes_FromStringAndSize((char *)NULL, 50);
1066	if (wf.str == NULL)
1067		return NULL;
1068	wf.ptr = PyBytes_AS_STRING((PyBytesObject *)wf.str);
1069	wf.end = wf.ptr + PyBytes_Size(wf.str);
1070	wf.error = 0;
1071	wf.depth = 0;
1072	wf.version = version;
1073	wf.strings = (version > 0) ? PyDict_New() : NULL;
1074	w_object(x, &wf);
1075	Py_XDECREF(wf.strings);
1076	if (wf.str != NULL) {
1077		char *base = PyBytes_AS_STRING((PyBytesObject *)wf.str);
1078		if (wf.ptr - base > PY_SSIZE_T_MAX) {
1079			Py_DECREF(wf.str);
1080			PyErr_SetString(PyExc_OverflowError,
1081					"too much marshal data for a string");
1082			return NULL;
1083		}
1084		if (_PyBytes_Resize(&wf.str, (Py_ssize_t)(wf.ptr - base)) < 0)
1085			return NULL;
1086	}
1087	if (wf.error) {
1088		Py_XDECREF(wf.str);
1089		PyErr_SetString(PyExc_ValueError,
1090				(wf.error==1)?"unmarshallable object"
1091				:"object too deeply nested to marshal");
1092		return NULL;
1093	}
1094	if (wf.str != NULL) {
1095		/* XXX Quick hack -- need to do this differently */
1096		res = PyByteArray_FromObject(wf.str);
1097		Py_DECREF(wf.str);
1098	}
1099	return res;
1100}
1101
1102/* And an interface for Python programs... */
1103
1104static PyObject *
1105marshal_dump(PyObject *self, PyObject *args)
1106{
1107	/* XXX Quick hack -- need to do this differently */
1108	PyObject *x;
1109	PyObject *f;
1110	int version = Py_MARSHAL_VERSION;
1111	PyObject *s;
1112	PyObject *res;
1113	if (!PyArg_ParseTuple(args, "OO|i:dump", &x, &f, &version))
1114		return NULL;
1115	s = PyMarshal_WriteObjectToString(x, version);
1116	if (s == NULL)
1117		return NULL;
1118	res = PyObject_CallMethod(f, "write", "O", s);
1119	Py_DECREF(s);
1120	return res;
1121}
1122
1123static PyObject *
1124marshal_load(PyObject *self, PyObject *f)
1125{
1126	/* XXX Quick hack -- need to do this differently */
1127	PyObject *data, *result;
1128	RFILE rf;
1129	data = PyObject_CallMethod(f, "read", "");
1130	if (data == NULL)
1131		return NULL;
1132	rf.fp = NULL;
1133	if (PyBytes_Check(data)) {
1134		rf.ptr = PyBytes_AS_STRING(data);
1135		rf.end = rf.ptr + PyBytes_GET_SIZE(data);
1136	}
1137	else if (PyByteArray_Check(data)) {
1138		rf.ptr = PyByteArray_AS_STRING(data);
1139		rf.end = rf.ptr + PyByteArray_GET_SIZE(data);
1140	}
1141	else {
1142		PyErr_Format(PyExc_TypeError,
1143			     "f.read() returned neither string "
1144			     "nor bytes but %.100s",
1145			     data->ob_type->tp_name);
1146		Py_DECREF(data);
1147		return NULL;
1148	}
1149	rf.strings = PyList_New(0);
1150	rf.depth = 0;
1151	result = read_object(&rf);
1152	Py_DECREF(rf.strings);
1153	Py_DECREF(data);
1154	return result;
1155}
1156
1157static PyObject *
1158marshal_dumps(PyObject *self, PyObject *args)
1159{
1160	PyObject *x;
1161	int version = Py_MARSHAL_VERSION;
1162	if (!PyArg_ParseTuple(args, "O|i:dumps", &x, &version))
1163		return NULL;
1164	return PyMarshal_WriteObjectToString(x, version);
1165}
1166
1167static PyObject *
1168marshal_loads(PyObject *self, PyObject *args)
1169{
1170	RFILE rf;
1171	char *s;
1172	Py_ssize_t n;
1173	PyObject* result;
1174	if (!PyArg_ParseTuple(args, "s#:loads", &s, &n))
1175		return NULL;
1176	rf.fp = NULL;
1177	rf.ptr = s;
1178	rf.end = s + n;
1179	rf.strings = PyList_New(0);
1180	rf.depth = 0;
1181	result = read_object(&rf);
1182	Py_DECREF(rf.strings);
1183	return result;
1184}
1185
1186static PyMethodDef marshal_methods[] = {
1187	{"dump",	marshal_dump,	METH_VARARGS},
1188	{"load",	marshal_load,	METH_O},
1189	{"dumps",	marshal_dumps,	METH_VARARGS},
1190	{"loads",	marshal_loads,	METH_VARARGS},
1191	{NULL,		NULL}		/* sentinel */
1192};
1193
1194PyMODINIT_FUNC
1195PyMarshal_Init(void)
1196{
1197	PyObject *mod = Py_InitModule("marshal", marshal_methods);
1198	if (mod == NULL)
1199		return;
1200	PyModule_AddIntConstant(mod, "version", Py_MARSHAL_VERSION);
1201}
1202