1 #include "Python.h"
2 #include "cStringIO.h"
3 #include "structmember.h"
4
5 PyDoc_STRVAR(cPickle_module_documentation,
6 "C implementation and optimization of the Python pickle module.");
7
8 #ifndef Py_eval_input
9 #include <graminit.h>
10 #define Py_eval_input eval_input
11 #endif /* Py_eval_input */
12
13 #define DEL_LIST_SLICE(list, from, to) (PyList_SetSlice(list, from, to, NULL))
14
15 #define WRITE_BUF_SIZE 256
16
17 /* Bump this when new opcodes are added to the pickle protocol. */
18 #define HIGHEST_PROTOCOL 2
19
20 /*
21 * Note: The UNICODE macro controls the TCHAR meaning of the win32 API. Since
22 * all headers have already been included here, we can safely redefine it.
23 */
24 #ifdef UNICODE
25 # undef UNICODE
26 #endif
27
28 /*
29 * Pickle opcodes. These must be kept in synch with pickle.py. Extensive
30 * docs are in pickletools.py.
31 */
32 #define MARK '('
33 #define STOP '.'
34 #define POP '0'
35 #define POP_MARK '1'
36 #define DUP '2'
37 #define FLOAT 'F'
38 #define BINFLOAT 'G'
39 #define INT 'I'
40 #define BININT 'J'
41 #define BININT1 'K'
42 #define LONG 'L'
43 #define BININT2 'M'
44 #define NONE 'N'
45 #define PERSID 'P'
46 #define BINPERSID 'Q'
47 #define REDUCE 'R'
48 #define STRING 'S'
49 #define BINSTRING 'T'
50 #define SHORT_BINSTRING 'U'
51 #define UNICODE 'V'
52 #define BINUNICODE 'X'
53 #define APPEND 'a'
54 #define BUILD 'b'
55 #define GLOBAL 'c'
56 #define DICT 'd'
57 #define EMPTY_DICT '}'
58 #define APPENDS 'e'
59 #define GET 'g'
60 #define BINGET 'h'
61 #define INST 'i'
62 #define LONG_BINGET 'j'
63 #define LIST 'l'
64 #define EMPTY_LIST ']'
65 #define OBJ 'o'
66 #define PUT 'p'
67 #define BINPUT 'q'
68 #define LONG_BINPUT 'r'
69 #define SETITEM 's'
70 #define TUPLE 't'
71 #define EMPTY_TUPLE ')'
72 #define SETITEMS 'u'
73
74 /* Protocol 2. */
75 #define PROTO '\x80' /* identify pickle protocol */
76 #define NEWOBJ '\x81' /* build object by applying cls.__new__ to argtuple */
77 #define EXT1 '\x82' /* push object from extension registry; 1-byte index */
78 #define EXT2 '\x83' /* ditto, but 2-byte index */
79 #define EXT4 '\x84' /* ditto, but 4-byte index */
80 #define TUPLE1 '\x85' /* build 1-tuple from stack top */
81 #define TUPLE2 '\x86' /* build 2-tuple from two topmost stack items */
82 #define TUPLE3 '\x87' /* build 3-tuple from three topmost stack items */
83 #define NEWTRUE '\x88' /* push True */
84 #define NEWFALSE '\x89' /* push False */
85 #define LONG1 '\x8a' /* push long from < 256 bytes */
86 #define LONG4 '\x8b' /* push really big long */
87
88 /* There aren't opcodes -- they're ways to pickle bools before protocol 2,
89 * so that unpicklers written before bools were introduced unpickle them
90 * as ints, but unpicklers after can recognize that bools were intended.
91 * Note that protocol 2 added direct ways to pickle bools.
92 */
93 #undef TRUE
94 #define TRUE "I01\n"
95 #undef FALSE
96 #define FALSE "I00\n"
97
98 /* Keep in synch with pickle.Pickler._BATCHSIZE. This is how many elements
99 * batch_list/dict() pumps out before doing APPENDS/SETITEMS. Nothing will
100 * break if this gets out of synch with pickle.py, but it's unclear that
101 * would help anything either.
102 */
103 #define BATCHSIZE 1000
104
105 static char MARKv = MARK;
106
107 static PyObject *PickleError;
108 static PyObject *PicklingError;
109 static PyObject *UnpickleableError;
110 static PyObject *UnpicklingError;
111 static PyObject *BadPickleGet;
112
113 /* As the name says, an empty tuple. */
114 static PyObject *empty_tuple;
115
116 /* copy_reg.dispatch_table, {type_object: pickling_function} */
117 static PyObject *dispatch_table;
118
119 /* For EXT[124] opcodes. */
120 /* copy_reg._extension_registry, {(module_name, function_name): code} */
121 static PyObject *extension_registry;
122 /* copy_reg._inverted_registry, {code: (module_name, function_name)} */
123 static PyObject *inverted_registry;
124 /* copy_reg._extension_cache, {code: object} */
125 static PyObject *extension_cache;
126
127 /* For looking up name pairs in copy_reg._extension_registry. */
128 static PyObject *two_tuple;
129
130 static PyObject *__class___str, *__getinitargs___str, *__dict___str,
131 *__getstate___str, *__setstate___str, *__name___str, *__reduce___str,
132 *__reduce_ex___str,
133 *write_str, *append_str,
134 *read_str, *readline_str, *__main___str,
135 *dispatch_table_str;
136
137 /*************************************************************************
138 Internal Data type for pickle data. */
139
140 typedef struct {
141 PyObject_HEAD
142 Py_ssize_t length; /* number of initial slots in data currently used */
143 Py_ssize_t size; /* number of slots in data allocated */
144 PyObject **data;
145 } Pdata;
146
147 static void
Pdata_dealloc(Pdata * self)148 Pdata_dealloc(Pdata *self)
149 {
150 Py_ssize_t i;
151 PyObject **p;
152
153 for (i = self->length, p = self->data; --i >= 0; p++) {
154 Py_DECREF(*p);
155 }
156 if (self->data)
157 free(self->data);
158 PyObject_Del(self);
159 }
160
161 static PyTypeObject PdataType = {
162 PyVarObject_HEAD_INIT(NULL, 0) "cPickle.Pdata", sizeof(Pdata), 0,
163 (destructor)Pdata_dealloc,
164 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0L,0L,0L,0L, ""
165 };
166
167 #define Pdata_Check(O) (Py_TYPE(O) == &PdataType)
168
169 static PyObject *
Pdata_New(void)170 Pdata_New(void)
171 {
172 Pdata *self;
173
174 if (!(self = PyObject_New(Pdata, &PdataType)))
175 return NULL;
176 self->size = 8;
177 self->length = 0;
178 self->data = malloc(self->size * sizeof(PyObject*));
179 if (self->data)
180 return (PyObject*)self;
181 Py_DECREF(self);
182 return PyErr_NoMemory();
183 }
184
185 static int
stackUnderflow(void)186 stackUnderflow(void)
187 {
188 PyErr_SetString(UnpicklingError, "unpickling stack underflow");
189 return -1;
190 }
191
192 /* Retain only the initial clearto items. If clearto >= the current
193 * number of items, this is a (non-erroneous) NOP.
194 */
195 static int
Pdata_clear(Pdata * self,Py_ssize_t clearto)196 Pdata_clear(Pdata *self, Py_ssize_t clearto)
197 {
198 Py_ssize_t i;
199 PyObject **p;
200
201 if (clearto < 0) return stackUnderflow();
202 if (clearto >= self->length) return 0;
203
204 for (i = self->length, p = self->data + clearto;
205 --i >= clearto;
206 p++) {
207 Py_CLEAR(*p);
208 }
209 self->length = clearto;
210
211 return 0;
212 }
213
214 static int
Pdata_grow(Pdata * self)215 Pdata_grow(Pdata *self)
216 {
217 Py_ssize_t bigger;
218 Py_ssize_t nbytes;
219
220 PyObject **tmp;
221
222 if (self->size > (PY_SSIZE_T_MAX >> 1))
223 goto nomemory;
224 bigger = self->size << 1;
225 if (bigger > (PY_SSIZE_T_MAX / sizeof(PyObject *)))
226 goto nomemory;
227 nbytes = bigger * sizeof(PyObject *);
228 tmp = realloc(self->data, nbytes);
229 if (tmp == NULL)
230 goto nomemory;
231 self->data = tmp;
232 self->size = bigger;
233 return 0;
234
235 nomemory:
236 PyErr_NoMemory();
237 return -1;
238 }
239
240 /* D is a Pdata*. Pop the topmost element and store it into V, which
241 * must be an lvalue holding PyObject*. On stack underflow, UnpicklingError
242 * is raised and V is set to NULL. D and V may be evaluated several times.
243 */
244 #define PDATA_POP(D, V) { \
245 if ((D)->length) \
246 (V) = (D)->data[--((D)->length)]; \
247 else { \
248 PyErr_SetString(UnpicklingError, "bad pickle data"); \
249 (V) = NULL; \
250 } \
251 }
252
253 /* PDATA_PUSH and PDATA_APPEND both push rvalue PyObject* O on to Pdata*
254 * D. If the Pdata stack can't be grown to hold the new value, both
255 * raise MemoryError and execute "return ER". The difference is in ownership
256 * of O after: _PUSH transfers ownership of O from the caller to the stack
257 * (no incref of O is done, and in case of error O is decrefed), while
258 * _APPEND pushes a new reference.
259 */
260
261 /* Push O on stack D, giving ownership of O to the stack. */
262 #define PDATA_PUSH(D, O, ER) { \
263 if (((Pdata*)(D))->length == ((Pdata*)(D))->size && \
264 Pdata_grow((Pdata*)(D)) < 0) { \
265 Py_DECREF(O); \
266 return ER; \
267 } \
268 ((Pdata*)(D))->data[((Pdata*)(D))->length++] = (O); \
269 }
270
271 /* Push O on stack D, pushing a new reference. */
272 #define PDATA_APPEND(D, O, ER) { \
273 if (((Pdata*)(D))->length == ((Pdata*)(D))->size && \
274 Pdata_grow((Pdata*)(D)) < 0) \
275 return ER; \
276 Py_INCREF(O); \
277 ((Pdata*)(D))->data[((Pdata*)(D))->length++] = (O); \
278 }
279
280
281 static PyObject *
Pdata_popTuple(Pdata * self,Py_ssize_t start)282 Pdata_popTuple(Pdata *self, Py_ssize_t start)
283 {
284 PyObject *r;
285 Py_ssize_t i, j, l;
286
287 l = self->length-start;
288 r = PyTuple_New(l);
289 if (r == NULL)
290 return NULL;
291 for (i = start, j = 0 ; j < l; i++, j++)
292 PyTuple_SET_ITEM(r, j, self->data[i]);
293
294 self->length = start;
295 return r;
296 }
297
298 static PyObject *
Pdata_popList(Pdata * self,Py_ssize_t start)299 Pdata_popList(Pdata *self, Py_ssize_t start)
300 {
301 PyObject *r;
302 Py_ssize_t i, j, l;
303
304 l=self->length-start;
305 if (!( r=PyList_New(l))) return NULL;
306 for (i=start, j=0 ; j < l; i++, j++)
307 PyList_SET_ITEM(r, j, self->data[i]);
308
309 self->length=start;
310 return r;
311 }
312
313 /*************************************************************************/
314
315 #define ARG_TUP(self, o) { \
316 if (self->arg || (self->arg=PyTuple_New(1))) { \
317 Py_XDECREF(PyTuple_GET_ITEM(self->arg,0)); \
318 PyTuple_SET_ITEM(self->arg,0,o); \
319 } \
320 else { \
321 Py_DECREF(o); \
322 } \
323 }
324
325 #define FREE_ARG_TUP(self) { \
326 if (Py_REFCNT(self->arg) > 1) { \
327 Py_CLEAR(self->arg); \
328 } \
329 }
330
331 typedef struct Picklerobject {
332 PyObject_HEAD
333 FILE *fp;
334 PyObject *write;
335 PyObject *file;
336 PyObject *memo;
337 PyObject *arg;
338 PyObject *pers_func;
339 PyObject *inst_pers_func;
340
341 /* pickle protocol number, >= 0 */
342 int proto;
343
344 /* bool, true if proto > 0 */
345 int bin;
346
347 int fast; /* Fast mode doesn't save in memo, don't use if circ ref */
348 Py_ssize_t (*write_func)(struct Picklerobject *, const char *, Py_ssize_t);
349 char *write_buf;
350 Py_ssize_t buf_size;
351 PyObject *dispatch_table;
352 int fast_container; /* count nested container dumps */
353 PyObject *fast_memo;
354 } Picklerobject;
355
356 #ifndef PY_CPICKLE_FAST_LIMIT
357 #define PY_CPICKLE_FAST_LIMIT 50
358 #endif
359
360 static PyTypeObject Picklertype;
361
362 typedef struct Unpicklerobject {
363 PyObject_HEAD
364 FILE *fp;
365 PyObject *file;
366 PyObject *readline;
367 PyObject *read;
368 PyObject *memo;
369 PyObject *arg;
370 Pdata *stack;
371 PyObject *mark;
372 PyObject *pers_func;
373 PyObject *last_string;
374 Py_ssize_t *marks;
375 Py_ssize_t num_marks;
376 Py_ssize_t marks_size;
377 Py_ssize_t (*read_func)(struct Unpicklerobject *, char **, Py_ssize_t);
378 Py_ssize_t (*readline_func)(struct Unpicklerobject *, char **);
379 Py_ssize_t buf_size;
380 char *buf;
381 PyObject *find_class;
382 } Unpicklerobject;
383
384 static PyTypeObject Unpicklertype;
385
386 /* Forward decls that need the above structs */
387 static int save(Picklerobject *, PyObject *, int);
388 static int put2(Picklerobject *, PyObject *);
389
390 static
391 PyObject *
cPickle_ErrFormat(PyObject * ErrType,char * stringformat,char * format,...)392 cPickle_ErrFormat(PyObject *ErrType, char *stringformat, char *format, ...)
393 {
394 va_list va;
395 PyObject *args=0, *retval=0;
396 va_start(va, format);
397
398 if (format) args = Py_VaBuildValue(format, va);
399 va_end(va);
400 if (format && ! args) return NULL;
401 if (stringformat && !(retval=PyString_FromString(stringformat)))
402 return NULL;
403
404 if (retval) {
405 if (args) {
406 PyObject *v;
407 v=PyString_Format(retval, args);
408 Py_DECREF(retval);
409 Py_DECREF(args);
410 if (! v) return NULL;
411 retval=v;
412 }
413 }
414 else
415 if (args) retval=args;
416 else {
417 PyErr_SetObject(ErrType,Py_None);
418 return NULL;
419 }
420 PyErr_SetObject(ErrType,retval);
421 Py_DECREF(retval);
422 return NULL;
423 }
424
425 static Py_ssize_t
write_file(Picklerobject * self,const char * s,Py_ssize_t n)426 write_file(Picklerobject *self, const char *s, Py_ssize_t n)
427 {
428 size_t nbyteswritten;
429
430 if (s == NULL) {
431 return 0;
432 }
433
434 PyFile_IncUseCount((PyFileObject *)self->file);
435 Py_BEGIN_ALLOW_THREADS
436 nbyteswritten = fwrite(s, sizeof(char), n, self->fp);
437 Py_END_ALLOW_THREADS
438 PyFile_DecUseCount((PyFileObject *)self->file);
439 if (nbyteswritten != (size_t)n) {
440 PyErr_SetFromErrno(PyExc_IOError);
441 return -1;
442 }
443
444 return n;
445 }
446
447 static Py_ssize_t
write_cStringIO(Picklerobject * self,const char * s,Py_ssize_t n)448 write_cStringIO(Picklerobject *self, const char *s, Py_ssize_t n)
449 {
450 Py_ssize_t len = n;
451
452 if (s == NULL) {
453 return 0;
454 }
455
456 while (n > INT_MAX) {
457 if (PycStringIO->cwrite((PyObject *)self->file, s, INT_MAX) != INT_MAX) {
458 return -1;
459 }
460 n -= INT_MAX;
461 }
462
463 if (PycStringIO->cwrite((PyObject *)self->file, s, n) != n) {
464 return -1;
465 }
466
467 return len;
468 }
469
470 static Py_ssize_t
write_none(Picklerobject * self,const char * s,Py_ssize_t n)471 write_none(Picklerobject *self, const char *s, Py_ssize_t n)
472 {
473 if (s == NULL) return 0;
474 return n;
475 }
476
477 static Py_ssize_t
write_other(Picklerobject * self,const char * s,Py_ssize_t n)478 write_other(Picklerobject *self, const char *s, Py_ssize_t n)
479 {
480 PyObject *py_str = 0, *junk = 0;
481
482 if (s == NULL) {
483 if (!( self->buf_size )) return 0;
484 py_str = PyString_FromStringAndSize(self->write_buf,
485 self->buf_size);
486 if (!py_str)
487 return -1;
488 }
489 else {
490 if (self->buf_size && n > WRITE_BUF_SIZE - self->buf_size) {
491 if (write_other(self, NULL, 0) < 0)
492 return -1;
493 }
494
495 if (n > WRITE_BUF_SIZE) {
496 if (!( py_str =
497 PyString_FromStringAndSize(s, n)))
498 return -1;
499 }
500 else {
501 memcpy(self->write_buf + self->buf_size, s, n);
502 self->buf_size += n;
503 return n;
504 }
505 }
506
507 if (self->write) {
508 /* object with write method */
509 ARG_TUP(self, py_str);
510 if (self->arg) {
511 junk = PyObject_Call(self->write, self->arg, NULL);
512 FREE_ARG_TUP(self);
513 }
514 if (junk) Py_DECREF(junk);
515 else return -1;
516 }
517 else
518 PDATA_PUSH(self->file, py_str, -1);
519
520 self->buf_size = 0;
521 return n;
522 }
523
524
525 static Py_ssize_t
read_file(Unpicklerobject * self,char ** s,Py_ssize_t n)526 read_file(Unpicklerobject *self, char **s, Py_ssize_t n)
527 {
528 size_t nbytesread;
529
530 if (self->buf_size == 0) {
531 Py_ssize_t size;
532
533 size = ((n < 32) ? 32 : n);
534 if (!( self->buf = (char *)malloc(size))) {
535 PyErr_NoMemory();
536 return -1;
537 }
538
539 self->buf_size = size;
540 }
541 else if (n > self->buf_size) {
542 char *newbuf = (char *)realloc(self->buf, n);
543 if (!newbuf) {
544 PyErr_NoMemory();
545 return -1;
546 }
547 self->buf = newbuf;
548 self->buf_size = n;
549 }
550
551 PyFile_IncUseCount((PyFileObject *)self->file);
552 Py_BEGIN_ALLOW_THREADS
553 nbytesread = fread(self->buf, sizeof(char), n, self->fp);
554 Py_END_ALLOW_THREADS
555 PyFile_DecUseCount((PyFileObject *)self->file);
556 if (nbytesread != (size_t)n) {
557 if (feof(self->fp)) {
558 PyErr_SetNone(PyExc_EOFError);
559 return -1;
560 }
561
562 PyErr_SetFromErrno(PyExc_IOError);
563 return -1;
564 }
565
566 *s = self->buf;
567
568 return n;
569 }
570
571
572 static Py_ssize_t
readline_file(Unpicklerobject * self,char ** s)573 readline_file(Unpicklerobject *self, char **s)
574 {
575 Py_ssize_t i;
576
577 if (self->buf_size == 0) {
578 if (!( self->buf = (char *)malloc(40))) {
579 PyErr_NoMemory();
580 return -1;
581 }
582 self->buf_size = 40;
583 }
584
585 i = 0;
586 while (1) {
587 Py_ssize_t bigger;
588 char *newbuf;
589 for (; i < (self->buf_size - 1); i++) {
590 if (feof(self->fp) ||
591 (self->buf[i] = getc(self->fp)) == '\n') {
592 self->buf[i + 1] = '\0';
593 *s = self->buf;
594 return i + 1;
595 }
596 }
597 if (self->buf_size > (PY_SSIZE_T_MAX >> 1)) {
598 PyErr_NoMemory();
599 return -1;
600 }
601 bigger = self->buf_size << 1;
602 newbuf = (char *)realloc(self->buf, bigger);
603 if (newbuf == NULL) {
604 PyErr_NoMemory();
605 return -1;
606 }
607 self->buf = newbuf;
608 self->buf_size = bigger;
609 }
610 }
611
612
613 static Py_ssize_t
read_cStringIO(Unpicklerobject * self,char ** s,Py_ssize_t n)614 read_cStringIO(Unpicklerobject *self, char **s, Py_ssize_t n)
615 {
616 Py_ssize_t len = n;
617 char *start, *end = NULL;
618
619 while (1) {
620 int k;
621 char *ptr;
622 if (n > INT_MAX)
623 k = INT_MAX;
624 else
625 k = (int)n;
626 if (PycStringIO->cread((PyObject *)self->file, &ptr, k) != k) {
627 PyErr_SetNone(PyExc_EOFError);
628 return -1;
629 }
630 if (end == NULL)
631 start = ptr;
632 else if (ptr != end) {
633 /* non-continuous area */
634 return -1;
635 }
636 if (n <= INT_MAX)
637 break;
638 end = ptr + INT_MAX;
639 n -= INT_MAX;
640 }
641
642 *s = start;
643
644 return len;
645 }
646
647
648 static Py_ssize_t
readline_cStringIO(Unpicklerobject * self,char ** s)649 readline_cStringIO(Unpicklerobject *self, char **s)
650 {
651 Py_ssize_t n = 0;
652 char *start = NULL, *end = NULL;
653
654 while (1) {
655 int k;
656 char *ptr;
657 if ((k = PycStringIO->creadline((PyObject *)self->file, &ptr)) < 0) {
658 return -1;
659 }
660 n += k;
661 if (end == NULL)
662 start = ptr;
663 else if (ptr != end) {
664 /* non-continuous area */
665 return -1;
666 }
667 if (k == 0 || ptr[k - 1] == '\n')
668 break;
669 end = ptr + k;
670 }
671
672 *s = start;
673
674 return n;
675 }
676
677
678 static Py_ssize_t
read_other(Unpicklerobject * self,char ** s,Py_ssize_t n)679 read_other(Unpicklerobject *self, char **s, Py_ssize_t n)
680 {
681 PyObject *bytes, *str=0;
682
683 if (!( bytes = PyInt_FromSsize_t(n))) return -1;
684
685 ARG_TUP(self, bytes);
686 if (self->arg) {
687 str = PyObject_Call(self->read, self->arg, NULL);
688 FREE_ARG_TUP(self);
689 }
690 if (! str) return -1;
691
692 Py_XSETREF(self->last_string, str);
693
694 if (! (*s = PyString_AsString(str))) return -1;
695
696 if (PyString_GET_SIZE(str) != n) {
697 PyErr_SetNone(PyExc_EOFError);
698 return -1;
699 }
700
701 return n;
702 }
703
704
705 static Py_ssize_t
readline_other(Unpicklerobject * self,char ** s)706 readline_other(Unpicklerobject *self, char **s)
707 {
708 PyObject *str;
709 Py_ssize_t str_size;
710
711 if (!( str = PyObject_CallObject(self->readline, empty_tuple))) {
712 return -1;
713 }
714
715 if ((str_size = PyString_Size(str)) < 0)
716 return -1;
717
718 Py_XSETREF(self->last_string, str);
719
720 if (! (*s = PyString_AsString(str)))
721 return -1;
722
723 return str_size;
724 }
725
726 /* Copy the first n bytes from s into newly malloc'ed memory, plus a
727 * trailing 0 byte. Return a pointer to that, or NULL if out of memory.
728 * The caller is responsible for free()'ing the return value.
729 */
730 static char *
pystrndup(const char * s,Py_ssize_t n)731 pystrndup(const char *s, Py_ssize_t n)
732 {
733 char *r = (char *)malloc(n+1);
734 if (r == NULL)
735 return (char*)PyErr_NoMemory();
736 memcpy(r, s, n);
737 r[n] = 0;
738 return r;
739 }
740
741
742 static int
get(Picklerobject * self,PyObject * id)743 get(Picklerobject *self, PyObject *id)
744 {
745 PyObject *value, *mv;
746 Py_ssize_t c_value;
747 char s[30];
748 size_t len;
749
750 if (!( mv = PyDict_GetItem(self->memo, id))) {
751 PyErr_SetObject(PyExc_KeyError, id);
752 return -1;
753 }
754
755 if (!( value = PyTuple_GetItem(mv, 0)))
756 return -1;
757
758 if (!( PyInt_Check(value))) {
759 PyErr_SetString(PicklingError, "no int where int expected in memo");
760 return -1;
761 }
762 c_value = PyInt_AS_LONG((PyIntObject*)value);
763
764 if (!self->bin) {
765 s[0] = GET;
766 PyOS_snprintf(s + 1, sizeof(s) - 1,
767 "%" PY_FORMAT_SIZE_T "d\n", c_value);
768 len = strlen(s);
769 }
770 else if (Pdata_Check(self->file)) {
771 if (write_other(self, NULL, 0) < 0) return -1;
772 PDATA_APPEND(self->file, mv, -1);
773 return 0;
774 }
775 else {
776 if (c_value < 256) {
777 s[0] = BINGET;
778 s[1] = (int)(c_value & 0xff);
779 len = 2;
780 }
781 else if (c_value < 0x7fffffffL) {
782 s[0] = LONG_BINGET;
783 s[1] = (int)(c_value & 0xff);
784 s[2] = (int)((c_value >> 8) & 0xff);
785 s[3] = (int)((c_value >> 16) & 0xff);
786 s[4] = (int)((c_value >> 24) & 0xff);
787 len = 5;
788 }
789 else { /* unlikely */
790 PyErr_SetString(PicklingError,
791 "memo id too large for LONG_BINGET");
792 return -1;
793 }
794 }
795
796 if (self->write_func(self, s, len) < 0)
797 return -1;
798
799 return 0;
800 }
801
802
803 static int
put(Picklerobject * self,PyObject * ob)804 put(Picklerobject *self, PyObject *ob)
805 {
806 if (Py_REFCNT(ob) < 2 || self->fast)
807 return 0;
808
809 return put2(self, ob);
810 }
811
812
813 static int
put2(Picklerobject * self,PyObject * ob)814 put2(Picklerobject *self, PyObject *ob)
815 {
816 char c_str[30];
817 Py_ssize_t len, p;
818 int res = -1;
819 PyObject *py_ob_id = 0, *memo_len = 0, *t = 0;
820
821 if (self->fast)
822 return 0;
823
824 if ((p = PyDict_Size(self->memo)) < 0)
825 goto finally;
826
827 /* Make sure memo keys are positive! */
828 /* XXX Why?
829 * XXX And does "positive" really mean non-negative?
830 * XXX pickle.py starts with PUT index 0, not 1. This makes for
831 * XXX gratuitous differences between the pickling modules.
832 */
833 p++;
834
835 if (!( py_ob_id = PyLong_FromVoidPtr(ob)))
836 goto finally;
837
838 if (!( memo_len = PyInt_FromLong(p)))
839 goto finally;
840
841 if (!( t = PyTuple_New(2)))
842 goto finally;
843
844 PyTuple_SET_ITEM(t, 0, memo_len);
845 Py_INCREF(memo_len);
846 PyTuple_SET_ITEM(t, 1, ob);
847 Py_INCREF(ob);
848
849 if (PyDict_SetItem(self->memo, py_ob_id, t) < 0)
850 goto finally;
851
852 if (!self->bin) {
853 c_str[0] = PUT;
854 PyOS_snprintf(c_str + 1, sizeof(c_str) - 1,
855 "%" PY_FORMAT_SIZE_T "d\n", p);
856 len = strlen(c_str);
857 }
858 else if (Pdata_Check(self->file)) {
859 if (write_other(self, NULL, 0) < 0) return -1;
860 PDATA_APPEND(self->file, memo_len, -1);
861 res=0; /* Job well done ;) */
862 goto finally;
863 }
864 else {
865 if (p < 256) {
866 c_str[0] = BINPUT;
867 c_str[1] = p;
868 len = 2;
869 }
870 else if (p < 0x7fffffffL) {
871 c_str[0] = LONG_BINPUT;
872 c_str[1] = (int)(p & 0xff);
873 c_str[2] = (int)((p >> 8) & 0xff);
874 c_str[3] = (int)((p >> 16) & 0xff);
875 c_str[4] = (int)((p >> 24) & 0xff);
876 len = 5;
877 }
878 else { /* unlikely */
879 PyErr_SetString(PicklingError,
880 "memo id too large for LONG_BINPUT");
881 goto finally;
882 }
883 }
884
885 if (self->write_func(self, c_str, len) < 0)
886 goto finally;
887
888 res = 0;
889
890 finally:
891 Py_XDECREF(py_ob_id);
892 Py_XDECREF(memo_len);
893 Py_XDECREF(t);
894
895 return res;
896 }
897
898 static PyObject *
whichmodule(PyObject * global,PyObject * global_name)899 whichmodule(PyObject *global, PyObject *global_name)
900 {
901 Py_ssize_t i, j;
902 PyObject *module = 0, *modules_dict = 0,
903 *global_name_attr = 0, *name = 0;
904
905 module = PyObject_GetAttrString(global, "__module__");
906 if (module)
907 return module;
908 if (PyErr_ExceptionMatches(PyExc_AttributeError))
909 PyErr_Clear();
910 else
911 return NULL;
912
913 if (!( modules_dict = PySys_GetObject("modules")))
914 return NULL;
915
916 i = 0;
917 while ((j = PyDict_Next(modules_dict, &i, &name, &module))) {
918
919 if (PyObject_Compare(name, __main___str)==0) continue;
920
921 global_name_attr = PyObject_GetAttr(module, global_name);
922 if (!global_name_attr) {
923 if (PyErr_ExceptionMatches(PyExc_AttributeError))
924 PyErr_Clear();
925 else
926 return NULL;
927 continue;
928 }
929
930 if (global_name_attr != global) {
931 Py_DECREF(global_name_attr);
932 continue;
933 }
934
935 Py_DECREF(global_name_attr);
936
937 break;
938 }
939
940 /* The following implements the rule in pickle.py added in 1.5
941 that used __main__ if no module is found. I don't actually
942 like this rule. jlf
943 */
944 if (!j) {
945 name=__main___str;
946 }
947
948 Py_INCREF(name);
949 return name;
950 }
951
952
953 static int
fast_save_enter(Picklerobject * self,PyObject * obj)954 fast_save_enter(Picklerobject *self, PyObject *obj)
955 {
956 /* if fast_container < 0, we're doing an error exit. */
957 if (++self->fast_container >= PY_CPICKLE_FAST_LIMIT) {
958 PyObject *key = NULL;
959 if (self->fast_memo == NULL) {
960 self->fast_memo = PyDict_New();
961 if (self->fast_memo == NULL) {
962 self->fast_container = -1;
963 return 0;
964 }
965 }
966 key = PyLong_FromVoidPtr(obj);
967 if (key == NULL)
968 return 0;
969 if (PyDict_GetItem(self->fast_memo, key)) {
970 Py_DECREF(key);
971 PyErr_Format(PyExc_ValueError,
972 "fast mode: can't pickle cyclic objects "
973 "including object type %s at %p",
974 Py_TYPE(obj)->tp_name, obj);
975 self->fast_container = -1;
976 return 0;
977 }
978 if (PyDict_SetItem(self->fast_memo, key, Py_None) < 0) {
979 Py_DECREF(key);
980 self->fast_container = -1;
981 return 0;
982 }
983 Py_DECREF(key);
984 }
985 return 1;
986 }
987
988 int
fast_save_leave(Picklerobject * self,PyObject * obj)989 fast_save_leave(Picklerobject *self, PyObject *obj)
990 {
991 if (self->fast_container-- >= PY_CPICKLE_FAST_LIMIT) {
992 PyObject *key = PyLong_FromVoidPtr(obj);
993 if (key == NULL)
994 return 0;
995 if (PyDict_DelItem(self->fast_memo, key) < 0) {
996 Py_DECREF(key);
997 return 0;
998 }
999 Py_DECREF(key);
1000 }
1001 return 1;
1002 }
1003
1004 static int
save_none(Picklerobject * self,PyObject * args)1005 save_none(Picklerobject *self, PyObject *args)
1006 {
1007 static char none = NONE;
1008 if (self->write_func(self, &none, 1) < 0)
1009 return -1;
1010
1011 return 0;
1012 }
1013
1014 static int
save_bool(Picklerobject * self,PyObject * args)1015 save_bool(Picklerobject *self, PyObject *args)
1016 {
1017 static const char *buf[2] = {FALSE, TRUE};
1018 static char len[2] = {sizeof(FALSE)-1, sizeof(TRUE)-1};
1019 long l = PyInt_AS_LONG((PyIntObject *)args);
1020
1021 if (self->proto >= 2) {
1022 char opcode = l ? NEWTRUE : NEWFALSE;
1023 if (self->write_func(self, &opcode, 1) < 0)
1024 return -1;
1025 }
1026 else if (self->write_func(self, buf[l], len[l]) < 0)
1027 return -1;
1028 return 0;
1029 }
1030
1031 static int
save_int(Picklerobject * self,PyObject * args)1032 save_int(Picklerobject *self, PyObject *args)
1033 {
1034 char c_str[32];
1035 long l = PyInt_AS_LONG((PyIntObject *)args);
1036 Py_ssize_t len = 0;
1037
1038 if (!self->bin
1039 #if SIZEOF_LONG > 4
1040 || l > 0x7fffffffL
1041 || l < -0x80000000L
1042 #endif
1043 ) {
1044 /* Text-mode pickle, or long too big to fit in the 4-byte
1045 * signed BININT format: store as a string.
1046 */
1047 c_str[0] = INT;
1048 PyOS_snprintf(c_str + 1, sizeof(c_str) - 1, "%ld\n", l);
1049 if (self->write_func(self, c_str, strlen(c_str)) < 0)
1050 return -1;
1051 }
1052 else {
1053 /* Binary pickle and l fits in a signed 4-byte int. */
1054 c_str[1] = (int)( l & 0xff);
1055 c_str[2] = (int)((l >> 8) & 0xff);
1056 c_str[3] = (int)((l >> 16) & 0xff);
1057 c_str[4] = (int)((l >> 24) & 0xff);
1058
1059 if ((c_str[4] == 0) && (c_str[3] == 0)) {
1060 if (c_str[2] == 0) {
1061 c_str[0] = BININT1;
1062 len = 2;
1063 }
1064 else {
1065 c_str[0] = BININT2;
1066 len = 3;
1067 }
1068 }
1069 else {
1070 c_str[0] = BININT;
1071 len = 5;
1072 }
1073
1074 if (self->write_func(self, c_str, len) < 0)
1075 return -1;
1076 }
1077
1078 return 0;
1079 }
1080
1081
1082 static int
save_long(Picklerobject * self,PyObject * args)1083 save_long(Picklerobject *self, PyObject *args)
1084 {
1085 Py_ssize_t size;
1086 int res = -1;
1087 PyObject *repr = NULL;
1088
1089 static char l = LONG;
1090
1091 if (self->proto >= 2) {
1092 /* Linear-time pickling. */
1093 size_t nbits;
1094 size_t nbytes;
1095 unsigned char *pdata;
1096 char c_str[5];
1097 int i;
1098 int sign = _PyLong_Sign(args);
1099
1100 if (sign == 0) {
1101 /* It's 0 -- an empty bytestring. */
1102 c_str[0] = LONG1;
1103 c_str[1] = 0;
1104 i = self->write_func(self, c_str, 2);
1105 if (i < 0) goto finally;
1106 res = 0;
1107 goto finally;
1108 }
1109 nbits = _PyLong_NumBits(args);
1110 if (nbits == (size_t)-1 && PyErr_Occurred())
1111 goto finally;
1112 /* How many bytes do we need? There are nbits >> 3 full
1113 * bytes of data, and nbits & 7 leftover bits. If there
1114 * are any leftover bits, then we clearly need another
1115 * byte. Wnat's not so obvious is that we *probably*
1116 * need another byte even if there aren't any leftovers:
1117 * the most-significant bit of the most-significant byte
1118 * acts like a sign bit, and it's usually got a sense
1119 * opposite of the one we need. The exception is longs
1120 * of the form -(2**(8*j-1)) for j > 0. Such a long is
1121 * its own 256's-complement, so has the right sign bit
1122 * even without the extra byte. That's a pain to check
1123 * for in advance, though, so we always grab an extra
1124 * byte at the start, and cut it back later if possible.
1125 */
1126 nbytes = (nbits >> 3) + 1;
1127 if (nbytes > INT_MAX) {
1128 PyErr_SetString(PyExc_OverflowError, "long too large "
1129 "to pickle");
1130 goto finally;
1131 }
1132 repr = PyString_FromStringAndSize(NULL, (int)nbytes);
1133 if (repr == NULL) goto finally;
1134 pdata = (unsigned char *)PyString_AS_STRING(repr);
1135 i = _PyLong_AsByteArray((PyLongObject *)args,
1136 pdata, nbytes,
1137 1 /* little endian */, 1 /* signed */);
1138 if (i < 0) goto finally;
1139 /* If the long is negative, this may be a byte more than
1140 * needed. This is so iff the MSB is all redundant sign
1141 * bits.
1142 */
1143 if (sign < 0 && nbytes > 1 && pdata[nbytes - 1] == 0xff &&
1144 (pdata[nbytes - 2] & 0x80) != 0)
1145 --nbytes;
1146
1147 if (nbytes < 256) {
1148 c_str[0] = LONG1;
1149 c_str[1] = (char)nbytes;
1150 size = 2;
1151 }
1152 else {
1153 c_str[0] = LONG4;
1154 size = (int)nbytes;
1155 for (i = 1; i < 5; i++) {
1156 c_str[i] = (char)(size & 0xff);
1157 size >>= 8;
1158 }
1159 size = 5;
1160 }
1161 i = self->write_func(self, c_str, size);
1162 if (i < 0) goto finally;
1163 i = self->write_func(self, (char *)pdata, (int)nbytes);
1164 if (i < 0) goto finally;
1165 res = 0;
1166 goto finally;
1167 }
1168
1169 /* proto < 2: write the repr and newline. This is quadratic-time
1170 * (in the number of digits), in both directions.
1171 */
1172 if (!( repr = PyObject_Repr(args)))
1173 goto finally;
1174
1175 if ((size = PyString_Size(repr)) < 0)
1176 goto finally;
1177
1178 if (self->write_func(self, &l, 1) < 0)
1179 goto finally;
1180
1181 if (self->write_func(self,
1182 PyString_AS_STRING((PyStringObject *)repr),
1183 size) < 0)
1184 goto finally;
1185
1186 if (self->write_func(self, "\n", 1) < 0)
1187 goto finally;
1188
1189 res = 0;
1190
1191 finally:
1192 Py_XDECREF(repr);
1193 return res;
1194 }
1195
1196
1197 static int
save_float(Picklerobject * self,PyObject * args)1198 save_float(Picklerobject *self, PyObject *args)
1199 {
1200 double x = PyFloat_AS_DOUBLE((PyFloatObject *)args);
1201
1202 if (self->bin) {
1203 char str[9];
1204 str[0] = BINFLOAT;
1205 if (_PyFloat_Pack8(x, (unsigned char *)&str[1], 0) < 0)
1206 return -1;
1207 if (self->write_func(self, str, 9) < 0)
1208 return -1;
1209 }
1210 else {
1211 int result = -1;
1212 char *buf = NULL;
1213 char op = FLOAT;
1214
1215 if (self->write_func(self, &op, 1) < 0)
1216 goto done;
1217
1218 buf = PyOS_double_to_string(x, 'g', 17, 0, NULL);
1219 if (!buf) {
1220 PyErr_NoMemory();
1221 goto done;
1222 }
1223
1224 if (self->write_func(self, buf, strlen(buf)) < 0)
1225 goto done;
1226
1227 if (self->write_func(self, "\n", 1) < 0)
1228 goto done;
1229
1230 result = 0;
1231 done:
1232 PyMem_Free(buf);
1233 return result;
1234 }
1235
1236 return 0;
1237 }
1238
1239
1240 static int
save_string(Picklerobject * self,PyObject * args,int doput)1241 save_string(Picklerobject *self, PyObject *args, int doput)
1242 {
1243 Py_ssize_t size, len;
1244 PyObject *repr=0;
1245
1246 if ((size = PyString_Size(args)) < 0)
1247 return -1;
1248
1249 if (!self->bin) {
1250 char *repr_str;
1251
1252 static char string = STRING;
1253
1254 if (!( repr = PyObject_Repr(args)))
1255 return -1;
1256
1257 if ((len = PyString_Size(repr)) < 0)
1258 goto err;
1259 repr_str = PyString_AS_STRING((PyStringObject *)repr);
1260
1261 if (self->write_func(self, &string, 1) < 0)
1262 goto err;
1263
1264 if (self->write_func(self, repr_str, len) < 0)
1265 goto err;
1266
1267 if (self->write_func(self, "\n", 1) < 0)
1268 goto err;
1269
1270 Py_XDECREF(repr);
1271 }
1272 else {
1273 int i;
1274 char c_str[5];
1275
1276 if (size < 256) {
1277 c_str[0] = SHORT_BINSTRING;
1278 c_str[1] = size;
1279 len = 2;
1280 }
1281 else if (size <= 0x7fffffffL) {
1282 c_str[0] = BINSTRING;
1283 for (i = 1; i < 5; i++)
1284 c_str[i] = (int)(size >> ((i - 1) * 8));
1285 len = 5;
1286 }
1287 else {
1288 PyErr_SetString(PyExc_OverflowError,
1289 "cannot serialize a string larger than 2 GiB");
1290 return -1; /* string too large */
1291 }
1292
1293 if (self->write_func(self, c_str, len) < 0)
1294 return -1;
1295
1296 if (size > 128 && Pdata_Check(self->file)) {
1297 if (write_other(self, NULL, 0) < 0) return -1;
1298 PDATA_APPEND(self->file, args, -1);
1299 }
1300 else {
1301 if (self->write_func(self,
1302 PyString_AS_STRING(
1303 (PyStringObject *)args),
1304 size) < 0)
1305 return -1;
1306 }
1307 }
1308
1309 if (doput)
1310 if (put(self, args) < 0)
1311 return -1;
1312
1313 return 0;
1314
1315 err:
1316 Py_XDECREF(repr);
1317 return -1;
1318 }
1319
1320
1321 #ifdef Py_USING_UNICODE
1322 /* A copy of PyUnicode_EncodeRawUnicodeEscape() that also translates
1323 backslash and newline characters to \uXXXX escapes. */
1324 static PyObject *
modified_EncodeRawUnicodeEscape(const Py_UNICODE * s,Py_ssize_t size)1325 modified_EncodeRawUnicodeEscape(const Py_UNICODE *s, Py_ssize_t size)
1326 {
1327 PyObject *repr;
1328 char *p;
1329 char *q;
1330
1331 static const char *hexdigit = "0123456789abcdef";
1332 #ifdef Py_UNICODE_WIDE
1333 const Py_ssize_t expandsize = 10;
1334 #else
1335 const Py_ssize_t expandsize = 6;
1336 #endif
1337
1338 if (size > PY_SSIZE_T_MAX / expandsize)
1339 return PyErr_NoMemory();
1340
1341 repr = PyString_FromStringAndSize(NULL, expandsize * size);
1342 if (repr == NULL)
1343 return NULL;
1344 if (size == 0)
1345 return repr;
1346
1347 p = q = PyString_AS_STRING(repr);
1348 while (size-- > 0) {
1349 Py_UNICODE ch = *s++;
1350 #ifdef Py_UNICODE_WIDE
1351 /* Map 32-bit characters to '\Uxxxxxxxx' */
1352 if (ch >= 0x10000) {
1353 *p++ = '\\';
1354 *p++ = 'U';
1355 *p++ = hexdigit[(ch >> 28) & 0xf];
1356 *p++ = hexdigit[(ch >> 24) & 0xf];
1357 *p++ = hexdigit[(ch >> 20) & 0xf];
1358 *p++ = hexdigit[(ch >> 16) & 0xf];
1359 *p++ = hexdigit[(ch >> 12) & 0xf];
1360 *p++ = hexdigit[(ch >> 8) & 0xf];
1361 *p++ = hexdigit[(ch >> 4) & 0xf];
1362 *p++ = hexdigit[ch & 15];
1363 }
1364 else
1365 #else
1366 /* Map UTF-16 surrogate pairs to '\U00xxxxxx' */
1367 if (ch >= 0xD800 && ch < 0xDC00) {
1368 Py_UNICODE ch2;
1369 Py_UCS4 ucs;
1370
1371 ch2 = *s++;
1372 size--;
1373 if (ch2 >= 0xDC00 && ch2 <= 0xDFFF) {
1374 ucs = (((ch & 0x03FF) << 10) | (ch2 & 0x03FF)) + 0x00010000;
1375 *p++ = '\\';
1376 *p++ = 'U';
1377 *p++ = hexdigit[(ucs >> 28) & 0xf];
1378 *p++ = hexdigit[(ucs >> 24) & 0xf];
1379 *p++ = hexdigit[(ucs >> 20) & 0xf];
1380 *p++ = hexdigit[(ucs >> 16) & 0xf];
1381 *p++ = hexdigit[(ucs >> 12) & 0xf];
1382 *p++ = hexdigit[(ucs >> 8) & 0xf];
1383 *p++ = hexdigit[(ucs >> 4) & 0xf];
1384 *p++ = hexdigit[ucs & 0xf];
1385 continue;
1386 }
1387 /* Fall through: isolated surrogates are copied as-is */
1388 s--;
1389 size++;
1390 }
1391 #endif
1392 /* Map 16-bit characters to '\uxxxx' */
1393 if (ch >= 256 || ch == '\\' || ch == '\n') {
1394 *p++ = '\\';
1395 *p++ = 'u';
1396 *p++ = hexdigit[(ch >> 12) & 0xf];
1397 *p++ = hexdigit[(ch >> 8) & 0xf];
1398 *p++ = hexdigit[(ch >> 4) & 0xf];
1399 *p++ = hexdigit[ch & 15];
1400 }
1401 /* Copy everything else as-is */
1402 else
1403 *p++ = (char) ch;
1404 }
1405 *p = '\0';
1406 _PyString_Resize(&repr, p - q);
1407 return repr;
1408 }
1409
1410 static int
save_unicode(Picklerobject * self,PyObject * args,int doput)1411 save_unicode(Picklerobject *self, PyObject *args, int doput)
1412 {
1413 Py_ssize_t size, len;
1414 PyObject *repr=0;
1415
1416 if (!PyUnicode_Check(args))
1417 return -1;
1418
1419 if (!self->bin) {
1420 char *repr_str;
1421 static char string = UNICODE;
1422
1423 repr = modified_EncodeRawUnicodeEscape(
1424 PyUnicode_AS_UNICODE(args), PyUnicode_GET_SIZE(args));
1425 if (!repr)
1426 return -1;
1427
1428 if ((len = PyString_Size(repr)) < 0)
1429 goto err;
1430 repr_str = PyString_AS_STRING((PyStringObject *)repr);
1431
1432 if (self->write_func(self, &string, 1) < 0)
1433 goto err;
1434
1435 if (self->write_func(self, repr_str, len) < 0)
1436 goto err;
1437
1438 if (self->write_func(self, "\n", 1) < 0)
1439 goto err;
1440
1441 Py_XDECREF(repr);
1442 }
1443 else {
1444 int i;
1445 char c_str[5];
1446
1447 if (!( repr = PyUnicode_AsUTF8String(args)))
1448 return -1;
1449
1450 if ((size = PyString_Size(repr)) < 0)
1451 goto err;
1452 if (size > 0x7fffffffL) {
1453 PyErr_SetString(PyExc_OverflowError,
1454 "cannot serialize a Unicode string larger than 2 GiB");
1455 goto err; /* string too large */
1456 }
1457
1458 c_str[0] = BINUNICODE;
1459 for (i = 1; i < 5; i++)
1460 c_str[i] = (int)(size >> ((i - 1) * 8));
1461 len = 5;
1462
1463 if (self->write_func(self, c_str, len) < 0)
1464 goto err;
1465
1466 if (size > 128 && Pdata_Check(self->file)) {
1467 if (write_other(self, NULL, 0) < 0)
1468 goto err;
1469 PDATA_APPEND(self->file, repr, -1);
1470 }
1471 else {
1472 if (self->write_func(self, PyString_AS_STRING(repr),
1473 size) < 0)
1474 goto err;
1475 }
1476
1477 Py_DECREF(repr);
1478 }
1479
1480 if (doput)
1481 if (put(self, args) < 0)
1482 return -1;
1483
1484 return 0;
1485
1486 err:
1487 Py_XDECREF(repr);
1488 return -1;
1489 }
1490 #endif
1491
1492 /* A helper for save_tuple. Push the len elements in tuple t on the stack. */
1493 static int
store_tuple_elements(Picklerobject * self,PyObject * t,int len)1494 store_tuple_elements(Picklerobject *self, PyObject *t, int len)
1495 {
1496 Py_ssize_t i;
1497 int res = -1; /* guilty until proved innocent */
1498
1499 assert(PyTuple_Size(t) == len);
1500
1501 for (i = 0; i < len; i++) {
1502 PyObject *element = PyTuple_GET_ITEM(t, i);
1503
1504 if (element == NULL)
1505 goto finally;
1506 if (save(self, element, 0) < 0)
1507 goto finally;
1508 }
1509 res = 0;
1510
1511 finally:
1512 return res;
1513 }
1514
1515 /* Tuples are ubiquitous in the pickle protocols, so many techniques are
1516 * used across protocols to minimize the space needed to pickle them.
1517 * Tuples are also the only builtin immutable type that can be recursive
1518 * (a tuple can be reached from itself), and that requires some subtle
1519 * magic so that it works in all cases. IOW, this is a long routine.
1520 */
1521 static int
save_tuple(Picklerobject * self,PyObject * args)1522 save_tuple(Picklerobject *self, PyObject *args)
1523 {
1524 PyObject *py_tuple_id = NULL;
1525 Py_ssize_t len, i;
1526 int res = -1;
1527
1528 static char tuple = TUPLE;
1529 static char pop = POP;
1530 static char pop_mark = POP_MARK;
1531 static char len2opcode[] = {EMPTY_TUPLE, TUPLE1, TUPLE2, TUPLE3};
1532
1533 if ((len = PyTuple_Size(args)) < 0)
1534 goto finally;
1535
1536 if (len == 0) {
1537 char c_str[2];
1538
1539 if (self->proto) {
1540 c_str[0] = EMPTY_TUPLE;
1541 len = 1;
1542 }
1543 else {
1544 c_str[0] = MARK;
1545 c_str[1] = TUPLE;
1546 len = 2;
1547 }
1548 if (self->write_func(self, c_str, len) >= 0)
1549 res = 0;
1550 /* Don't memoize an empty tuple. */
1551 goto finally;
1552 }
1553
1554 /* A non-empty tuple. */
1555
1556 /* id(tuple) isn't in the memo now. If it shows up there after
1557 * saving the tuple elements, the tuple must be recursive, in
1558 * which case we'll pop everything we put on the stack, and fetch
1559 * its value from the memo.
1560 */
1561 py_tuple_id = PyLong_FromVoidPtr(args);
1562 if (py_tuple_id == NULL)
1563 goto finally;
1564
1565 if (len <= 3 && self->proto >= 2) {
1566 /* Use TUPLE{1,2,3} opcodes. */
1567 if (store_tuple_elements(self, args, len) < 0)
1568 goto finally;
1569 if (PyDict_GetItem(self->memo, py_tuple_id)) {
1570 /* pop the len elements */
1571 for (i = 0; i < len; ++i)
1572 if (self->write_func(self, &pop, 1) < 0)
1573 goto finally;
1574 /* fetch from memo */
1575 if (get(self, py_tuple_id) < 0)
1576 goto finally;
1577 res = 0;
1578 goto finally;
1579 }
1580 /* Not recursive. */
1581 if (self->write_func(self, len2opcode + len, 1) < 0)
1582 goto finally;
1583 goto memoize;
1584 }
1585
1586 /* proto < 2 and len > 0, or proto >= 2 and len > 3.
1587 * Generate MARK elt1 elt2 ... TUPLE
1588 */
1589 if (self->write_func(self, &MARKv, 1) < 0)
1590 goto finally;
1591
1592 if (store_tuple_elements(self, args, len) < 0)
1593 goto finally;
1594
1595 if (PyDict_GetItem(self->memo, py_tuple_id)) {
1596 /* pop the stack stuff we pushed */
1597 if (self->bin) {
1598 if (self->write_func(self, &pop_mark, 1) < 0)
1599 goto finally;
1600 }
1601 else {
1602 /* Note that we pop one more than len, to remove
1603 * the MARK too.
1604 */
1605 for (i = 0; i <= len; i++)
1606 if (self->write_func(self, &pop, 1) < 0)
1607 goto finally;
1608 }
1609 /* fetch from memo */
1610 if (get(self, py_tuple_id) >= 0)
1611 res = 0;
1612 goto finally;
1613 }
1614
1615 /* Not recursive. */
1616 if (self->write_func(self, &tuple, 1) < 0)
1617 goto finally;
1618
1619 memoize:
1620 if (put(self, args) >= 0)
1621 res = 0;
1622
1623 finally:
1624 Py_XDECREF(py_tuple_id);
1625 return res;
1626 }
1627
1628 /* iter is an iterator giving items, and we batch up chunks of
1629 * MARK item item ... item APPENDS
1630 * opcode sequences. Calling code should have arranged to first create an
1631 * empty list, or list-like object, for the APPENDS to operate on.
1632 * Returns 0 on success, <0 on error.
1633 */
1634 static int
batch_list(Picklerobject * self,PyObject * iter)1635 batch_list(Picklerobject *self, PyObject *iter)
1636 {
1637 PyObject *obj = NULL;
1638 PyObject *firstitem = NULL;
1639 int i, n;
1640
1641 static char append = APPEND;
1642 static char appends = APPENDS;
1643
1644 assert(iter != NULL);
1645
1646 if (self->proto == 0) {
1647 /* APPENDS isn't available; do one at a time. */
1648 for (;;) {
1649 obj = PyIter_Next(iter);
1650 if (obj == NULL) {
1651 if (PyErr_Occurred())
1652 return -1;
1653 break;
1654 }
1655 i = save(self, obj, 0);
1656 Py_DECREF(obj);
1657 if (i < 0)
1658 return -1;
1659 if (self->write_func(self, &append, 1) < 0)
1660 return -1;
1661 }
1662 return 0;
1663 }
1664
1665 /* proto > 0: write in batches of BATCHSIZE. */
1666 do {
1667 /* Get first item */
1668 firstitem = PyIter_Next(iter);
1669 if (firstitem == NULL) {
1670 if (PyErr_Occurred())
1671 goto BatchFailed;
1672
1673 /* nothing more to add */
1674 break;
1675 }
1676
1677 /* Try to get a second item */
1678 obj = PyIter_Next(iter);
1679 if (obj == NULL) {
1680 if (PyErr_Occurred())
1681 goto BatchFailed;
1682
1683 /* Only one item to write */
1684 if (save(self, firstitem, 0) < 0)
1685 goto BatchFailed;
1686 if (self->write_func(self, &append, 1) < 0)
1687 goto BatchFailed;
1688 Py_CLEAR(firstitem);
1689 break;
1690 }
1691
1692 /* More than one item to write */
1693
1694 /* Pump out MARK, items, APPENDS. */
1695 if (self->write_func(self, &MARKv, 1) < 0)
1696 goto BatchFailed;
1697
1698 if (save(self, firstitem, 0) < 0)
1699 goto BatchFailed;
1700 Py_CLEAR(firstitem);
1701 n = 1;
1702
1703 /* Fetch and save up to BATCHSIZE items */
1704 while (obj) {
1705 if (save(self, obj, 0) < 0)
1706 goto BatchFailed;
1707 Py_CLEAR(obj);
1708 n += 1;
1709
1710 if (n == BATCHSIZE)
1711 break;
1712
1713 obj = PyIter_Next(iter);
1714 if (obj == NULL) {
1715 if (PyErr_Occurred())
1716 goto BatchFailed;
1717 break;
1718 }
1719 }
1720
1721 if (self->write_func(self, &appends, 1) < 0)
1722 goto BatchFailed;
1723
1724 } while (n == BATCHSIZE);
1725 return 0;
1726
1727 BatchFailed:
1728 Py_XDECREF(firstitem);
1729 Py_XDECREF(obj);
1730 return -1;
1731 }
1732
1733 static int
save_list(Picklerobject * self,PyObject * args)1734 save_list(Picklerobject *self, PyObject *args)
1735 {
1736 int res = -1;
1737 char s[3];
1738 Py_ssize_t len;
1739 PyObject *iter;
1740
1741 if (self->fast && !fast_save_enter(self, args))
1742 goto finally;
1743
1744 /* Create an empty list. */
1745 if (self->bin) {
1746 s[0] = EMPTY_LIST;
1747 len = 1;
1748 }
1749 else {
1750 s[0] = MARK;
1751 s[1] = LIST;
1752 len = 2;
1753 }
1754
1755 if (self->write_func(self, s, len) < 0)
1756 goto finally;
1757
1758 /* Get list length, and bow out early if empty. */
1759 if ((len = PyList_Size(args)) < 0)
1760 goto finally;
1761
1762 /* Memoize. */
1763 if (len == 0) {
1764 if (put(self, args) >= 0)
1765 res = 0;
1766 goto finally;
1767 }
1768 if (put2(self, args) < 0)
1769 goto finally;
1770
1771 /* Materialize the list elements. */
1772 iter = PyObject_GetIter(args);
1773 if (iter == NULL)
1774 goto finally;
1775
1776 if (Py_EnterRecursiveCall(" while pickling an object") == 0)
1777 {
1778 res = batch_list(self, iter);
1779 Py_LeaveRecursiveCall();
1780 }
1781 Py_DECREF(iter);
1782
1783 finally:
1784 if (self->fast && !fast_save_leave(self, args))
1785 res = -1;
1786
1787 return res;
1788 }
1789
1790
1791 /* iter is an iterator giving (key, value) pairs, and we batch up chunks of
1792 * MARK key value ... key value SETITEMS
1793 * opcode sequences. Calling code should have arranged to first create an
1794 * empty dict, or dict-like object, for the SETITEMS to operate on.
1795 * Returns 0 on success, <0 on error.
1796 *
1797 * This is very much like batch_list(). The difference between saving
1798 * elements directly, and picking apart two-tuples, is so long-winded at
1799 * the C level, though, that attempts to combine these routines were too
1800 * ugly to bear.
1801 */
1802 static int
batch_dict(Picklerobject * self,PyObject * iter)1803 batch_dict(Picklerobject *self, PyObject *iter)
1804 {
1805 PyObject *p = NULL;
1806 PyObject *firstitem = NULL;
1807 int i, n;
1808
1809 static char setitem = SETITEM;
1810 static char setitems = SETITEMS;
1811
1812 assert(iter != NULL);
1813
1814 if (self->proto == 0) {
1815 /* SETITEMS isn't available; do one at a time. */
1816 for (;;) {
1817 p = PyIter_Next(iter);
1818 if (p == NULL) {
1819 if (PyErr_Occurred())
1820 return -1;
1821 break;
1822 }
1823 if (!PyTuple_Check(p) || PyTuple_Size(p) != 2) {
1824 PyErr_SetString(PyExc_TypeError, "dict items "
1825 "iterator must return 2-tuples");
1826 return -1;
1827 }
1828 i = save(self, PyTuple_GET_ITEM(p, 0), 0);
1829 if (i >= 0)
1830 i = save(self, PyTuple_GET_ITEM(p, 1), 0);
1831 Py_DECREF(p);
1832 if (i < 0)
1833 return -1;
1834 if (self->write_func(self, &setitem, 1) < 0)
1835 return -1;
1836 }
1837 return 0;
1838 }
1839
1840 /* proto > 0: write in batches of BATCHSIZE. */
1841 do {
1842 /* Get first item */
1843 firstitem = PyIter_Next(iter);
1844 if (firstitem == NULL) {
1845 if (PyErr_Occurred())
1846 goto BatchFailed;
1847
1848 /* nothing more to add */
1849 break;
1850 }
1851 if (!PyTuple_Check(firstitem) || PyTuple_Size(firstitem) != 2) {
1852 PyErr_SetString(PyExc_TypeError, "dict items "
1853 "iterator must return 2-tuples");
1854 goto BatchFailed;
1855 }
1856
1857 /* Try to get a second item */
1858 p = PyIter_Next(iter);
1859 if (p == NULL) {
1860 if (PyErr_Occurred())
1861 goto BatchFailed;
1862
1863 /* Only one item to write */
1864 if (save(self, PyTuple_GET_ITEM(firstitem, 0), 0) < 0)
1865 goto BatchFailed;
1866 if (save(self, PyTuple_GET_ITEM(firstitem, 1), 0) < 0)
1867 goto BatchFailed;
1868 if (self->write_func(self, &setitem, 1) < 0)
1869 goto BatchFailed;
1870 Py_CLEAR(firstitem);
1871 break;
1872 }
1873
1874 /* More than one item to write */
1875
1876 /* Pump out MARK, items, SETITEMS. */
1877 if (self->write_func(self, &MARKv, 1) < 0)
1878 goto BatchFailed;
1879
1880 if (save(self, PyTuple_GET_ITEM(firstitem, 0), 0) < 0)
1881 goto BatchFailed;
1882 if (save(self, PyTuple_GET_ITEM(firstitem, 1), 0) < 0)
1883 goto BatchFailed;
1884 Py_CLEAR(firstitem);
1885 n = 1;
1886
1887 /* Fetch and save up to BATCHSIZE items */
1888 while (p) {
1889 if (!PyTuple_Check(p) || PyTuple_Size(p) != 2) {
1890 PyErr_SetString(PyExc_TypeError, "dict items "
1891 "iterator must return 2-tuples");
1892 goto BatchFailed;
1893 }
1894 if (save(self, PyTuple_GET_ITEM(p, 0), 0) < 0)
1895 goto BatchFailed;
1896 if (save(self, PyTuple_GET_ITEM(p, 1), 0) < 0)
1897 goto BatchFailed;
1898 Py_CLEAR(p);
1899 n += 1;
1900
1901 if (n == BATCHSIZE)
1902 break;
1903
1904 p = PyIter_Next(iter);
1905 if (p == NULL) {
1906 if (PyErr_Occurred())
1907 goto BatchFailed;
1908 break;
1909 }
1910 }
1911
1912 if (self->write_func(self, &setitems, 1) < 0)
1913 goto BatchFailed;
1914
1915 } while (n == BATCHSIZE);
1916 return 0;
1917
1918 BatchFailed:
1919 Py_XDECREF(firstitem);
1920 Py_XDECREF(p);
1921 return -1;
1922 }
1923
1924 /* This is a variant of batch_dict() above that specializes for dicts, with no
1925 * support for dict subclasses. Like batch_dict(), we batch up chunks of
1926 * MARK key value ... key value SETITEMS
1927 * opcode sequences. Calling code should have arranged to first create an
1928 * empty dict, or dict-like object, for the SETITEMS to operate on.
1929 * Returns 0 on success, -1 on error.
1930 *
1931 * Note that this currently doesn't work for protocol 0.
1932 */
1933 static int
batch_dict_exact(Picklerobject * self,PyObject * obj)1934 batch_dict_exact(Picklerobject *self, PyObject *obj)
1935 {
1936 PyObject *key = NULL, *value = NULL;
1937 int i;
1938 Py_ssize_t dict_size, ppos = 0;
1939
1940 static char setitem = SETITEM;
1941 static char setitems = SETITEMS;
1942
1943 assert(obj != NULL);
1944 assert(self->proto > 0);
1945
1946 dict_size = PyDict_Size(obj);
1947
1948 /* Special-case len(d) == 1 to save space. */
1949 if (dict_size == 1) {
1950 PyDict_Next(obj, &ppos, &key, &value);
1951 if (save(self, key, 0) < 0)
1952 return -1;
1953 if (save(self, value, 0) < 0)
1954 return -1;
1955 if (self->write_func(self, &setitem, 1) < 0)
1956 return -1;
1957 return 0;
1958 }
1959
1960 /* Write in batches of BATCHSIZE. */
1961 do {
1962 i = 0;
1963 if (self->write_func(self, &MARKv, 1) < 0)
1964 return -1;
1965 while (PyDict_Next(obj, &ppos, &key, &value)) {
1966 if (save(self, key, 0) < 0)
1967 return -1;
1968 if (save(self, value, 0) < 0)
1969 return -1;
1970 if (++i == BATCHSIZE)
1971 break;
1972 }
1973 if (self->write_func(self, &setitems, 1) < 0)
1974 return -1;
1975 if (PyDict_Size(obj) != dict_size) {
1976 PyErr_Format(
1977 PyExc_RuntimeError,
1978 "dictionary changed size during iteration");
1979 return -1;
1980 }
1981
1982 } while (i == BATCHSIZE);
1983 return 0;
1984 }
1985
1986 static int
save_dict(Picklerobject * self,PyObject * args)1987 save_dict(Picklerobject *self, PyObject *args)
1988 {
1989 int res = -1;
1990 char s[3];
1991 Py_ssize_t len;
1992
1993 if (self->fast && !fast_save_enter(self, args))
1994 goto finally;
1995
1996 /* Create an empty dict. */
1997 if (self->bin) {
1998 s[0] = EMPTY_DICT;
1999 len = 1;
2000 }
2001 else {
2002 s[0] = MARK;
2003 s[1] = DICT;
2004 len = 2;
2005 }
2006
2007 if (self->write_func(self, s, len) < 0)
2008 goto finally;
2009
2010 /* Get dict size, and bow out early if empty. */
2011 if ((len = PyDict_Size(args)) < 0)
2012 goto finally;
2013
2014 if (len == 0) {
2015 if (put(self, args) >= 0)
2016 res = 0;
2017 goto finally;
2018 }
2019 if (put2(self, args) < 0)
2020 goto finally;
2021
2022 /* Materialize the dict items. */
2023 if (PyDict_CheckExact(args) && self->proto > 0) {
2024 /* We can take certain shortcuts if we know this is a dict and
2025 not a dict subclass. */
2026 if (Py_EnterRecursiveCall(" while pickling an object") == 0) {
2027 res = batch_dict_exact(self, args);
2028 Py_LeaveRecursiveCall();
2029 }
2030 } else {
2031 PyObject *iter = PyObject_CallMethod(args, "iteritems", "()");
2032 if (iter == NULL)
2033 goto finally;
2034 if (Py_EnterRecursiveCall(" while pickling an object") == 0) {
2035 res = batch_dict(self, iter);
2036 Py_LeaveRecursiveCall();
2037 }
2038 Py_DECREF(iter);
2039 }
2040
2041 finally:
2042 if (self->fast && !fast_save_leave(self, args))
2043 res = -1;
2044
2045 return res;
2046 }
2047
2048
2049 static int
save_inst(Picklerobject * self,PyObject * args)2050 save_inst(Picklerobject *self, PyObject *args)
2051 {
2052 PyObject *class = 0, *module = 0, *name = 0, *state = 0,
2053 *getinitargs_func = 0, *getstate_func = 0, *class_args = 0;
2054 char *module_str, *name_str;
2055 int module_size, name_size, res = -1;
2056
2057 static char inst = INST, obj = OBJ, build = BUILD;
2058
2059 if (self->fast && !fast_save_enter(self, args))
2060 goto finally;
2061
2062 if (self->write_func(self, &MARKv, 1) < 0)
2063 goto finally;
2064
2065 if (!( class = PyObject_GetAttr(args, __class___str)))
2066 goto finally;
2067
2068 if (self->bin) {
2069 if (save(self, class, 0) < 0)
2070 goto finally;
2071 }
2072
2073 if ((getinitargs_func = PyObject_GetAttr(args, __getinitargs___str))) {
2074 PyObject *element = 0;
2075 Py_ssize_t i, len;
2076
2077 if (!( class_args =
2078 PyObject_Call(getinitargs_func, empty_tuple, NULL)))
2079 goto finally;
2080
2081 if ((len = PyObject_Size(class_args)) < 0)
2082 goto finally;
2083
2084 for (i = 0; i < len; i++) {
2085 if (!( element = PySequence_GetItem(class_args, i)))
2086 goto finally;
2087
2088 if (save(self, element, 0) < 0) {
2089 Py_DECREF(element);
2090 goto finally;
2091 }
2092
2093 Py_DECREF(element);
2094 }
2095 }
2096 else {
2097 if (PyErr_ExceptionMatches(PyExc_AttributeError))
2098 PyErr_Clear();
2099 else
2100 goto finally;
2101 }
2102
2103 if (!self->bin) {
2104 if (!( name = ((PyClassObject *)class)->cl_name )) {
2105 PyErr_SetString(PicklingError, "class has no name");
2106 goto finally;
2107 }
2108
2109 if (!( module = whichmodule(class, name)))
2110 goto finally;
2111
2112
2113 if ((module_size = PyString_Size(module)) < 0 ||
2114 (name_size = PyString_Size(name)) < 0)
2115 goto finally;
2116
2117 module_str = PyString_AS_STRING((PyStringObject *)module);
2118 name_str = PyString_AS_STRING((PyStringObject *)name);
2119
2120 if (self->write_func(self, &inst, 1) < 0)
2121 goto finally;
2122
2123 if (self->write_func(self, module_str, module_size) < 0)
2124 goto finally;
2125
2126 if (self->write_func(self, "\n", 1) < 0)
2127 goto finally;
2128
2129 if (self->write_func(self, name_str, name_size) < 0)
2130 goto finally;
2131
2132 if (self->write_func(self, "\n", 1) < 0)
2133 goto finally;
2134 }
2135 else if (self->write_func(self, &obj, 1) < 0) {
2136 goto finally;
2137 }
2138
2139 if ((getstate_func = PyObject_GetAttr(args, __getstate___str))) {
2140 state = PyObject_Call(getstate_func, empty_tuple, NULL);
2141 if (!state)
2142 goto finally;
2143 }
2144 else {
2145 if (PyErr_ExceptionMatches(PyExc_AttributeError))
2146 PyErr_Clear();
2147 else
2148 goto finally;
2149
2150 if (!( state = PyObject_GetAttr(args, __dict___str))) {
2151 if (PyErr_ExceptionMatches(PyExc_AttributeError))
2152 PyErr_Clear();
2153 else
2154 goto finally;
2155 res = 0;
2156 goto finally;
2157 }
2158 }
2159
2160 if (!PyDict_Check(state)) {
2161 if (put2(self, args) < 0)
2162 goto finally;
2163 }
2164 else {
2165 if (put(self, args) < 0)
2166 goto finally;
2167 }
2168
2169 if (save(self, state, 0) < 0)
2170 goto finally;
2171
2172 if (self->write_func(self, &build, 1) < 0)
2173 goto finally;
2174
2175 res = 0;
2176
2177 finally:
2178 if (self->fast && !fast_save_leave(self, args))
2179 res = -1;
2180
2181 Py_XDECREF(module);
2182 Py_XDECREF(class);
2183 Py_XDECREF(state);
2184 Py_XDECREF(getinitargs_func);
2185 Py_XDECREF(getstate_func);
2186 Py_XDECREF(class_args);
2187
2188 return res;
2189 }
2190
2191
2192 static int
save_global(Picklerobject * self,PyObject * args,PyObject * name)2193 save_global(Picklerobject *self, PyObject *args, PyObject *name)
2194 {
2195 PyObject *global_name = 0, *module = 0, *mod = 0, *klass = 0;
2196 char *name_str, *module_str;
2197 int module_size, name_size, res = -1;
2198
2199 static char global = GLOBAL;
2200
2201 if (name) {
2202 global_name = name;
2203 Py_INCREF(global_name);
2204 }
2205 else {
2206 if (!( global_name = PyObject_GetAttr(args, __name___str)))
2207 goto finally;
2208 }
2209
2210 if (!( module = whichmodule(args, global_name)))
2211 goto finally;
2212
2213 if ((module_size = PyString_Size(module)) < 0 ||
2214 (name_size = PyString_Size(global_name)) < 0)
2215 goto finally;
2216
2217 module_str = PyString_AS_STRING((PyStringObject *)module);
2218 name_str = PyString_AS_STRING((PyStringObject *)global_name);
2219
2220 /* XXX This can be doing a relative import. Clearly it shouldn't,
2221 but I don't know how to stop it. :-( */
2222 mod = PyImport_ImportModule(module_str);
2223 if (mod == NULL) {
2224 cPickle_ErrFormat(PicklingError,
2225 "Can't pickle %s: import of module %s "
2226 "failed",
2227 "OS", args, module);
2228 goto finally;
2229 }
2230 klass = PyObject_GetAttrString(mod, name_str);
2231 if (klass == NULL) {
2232 cPickle_ErrFormat(PicklingError,
2233 "Can't pickle %s: attribute lookup %s.%s "
2234 "failed",
2235 "OSS", args, module, global_name);
2236 goto finally;
2237 }
2238 if (klass != args) {
2239 Py_DECREF(klass);
2240 cPickle_ErrFormat(PicklingError,
2241 "Can't pickle %s: it's not the same object "
2242 "as %s.%s",
2243 "OSS", args, module, global_name);
2244 goto finally;
2245 }
2246 Py_DECREF(klass);
2247
2248 if (self->proto >= 2) {
2249 /* See whether this is in the extension registry, and if
2250 * so generate an EXT opcode.
2251 */
2252 PyObject *py_code; /* extension code as Python object */
2253 long code; /* extension code as C value */
2254 char c_str[5];
2255 int n;
2256
2257 PyTuple_SET_ITEM(two_tuple, 0, module);
2258 PyTuple_SET_ITEM(two_tuple, 1, global_name);
2259 py_code = PyDict_GetItem(extension_registry, two_tuple);
2260 if (py_code == NULL)
2261 goto gen_global; /* not registered */
2262
2263 /* Verify py_code has the right type and value. */
2264 if (!PyInt_Check(py_code)) {
2265 cPickle_ErrFormat(PicklingError, "Can't pickle %s: "
2266 "extension code %s isn't an integer",
2267 "OO", args, py_code);
2268 goto finally;
2269 }
2270 code = PyInt_AS_LONG(py_code);
2271 if (code <= 0 || code > 0x7fffffffL) {
2272 cPickle_ErrFormat(PicklingError, "Can't pickle %s: "
2273 "extension code %ld is out of range",
2274 "Ol", args, code);
2275 goto finally;
2276 }
2277
2278 /* Generate an EXT opcode. */
2279 if (code <= 0xff) {
2280 c_str[0] = EXT1;
2281 c_str[1] = (char)code;
2282 n = 2;
2283 }
2284 else if (code <= 0xffff) {
2285 c_str[0] = EXT2;
2286 c_str[1] = (char)(code & 0xff);
2287 c_str[2] = (char)((code >> 8) & 0xff);
2288 n = 3;
2289 }
2290 else {
2291 c_str[0] = EXT4;
2292 c_str[1] = (char)(code & 0xff);
2293 c_str[2] = (char)((code >> 8) & 0xff);
2294 c_str[3] = (char)((code >> 16) & 0xff);
2295 c_str[4] = (char)((code >> 24) & 0xff);
2296 n = 5;
2297 }
2298
2299 if (self->write_func(self, c_str, n) >= 0)
2300 res = 0;
2301 goto finally; /* and don't memoize */
2302 }
2303
2304 gen_global:
2305 if (self->write_func(self, &global, 1) < 0)
2306 goto finally;
2307
2308 if (self->write_func(self, module_str, module_size) < 0)
2309 goto finally;
2310
2311 if (self->write_func(self, "\n", 1) < 0)
2312 goto finally;
2313
2314 if (self->write_func(self, name_str, name_size) < 0)
2315 goto finally;
2316
2317 if (self->write_func(self, "\n", 1) < 0)
2318 goto finally;
2319
2320 if (put(self, args) < 0)
2321 goto finally;
2322
2323 res = 0;
2324
2325 finally:
2326 Py_XDECREF(module);
2327 Py_XDECREF(global_name);
2328 Py_XDECREF(mod);
2329
2330 return res;
2331 }
2332
2333 static int
save_pers(Picklerobject * self,PyObject * args,PyObject * f)2334 save_pers(Picklerobject *self, PyObject *args, PyObject *f)
2335 {
2336 PyObject *pid = 0;
2337 Py_ssize_t size;
2338 int res = -1;
2339
2340 static char persid = PERSID, binpersid = BINPERSID;
2341
2342 Py_INCREF(args);
2343 ARG_TUP(self, args);
2344 if (self->arg) {
2345 pid = PyObject_Call(f, self->arg, NULL);
2346 FREE_ARG_TUP(self);
2347 }
2348 if (! pid) return -1;
2349
2350 if (pid != Py_None) {
2351 if (!self->bin) {
2352 if (!PyString_Check(pid)) {
2353 PyErr_SetString(PicklingError,
2354 "persistent id must be string");
2355 goto finally;
2356 }
2357
2358 if (self->write_func(self, &persid, 1) < 0)
2359 goto finally;
2360
2361 if ((size = PyString_Size(pid)) < 0)
2362 goto finally;
2363
2364 if (self->write_func(self,
2365 PyString_AS_STRING(
2366 (PyStringObject *)pid),
2367 size) < 0)
2368 goto finally;
2369
2370 if (self->write_func(self, "\n", 1) < 0)
2371 goto finally;
2372
2373 res = 1;
2374 goto finally;
2375 }
2376 else if (save(self, pid, 1) >= 0) {
2377 if (self->write_func(self, &binpersid, 1) < 0)
2378 res = -1;
2379 else
2380 res = 1;
2381 }
2382
2383 goto finally;
2384 }
2385
2386 res = 0;
2387
2388 finally:
2389 Py_XDECREF(pid);
2390
2391 return res;
2392 }
2393
2394 /* We're saving ob, and args is the 2-thru-5 tuple returned by the
2395 * appropriate __reduce__ method for ob.
2396 */
2397 static int
save_reduce(Picklerobject * self,PyObject * args,PyObject * fn,PyObject * ob)2398 save_reduce(Picklerobject *self, PyObject *args, PyObject *fn, PyObject *ob)
2399 {
2400 PyObject *callable;
2401 PyObject *argtup;
2402 PyObject *state = NULL;
2403 PyObject *listitems = Py_None;
2404 PyObject *dictitems = Py_None;
2405 Py_ssize_t size;
2406
2407 int use_newobj = self->proto >= 2;
2408
2409 static char reduce = REDUCE;
2410 static char build = BUILD;
2411 static char newobj = NEWOBJ;
2412
2413 size = PyTuple_Size(args);
2414 if (size < 2 || size > 5) {
2415 cPickle_ErrFormat(PicklingError, "tuple returned by "
2416 "%s must contain 2 through 5 elements",
2417 "O", fn);
2418 return -1;
2419 }
2420
2421 if (! PyArg_UnpackTuple(args, "save_reduce", 2, 5,
2422 &callable,
2423 &argtup,
2424 &state,
2425 &listitems,
2426 &dictitems))
2427 return -1;
2428
2429 if (!PyTuple_Check(argtup)) {
2430 cPickle_ErrFormat(PicklingError, "Second element of "
2431 "tuple returned by %s must be a tuple",
2432 "O", fn);
2433 return -1;
2434 }
2435
2436 if (state == Py_None)
2437 state = NULL;
2438
2439 if (listitems == Py_None)
2440 listitems = NULL;
2441 else if (!PyIter_Check(listitems)) {
2442 cPickle_ErrFormat(PicklingError, "Fourth element of "
2443 "tuple returned by %s must be an iterator, not %s",
2444 "Os", fn, Py_TYPE(listitems)->tp_name);
2445 return -1;
2446 }
2447
2448 if (dictitems == Py_None)
2449 dictitems = NULL;
2450 else if (!PyIter_Check(dictitems)) {
2451 cPickle_ErrFormat(PicklingError, "Fifth element of "
2452 "tuple returned by %s must be an iterator, not %s",
2453 "Os", fn, Py_TYPE(dictitems)->tp_name);
2454 return -1;
2455 }
2456
2457 /* Protocol 2 special case: if callable's name is __newobj__, use
2458 * NEWOBJ. This consumes a lot of code.
2459 */
2460 if (use_newobj) {
2461 PyObject *temp = PyObject_GetAttr(callable, __name___str);
2462
2463 if (temp == NULL) {
2464 if (PyErr_ExceptionMatches(PyExc_AttributeError))
2465 PyErr_Clear();
2466 else
2467 return -1;
2468 use_newobj = 0;
2469 }
2470 else {
2471 use_newobj = PyString_Check(temp) &&
2472 strcmp(PyString_AS_STRING(temp),
2473 "__newobj__") == 0;
2474 Py_DECREF(temp);
2475 }
2476 }
2477 if (use_newobj) {
2478 PyObject *cls;
2479 PyObject *newargtup;
2480 Py_ssize_t n, i;
2481
2482 /* Sanity checks. */
2483 n = PyTuple_Size(argtup);
2484 if (n < 1) {
2485 PyErr_SetString(PicklingError, "__newobj__ arglist "
2486 "is empty");
2487 return -1;
2488 }
2489
2490 cls = PyTuple_GET_ITEM(argtup, 0);
2491 if (! PyObject_HasAttrString(cls, "__new__")) {
2492 PyErr_SetString(PicklingError, "args[0] from "
2493 "__newobj__ args has no __new__");
2494 return -1;
2495 }
2496
2497 /* XXX How could ob be NULL? */
2498 if (ob != NULL) {
2499 PyObject *ob_dot_class;
2500
2501 ob_dot_class = PyObject_GetAttr(ob, __class___str);
2502 if (ob_dot_class == NULL) {
2503 if (PyErr_ExceptionMatches(
2504 PyExc_AttributeError))
2505 PyErr_Clear();
2506 else
2507 return -1;
2508 }
2509 i = ob_dot_class != cls; /* true iff a problem */
2510 Py_XDECREF(ob_dot_class);
2511 if (i) {
2512 PyErr_SetString(PicklingError, "args[0] from "
2513 "__newobj__ args has the wrong class");
2514 return -1;
2515 }
2516 }
2517
2518 /* Save the class and its __new__ arguments. */
2519 if (save(self, cls, 0) < 0)
2520 return -1;
2521
2522 newargtup = PyTuple_New(n-1); /* argtup[1:] */
2523 if (newargtup == NULL)
2524 return -1;
2525 for (i = 1; i < n; ++i) {
2526 PyObject *temp = PyTuple_GET_ITEM(argtup, i);
2527 Py_INCREF(temp);
2528 PyTuple_SET_ITEM(newargtup, i-1, temp);
2529 }
2530 i = save(self, newargtup, 0);
2531 Py_DECREF(newargtup);
2532 if (i < 0)
2533 return -1;
2534
2535 /* Add NEWOBJ opcode. */
2536 if (self->write_func(self, &newobj, 1) < 0)
2537 return -1;
2538 }
2539 else {
2540 /* Not using NEWOBJ. */
2541 if (save(self, callable, 0) < 0 ||
2542 save(self, argtup, 0) < 0 ||
2543 self->write_func(self, &reduce, 1) < 0)
2544 return -1;
2545 }
2546
2547 /* Memoize. */
2548 /* XXX How can ob be NULL? */
2549 if (ob != NULL) {
2550 /* If the object is already in the memo, this means it is
2551 recursive. In this case, throw away everything we put on the
2552 stack, and fetch the object back from the memo. */
2553 if (Py_REFCNT(ob) > 1 && !self->fast) {
2554 PyObject *py_ob_id = PyLong_FromVoidPtr(ob);
2555 if (!py_ob_id)
2556 return -1;
2557 if (PyDict_GetItem(self->memo, py_ob_id)) {
2558 const char pop_op = POP;
2559 if (self->write_func(self, &pop_op, 1) < 0 ||
2560 get(self, py_ob_id) < 0) {
2561 Py_DECREF(py_ob_id);
2562 return -1;
2563 }
2564 Py_DECREF(py_ob_id);
2565 return 0;
2566 }
2567 Py_DECREF(py_ob_id);
2568 if (PyErr_Occurred())
2569 return -1;
2570 }
2571 if (state && !PyDict_Check(state)) {
2572 if (put2(self, ob) < 0)
2573 return -1;
2574 }
2575 else if (put(self, ob) < 0)
2576 return -1;
2577 }
2578
2579
2580 if (listitems && batch_list(self, listitems) < 0)
2581 return -1;
2582
2583 if (dictitems && batch_dict(self, dictitems) < 0)
2584 return -1;
2585
2586 if (state) {
2587 if (save(self, state, 0) < 0 ||
2588 self->write_func(self, &build, 1) < 0)
2589 return -1;
2590 }
2591
2592 return 0;
2593 }
2594
2595 static int
save(Picklerobject * self,PyObject * args,int pers_save)2596 save(Picklerobject *self, PyObject *args, int pers_save)
2597 {
2598 PyTypeObject *type;
2599 PyObject *py_ob_id = 0, *__reduce__ = 0, *t = 0;
2600 int res = -1;
2601 int tmp;
2602
2603 if (Py_EnterRecursiveCall(" while pickling an object"))
2604 return -1;
2605
2606 if (!pers_save && self->pers_func) {
2607 if ((tmp = save_pers(self, args, self->pers_func)) != 0) {
2608 res = tmp;
2609 goto finally;
2610 }
2611 }
2612
2613 if (args == Py_None) {
2614 res = save_none(self, args);
2615 goto finally;
2616 }
2617
2618 type = Py_TYPE(args);
2619
2620 switch (type->tp_name[0]) {
2621 case 'b':
2622 if (args == Py_False || args == Py_True) {
2623 res = save_bool(self, args);
2624 goto finally;
2625 }
2626 break;
2627 case 'i':
2628 if (type == &PyInt_Type) {
2629 res = save_int(self, args);
2630 goto finally;
2631 }
2632 break;
2633
2634 case 'l':
2635 if (type == &PyLong_Type) {
2636 res = save_long(self, args);
2637 goto finally;
2638 }
2639 break;
2640
2641 case 'f':
2642 if (type == &PyFloat_Type) {
2643 res = save_float(self, args);
2644 goto finally;
2645 }
2646 break;
2647
2648 case 't':
2649 if (type == &PyTuple_Type && PyTuple_Size(args) == 0) {
2650 res = save_tuple(self, args);
2651 goto finally;
2652 }
2653 break;
2654
2655 case 's':
2656 if ((type == &PyString_Type) && (PyString_GET_SIZE(args) < 2)) {
2657 res = save_string(self, args, 0);
2658 goto finally;
2659 }
2660 break;
2661
2662 #ifdef Py_USING_UNICODE
2663 case 'u':
2664 if ((type == &PyUnicode_Type) && (PyString_GET_SIZE(args) < 2)) {
2665 res = save_unicode(self, args, 0);
2666 goto finally;
2667 }
2668 break;
2669 #endif
2670 }
2671
2672 if (Py_REFCNT(args) > 1) {
2673 if (!( py_ob_id = PyLong_FromVoidPtr(args)))
2674 goto finally;
2675
2676 if (PyDict_GetItem(self->memo, py_ob_id)) {
2677 if (get(self, py_ob_id) < 0)
2678 goto finally;
2679
2680 res = 0;
2681 goto finally;
2682 }
2683 }
2684
2685 switch (type->tp_name[0]) {
2686 case 's':
2687 if (type == &PyString_Type) {
2688 res = save_string(self, args, 1);
2689 goto finally;
2690 }
2691 break;
2692
2693 #ifdef Py_USING_UNICODE
2694 case 'u':
2695 if (type == &PyUnicode_Type) {
2696 res = save_unicode(self, args, 1);
2697 goto finally;
2698 }
2699 break;
2700 #endif
2701
2702 case 't':
2703 if (type == &PyTuple_Type) {
2704 res = save_tuple(self, args);
2705 goto finally;
2706 }
2707 if (type == &PyType_Type) {
2708 res = save_global(self, args, NULL);
2709 goto finally;
2710 }
2711 break;
2712
2713 case 'l':
2714 if (type == &PyList_Type) {
2715 res = save_list(self, args);
2716 goto finally;
2717 }
2718 break;
2719
2720 case 'd':
2721 if (type == &PyDict_Type) {
2722 res = save_dict(self, args);
2723 goto finally;
2724 }
2725 break;
2726
2727 case 'i':
2728 if (type == &PyInstance_Type) {
2729 res = save_inst(self, args);
2730 goto finally;
2731 }
2732 break;
2733
2734 case 'c':
2735 if (type == &PyClass_Type) {
2736 res = save_global(self, args, NULL);
2737 goto finally;
2738 }
2739 break;
2740
2741 case 'f':
2742 if (type == &PyFunction_Type) {
2743 res = save_global(self, args, NULL);
2744 if (res && PyErr_ExceptionMatches(PickleError)) {
2745 /* fall back to reduce */
2746 PyErr_Clear();
2747 break;
2748 }
2749 goto finally;
2750 }
2751 break;
2752
2753 case 'b':
2754 if (type == &PyCFunction_Type) {
2755 res = save_global(self, args, NULL);
2756 goto finally;
2757 }
2758 }
2759
2760 if (!pers_save && self->inst_pers_func) {
2761 if ((tmp = save_pers(self, args, self->inst_pers_func)) != 0) {
2762 res = tmp;
2763 goto finally;
2764 }
2765 }
2766
2767 /* Get a reduction callable, and call it. This may come from
2768 * copy_reg.dispatch_table, the object's __reduce_ex__ method,
2769 * or the object's __reduce__ method.
2770 */
2771 __reduce__ = PyDict_GetItem(dispatch_table, (PyObject *)type);
2772 if (__reduce__ != NULL) {
2773 Py_INCREF(__reduce__);
2774 Py_INCREF(args);
2775 ARG_TUP(self, args);
2776 if (self->arg) {
2777 t = PyObject_Call(__reduce__, self->arg, NULL);
2778 FREE_ARG_TUP(self);
2779 }
2780 }
2781 else {
2782 if (PyType_IsSubtype(type, &PyType_Type)) {
2783 res = save_global(self, args, NULL);
2784 goto finally;
2785 }
2786
2787 /* Check for a __reduce_ex__ method. */
2788 __reduce__ = PyObject_GetAttr(args, __reduce_ex___str);
2789 if (__reduce__ != NULL) {
2790 t = PyInt_FromLong(self->proto);
2791 if (t != NULL) {
2792 ARG_TUP(self, t);
2793 t = NULL;
2794 if (self->arg) {
2795 t = PyObject_Call(__reduce__,
2796 self->arg, NULL);
2797 FREE_ARG_TUP(self);
2798 }
2799 }
2800 }
2801 else {
2802 if (PyErr_ExceptionMatches(PyExc_AttributeError))
2803 PyErr_Clear();
2804 else
2805 goto finally;
2806 /* Check for a __reduce__ method. */
2807 __reduce__ = PyObject_GetAttr(args, __reduce___str);
2808 if (__reduce__ != NULL) {
2809 t = PyObject_Call(__reduce__,
2810 empty_tuple, NULL);
2811 }
2812 else {
2813 PyErr_SetObject(UnpickleableError, args);
2814 goto finally;
2815 }
2816 }
2817 }
2818
2819 if (t == NULL)
2820 goto finally;
2821
2822 if (PyString_Check(t)) {
2823 res = save_global(self, args, t);
2824 goto finally;
2825 }
2826
2827 if (!PyTuple_Check(t)) {
2828 cPickle_ErrFormat(PicklingError, "Value returned by "
2829 "%s must be string or tuple",
2830 "O", __reduce__);
2831 goto finally;
2832 }
2833
2834 res = save_reduce(self, t, __reduce__, args);
2835
2836 finally:
2837 Py_LeaveRecursiveCall();
2838 Py_XDECREF(py_ob_id);
2839 Py_XDECREF(__reduce__);
2840 Py_XDECREF(t);
2841
2842 return res;
2843 }
2844
2845
2846 static int
dump(Picklerobject * self,PyObject * args)2847 dump(Picklerobject *self, PyObject *args)
2848 {
2849 static char stop = STOP;
2850
2851 if (self->proto >= 2) {
2852 char bytes[2];
2853
2854 bytes[0] = PROTO;
2855 assert(self->proto >= 0 && self->proto < 256);
2856 bytes[1] = (char)self->proto;
2857 if (self->write_func(self, bytes, 2) < 0)
2858 return -1;
2859 }
2860
2861 if (save(self, args, 0) < 0)
2862 return -1;
2863
2864 if (self->write_func(self, &stop, 1) < 0)
2865 return -1;
2866
2867 if (self->write_func(self, NULL, 0) < 0)
2868 return -1;
2869
2870 return 0;
2871 }
2872
2873 static PyObject *
Pickle_clear_memo(Picklerobject * self,PyObject * args)2874 Pickle_clear_memo(Picklerobject *self, PyObject *args)
2875 {
2876 if (self->memo)
2877 PyDict_Clear(self->memo);
2878 Py_INCREF(Py_None);
2879 return Py_None;
2880 }
2881
2882 static PyObject *
Pickle_getvalue(Picklerobject * self,PyObject * args)2883 Pickle_getvalue(Picklerobject *self, PyObject *args)
2884 {
2885 Py_ssize_t l, i, rsize, ssize, clear=1, lm;
2886 long ik;
2887 PyObject *k, *r;
2888 char *s, *p, *have_get;
2889 Pdata *data;
2890
2891 /* Can be called by Python code or C code */
2892 if (args && !PyArg_ParseTuple(args, "|i:getvalue", &clear))
2893 return NULL;
2894
2895 /* Check to make sure we are based on a list */
2896 if (! Pdata_Check(self->file)) {
2897 PyErr_SetString(PicklingError,
2898 "Attempt to getvalue() a non-list-based pickler");
2899 return NULL;
2900 }
2901
2902 /* flush write buffer */
2903 if (write_other(self, NULL, 0) < 0) return NULL;
2904
2905 data=(Pdata*)self->file;
2906 l=data->length;
2907
2908 /* set up an array to hold get/put status */
2909 lm = PyDict_Size(self->memo);
2910 if (lm < 0) return NULL;
2911 lm++;
2912 have_get = malloc(lm);
2913 if (have_get == NULL) return PyErr_NoMemory();
2914 memset(have_get, 0, lm);
2915
2916 /* Scan for gets. */
2917 for (rsize = 0, i = l; --i >= 0; ) {
2918 k = data->data[i];
2919
2920 if (PyString_Check(k))
2921 rsize += PyString_GET_SIZE(k);
2922
2923 else if (PyInt_Check(k)) { /* put */
2924 ik = PyInt_AS_LONG((PyIntObject*)k);
2925 if (ik >= lm || ik == 0) {
2926 PyErr_SetString(PicklingError,
2927 "Invalid get data");
2928 goto err;
2929 }
2930 if (have_get[ik]) /* with matching get */
2931 rsize += ik < 256 ? 2 : 5;
2932 }
2933
2934 else if (! (PyTuple_Check(k) &&
2935 PyTuple_GET_SIZE(k) == 2 &&
2936 PyInt_Check((k = PyTuple_GET_ITEM(k, 0))))
2937 ) {
2938 PyErr_SetString(PicklingError,
2939 "Unexpected data in internal list");
2940 goto err;
2941 }
2942
2943 else { /* put */
2944 ik = PyInt_AS_LONG((PyIntObject *)k);
2945 if (ik >= lm || ik == 0) {
2946 PyErr_SetString(PicklingError,
2947 "Invalid get data");
2948 goto err;
2949 }
2950 have_get[ik] = 1;
2951 rsize += ik < 256 ? 2 : 5;
2952 }
2953 }
2954
2955 /* Now generate the result */
2956 r = PyString_FromStringAndSize(NULL, rsize);
2957 if (r == NULL) goto err;
2958 s = PyString_AS_STRING((PyStringObject *)r);
2959
2960 for (i = 0; i < l; i++) {
2961 k = data->data[i];
2962
2963 if (PyString_Check(k)) {
2964 ssize = PyString_GET_SIZE(k);
2965 if (ssize) {
2966 p=PyString_AS_STRING((PyStringObject *)k);
2967 while (--ssize >= 0)
2968 *s++ = *p++;
2969 }
2970 }
2971
2972 else if (PyTuple_Check(k)) { /* get */
2973 ik = PyInt_AS_LONG((PyIntObject *)
2974 PyTuple_GET_ITEM(k, 0));
2975 if (ik < 256) {
2976 *s++ = BINGET;
2977 *s++ = (int)(ik & 0xff);
2978 }
2979 else {
2980 *s++ = LONG_BINGET;
2981 *s++ = (int)(ik & 0xff);
2982 *s++ = (int)((ik >> 8) & 0xff);
2983 *s++ = (int)((ik >> 16) & 0xff);
2984 *s++ = (int)((ik >> 24) & 0xff);
2985 }
2986 }
2987
2988 else { /* put */
2989 ik = PyInt_AS_LONG((PyIntObject*)k);
2990
2991 if (have_get[ik]) { /* with matching get */
2992 if (ik < 256) {
2993 *s++ = BINPUT;
2994 *s++ = (int)(ik & 0xff);
2995 }
2996 else {
2997 *s++ = LONG_BINPUT;
2998 *s++ = (int)(ik & 0xff);
2999 *s++ = (int)((ik >> 8) & 0xff);
3000 *s++ = (int)((ik >> 16) & 0xff);
3001 *s++ = (int)((ik >> 24) & 0xff);
3002 }
3003 }
3004 }
3005 }
3006
3007 if (clear) {
3008 PyDict_Clear(self->memo);
3009 Pdata_clear(data, 0);
3010 }
3011
3012 free(have_get);
3013 return r;
3014 err:
3015 free(have_get);
3016 return NULL;
3017 }
3018
3019 static PyObject *
Pickler_dump(Picklerobject * self,PyObject * args)3020 Pickler_dump(Picklerobject *self, PyObject *args)
3021 {
3022 PyObject *ob;
3023 int get=0;
3024
3025 if (!( PyArg_ParseTuple(args, "O|i:dump", &ob, &get)))
3026 return NULL;
3027
3028 if (dump(self, ob) < 0)
3029 return NULL;
3030
3031 if (get) return Pickle_getvalue(self, NULL);
3032
3033 /* XXX Why does dump() return self? */
3034 Py_INCREF(self);
3035 return (PyObject*)self;
3036 }
3037
3038
3039 static struct PyMethodDef Pickler_methods[] =
3040 {
3041 {"dump", (PyCFunction)Pickler_dump, METH_VARARGS,
3042 PyDoc_STR("dump(object) -- "
3043 "Write an object in pickle format to the object's pickle stream")},
3044 {"clear_memo", (PyCFunction)Pickle_clear_memo, METH_NOARGS,
3045 PyDoc_STR("clear_memo() -- Clear the picklers memo")},
3046 {"getvalue", (PyCFunction)Pickle_getvalue, METH_VARARGS,
3047 PyDoc_STR("getvalue() -- Finish picking a list-based pickle")},
3048 {NULL, NULL} /* sentinel */
3049 };
3050
3051
3052 static Picklerobject *
newPicklerobject(PyObject * file,int proto)3053 newPicklerobject(PyObject *file, int proto)
3054 {
3055 Picklerobject *self;
3056
3057 if (proto < 0)
3058 proto = HIGHEST_PROTOCOL;
3059 if (proto > HIGHEST_PROTOCOL) {
3060 PyErr_Format(PyExc_ValueError, "pickle protocol %d asked for; "
3061 "the highest available protocol is %d",
3062 proto, HIGHEST_PROTOCOL);
3063 return NULL;
3064 }
3065
3066 self = PyObject_GC_New(Picklerobject, &Picklertype);
3067 if (self == NULL)
3068 return NULL;
3069 self->proto = proto;
3070 self->bin = proto > 0;
3071 self->fp = NULL;
3072 self->write = NULL;
3073 self->memo = NULL;
3074 self->arg = NULL;
3075 self->pers_func = NULL;
3076 self->inst_pers_func = NULL;
3077 self->write_buf = NULL;
3078 self->fast = 0;
3079 self->fast_container = 0;
3080 self->fast_memo = NULL;
3081 self->buf_size = 0;
3082 self->dispatch_table = NULL;
3083
3084 self->file = NULL;
3085 if (file)
3086 Py_INCREF(file);
3087 else {
3088 file = Pdata_New();
3089 if (file == NULL)
3090 goto err;
3091 }
3092 self->file = file;
3093
3094 if (!( self->memo = PyDict_New()))
3095 goto err;
3096
3097 if (PyFile_Check(file)) {
3098 self->fp = PyFile_AsFile(file);
3099 if (self->fp == NULL) {
3100 PyErr_SetString(PyExc_ValueError,
3101 "I/O operation on closed file");
3102 goto err;
3103 }
3104 self->write_func = write_file;
3105 }
3106 else if (PycStringIO_OutputCheck(file)) {
3107 self->write_func = write_cStringIO;
3108 }
3109 else if (file == Py_None) {
3110 self->write_func = write_none;
3111 }
3112 else {
3113 self->write_func = write_other;
3114
3115 if (! Pdata_Check(file)) {
3116 self->write = PyObject_GetAttr(file, write_str);
3117 if (!self->write) {
3118 PyErr_Clear();
3119 PyErr_SetString(PyExc_TypeError,
3120 "argument must have 'write' "
3121 "attribute");
3122 goto err;
3123 }
3124 }
3125
3126 self->write_buf = (char *)PyMem_Malloc(WRITE_BUF_SIZE);
3127 if (self->write_buf == NULL) {
3128 PyErr_NoMemory();
3129 goto err;
3130 }
3131 }
3132
3133 if (PyEval_GetRestricted()) {
3134 /* Restricted execution, get private tables */
3135 PyObject *m = PyImport_ImportModule("copy_reg");
3136
3137 if (m == NULL)
3138 goto err;
3139 self->dispatch_table = PyObject_GetAttr(m, dispatch_table_str);
3140 Py_DECREF(m);
3141 if (self->dispatch_table == NULL)
3142 goto err;
3143 }
3144 else {
3145 self->dispatch_table = dispatch_table;
3146 Py_INCREF(dispatch_table);
3147 }
3148 PyObject_GC_Track(self);
3149
3150 return self;
3151
3152 err:
3153 Py_DECREF(self);
3154 return NULL;
3155 }
3156
3157
3158 static PyObject *
get_Pickler(PyObject * self,PyObject * args,PyObject * kwds)3159 get_Pickler(PyObject *self, PyObject *args, PyObject *kwds)
3160 {
3161 static char *kwlist[] = {"file", "protocol", NULL};
3162 PyObject *file = NULL;
3163 int proto = 0;
3164
3165 /* XXX
3166 * The documented signature is Pickler(file, protocol=0), but this
3167 * accepts Pickler() and Pickler(integer) too. The meaning then
3168 * is clear as mud, undocumented, and not supported by pickle.py.
3169 * I'm told Zope uses this, but I haven't traced into this code
3170 * far enough to figure out what it means.
3171 */
3172 if (!PyArg_ParseTuple(args, "|i:Pickler", &proto)) {
3173 PyErr_Clear();
3174 proto = 0;
3175 if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|i:Pickler",
3176 kwlist, &file, &proto))
3177 return NULL;
3178 }
3179 return (PyObject *)newPicklerobject(file, proto);
3180 }
3181
3182
3183 static void
Pickler_dealloc(Picklerobject * self)3184 Pickler_dealloc(Picklerobject *self)
3185 {
3186 PyObject_GC_UnTrack(self);
3187 Py_XDECREF(self->write);
3188 Py_XDECREF(self->memo);
3189 Py_XDECREF(self->fast_memo);
3190 Py_XDECREF(self->arg);
3191 Py_XDECREF(self->file);
3192 Py_XDECREF(self->pers_func);
3193 Py_XDECREF(self->inst_pers_func);
3194 Py_XDECREF(self->dispatch_table);
3195 PyMem_Free(self->write_buf);
3196 Py_TYPE(self)->tp_free((PyObject *)self);
3197 }
3198
3199 static int
Pickler_traverse(Picklerobject * self,visitproc visit,void * arg)3200 Pickler_traverse(Picklerobject *self, visitproc visit, void *arg)
3201 {
3202 Py_VISIT(self->write);
3203 Py_VISIT(self->memo);
3204 Py_VISIT(self->fast_memo);
3205 Py_VISIT(self->arg);
3206 Py_VISIT(self->file);
3207 Py_VISIT(self->pers_func);
3208 Py_VISIT(self->inst_pers_func);
3209 Py_VISIT(self->dispatch_table);
3210 return 0;
3211 }
3212
3213 static int
Pickler_clear(Picklerobject * self)3214 Pickler_clear(Picklerobject *self)
3215 {
3216 Py_CLEAR(self->write);
3217 Py_CLEAR(self->memo);
3218 Py_CLEAR(self->fast_memo);
3219 Py_CLEAR(self->arg);
3220 Py_CLEAR(self->file);
3221 Py_CLEAR(self->pers_func);
3222 Py_CLEAR(self->inst_pers_func);
3223 Py_CLEAR(self->dispatch_table);
3224 return 0;
3225 }
3226
3227 static PyObject *
Pickler_get_pers_func(Picklerobject * p)3228 Pickler_get_pers_func(Picklerobject *p)
3229 {
3230 if (p->pers_func == NULL)
3231 PyErr_SetString(PyExc_AttributeError, "persistent_id");
3232 else
3233 Py_INCREF(p->pers_func);
3234 return p->pers_func;
3235 }
3236
3237 static int
Pickler_set_pers_func(Picklerobject * p,PyObject * v)3238 Pickler_set_pers_func(Picklerobject *p, PyObject *v)
3239 {
3240 if (v == NULL) {
3241 PyErr_SetString(PyExc_TypeError,
3242 "attribute deletion is not supported");
3243 return -1;
3244 }
3245 Py_INCREF(v);
3246 Py_XSETREF(p->pers_func, v);
3247 return 0;
3248 }
3249
3250 static int
Pickler_set_inst_pers_func(Picklerobject * p,PyObject * v)3251 Pickler_set_inst_pers_func(Picklerobject *p, PyObject *v)
3252 {
3253 if (v == NULL) {
3254 PyErr_SetString(PyExc_TypeError,
3255 "attribute deletion is not supported");
3256 return -1;
3257 }
3258 Py_INCREF(v);
3259 Py_XSETREF(p->inst_pers_func, v);
3260 return 0;
3261 }
3262
3263 static PyObject *
Pickler_get_memo(Picklerobject * p)3264 Pickler_get_memo(Picklerobject *p)
3265 {
3266 if (p->memo == NULL)
3267 PyErr_SetString(PyExc_AttributeError, "memo");
3268 else
3269 Py_INCREF(p->memo);
3270 return p->memo;
3271 }
3272
3273 static int
Pickler_set_memo(Picklerobject * p,PyObject * v)3274 Pickler_set_memo(Picklerobject *p, PyObject *v)
3275 {
3276 if (v == NULL) {
3277 PyErr_SetString(PyExc_TypeError,
3278 "attribute deletion is not supported");
3279 return -1;
3280 }
3281 if (!PyDict_Check(v)) {
3282 PyErr_SetString(PyExc_TypeError, "memo must be a dictionary");
3283 return -1;
3284 }
3285 Py_INCREF(v);
3286 Py_XSETREF(p->memo, v);
3287 return 0;
3288 }
3289
3290 static PyObject *
Pickler_get_error(Picklerobject * p)3291 Pickler_get_error(Picklerobject *p)
3292 {
3293 /* why is this an attribute on the Pickler? */
3294 Py_INCREF(PicklingError);
3295 return PicklingError;
3296 }
3297
3298 static PyMemberDef Pickler_members[] = {
3299 {"binary", T_INT, offsetof(Picklerobject, bin)},
3300 {"fast", T_INT, offsetof(Picklerobject, fast)},
3301 {NULL}
3302 };
3303
3304 static PyGetSetDef Pickler_getsets[] = {
3305 {"persistent_id", (getter)Pickler_get_pers_func,
3306 (setter)Pickler_set_pers_func},
3307 {"inst_persistent_id", NULL, (setter)Pickler_set_inst_pers_func},
3308 {"memo", (getter)Pickler_get_memo, (setter)Pickler_set_memo},
3309 {"PicklingError", (getter)Pickler_get_error, NULL},
3310 {NULL}
3311 };
3312
3313 PyDoc_STRVAR(Picklertype__doc__,
3314 "Objects that know how to pickle objects\n");
3315
3316 static PyTypeObject Picklertype = {
3317 PyVarObject_HEAD_INIT(NULL, 0)
3318 "cPickle.Pickler", /*tp_name*/
3319 sizeof(Picklerobject), /*tp_basicsize*/
3320 0,
3321 (destructor)Pickler_dealloc, /* tp_dealloc */
3322 0, /* tp_print */
3323 0, /* tp_getattr */
3324 0, /* tp_setattr */
3325 0, /* tp_compare */
3326 0, /* tp_repr */
3327 0, /* tp_as_number */
3328 0, /* tp_as_sequence */
3329 0, /* tp_as_mapping */
3330 0, /* tp_hash */
3331 0, /* tp_call */
3332 0, /* tp_str */
3333 PyObject_GenericGetAttr, /* tp_getattro */
3334 PyObject_GenericSetAttr, /* tp_setattro */
3335 0, /* tp_as_buffer */
3336 Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC,
3337 Picklertype__doc__, /* tp_doc */
3338 (traverseproc)Pickler_traverse, /* tp_traverse */
3339 (inquiry)Pickler_clear, /* tp_clear */
3340 0, /* tp_richcompare */
3341 0, /* tp_weaklistoffset */
3342 0, /* tp_iter */
3343 0, /* tp_iternext */
3344 Pickler_methods, /* tp_methods */
3345 Pickler_members, /* tp_members */
3346 Pickler_getsets, /* tp_getset */
3347 };
3348
3349 static PyObject *
find_class(PyObject * py_module_name,PyObject * py_global_name,PyObject * fc)3350 find_class(PyObject *py_module_name, PyObject *py_global_name, PyObject *fc)
3351 {
3352 PyObject *global = 0, *module;
3353
3354 if (fc) {
3355 if (fc==Py_None) {
3356 PyErr_SetString(UnpicklingError, "Global and instance "
3357 "pickles are not supported.");
3358 return NULL;
3359 }
3360 return PyObject_CallFunctionObjArgs(fc, py_module_name,
3361 py_global_name, NULL);
3362 }
3363
3364 module = PySys_GetObject("modules");
3365 if (module == NULL)
3366 return NULL;
3367
3368 module = PyDict_GetItem(module, py_module_name);
3369 if (module == NULL) {
3370 module = PyImport_Import(py_module_name);
3371 if (!module)
3372 return NULL;
3373 global = PyObject_GetAttr(module, py_global_name);
3374 Py_DECREF(module);
3375 }
3376 else
3377 global = PyObject_GetAttr(module, py_global_name);
3378 return global;
3379 }
3380
3381 static Py_ssize_t
marker(Unpicklerobject * self)3382 marker(Unpicklerobject *self)
3383 {
3384 if (self->num_marks < 1) {
3385 PyErr_SetString(UnpicklingError, "could not find MARK");
3386 return -1;
3387 }
3388
3389 return self->marks[--self->num_marks];
3390 }
3391
3392
3393 static int
load_none(Unpicklerobject * self)3394 load_none(Unpicklerobject *self)
3395 {
3396 PDATA_APPEND(self->stack, Py_None, -1);
3397 return 0;
3398 }
3399
3400 static int
bad_readline(void)3401 bad_readline(void)
3402 {
3403 PyErr_SetString(UnpicklingError, "pickle data was truncated");
3404 return -1;
3405 }
3406
3407 static int
load_int(Unpicklerobject * self)3408 load_int(Unpicklerobject *self)
3409 {
3410 PyObject *py_int = 0;
3411 char *endptr, *s;
3412 Py_ssize_t len;
3413 int res = -1;
3414 long l;
3415
3416 if ((len = self->readline_func(self, &s)) < 0) return -1;
3417 if (len < 2) return bad_readline();
3418 if (!( s=pystrndup(s,len))) return -1;
3419
3420 errno = 0;
3421 l = strtol(s, &endptr, 0);
3422
3423 if (errno || (*endptr != '\n') || (endptr[1] != '\0')) {
3424 /* Hm, maybe we've got something long. Let's try reading
3425 it as a Python long object. */
3426 errno = 0;
3427 py_int = PyLong_FromString(s, NULL, 0);
3428 if (py_int == NULL) {
3429 PyErr_SetString(PyExc_ValueError,
3430 "could not convert string to int");
3431 goto finally;
3432 }
3433 }
3434 else {
3435 if (len == 3 && (l == 0 || l == 1)) {
3436 if (!( py_int = PyBool_FromLong(l))) goto finally;
3437 }
3438 else {
3439 if (!( py_int = PyInt_FromLong(l))) goto finally;
3440 }
3441 }
3442
3443 free(s);
3444 PDATA_PUSH(self->stack, py_int, -1);
3445 return 0;
3446
3447 finally:
3448 free(s);
3449
3450 return res;
3451 }
3452
3453 static int
load_bool(Unpicklerobject * self,PyObject * boolean)3454 load_bool(Unpicklerobject *self, PyObject *boolean)
3455 {
3456 assert(boolean == Py_True || boolean == Py_False);
3457 PDATA_APPEND(self->stack, boolean, -1);
3458 return 0;
3459 }
3460
3461 /* s contains x bytes of a little-endian integer. Return its value as a
3462 * C int. Obscure: when x is 1 or 2, this is an unsigned little-endian
3463 * int, but when x is 4 it's a signed one. This is a historical source
3464 * of x-platform bugs.
3465 */
3466 static long
calc_binint(char * s,int x)3467 calc_binint(char *s, int x)
3468 {
3469 unsigned char c;
3470 int i;
3471 long l;
3472
3473 for (i = 0, l = 0L; i < x; i++) {
3474 c = (unsigned char)s[i];
3475 l |= (long)c << (i * 8);
3476 }
3477 #if SIZEOF_LONG > 4
3478 /* Unlike BININT1 and BININT2, BININT (more accurately BININT4)
3479 * is signed, so on a box with longs bigger than 4 bytes we need
3480 * to extend a BININT's sign bit to the full width.
3481 */
3482 if (x == 4 && l & (1L << 31))
3483 l |= (~0UL) << 32;
3484 #endif
3485 return l;
3486 }
3487
3488
3489 static int
load_binintx(Unpicklerobject * self,char * s,int x)3490 load_binintx(Unpicklerobject *self, char *s, int x)
3491 {
3492 PyObject *py_int = 0;
3493 long l;
3494
3495 l = calc_binint(s, x);
3496
3497 if (!( py_int = PyInt_FromLong(l)))
3498 return -1;
3499
3500 PDATA_PUSH(self->stack, py_int, -1);
3501 return 0;
3502 }
3503
3504
3505 static int
load_binint(Unpicklerobject * self)3506 load_binint(Unpicklerobject *self)
3507 {
3508 char *s;
3509
3510 if (self->read_func(self, &s, 4) < 0)
3511 return -1;
3512
3513 return load_binintx(self, s, 4);
3514 }
3515
3516
3517 static int
load_binint1(Unpicklerobject * self)3518 load_binint1(Unpicklerobject *self)
3519 {
3520 char *s;
3521
3522 if (self->read_func(self, &s, 1) < 0)
3523 return -1;
3524
3525 return load_binintx(self, s, 1);
3526 }
3527
3528
3529 static int
load_binint2(Unpicklerobject * self)3530 load_binint2(Unpicklerobject *self)
3531 {
3532 char *s;
3533
3534 if (self->read_func(self, &s, 2) < 0)
3535 return -1;
3536
3537 return load_binintx(self, s, 2);
3538 }
3539
3540 static int
load_long(Unpicklerobject * self)3541 load_long(Unpicklerobject *self)
3542 {
3543 PyObject *l = 0;
3544 char *end, *s;
3545 Py_ssize_t len;
3546 int res = -1;
3547
3548 if ((len = self->readline_func(self, &s)) < 0) return -1;
3549 if (len < 2) return bad_readline();
3550 if (!( s=pystrndup(s,len))) return -1;
3551
3552 if (!( l = PyLong_FromString(s, &end, 0)))
3553 goto finally;
3554
3555 free(s);
3556 PDATA_PUSH(self->stack, l, -1);
3557 return 0;
3558
3559 finally:
3560 free(s);
3561
3562 return res;
3563 }
3564
3565 /* 'size' bytes contain the # of bytes of little-endian 256's-complement
3566 * data following.
3567 */
3568 static int
load_counted_long(Unpicklerobject * self,int size)3569 load_counted_long(Unpicklerobject *self, int size)
3570 {
3571 Py_ssize_t i;
3572 char *nbytes;
3573 unsigned char *pdata;
3574 PyObject *along;
3575
3576 assert(size == 1 || size == 4);
3577 i = self->read_func(self, &nbytes, size);
3578 if (i < 0) return -1;
3579
3580 size = calc_binint(nbytes, size);
3581 if (size < 0) {
3582 /* Corrupt or hostile pickle -- we never write one like
3583 * this.
3584 */
3585 PyErr_SetString(UnpicklingError, "LONG pickle has negative "
3586 "byte count");
3587 return -1;
3588 }
3589
3590 if (size == 0)
3591 along = PyLong_FromLong(0L);
3592 else {
3593 /* Read the raw little-endian bytes & convert. */
3594 i = self->read_func(self, (char **)&pdata, size);
3595 if (i < 0) return -1;
3596 along = _PyLong_FromByteArray(pdata, (size_t)size,
3597 1 /* little endian */, 1 /* signed */);
3598 }
3599 if (along == NULL)
3600 return -1;
3601 PDATA_PUSH(self->stack, along, -1);
3602 return 0;
3603 }
3604
3605 static int
load_float(Unpicklerobject * self)3606 load_float(Unpicklerobject *self)
3607 {
3608 PyObject *py_float = 0;
3609 char *endptr, *s;
3610 Py_ssize_t len;
3611 int res = -1;
3612 double d;
3613
3614 if ((len = self->readline_func(self, &s)) < 0) return -1;
3615 if (len < 2) return bad_readline();
3616 if (!( s=pystrndup(s,len))) return -1;
3617
3618 d = PyOS_string_to_double(s, &endptr, PyExc_OverflowError);
3619
3620 if (d == -1.0 && PyErr_Occurred()) {
3621 goto finally;
3622 } else if ((endptr[0] != '\n') || (endptr[1] != '\0')) {
3623 PyErr_SetString(PyExc_ValueError,
3624 "could not convert string to float");
3625 goto finally;
3626 }
3627
3628 if (!( py_float = PyFloat_FromDouble(d)))
3629 goto finally;
3630
3631 free(s);
3632 PDATA_PUSH(self->stack, py_float, -1);
3633 return 0;
3634
3635 finally:
3636 free(s);
3637
3638 return res;
3639 }
3640
3641 static int
load_binfloat(Unpicklerobject * self)3642 load_binfloat(Unpicklerobject *self)
3643 {
3644 PyObject *py_float;
3645 double x;
3646 char *p;
3647
3648 if (self->read_func(self, &p, 8) < 0)
3649 return -1;
3650
3651 x = _PyFloat_Unpack8((unsigned char *)p, 0);
3652 if (x == -1.0 && PyErr_Occurred())
3653 return -1;
3654
3655 py_float = PyFloat_FromDouble(x);
3656 if (py_float == NULL)
3657 return -1;
3658
3659 PDATA_PUSH(self->stack, py_float, -1);
3660 return 0;
3661 }
3662
3663 static int
load_string(Unpicklerobject * self)3664 load_string(Unpicklerobject *self)
3665 {
3666 PyObject *str = 0;
3667 Py_ssize_t len;
3668 int res = -1;
3669 char *s, *p;
3670
3671 if ((len = self->readline_func(self, &s)) < 0) return -1;
3672 if (len < 2) return bad_readline();
3673 if (!( s=pystrndup(s,len))) return -1;
3674
3675
3676 /* Strip outermost quotes */
3677 while (len > 0 && s[len-1] <= ' ')
3678 len--;
3679 if (len > 1 && s[0]=='"' && s[len-1]=='"') {
3680 s[len-1] = '\0';
3681 p = s + 1 ;
3682 len -= 2;
3683 }
3684 else if (len > 1 && s[0]=='\'' && s[len-1]=='\'') {
3685 s[len-1] = '\0';
3686 p = s + 1 ;
3687 len -= 2;
3688 }
3689 else
3690 goto insecure;
3691 /********************************************/
3692
3693 str = PyString_DecodeEscape(p, len, NULL, 0, NULL);
3694 free(s);
3695 if (str) {
3696 PDATA_PUSH(self->stack, str, -1);
3697 res = 0;
3698 }
3699 return res;
3700
3701 insecure:
3702 free(s);
3703 PyErr_SetString(PyExc_ValueError,"insecure string pickle");
3704 return -1;
3705 }
3706
3707
3708 static int
load_binstring(Unpicklerobject * self)3709 load_binstring(Unpicklerobject *self)
3710 {
3711 PyObject *py_string = 0;
3712 Py_ssize_t l;
3713 char *s;
3714
3715 if (self->read_func(self, &s, 4) < 0) return -1;
3716
3717 l = calc_binint(s, 4);
3718 if (l < 0) {
3719 /* Corrupt or hostile pickle -- we never write one like
3720 * this.
3721 */
3722 PyErr_SetString(UnpicklingError,
3723 "BINSTRING pickle has negative byte count");
3724 return -1;
3725 }
3726
3727 if (self->read_func(self, &s, l) < 0)
3728 return -1;
3729
3730 if (!( py_string = PyString_FromStringAndSize(s, l)))
3731 return -1;
3732
3733 PDATA_PUSH(self->stack, py_string, -1);
3734 return 0;
3735 }
3736
3737
3738 static int
load_short_binstring(Unpicklerobject * self)3739 load_short_binstring(Unpicklerobject *self)
3740 {
3741 PyObject *py_string = 0;
3742 unsigned char l;
3743 char *s;
3744
3745 if (self->read_func(self, &s, 1) < 0)
3746 return -1;
3747
3748 l = (unsigned char)s[0];
3749
3750 if (self->read_func(self, &s, l) < 0) return -1;
3751
3752 if (!( py_string = PyString_FromStringAndSize(s, l))) return -1;
3753
3754 PDATA_PUSH(self->stack, py_string, -1);
3755 return 0;
3756 }
3757
3758
3759 #ifdef Py_USING_UNICODE
3760 static int
load_unicode(Unpicklerobject * self)3761 load_unicode(Unpicklerobject *self)
3762 {
3763 PyObject *str = 0;
3764 Py_ssize_t len;
3765 char *s;
3766
3767 if ((len = self->readline_func(self, &s)) < 0) return -1;
3768 if (len < 1) return bad_readline();
3769
3770 if (!( str = PyUnicode_DecodeRawUnicodeEscape(s, len - 1, NULL)))
3771 return -1;
3772
3773 PDATA_PUSH(self->stack, str, -1);
3774 return 0;
3775 }
3776 #endif
3777
3778
3779 #ifdef Py_USING_UNICODE
3780 static int
load_binunicode(Unpicklerobject * self)3781 load_binunicode(Unpicklerobject *self)
3782 {
3783 PyObject *unicode;
3784 Py_ssize_t l;
3785 char *s;
3786
3787 if (self->read_func(self, &s, 4) < 0) return -1;
3788
3789 l = calc_binint(s, 4);
3790 if (l < 0) {
3791 /* Corrupt or hostile pickle -- we never write one like
3792 * this.
3793 */
3794 PyErr_SetString(UnpicklingError,
3795 "BINUNICODE pickle has negative byte count");
3796 return -1;
3797 }
3798
3799 if (self->read_func(self, &s, l) < 0)
3800 return -1;
3801
3802 if (!( unicode = PyUnicode_DecodeUTF8(s, l, NULL)))
3803 return -1;
3804
3805 PDATA_PUSH(self->stack, unicode, -1);
3806 return 0;
3807 }
3808 #endif
3809
3810
3811 static int
load_counted_tuple(Unpicklerobject * self,int len)3812 load_counted_tuple(Unpicklerobject *self, int len)
3813 {
3814 PyObject *tup;
3815
3816 if (self->stack->length < len)
3817 return stackUnderflow();
3818
3819 if (!(tup = Pdata_popTuple(self->stack, self->stack->length - len)))
3820 return -1;
3821 PDATA_PUSH(self->stack, tup, -1);
3822 return 0;
3823 }
3824
3825 static int
load_tuple(Unpicklerobject * self)3826 load_tuple(Unpicklerobject *self)
3827 {
3828 Py_ssize_t i;
3829
3830 if ((i = marker(self)) < 0) return -1;
3831 return load_counted_tuple(self, self->stack->length - i);
3832 }
3833
3834 static int
load_empty_list(Unpicklerobject * self)3835 load_empty_list(Unpicklerobject *self)
3836 {
3837 PyObject *list;
3838
3839 if (!( list=PyList_New(0))) return -1;
3840 PDATA_PUSH(self->stack, list, -1);
3841 return 0;
3842 }
3843
3844 static int
load_empty_dict(Unpicklerobject * self)3845 load_empty_dict(Unpicklerobject *self)
3846 {
3847 PyObject *dict;
3848
3849 if (!( dict=PyDict_New())) return -1;
3850 PDATA_PUSH(self->stack, dict, -1);
3851 return 0;
3852 }
3853
3854
3855 static int
load_list(Unpicklerobject * self)3856 load_list(Unpicklerobject *self)
3857 {
3858 PyObject *list = 0;
3859 Py_ssize_t i;
3860
3861 if ((i = marker(self)) < 0) return -1;
3862 if (!( list=Pdata_popList(self->stack, i))) return -1;
3863 PDATA_PUSH(self->stack, list, -1);
3864 return 0;
3865 }
3866
3867 static int
load_dict(Unpicklerobject * self)3868 load_dict(Unpicklerobject *self)
3869 {
3870 PyObject *dict, *key, *value;
3871 Py_ssize_t i, j, k;
3872
3873 if ((i = marker(self)) < 0) return -1;
3874 j=self->stack->length;
3875
3876 if (!( dict = PyDict_New())) return -1;
3877
3878 for (k = i+1; k < j; k += 2) {
3879 key =self->stack->data[k-1];
3880 value=self->stack->data[k ];
3881 if (PyDict_SetItem(dict, key, value) < 0) {
3882 Py_DECREF(dict);
3883 return -1;
3884 }
3885 }
3886 Pdata_clear(self->stack, i);
3887 PDATA_PUSH(self->stack, dict, -1);
3888 return 0;
3889 }
3890
3891 static PyObject *
Instance_New(PyObject * cls,PyObject * args)3892 Instance_New(PyObject *cls, PyObject *args)
3893 {
3894 if (PyClass_Check(cls)) {
3895 int l;
3896
3897 if ((l=PyObject_Size(args)) < 0) return NULL;
3898 if (!( l )) {
3899 if (!PyObject_HasAttr(cls, __getinitargs___str)) {
3900 /* We have a class with no __getinitargs__,
3901 so bypass usual construction */
3902 return PyInstance_NewRaw(cls, NULL);
3903 }
3904 }
3905
3906 return PyInstance_New(cls, args, NULL);
3907 }
3908
3909 return PyObject_CallObject(cls, args);
3910 }
3911
3912
3913 static int
load_obj(Unpicklerobject * self)3914 load_obj(Unpicklerobject *self)
3915 {
3916 PyObject *class, *tup, *obj=0;
3917 Py_ssize_t i;
3918
3919 if ((i = marker(self)) < 0) return -1;
3920
3921 if (self->stack->length - i < 1)
3922 return stackUnderflow();
3923
3924 if (!( tup=Pdata_popTuple(self->stack, i+1))) return -1;
3925 PDATA_POP(self->stack, class);
3926 if (class) {
3927 obj = Instance_New(class, tup);
3928 Py_DECREF(class);
3929 }
3930 Py_DECREF(tup);
3931
3932 if (! obj) return -1;
3933 PDATA_PUSH(self->stack, obj, -1);
3934 return 0;
3935 }
3936
3937
3938 static int
load_inst(Unpicklerobject * self)3939 load_inst(Unpicklerobject *self)
3940 {
3941 PyObject *tup, *class=0, *obj=0, *module_name, *class_name;
3942 Py_ssize_t i, len;
3943 char *s;
3944
3945 if ((i = marker(self)) < 0) return -1;
3946
3947 if ((len = self->readline_func(self, &s)) < 0) return -1;
3948 if (len < 2) return bad_readline();
3949 module_name = PyString_FromStringAndSize(s, len - 1);
3950 if (!module_name) return -1;
3951
3952 if ((len = self->readline_func(self, &s)) >= 0) {
3953 if (len < 2) {
3954 Py_DECREF(module_name);
3955 return bad_readline();
3956 }
3957 if ((class_name = PyString_FromStringAndSize(s, len - 1))) {
3958 class = find_class(module_name, class_name,
3959 self->find_class);
3960 Py_DECREF(class_name);
3961 }
3962 }
3963 Py_DECREF(module_name);
3964
3965 if (! class) return -1;
3966
3967 if ((tup=Pdata_popTuple(self->stack, i))) {
3968 obj = Instance_New(class, tup);
3969 Py_DECREF(tup);
3970 }
3971 Py_DECREF(class);
3972
3973 if (! obj) return -1;
3974
3975 PDATA_PUSH(self->stack, obj, -1);
3976 return 0;
3977 }
3978
3979 static int
load_newobj(Unpicklerobject * self)3980 load_newobj(Unpicklerobject *self)
3981 {
3982 PyObject *args = NULL;
3983 PyObject *clsraw = NULL;
3984 PyTypeObject *cls; /* clsraw cast to its true type */
3985 PyObject *obj;
3986
3987 /* Stack is ... cls argtuple, and we want to call
3988 * cls.__new__(cls, *argtuple).
3989 */
3990 PDATA_POP(self->stack, args);
3991 if (args == NULL) goto Fail;
3992 if (! PyTuple_Check(args)) {
3993 PyErr_SetString(UnpicklingError, "NEWOBJ expected an arg "
3994 "tuple.");
3995 goto Fail;
3996 }
3997
3998 PDATA_POP(self->stack, clsraw);
3999 cls = (PyTypeObject *)clsraw;
4000 if (cls == NULL) goto Fail;
4001 if (! PyType_Check(cls)) {
4002 PyErr_SetString(UnpicklingError, "NEWOBJ class argument "
4003 "isn't a type object");
4004 goto Fail;
4005 }
4006 if (cls->tp_new == NULL) {
4007 PyErr_SetString(UnpicklingError, "NEWOBJ class argument "
4008 "has NULL tp_new");
4009 goto Fail;
4010 }
4011
4012 /* Call __new__. */
4013 obj = cls->tp_new(cls, args, NULL);
4014 if (obj == NULL) goto Fail;
4015
4016 Py_DECREF(args);
4017 Py_DECREF(clsraw);
4018 PDATA_PUSH(self->stack, obj, -1);
4019 return 0;
4020
4021 Fail:
4022 Py_XDECREF(args);
4023 Py_XDECREF(clsraw);
4024 return -1;
4025 }
4026
4027 static int
load_global(Unpicklerobject * self)4028 load_global(Unpicklerobject *self)
4029 {
4030 PyObject *class = 0, *module_name = 0, *class_name = 0;
4031 Py_ssize_t len;
4032 char *s;
4033
4034 if ((len = self->readline_func(self, &s)) < 0) return -1;
4035 if (len < 2) return bad_readline();
4036 module_name = PyString_FromStringAndSize(s, len - 1);
4037 if (!module_name) return -1;
4038
4039 if ((len = self->readline_func(self, &s)) >= 0) {
4040 if (len < 2) {
4041 Py_DECREF(module_name);
4042 return bad_readline();
4043 }
4044 if ((class_name = PyString_FromStringAndSize(s, len - 1))) {
4045 class = find_class(module_name, class_name,
4046 self->find_class);
4047 Py_DECREF(class_name);
4048 }
4049 }
4050 Py_DECREF(module_name);
4051
4052 if (! class) return -1;
4053 PDATA_PUSH(self->stack, class, -1);
4054 return 0;
4055 }
4056
4057
4058 static int
load_persid(Unpicklerobject * self)4059 load_persid(Unpicklerobject *self)
4060 {
4061 PyObject *pid = 0;
4062 Py_ssize_t len;
4063 char *s;
4064
4065 if (self->pers_func) {
4066 if ((len = self->readline_func(self, &s)) < 0) return -1;
4067 if (len < 2) return bad_readline();
4068
4069 pid = PyString_FromStringAndSize(s, len - 1);
4070 if (!pid) return -1;
4071
4072 if (PyList_Check(self->pers_func)) {
4073 if (PyList_Append(self->pers_func, pid) < 0) {
4074 Py_DECREF(pid);
4075 return -1;
4076 }
4077 }
4078 else {
4079 ARG_TUP(self, pid);
4080 if (self->arg) {
4081 pid = PyObject_Call(self->pers_func, self->arg,
4082 NULL);
4083 FREE_ARG_TUP(self);
4084 }
4085 }
4086
4087 if (! pid) return -1;
4088
4089 PDATA_PUSH(self->stack, pid, -1);
4090 return 0;
4091 }
4092 else {
4093 PyErr_SetString(UnpicklingError,
4094 "A load persistent id instruction was encountered,\n"
4095 "but no persistent_load function was specified.");
4096 return -1;
4097 }
4098 }
4099
4100 static int
load_binpersid(Unpicklerobject * self)4101 load_binpersid(Unpicklerobject *self)
4102 {
4103 PyObject *pid = 0;
4104
4105 if (self->pers_func) {
4106 PDATA_POP(self->stack, pid);
4107 if (! pid) return -1;
4108
4109 if (PyList_Check(self->pers_func)) {
4110 if (PyList_Append(self->pers_func, pid) < 0) {
4111 Py_DECREF(pid);
4112 return -1;
4113 }
4114 }
4115 else {
4116 ARG_TUP(self, pid);
4117 if (self->arg) {
4118 pid = PyObject_Call(self->pers_func, self->arg,
4119 NULL);
4120 FREE_ARG_TUP(self);
4121 }
4122 if (! pid) return -1;
4123 }
4124
4125 PDATA_PUSH(self->stack, pid, -1);
4126 return 0;
4127 }
4128 else {
4129 PyErr_SetString(UnpicklingError,
4130 "A load persistent id instruction was encountered,\n"
4131 "but no persistent_load function was specified.");
4132 return -1;
4133 }
4134 }
4135
4136
4137 static int
load_pop(Unpicklerobject * self)4138 load_pop(Unpicklerobject *self)
4139 {
4140 Py_ssize_t len = self->stack->length;
4141
4142 /* Note that we split the (pickle.py) stack into two stacks,
4143 an object stack and a mark stack. We have to be clever and
4144 pop the right one. We do this by looking at the top of the
4145 mark stack first, and only signalling a stack underflow if
4146 the object stack is empty and the mark stack doesn't match
4147 our expectations.
4148 */
4149 if (self->num_marks > 0 && self->marks[self->num_marks - 1] == len) {
4150 self->num_marks--;
4151 } else if (len > 0) {
4152 len--;
4153 Py_DECREF(self->stack->data[len]);
4154 self->stack->length = len;
4155 } else {
4156 return stackUnderflow();
4157 }
4158 return 0;
4159 }
4160
4161
4162 static int
load_pop_mark(Unpicklerobject * self)4163 load_pop_mark(Unpicklerobject *self)
4164 {
4165 Py_ssize_t i;
4166
4167 if ((i = marker(self)) < 0)
4168 return -1;
4169
4170 Pdata_clear(self->stack, i);
4171
4172 return 0;
4173 }
4174
4175
4176 static int
load_dup(Unpicklerobject * self)4177 load_dup(Unpicklerobject *self)
4178 {
4179 PyObject *last;
4180 Py_ssize_t len;
4181
4182 if ((len = self->stack->length) <= 0) return stackUnderflow();
4183 last=self->stack->data[len-1];
4184 Py_INCREF(last);
4185 PDATA_PUSH(self->stack, last, -1);
4186 return 0;
4187 }
4188
4189
4190 static int
load_get(Unpicklerobject * self)4191 load_get(Unpicklerobject *self)
4192 {
4193 PyObject *py_str = 0, *value = 0;
4194 Py_ssize_t len;
4195 char *s;
4196 int rc;
4197
4198 if ((len = self->readline_func(self, &s)) < 0) return -1;
4199 if (len < 2) return bad_readline();
4200
4201 if (!( py_str = PyString_FromStringAndSize(s, len - 1))) return -1;
4202
4203 value = PyDict_GetItem(self->memo, py_str);
4204 if (! value) {
4205 PyErr_SetObject(BadPickleGet, py_str);
4206 rc = -1;
4207 }
4208 else {
4209 PDATA_APPEND(self->stack, value, -1);
4210 rc = 0;
4211 }
4212
4213 Py_DECREF(py_str);
4214 return rc;
4215 }
4216
4217
4218 static int
load_binget(Unpicklerobject * self)4219 load_binget(Unpicklerobject *self)
4220 {
4221 PyObject *py_key = 0, *value = 0;
4222 unsigned char key;
4223 char *s;
4224 int rc;
4225
4226 if (self->read_func(self, &s, 1) < 0) return -1;
4227
4228 key = (unsigned char)s[0];
4229 if (!( py_key = PyInt_FromLong((long)key))) return -1;
4230
4231 value = PyDict_GetItem(self->memo, py_key);
4232 if (! value) {
4233 PyErr_SetObject(BadPickleGet, py_key);
4234 rc = -1;
4235 }
4236 else {
4237 PDATA_APPEND(self->stack, value, -1);
4238 rc = 0;
4239 }
4240
4241 Py_DECREF(py_key);
4242 return rc;
4243 }
4244
4245
4246 static int
load_long_binget(Unpicklerobject * self)4247 load_long_binget(Unpicklerobject *self)
4248 {
4249 PyObject *py_key = 0, *value = 0;
4250 unsigned char c;
4251 char *s;
4252 Py_ssize_t key;
4253 int rc;
4254
4255 if (self->read_func(self, &s, 4) < 0) return -1;
4256
4257 c = (unsigned char)s[0];
4258 key = (long)c;
4259 c = (unsigned char)s[1];
4260 key |= (long)c << 8;
4261 c = (unsigned char)s[2];
4262 key |= (long)c << 16;
4263 c = (unsigned char)s[3];
4264 key |= (long)c << 24;
4265
4266 if (!( py_key = PyInt_FromLong((long)key))) return -1;
4267
4268 value = PyDict_GetItem(self->memo, py_key);
4269 if (! value) {
4270 PyErr_SetObject(BadPickleGet, py_key);
4271 rc = -1;
4272 }
4273 else {
4274 PDATA_APPEND(self->stack, value, -1);
4275 rc = 0;
4276 }
4277
4278 Py_DECREF(py_key);
4279 return rc;
4280 }
4281
4282 /* Push an object from the extension registry (EXT[124]). nbytes is
4283 * the number of bytes following the opcode, holding the index (code) value.
4284 */
4285 static int
load_extension(Unpicklerobject * self,int nbytes)4286 load_extension(Unpicklerobject *self, int nbytes)
4287 {
4288 char *codebytes; /* the nbytes bytes after the opcode */
4289 long code; /* calc_binint returns long */
4290 PyObject *py_code; /* code as a Python int */
4291 PyObject *obj; /* the object to push */
4292 PyObject *pair; /* (module_name, class_name) */
4293 PyObject *module_name, *class_name;
4294
4295 assert(nbytes == 1 || nbytes == 2 || nbytes == 4);
4296 if (self->read_func(self, &codebytes, nbytes) < 0) return -1;
4297 code = calc_binint(codebytes, nbytes);
4298 if (code <= 0) { /* note that 0 is forbidden */
4299 /* Corrupt or hostile pickle. */
4300 PyErr_SetString(UnpicklingError, "EXT specifies code <= 0");
4301 return -1;
4302 }
4303
4304 /* Look for the code in the cache. */
4305 py_code = PyInt_FromLong(code);
4306 if (py_code == NULL) return -1;
4307 obj = PyDict_GetItem(extension_cache, py_code);
4308 if (obj != NULL) {
4309 /* Bingo. */
4310 Py_DECREF(py_code);
4311 PDATA_APPEND(self->stack, obj, -1);
4312 return 0;
4313 }
4314
4315 /* Look up the (module_name, class_name) pair. */
4316 pair = PyDict_GetItem(inverted_registry, py_code);
4317 if (pair == NULL) {
4318 Py_DECREF(py_code);
4319 PyErr_Format(PyExc_ValueError, "unregistered extension "
4320 "code %ld", code);
4321 return -1;
4322 }
4323 /* Since the extension registry is manipulable via Python code,
4324 * confirm that pair is really a 2-tuple of strings.
4325 */
4326 if (!PyTuple_Check(pair) || PyTuple_Size(pair) != 2 ||
4327 !PyString_Check(module_name = PyTuple_GET_ITEM(pair, 0)) ||
4328 !PyString_Check(class_name = PyTuple_GET_ITEM(pair, 1))) {
4329 Py_DECREF(py_code);
4330 PyErr_Format(PyExc_ValueError, "_inverted_registry[%ld] "
4331 "isn't a 2-tuple of strings", code);
4332 return -1;
4333 }
4334 /* Load the object. */
4335 obj = find_class(module_name, class_name, self->find_class);
4336 if (obj == NULL) {
4337 Py_DECREF(py_code);
4338 return -1;
4339 }
4340 /* Cache code -> obj. */
4341 code = PyDict_SetItem(extension_cache, py_code, obj);
4342 Py_DECREF(py_code);
4343 if (code < 0) {
4344 Py_DECREF(obj);
4345 return -1;
4346 }
4347 PDATA_PUSH(self->stack, obj, -1);
4348 return 0;
4349 }
4350
4351 static int
load_put(Unpicklerobject * self)4352 load_put(Unpicklerobject *self)
4353 {
4354 PyObject *py_str = 0, *value = 0;
4355 Py_ssize_t len, l;
4356 char *s;
4357
4358 if ((l = self->readline_func(self, &s)) < 0) return -1;
4359 if (l < 2) return bad_readline();
4360 if (!( len=self->stack->length )) return stackUnderflow();
4361 if (!( py_str = PyString_FromStringAndSize(s, l - 1))) return -1;
4362 value=self->stack->data[len-1];
4363 l=PyDict_SetItem(self->memo, py_str, value);
4364 Py_DECREF(py_str);
4365 return l;
4366 }
4367
4368
4369 static int
load_binput(Unpicklerobject * self)4370 load_binput(Unpicklerobject *self)
4371 {
4372 PyObject *py_key = 0, *value = 0;
4373 unsigned char key;
4374 char *s;
4375 Py_ssize_t len;
4376
4377 if (self->read_func(self, &s, 1) < 0) return -1;
4378 if (!( (len=self->stack->length) > 0 )) return stackUnderflow();
4379
4380 key = (unsigned char)s[0];
4381
4382 if (!( py_key = PyInt_FromLong((long)key))) return -1;
4383 value=self->stack->data[len-1];
4384 len=PyDict_SetItem(self->memo, py_key, value);
4385 Py_DECREF(py_key);
4386 return len;
4387 }
4388
4389
4390 static int
load_long_binput(Unpicklerobject * self)4391 load_long_binput(Unpicklerobject *self)
4392 {
4393 PyObject *py_key = 0, *value = 0;
4394 Py_ssize_t key;
4395 unsigned char c;
4396 char *s;
4397 Py_ssize_t len;
4398
4399 if (self->read_func(self, &s, 4) < 0) return -1;
4400 if (!( len=self->stack->length )) return stackUnderflow();
4401
4402 c = (unsigned char)s[0];
4403 key = (long)c;
4404 c = (unsigned char)s[1];
4405 key |= (long)c << 8;
4406 c = (unsigned char)s[2];
4407 key |= (long)c << 16;
4408 c = (unsigned char)s[3];
4409 key |= (long)c << 24;
4410
4411 if (!( py_key = PyInt_FromLong(key))) return -1;
4412 value=self->stack->data[len-1];
4413 len=PyDict_SetItem(self->memo, py_key, value);
4414 Py_DECREF(py_key);
4415 return len;
4416 }
4417
4418
4419 static int
do_append(Unpicklerobject * self,Py_ssize_t x)4420 do_append(Unpicklerobject *self, Py_ssize_t x)
4421 {
4422 PyObject *value = 0, *list = 0, *append_method = 0;
4423 Py_ssize_t len, i;
4424
4425 len=self->stack->length;
4426 if (!( len >= x && x > 0 )) return stackUnderflow();
4427 /* nothing to do */
4428 if (len==x) return 0;
4429
4430 list=self->stack->data[x-1];
4431
4432 if (PyList_Check(list)) {
4433 PyObject *slice;
4434 int list_len;
4435
4436 slice=Pdata_popList(self->stack, x);
4437 if (! slice) return -1;
4438 list_len = PyList_GET_SIZE(list);
4439 i=PyList_SetSlice(list, list_len, list_len, slice);
4440 Py_DECREF(slice);
4441 return i;
4442 }
4443 else {
4444
4445 if (!( append_method = PyObject_GetAttr(list, append_str)))
4446 return -1;
4447
4448 for (i = x; i < len; i++) {
4449 PyObject *junk;
4450
4451 value=self->stack->data[i];
4452 junk=0;
4453 ARG_TUP(self, value);
4454 if (self->arg) {
4455 junk = PyObject_Call(append_method, self->arg,
4456 NULL);
4457 FREE_ARG_TUP(self);
4458 }
4459 if (! junk) {
4460 Pdata_clear(self->stack, i+1);
4461 self->stack->length=x;
4462 Py_DECREF(append_method);
4463 return -1;
4464 }
4465 Py_DECREF(junk);
4466 }
4467 self->stack->length=x;
4468 Py_DECREF(append_method);
4469 }
4470
4471 return 0;
4472 }
4473
4474
4475 static int
load_append(Unpicklerobject * self)4476 load_append(Unpicklerobject *self)
4477 {
4478 if (self->stack->length - 1 <= 0)
4479 return stackUnderflow();
4480 return do_append(self, self->stack->length - 1);
4481 }
4482
4483
4484 static int
load_appends(Unpicklerobject * self)4485 load_appends(Unpicklerobject *self)
4486 {
4487 Py_ssize_t i = marker(self);
4488 if (i < 0)
4489 return -1;
4490 return do_append(self, i);
4491 }
4492
4493
4494 static Py_ssize_t
do_setitems(Unpicklerobject * self,Py_ssize_t x)4495 do_setitems(Unpicklerobject *self, Py_ssize_t x)
4496 {
4497 PyObject *value = 0, *key = 0, *dict = 0;
4498 Py_ssize_t len, i, r=0;
4499
4500 if (!( (len=self->stack->length) >= x
4501 && x > 0 )) return stackUnderflow();
4502 if (len == x) /* nothing to do */
4503 return 0;
4504 if ((len - x) % 2 != 0) {
4505 /* Currupt or hostile pickle -- we never write one like this. */
4506 PyErr_SetString(UnpicklingError,
4507 "odd number of items for SETITEMS");
4508 return -1;
4509 }
4510
4511 dict=self->stack->data[x-1];
4512
4513 for (i = x+1; i < len; i += 2) {
4514 key =self->stack->data[i-1];
4515 value=self->stack->data[i ];
4516 if (PyObject_SetItem(dict, key, value) < 0) {
4517 r=-1;
4518 break;
4519 }
4520 }
4521
4522 Pdata_clear(self->stack, x);
4523
4524 return r;
4525 }
4526
4527
4528 static int
load_setitem(Unpicklerobject * self)4529 load_setitem(Unpicklerobject *self)
4530 {
4531 return do_setitems(self, self->stack->length - 2);
4532 }
4533
4534 static int
load_setitems(Unpicklerobject * self)4535 load_setitems(Unpicklerobject *self)
4536 {
4537 Py_ssize_t i = marker(self);
4538 if (i < 0)
4539 return -1;
4540 return do_setitems(self, i);
4541 }
4542
4543
4544 static int
load_build(Unpicklerobject * self)4545 load_build(Unpicklerobject *self)
4546 {
4547 PyObject *state, *inst, *slotstate;
4548 PyObject *__setstate__;
4549 PyObject *d_key, *d_value;
4550 int res = -1;
4551 Py_ssize_t i;
4552
4553 /* Stack is ... instance, state. We want to leave instance at
4554 * the stack top, possibly mutated via instance.__setstate__(state).
4555 */
4556 if (self->stack->length < 2)
4557 return stackUnderflow();
4558 PDATA_POP(self->stack, state);
4559 if (state == NULL)
4560 return -1;
4561 inst = self->stack->data[self->stack->length - 1];
4562
4563 __setstate__ = PyObject_GetAttr(inst, __setstate___str);
4564 if (__setstate__ != NULL) {
4565 PyObject *junk = NULL;
4566
4567 /* The explicit __setstate__ is responsible for everything. */
4568 ARG_TUP(self, state);
4569 if (self->arg) {
4570 junk = PyObject_Call(__setstate__, self->arg, NULL);
4571 FREE_ARG_TUP(self);
4572 }
4573 Py_DECREF(__setstate__);
4574 if (junk == NULL)
4575 return -1;
4576 Py_DECREF(junk);
4577 return 0;
4578 }
4579 if (!PyErr_ExceptionMatches(PyExc_AttributeError))
4580 return -1;
4581 PyErr_Clear();
4582
4583 /* A default __setstate__. First see whether state embeds a
4584 * slot state dict too (a proto 2 addition).
4585 */
4586 if (PyTuple_Check(state) && PyTuple_Size(state) == 2) {
4587 PyObject *temp = state;
4588 state = PyTuple_GET_ITEM(temp, 0);
4589 slotstate = PyTuple_GET_ITEM(temp, 1);
4590 Py_INCREF(state);
4591 Py_INCREF(slotstate);
4592 Py_DECREF(temp);
4593 }
4594 else
4595 slotstate = NULL;
4596
4597 /* Set inst.__dict__ from the state dict (if any). */
4598 if (state != Py_None) {
4599 PyObject *dict;
4600 if (! PyDict_Check(state)) {
4601 PyErr_SetString(UnpicklingError, "state is not a "
4602 "dictionary");
4603 goto finally;
4604 }
4605 dict = PyObject_GetAttr(inst, __dict___str);
4606 if (dict == NULL)
4607 goto finally;
4608
4609 i = 0;
4610 while (PyDict_Next(state, &i, &d_key, &d_value)) {
4611 /* normally the keys for instance attributes are
4612 interned. we should try to do that here. */
4613 Py_INCREF(d_key);
4614 if (PyString_CheckExact(d_key))
4615 PyString_InternInPlace(&d_key);
4616 if (PyObject_SetItem(dict, d_key, d_value) < 0) {
4617 Py_DECREF(d_key);
4618 goto finally;
4619 }
4620 Py_DECREF(d_key);
4621 }
4622 Py_DECREF(dict);
4623 }
4624
4625 /* Also set instance attributes from the slotstate dict (if any). */
4626 if (slotstate != NULL) {
4627 if (! PyDict_Check(slotstate)) {
4628 PyErr_SetString(UnpicklingError, "slot state is not "
4629 "a dictionary");
4630 goto finally;
4631 }
4632 i = 0;
4633 while (PyDict_Next(slotstate, &i, &d_key, &d_value)) {
4634 if (PyObject_SetAttr(inst, d_key, d_value) < 0)
4635 goto finally;
4636 }
4637 }
4638 res = 0;
4639
4640 finally:
4641 Py_DECREF(state);
4642 Py_XDECREF(slotstate);
4643 return res;
4644 }
4645
4646
4647 static int
load_mark(Unpicklerobject * self)4648 load_mark(Unpicklerobject *self)
4649 {
4650 Py_ssize_t s;
4651
4652 /* Note that we split the (pickle.py) stack into two stacks, an
4653 object stack and a mark stack. Here we push a mark onto the
4654 mark stack.
4655 */
4656
4657 if ((self->num_marks + 1) >= self->marks_size) {
4658 Py_ssize_t *marks;
4659 s=self->marks_size+20;
4660 if (s <= self->num_marks) s=self->num_marks + 1;
4661 if (self->marks == NULL)
4662 marks=(Py_ssize_t *)malloc(s * sizeof(Py_ssize_t));
4663 else
4664 marks=(Py_ssize_t *)realloc(self->marks,
4665 s * sizeof(Py_ssize_t));
4666 if (!marks) {
4667 PyErr_NoMemory();
4668 return -1;
4669 }
4670 self->marks = marks;
4671 self->marks_size = s;
4672 }
4673
4674 self->marks[self->num_marks++] = self->stack->length;
4675
4676 return 0;
4677 }
4678
4679 static int
load_reduce(Unpicklerobject * self)4680 load_reduce(Unpicklerobject *self)
4681 {
4682 PyObject *callable = 0, *arg_tup = 0, *ob = 0;
4683
4684 PDATA_POP(self->stack, arg_tup);
4685 if (! arg_tup) return -1;
4686 PDATA_POP(self->stack, callable);
4687 if (callable) {
4688 ob = Instance_New(callable, arg_tup);
4689 Py_DECREF(callable);
4690 }
4691 Py_DECREF(arg_tup);
4692
4693 if (! ob) return -1;
4694
4695 PDATA_PUSH(self->stack, ob, -1);
4696 return 0;
4697 }
4698
4699 /* Just raises an error if we don't know the protocol specified. PROTO
4700 * is the first opcode for protocols >= 2.
4701 */
4702 static int
load_proto(Unpicklerobject * self)4703 load_proto(Unpicklerobject *self)
4704 {
4705 int i;
4706 char *protobyte;
4707
4708 i = self->read_func(self, &protobyte, 1);
4709 if (i < 0)
4710 return -1;
4711
4712 i = calc_binint(protobyte, 1);
4713 /* No point checking for < 0, since calc_binint returns an unsigned
4714 * int when chewing on 1 byte.
4715 */
4716 assert(i >= 0);
4717 if (i <= HIGHEST_PROTOCOL)
4718 return 0;
4719
4720 PyErr_Format(PyExc_ValueError, "unsupported pickle protocol: %d", i);
4721 return -1;
4722 }
4723
4724 static PyObject *
load(Unpicklerobject * self)4725 load(Unpicklerobject *self)
4726 {
4727 PyObject *err = 0, *val = 0;
4728 char *s;
4729
4730 self->num_marks = 0;
4731 if (self->stack->length) Pdata_clear(self->stack, 0);
4732
4733 while (1) {
4734 if (self->read_func(self, &s, 1) < 0)
4735 break;
4736
4737 switch (s[0]) {
4738 case NONE:
4739 if (load_none(self) < 0)
4740 break;
4741 continue;
4742
4743 case BININT:
4744 if (load_binint(self) < 0)
4745 break;
4746 continue;
4747
4748 case BININT1:
4749 if (load_binint1(self) < 0)
4750 break;
4751 continue;
4752
4753 case BININT2:
4754 if (load_binint2(self) < 0)
4755 break;
4756 continue;
4757
4758 case INT:
4759 if (load_int(self) < 0)
4760 break;
4761 continue;
4762
4763 case LONG:
4764 if (load_long(self) < 0)
4765 break;
4766 continue;
4767
4768 case LONG1:
4769 if (load_counted_long(self, 1) < 0)
4770 break;
4771 continue;
4772
4773 case LONG4:
4774 if (load_counted_long(self, 4) < 0)
4775 break;
4776 continue;
4777
4778 case FLOAT:
4779 if (load_float(self) < 0)
4780 break;
4781 continue;
4782
4783 case BINFLOAT:
4784 if (load_binfloat(self) < 0)
4785 break;
4786 continue;
4787
4788 case BINSTRING:
4789 if (load_binstring(self) < 0)
4790 break;
4791 continue;
4792
4793 case SHORT_BINSTRING:
4794 if (load_short_binstring(self) < 0)
4795 break;
4796 continue;
4797
4798 case STRING:
4799 if (load_string(self) < 0)
4800 break;
4801 continue;
4802
4803 #ifdef Py_USING_UNICODE
4804 case UNICODE:
4805 if (load_unicode(self) < 0)
4806 break;
4807 continue;
4808
4809 case BINUNICODE:
4810 if (load_binunicode(self) < 0)
4811 break;
4812 continue;
4813 #endif
4814
4815 case EMPTY_TUPLE:
4816 if (load_counted_tuple(self, 0) < 0)
4817 break;
4818 continue;
4819
4820 case TUPLE1:
4821 if (load_counted_tuple(self, 1) < 0)
4822 break;
4823 continue;
4824
4825 case TUPLE2:
4826 if (load_counted_tuple(self, 2) < 0)
4827 break;
4828 continue;
4829
4830 case TUPLE3:
4831 if (load_counted_tuple(self, 3) < 0)
4832 break;
4833 continue;
4834
4835 case TUPLE:
4836 if (load_tuple(self) < 0)
4837 break;
4838 continue;
4839
4840 case EMPTY_LIST:
4841 if (load_empty_list(self) < 0)
4842 break;
4843 continue;
4844
4845 case LIST:
4846 if (load_list(self) < 0)
4847 break;
4848 continue;
4849
4850 case EMPTY_DICT:
4851 if (load_empty_dict(self) < 0)
4852 break;
4853 continue;
4854
4855 case DICT:
4856 if (load_dict(self) < 0)
4857 break;
4858 continue;
4859
4860 case OBJ:
4861 if (load_obj(self) < 0)
4862 break;
4863 continue;
4864
4865 case INST:
4866 if (load_inst(self) < 0)
4867 break;
4868 continue;
4869
4870 case NEWOBJ:
4871 if (load_newobj(self) < 0)
4872 break;
4873 continue;
4874
4875 case GLOBAL:
4876 if (load_global(self) < 0)
4877 break;
4878 continue;
4879
4880 case APPEND:
4881 if (load_append(self) < 0)
4882 break;
4883 continue;
4884
4885 case APPENDS:
4886 if (load_appends(self) < 0)
4887 break;
4888 continue;
4889
4890 case BUILD:
4891 if (load_build(self) < 0)
4892 break;
4893 continue;
4894
4895 case DUP:
4896 if (load_dup(self) < 0)
4897 break;
4898 continue;
4899
4900 case BINGET:
4901 if (load_binget(self) < 0)
4902 break;
4903 continue;
4904
4905 case LONG_BINGET:
4906 if (load_long_binget(self) < 0)
4907 break;
4908 continue;
4909
4910 case GET:
4911 if (load_get(self) < 0)
4912 break;
4913 continue;
4914
4915 case EXT1:
4916 if (load_extension(self, 1) < 0)
4917 break;
4918 continue;
4919
4920 case EXT2:
4921 if (load_extension(self, 2) < 0)
4922 break;
4923 continue;
4924
4925 case EXT4:
4926 if (load_extension(self, 4) < 0)
4927 break;
4928 continue;
4929 case MARK:
4930 if (load_mark(self) < 0)
4931 break;
4932 continue;
4933
4934 case BINPUT:
4935 if (load_binput(self) < 0)
4936 break;
4937 continue;
4938
4939 case LONG_BINPUT:
4940 if (load_long_binput(self) < 0)
4941 break;
4942 continue;
4943
4944 case PUT:
4945 if (load_put(self) < 0)
4946 break;
4947 continue;
4948
4949 case POP:
4950 if (load_pop(self) < 0)
4951 break;
4952 continue;
4953
4954 case POP_MARK:
4955 if (load_pop_mark(self) < 0)
4956 break;
4957 continue;
4958
4959 case SETITEM:
4960 if (load_setitem(self) < 0)
4961 break;
4962 continue;
4963
4964 case SETITEMS:
4965 if (load_setitems(self) < 0)
4966 break;
4967 continue;
4968
4969 case STOP:
4970 break;
4971
4972 case PERSID:
4973 if (load_persid(self) < 0)
4974 break;
4975 continue;
4976
4977 case BINPERSID:
4978 if (load_binpersid(self) < 0)
4979 break;
4980 continue;
4981
4982 case REDUCE:
4983 if (load_reduce(self) < 0)
4984 break;
4985 continue;
4986
4987 case PROTO:
4988 if (load_proto(self) < 0)
4989 break;
4990 continue;
4991
4992 case NEWTRUE:
4993 if (load_bool(self, Py_True) < 0)
4994 break;
4995 continue;
4996
4997 case NEWFALSE:
4998 if (load_bool(self, Py_False) < 0)
4999 break;
5000 continue;
5001
5002 case '\0':
5003 /* end of file */
5004 PyErr_SetNone(PyExc_EOFError);
5005 break;
5006
5007 default:
5008 cPickle_ErrFormat(UnpicklingError,
5009 "invalid load key, '%s'.",
5010 "c", s[0]);
5011 return NULL;
5012 }
5013
5014 break;
5015 }
5016
5017 if ((err = PyErr_Occurred())) {
5018 if (err == PyExc_EOFError) {
5019 PyErr_SetNone(PyExc_EOFError);
5020 }
5021 return NULL;
5022 }
5023
5024 PDATA_POP(self->stack, val);
5025 return val;
5026 }
5027
5028
5029 /* No-load functions to support noload, which is used to
5030 find persistent references. */
5031
5032 static int
noload_obj(Unpicklerobject * self)5033 noload_obj(Unpicklerobject *self)
5034 {
5035 Py_ssize_t i;
5036
5037 if ((i = marker(self)) < 0) return -1;
5038 return Pdata_clear(self->stack, i+1);
5039 }
5040
5041
5042 static int
noload_inst(Unpicklerobject * self)5043 noload_inst(Unpicklerobject *self)
5044 {
5045 Py_ssize_t i;
5046 char *s;
5047
5048 if ((i = marker(self)) < 0) return -1;
5049 Pdata_clear(self->stack, i);
5050 if (self->readline_func(self, &s) < 0) return -1;
5051 if (self->readline_func(self, &s) < 0) return -1;
5052 PDATA_APPEND(self->stack, Py_None, -1);
5053 return 0;
5054 }
5055
5056 static int
noload_newobj(Unpicklerobject * self)5057 noload_newobj(Unpicklerobject *self)
5058 {
5059 PyObject *obj;
5060
5061 PDATA_POP(self->stack, obj); /* pop argtuple */
5062 if (obj == NULL) return -1;
5063 Py_DECREF(obj);
5064
5065 PDATA_POP(self->stack, obj); /* pop cls */
5066 if (obj == NULL) return -1;
5067 Py_DECREF(obj);
5068
5069 PDATA_APPEND(self->stack, Py_None, -1);
5070 return 0;
5071 }
5072
5073 static int
noload_global(Unpicklerobject * self)5074 noload_global(Unpicklerobject *self)
5075 {
5076 char *s;
5077
5078 if (self->readline_func(self, &s) < 0) return -1;
5079 if (self->readline_func(self, &s) < 0) return -1;
5080 PDATA_APPEND(self->stack, Py_None,-1);
5081 return 0;
5082 }
5083
5084 static int
noload_reduce(Unpicklerobject * self)5085 noload_reduce(Unpicklerobject *self)
5086 {
5087
5088 if (self->stack->length < 2) return stackUnderflow();
5089 Pdata_clear(self->stack, self->stack->length-2);
5090 PDATA_APPEND(self->stack, Py_None,-1);
5091 return 0;
5092 }
5093
5094 static int
noload_build(Unpicklerobject * self)5095 noload_build(Unpicklerobject *self) {
5096
5097 if (self->stack->length < 1) return stackUnderflow();
5098 Pdata_clear(self->stack, self->stack->length-1);
5099 return 0;
5100 }
5101
5102 static int
noload_extension(Unpicklerobject * self,int nbytes)5103 noload_extension(Unpicklerobject *self, int nbytes)
5104 {
5105 char *codebytes;
5106
5107 assert(nbytes == 1 || nbytes == 2 || nbytes == 4);
5108 if (self->read_func(self, &codebytes, nbytes) < 0) return -1;
5109 PDATA_APPEND(self->stack, Py_None, -1);
5110 return 0;
5111 }
5112
5113 static int
noload_append(Unpicklerobject * self)5114 noload_append(Unpicklerobject *self)
5115 {
5116 return Pdata_clear(self->stack, self->stack->length - 1);
5117 }
5118
5119 static int
noload_appends(Unpicklerobject * self)5120 noload_appends(Unpicklerobject *self)
5121 {
5122 Py_ssize_t i;
5123 if ((i = marker(self)) < 0) return -1;
5124 return Pdata_clear(self->stack, i);
5125 }
5126
5127 static int
noload_setitem(Unpicklerobject * self)5128 noload_setitem(Unpicklerobject *self)
5129 {
5130 return Pdata_clear(self->stack, self->stack->length - 2);
5131 }
5132
5133 static int
noload_setitems(Unpicklerobject * self)5134 noload_setitems(Unpicklerobject *self)
5135 {
5136 Py_ssize_t i;
5137 if ((i = marker(self)) < 0) return -1;
5138 return Pdata_clear(self->stack, i);
5139 }
5140
5141 static PyObject *
noload(Unpicklerobject * self)5142 noload(Unpicklerobject *self)
5143 {
5144 PyObject *err = 0, *val = 0;
5145 char *s;
5146
5147 self->num_marks = 0;
5148 Pdata_clear(self->stack, 0);
5149
5150 while (1) {
5151 if (self->read_func(self, &s, 1) < 0)
5152 break;
5153
5154 switch (s[0]) {
5155 case NONE:
5156 if (load_none(self) < 0)
5157 break;
5158 continue;
5159
5160 case BININT:
5161 if (load_binint(self) < 0)
5162 break;
5163 continue;
5164
5165 case BININT1:
5166 if (load_binint1(self) < 0)
5167 break;
5168 continue;
5169
5170 case BININT2:
5171 if (load_binint2(self) < 0)
5172 break;
5173 continue;
5174
5175 case INT:
5176 if (load_int(self) < 0)
5177 break;
5178 continue;
5179
5180 case LONG:
5181 if (load_long(self) < 0)
5182 break;
5183 continue;
5184
5185 case LONG1:
5186 if (load_counted_long(self, 1) < 0)
5187 break;
5188 continue;
5189
5190 case LONG4:
5191 if (load_counted_long(self, 4) < 0)
5192 break;
5193 continue;
5194
5195 case FLOAT:
5196 if (load_float(self) < 0)
5197 break;
5198 continue;
5199
5200 case BINFLOAT:
5201 if (load_binfloat(self) < 0)
5202 break;
5203 continue;
5204
5205 case BINSTRING:
5206 if (load_binstring(self) < 0)
5207 break;
5208 continue;
5209
5210 case SHORT_BINSTRING:
5211 if (load_short_binstring(self) < 0)
5212 break;
5213 continue;
5214
5215 case STRING:
5216 if (load_string(self) < 0)
5217 break;
5218 continue;
5219
5220 #ifdef Py_USING_UNICODE
5221 case UNICODE:
5222 if (load_unicode(self) < 0)
5223 break;
5224 continue;
5225
5226 case BINUNICODE:
5227 if (load_binunicode(self) < 0)
5228 break;
5229 continue;
5230 #endif
5231
5232 case EMPTY_TUPLE:
5233 if (load_counted_tuple(self, 0) < 0)
5234 break;
5235 continue;
5236
5237 case TUPLE1:
5238 if (load_counted_tuple(self, 1) < 0)
5239 break;
5240 continue;
5241
5242 case TUPLE2:
5243 if (load_counted_tuple(self, 2) < 0)
5244 break;
5245 continue;
5246
5247 case TUPLE3:
5248 if (load_counted_tuple(self, 3) < 0)
5249 break;
5250 continue;
5251
5252 case TUPLE:
5253 if (load_tuple(self) < 0)
5254 break;
5255 continue;
5256
5257 case EMPTY_LIST:
5258 if (load_empty_list(self) < 0)
5259 break;
5260 continue;
5261
5262 case LIST:
5263 if (load_list(self) < 0)
5264 break;
5265 continue;
5266
5267 case EMPTY_DICT:
5268 if (load_empty_dict(self) < 0)
5269 break;
5270 continue;
5271
5272 case DICT:
5273 if (load_dict(self) < 0)
5274 break;
5275 continue;
5276
5277 case OBJ:
5278 if (noload_obj(self) < 0)
5279 break;
5280 continue;
5281
5282 case INST:
5283 if (noload_inst(self) < 0)
5284 break;
5285 continue;
5286
5287 case NEWOBJ:
5288 if (noload_newobj(self) < 0)
5289 break;
5290 continue;
5291
5292 case GLOBAL:
5293 if (noload_global(self) < 0)
5294 break;
5295 continue;
5296
5297 case APPEND:
5298 if (noload_append(self) < 0)
5299 break;
5300 continue;
5301
5302 case APPENDS:
5303 if (noload_appends(self) < 0)
5304 break;
5305 continue;
5306
5307 case BUILD:
5308 if (noload_build(self) < 0)
5309 break;
5310 continue;
5311
5312 case DUP:
5313 if (load_dup(self) < 0)
5314 break;
5315 continue;
5316
5317 case BINGET:
5318 if (load_binget(self) < 0)
5319 break;
5320 continue;
5321
5322 case LONG_BINGET:
5323 if (load_long_binget(self) < 0)
5324 break;
5325 continue;
5326
5327 case GET:
5328 if (load_get(self) < 0)
5329 break;
5330 continue;
5331
5332 case EXT1:
5333 if (noload_extension(self, 1) < 0)
5334 break;
5335 continue;
5336
5337 case EXT2:
5338 if (noload_extension(self, 2) < 0)
5339 break;
5340 continue;
5341
5342 case EXT4:
5343 if (noload_extension(self, 4) < 0)
5344 break;
5345 continue;
5346
5347 case MARK:
5348 if (load_mark(self) < 0)
5349 break;
5350 continue;
5351
5352 case BINPUT:
5353 if (load_binput(self) < 0)
5354 break;
5355 continue;
5356
5357 case LONG_BINPUT:
5358 if (load_long_binput(self) < 0)
5359 break;
5360 continue;
5361
5362 case PUT:
5363 if (load_put(self) < 0)
5364 break;
5365 continue;
5366
5367 case POP:
5368 if (load_pop(self) < 0)
5369 break;
5370 continue;
5371
5372 case POP_MARK:
5373 if (load_pop_mark(self) < 0)
5374 break;
5375 continue;
5376
5377 case SETITEM:
5378 if (noload_setitem(self) < 0)
5379 break;
5380 continue;
5381
5382 case SETITEMS:
5383 if (noload_setitems(self) < 0)
5384 break;
5385 continue;
5386
5387 case STOP:
5388 break;
5389
5390 case PERSID:
5391 if (load_persid(self) < 0)
5392 break;
5393 continue;
5394
5395 case BINPERSID:
5396 if (load_binpersid(self) < 0)
5397 break;
5398 continue;
5399
5400 case REDUCE:
5401 if (noload_reduce(self) < 0)
5402 break;
5403 continue;
5404
5405 case PROTO:
5406 if (load_proto(self) < 0)
5407 break;
5408 continue;
5409
5410 case NEWTRUE:
5411 if (load_bool(self, Py_True) < 0)
5412 break;
5413 continue;
5414
5415 case NEWFALSE:
5416 if (load_bool(self, Py_False) < 0)
5417 break;
5418 continue;
5419 default:
5420 cPickle_ErrFormat(UnpicklingError,
5421 "invalid load key, '%s'.",
5422 "c", s[0]);
5423 return NULL;
5424 }
5425
5426 break;
5427 }
5428
5429 if ((err = PyErr_Occurred())) {
5430 if (err == PyExc_EOFError) {
5431 PyErr_SetNone(PyExc_EOFError);
5432 }
5433 return NULL;
5434 }
5435
5436 PDATA_POP(self->stack, val);
5437 return val;
5438 }
5439
5440
5441 static PyObject *
Unpickler_load(Unpicklerobject * self,PyObject * unused)5442 Unpickler_load(Unpicklerobject *self, PyObject *unused)
5443 {
5444 return load(self);
5445 }
5446
5447 static PyObject *
Unpickler_noload(Unpicklerobject * self,PyObject * unused)5448 Unpickler_noload(Unpicklerobject *self, PyObject *unused)
5449 {
5450 return noload(self);
5451 }
5452
5453
5454 static struct PyMethodDef Unpickler_methods[] = {
5455 {"load", (PyCFunction)Unpickler_load, METH_NOARGS,
5456 PyDoc_STR("load() -- Load a pickle")
5457 },
5458 {"noload", (PyCFunction)Unpickler_noload, METH_NOARGS,
5459 PyDoc_STR(
5460 "noload() -- not load a pickle, but go through most of the motions\n"
5461 "\n"
5462 "This function can be used to read past a pickle without instantiating\n"
5463 "any objects or importing any modules. It can also be used to find all\n"
5464 "persistent references without instantiating any objects or importing\n"
5465 "any modules.\n")
5466 },
5467 {NULL, NULL} /* sentinel */
5468 };
5469
5470
5471 static Unpicklerobject *
newUnpicklerobject(PyObject * f)5472 newUnpicklerobject(PyObject *f)
5473 {
5474 Unpicklerobject *self;
5475
5476 if (!( self = PyObject_GC_New(Unpicklerobject, &Unpicklertype)))
5477 return NULL;
5478
5479 self->file = NULL;
5480 self->arg = NULL;
5481 self->stack = (Pdata*)Pdata_New();
5482 self->pers_func = NULL;
5483 self->last_string = NULL;
5484 self->marks = NULL;
5485 self->num_marks = 0;
5486 self->marks_size = 0;
5487 self->buf_size = 0;
5488 self->read = NULL;
5489 self->readline = NULL;
5490 self->find_class = NULL;
5491
5492 if (!( self->memo = PyDict_New()))
5493 goto err;
5494
5495 if (!self->stack)
5496 goto err;
5497
5498 Py_INCREF(f);
5499 self->file = f;
5500
5501 /* Set read, readline based on type of f */
5502 if (PyFile_Check(f)) {
5503 self->fp = PyFile_AsFile(f);
5504 if (self->fp == NULL) {
5505 PyErr_SetString(PyExc_ValueError,
5506 "I/O operation on closed file");
5507 goto err;
5508 }
5509 self->read_func = read_file;
5510 self->readline_func = readline_file;
5511 }
5512 else if (PycStringIO_InputCheck(f)) {
5513 self->fp = NULL;
5514 self->read_func = read_cStringIO;
5515 self->readline_func = readline_cStringIO;
5516 }
5517 else {
5518
5519 self->fp = NULL;
5520 self->read_func = read_other;
5521 self->readline_func = readline_other;
5522
5523 if (!( (self->readline = PyObject_GetAttr(f, readline_str)) &&
5524 (self->read = PyObject_GetAttr(f, read_str)))) {
5525 PyErr_Clear();
5526 PyErr_SetString( PyExc_TypeError,
5527 "argument must have 'read' and "
5528 "'readline' attributes" );
5529 goto err;
5530 }
5531 }
5532 PyObject_GC_Track(self);
5533
5534 return self;
5535
5536 err:
5537 Py_DECREF((PyObject *)self);
5538 return NULL;
5539 }
5540
5541
5542 static PyObject *
get_Unpickler(PyObject * self,PyObject * file)5543 get_Unpickler(PyObject *self, PyObject *file)
5544 {
5545 return (PyObject *)newUnpicklerobject(file);
5546 }
5547
5548
5549 static void
Unpickler_dealloc(Unpicklerobject * self)5550 Unpickler_dealloc(Unpicklerobject *self)
5551 {
5552 PyObject_GC_UnTrack((PyObject *)self);
5553 Py_XDECREF(self->readline);
5554 Py_XDECREF(self->read);
5555 Py_XDECREF(self->file);
5556 Py_XDECREF(self->memo);
5557 Py_XDECREF(self->stack);
5558 Py_XDECREF(self->pers_func);
5559 Py_XDECREF(self->arg);
5560 Py_XDECREF(self->last_string);
5561 Py_XDECREF(self->find_class);
5562
5563 if (self->marks) {
5564 free(self->marks);
5565 }
5566
5567 if (self->buf_size) {
5568 free(self->buf);
5569 }
5570
5571 Py_TYPE(self)->tp_free((PyObject *)self);
5572 }
5573
5574 static int
Unpickler_traverse(Unpicklerobject * self,visitproc visit,void * arg)5575 Unpickler_traverse(Unpicklerobject *self, visitproc visit, void *arg)
5576 {
5577 Py_VISIT(self->readline);
5578 Py_VISIT(self->read);
5579 Py_VISIT(self->file);
5580 Py_VISIT(self->memo);
5581 Py_VISIT(self->stack);
5582 Py_VISIT(self->pers_func);
5583 Py_VISIT(self->arg);
5584 Py_VISIT(self->last_string);
5585 Py_VISIT(self->find_class);
5586 return 0;
5587 }
5588
5589 static int
Unpickler_clear(Unpicklerobject * self)5590 Unpickler_clear(Unpicklerobject *self)
5591 {
5592 Py_CLEAR(self->readline);
5593 Py_CLEAR(self->read);
5594 Py_CLEAR(self->file);
5595 Py_CLEAR(self->memo);
5596 Py_CLEAR(self->stack);
5597 Py_CLEAR(self->pers_func);
5598 Py_CLEAR(self->arg);
5599 Py_CLEAR(self->last_string);
5600 Py_CLEAR(self->find_class);
5601 return 0;
5602 }
5603
5604 static PyObject *
Unpickler_getattr(Unpicklerobject * self,char * name)5605 Unpickler_getattr(Unpicklerobject *self, char *name)
5606 {
5607 if (!strcmp(name, "persistent_load")) {
5608 if (!self->pers_func) {
5609 PyErr_SetString(PyExc_AttributeError, name);
5610 return NULL;
5611 }
5612
5613 Py_INCREF(self->pers_func);
5614 return self->pers_func;
5615 }
5616
5617 if (!strcmp(name, "find_global")) {
5618 if (!self->find_class) {
5619 PyErr_SetString(PyExc_AttributeError, name);
5620 return NULL;
5621 }
5622
5623 Py_INCREF(self->find_class);
5624 return self->find_class;
5625 }
5626
5627 if (!strcmp(name, "memo")) {
5628 if (!self->memo) {
5629 PyErr_SetString(PyExc_AttributeError, name);
5630 return NULL;
5631 }
5632
5633 Py_INCREF(self->memo);
5634 return self->memo;
5635 }
5636
5637 if (!strcmp(name, "UnpicklingError")) {
5638 Py_INCREF(UnpicklingError);
5639 return UnpicklingError;
5640 }
5641
5642 return Py_FindMethod(Unpickler_methods, (PyObject *)self, name);
5643 }
5644
5645
5646 static int
Unpickler_setattr(Unpicklerobject * self,char * name,PyObject * value)5647 Unpickler_setattr(Unpicklerobject *self, char *name, PyObject *value)
5648 {
5649
5650 if (!strcmp(name, "persistent_load")) {
5651 Py_XINCREF(value);
5652 Py_XSETREF(self->pers_func, value);
5653 return 0;
5654 }
5655
5656 if (!strcmp(name, "find_global")) {
5657 Py_XINCREF(value);
5658 Py_XSETREF(self->find_class, value);
5659 return 0;
5660 }
5661
5662 if (! value) {
5663 PyErr_SetString(PyExc_TypeError,
5664 "attribute deletion is not supported");
5665 return -1;
5666 }
5667
5668 if (strcmp(name, "memo") == 0) {
5669 if (!PyDict_Check(value)) {
5670 PyErr_SetString(PyExc_TypeError,
5671 "memo must be a dictionary");
5672 return -1;
5673 }
5674 Py_INCREF(value);
5675 Py_XSETREF(self->memo, value);
5676 return 0;
5677 }
5678
5679 PyErr_SetString(PyExc_AttributeError, name);
5680 return -1;
5681 }
5682
5683 /* ---------------------------------------------------------------------------
5684 * Module-level functions.
5685 */
5686
5687 /* dump(obj, file, protocol=0). */
5688 static PyObject *
cpm_dump(PyObject * self,PyObject * args,PyObject * kwds)5689 cpm_dump(PyObject *self, PyObject *args, PyObject *kwds)
5690 {
5691 static char *kwlist[] = {"obj", "file", "protocol", NULL};
5692 PyObject *ob, *file, *res = NULL;
5693 Picklerobject *pickler = 0;
5694 int proto = 0;
5695
5696 if (!( PyArg_ParseTupleAndKeywords(args, kwds, "OO|i", kwlist,
5697 &ob, &file, &proto)))
5698 goto finally;
5699
5700 if (!( pickler = newPicklerobject(file, proto)))
5701 goto finally;
5702
5703 if (dump(pickler, ob) < 0)
5704 goto finally;
5705
5706 Py_INCREF(Py_None);
5707 res = Py_None;
5708
5709 finally:
5710 Py_XDECREF(pickler);
5711
5712 return res;
5713 }
5714
5715
5716 /* dumps(obj, protocol=0). */
5717 static PyObject *
cpm_dumps(PyObject * self,PyObject * args,PyObject * kwds)5718 cpm_dumps(PyObject *self, PyObject *args, PyObject *kwds)
5719 {
5720 static char *kwlist[] = {"obj", "protocol", NULL};
5721 PyObject *ob, *file = 0, *res = NULL;
5722 Picklerobject *pickler = 0;
5723 int proto = 0;
5724
5725 if (!( PyArg_ParseTupleAndKeywords(args, kwds, "O|i:dumps", kwlist,
5726 &ob, &proto)))
5727 goto finally;
5728
5729 if (!( file = PycStringIO->NewOutput(128)))
5730 goto finally;
5731
5732 if (!( pickler = newPicklerobject(file, proto)))
5733 goto finally;
5734
5735 if (dump(pickler, ob) < 0)
5736 goto finally;
5737
5738 res = PycStringIO->cgetvalue(file);
5739
5740 finally:
5741 Py_XDECREF(pickler);
5742 Py_XDECREF(file);
5743
5744 return res;
5745 }
5746
5747
5748 /* load(fileobj). */
5749 static PyObject *
cpm_load(PyObject * self,PyObject * ob)5750 cpm_load(PyObject *self, PyObject *ob)
5751 {
5752 Unpicklerobject *unpickler = 0;
5753 PyObject *res = NULL;
5754
5755 if (!( unpickler = newUnpicklerobject(ob)))
5756 goto finally;
5757
5758 res = load(unpickler);
5759
5760 finally:
5761 Py_XDECREF(unpickler);
5762
5763 return res;
5764 }
5765
5766
5767 /* loads(string) */
5768 static PyObject *
cpm_loads(PyObject * self,PyObject * args)5769 cpm_loads(PyObject *self, PyObject *args)
5770 {
5771 PyObject *ob, *file = 0, *res = NULL;
5772 Unpicklerobject *unpickler = 0;
5773
5774 if (!( PyArg_ParseTuple(args, "S:loads", &ob)))
5775 goto finally;
5776
5777 if (!( file = PycStringIO->NewInput(ob)))
5778 goto finally;
5779
5780 if (!( unpickler = newUnpicklerobject(file)))
5781 goto finally;
5782
5783 res = load(unpickler);
5784
5785 finally:
5786 Py_XDECREF(file);
5787 Py_XDECREF(unpickler);
5788
5789 return res;
5790 }
5791
5792
5793 PyDoc_STRVAR(Unpicklertype__doc__,
5794 "Objects that know how to unpickle");
5795
5796 static PyTypeObject Unpicklertype = {
5797 PyVarObject_HEAD_INIT(NULL, 0)
5798 "cPickle.Unpickler", /*tp_name*/
5799 sizeof(Unpicklerobject), /*tp_basicsize*/
5800 0,
5801 (destructor)Unpickler_dealloc, /* tp_dealloc */
5802 0, /* tp_print */
5803 (getattrfunc)Unpickler_getattr, /* tp_getattr */
5804 (setattrfunc)Unpickler_setattr, /* tp_setattr */
5805 0, /* tp_compare */
5806 0, /* tp_repr */
5807 0, /* tp_as_number */
5808 0, /* tp_as_sequence */
5809 0, /* tp_as_mapping */
5810 0, /* tp_hash */
5811 0, /* tp_call */
5812 0, /* tp_str */
5813 0, /* tp_getattro */
5814 0, /* tp_setattro */
5815 0, /* tp_as_buffer */
5816 Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC,
5817 Unpicklertype__doc__, /* tp_doc */
5818 (traverseproc)Unpickler_traverse, /* tp_traverse */
5819 (inquiry)Unpickler_clear, /* tp_clear */
5820 };
5821
5822 static struct PyMethodDef cPickle_methods[] = {
5823 {"dump", (PyCFunction)cpm_dump, METH_VARARGS | METH_KEYWORDS,
5824 PyDoc_STR("dump(obj, file, protocol=0) -- "
5825 "Write an object in pickle format to the given file.\n"
5826 "\n"
5827 "See the Pickler docstring for the meaning of optional argument proto.")
5828 },
5829
5830 {"dumps", (PyCFunction)cpm_dumps, METH_VARARGS | METH_KEYWORDS,
5831 PyDoc_STR("dumps(obj, protocol=0) -- "
5832 "Return a string containing an object in pickle format.\n"
5833 "\n"
5834 "See the Pickler docstring for the meaning of optional argument proto.")
5835 },
5836
5837 {"load", (PyCFunction)cpm_load, METH_O,
5838 PyDoc_STR("load(file) -- Load a pickle from the given file")},
5839
5840 {"loads", (PyCFunction)cpm_loads, METH_VARARGS,
5841 PyDoc_STR("loads(string) -- Load a pickle from the given string")},
5842
5843 {"Pickler", (PyCFunction)get_Pickler, METH_VARARGS | METH_KEYWORDS,
5844 PyDoc_STR("Pickler(file, protocol=0) -- Create a pickler.\n"
5845 "\n"
5846 "This takes a file-like object for writing a pickle data stream.\n"
5847 "The optional proto argument tells the pickler to use the given\n"
5848 "protocol; supported protocols are 0, 1, 2. The default\n"
5849 "protocol is 0, to be backwards compatible. (Protocol 0 is the\n"
5850 "only protocol that can be written to a file opened in text\n"
5851 "mode and read back successfully. When using a protocol higher\n"
5852 "than 0, make sure the file is opened in binary mode, both when\n"
5853 "pickling and unpickling.)\n"
5854 "\n"
5855 "Protocol 1 is more efficient than protocol 0; protocol 2 is\n"
5856 "more efficient than protocol 1.\n"
5857 "\n"
5858 "Specifying a negative protocol version selects the highest\n"
5859 "protocol version supported. The higher the protocol used, the\n"
5860 "more recent the version of Python needed to read the pickle\n"
5861 "produced.\n"
5862 "\n"
5863 "The file parameter must have a write() method that accepts a single\n"
5864 "string argument. It can thus be an open file object, a StringIO\n"
5865 "object, or any other custom object that meets this interface.\n")
5866 },
5867
5868 {"Unpickler", (PyCFunction)get_Unpickler, METH_O,
5869 PyDoc_STR("Unpickler(file) -- Create an unpickler.")},
5870
5871 { NULL, NULL }
5872 };
5873
5874 static int
init_stuff(PyObject * module_dict)5875 init_stuff(PyObject *module_dict)
5876 {
5877 PyObject *copyreg, *t, *r;
5878
5879 #define INIT_STR(S) if (!( S ## _str=PyString_InternFromString(#S))) return -1;
5880
5881 if (PyType_Ready(&Unpicklertype) < 0)
5882 return -1;
5883 if (PyType_Ready(&Picklertype) < 0)
5884 return -1;
5885
5886 INIT_STR(__class__);
5887 INIT_STR(__getinitargs__);
5888 INIT_STR(__dict__);
5889 INIT_STR(__getstate__);
5890 INIT_STR(__setstate__);
5891 INIT_STR(__name__);
5892 INIT_STR(__main__);
5893 INIT_STR(__reduce__);
5894 INIT_STR(__reduce_ex__);
5895 INIT_STR(write);
5896 INIT_STR(append);
5897 INIT_STR(read);
5898 INIT_STR(readline);
5899 INIT_STR(dispatch_table);
5900
5901 if (!( copyreg = PyImport_ImportModule("copy_reg")))
5902 return -1;
5903
5904 /* This is special because we want to use a different
5905 one in restricted mode. */
5906 dispatch_table = PyObject_GetAttr(copyreg, dispatch_table_str);
5907 if (!dispatch_table) return -1;
5908
5909 extension_registry = PyObject_GetAttrString(copyreg,
5910 "_extension_registry");
5911 if (!extension_registry) return -1;
5912
5913 inverted_registry = PyObject_GetAttrString(copyreg,
5914 "_inverted_registry");
5915 if (!inverted_registry) return -1;
5916
5917 extension_cache = PyObject_GetAttrString(copyreg,
5918 "_extension_cache");
5919 if (!extension_cache) return -1;
5920
5921 Py_DECREF(copyreg);
5922
5923 if (!(empty_tuple = PyTuple_New(0)))
5924 return -1;
5925
5926 two_tuple = PyTuple_New(2);
5927 if (two_tuple == NULL)
5928 return -1;
5929 /* We use this temp container with no regard to refcounts, or to
5930 * keeping containees alive. Exempt from GC, because we don't
5931 * want anything looking at two_tuple() by magic.
5932 */
5933 PyObject_GC_UnTrack(two_tuple);
5934
5935 /* Ugh */
5936 if (!( t=PyImport_ImportModule("__builtin__"))) return -1;
5937 if (PyDict_SetItemString(module_dict, "__builtins__", t) < 0)
5938 return -1;
5939
5940 if (!( t=PyDict_New())) return -1;
5941 if (!( r=PyRun_String(
5942 "def __str__(self):\n"
5943 " return self.args and ('%s' % self.args[0]) or '(what)'\n",
5944 Py_file_input,
5945 module_dict, t) )) return -1;
5946 Py_DECREF(r);
5947
5948 PickleError = PyErr_NewException("cPickle.PickleError", NULL, t);
5949 if (!PickleError)
5950 return -1;
5951
5952 Py_DECREF(t);
5953
5954 PicklingError = PyErr_NewException("cPickle.PicklingError",
5955 PickleError, NULL);
5956 if (!PicklingError)
5957 return -1;
5958
5959 if (!( t=PyDict_New())) return -1;
5960 if (!( r=PyRun_String(
5961 "def __str__(self):\n"
5962 " a=self.args\n"
5963 " a=a and type(a[0]) or '(what)'\n"
5964 " return 'Cannot pickle %s objects' % a\n"
5965 , Py_file_input,
5966 module_dict, t) )) return -1;
5967 Py_DECREF(r);
5968
5969 if (!( UnpickleableError = PyErr_NewException(
5970 "cPickle.UnpickleableError", PicklingError, t)))
5971 return -1;
5972
5973 Py_DECREF(t);
5974
5975 if (!( UnpicklingError = PyErr_NewException("cPickle.UnpicklingError",
5976 PickleError, NULL)))
5977 return -1;
5978
5979 if (!( BadPickleGet = PyErr_NewException("cPickle.BadPickleGet",
5980 UnpicklingError, NULL)))
5981 return -1;
5982
5983 if (PyDict_SetItemString(module_dict, "PickleError",
5984 PickleError) < 0)
5985 return -1;
5986
5987 if (PyDict_SetItemString(module_dict, "PicklingError",
5988 PicklingError) < 0)
5989 return -1;
5990
5991 if (PyDict_SetItemString(module_dict, "UnpicklingError",
5992 UnpicklingError) < 0)
5993 return -1;
5994
5995 if (PyDict_SetItemString(module_dict, "UnpickleableError",
5996 UnpickleableError) < 0)
5997 return -1;
5998
5999 if (PyDict_SetItemString(module_dict, "BadPickleGet",
6000 BadPickleGet) < 0)
6001 return -1;
6002
6003 PycString_IMPORT;
6004
6005 return 0;
6006 }
6007
6008 #ifndef PyMODINIT_FUNC /* declarations for DLL import/export */
6009 #define PyMODINIT_FUNC void
6010 #endif
6011 PyMODINIT_FUNC
initcPickle(void)6012 initcPickle(void)
6013 {
6014 PyObject *m, *d, *di, *v, *k;
6015 Py_ssize_t i;
6016 char *rev = "1.71"; /* XXX when does this change? */
6017 PyObject *format_version;
6018 PyObject *compatible_formats;
6019
6020 Py_TYPE(&Picklertype) = &PyType_Type;
6021 Py_TYPE(&Unpicklertype) = &PyType_Type;
6022 Py_TYPE(&PdataType) = &PyType_Type;
6023
6024 /* Initialize some pieces. We need to do this before module creation,
6025 * so we're forced to use a temporary dictionary. :(
6026 */
6027 di = PyDict_New();
6028 if (!di) return;
6029 if (init_stuff(di) < 0) return;
6030
6031 /* Create the module and add the functions */
6032 m = Py_InitModule4("cPickle", cPickle_methods,
6033 cPickle_module_documentation,
6034 (PyObject*)NULL,PYTHON_API_VERSION);
6035 if (m == NULL)
6036 return;
6037
6038 /* Add some symbolic constants to the module */
6039 d = PyModule_GetDict(m);
6040 v = PyString_FromString(rev);
6041 PyDict_SetItemString(d, "__version__", v);
6042 Py_XDECREF(v);
6043
6044 /* Copy data from di. Waaa. */
6045 for (i=0; PyDict_Next(di, &i, &k, &v); ) {
6046 if (PyObject_SetItem(d, k, v) < 0) {
6047 Py_DECREF(di);
6048 return;
6049 }
6050 }
6051 Py_DECREF(di);
6052
6053 i = PyModule_AddIntConstant(m, "HIGHEST_PROTOCOL", HIGHEST_PROTOCOL);
6054 if (i < 0)
6055 return;
6056
6057 /* These are purely informational; no code uses them. */
6058 /* File format version we write. */
6059 format_version = PyString_FromString("2.0");
6060 /* Format versions we can read. */
6061 compatible_formats = Py_BuildValue("[sssss]",
6062 "1.0", /* Original protocol 0 */
6063 "1.1", /* Protocol 0 + INST */
6064 "1.2", /* Original protocol 1 */
6065 "1.3", /* Protocol 1 + BINFLOAT */
6066 "2.0"); /* Original protocol 2 */
6067 PyDict_SetItemString(d, "format_version", format_version);
6068 PyDict_SetItemString(d, "compatible_formats", compatible_formats);
6069 Py_XDECREF(format_version);
6070 Py_XDECREF(compatible_formats);
6071 }
6072