Mercurial > cpython-lru_cache-weakref

comparison Python/marshal.c @ 82191:b48e1cd2d3be 3.3

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Issue #5308: Raise ValueError when marshalling too large object (a sequence with size >= 2**31), instead of producing illegal marshal data.
author Serhiy Storchaka <storchaka@gmail.com>
date Wed, 13 Feb 2013 12:11:03 +0200
parents 461e84fc8c60 e0464fa28c85
children 8b99f2224c3a 01372117a5b4
comparison
equal deleted inserted replaced
82187:2fb03fe354e3 82191:b48e1cd2d3be
93 *p->ptr++ = Py_SAFE_DOWNCAST(c, int, char); 93 *p->ptr++ = Py_SAFE_DOWNCAST(c, int, char);
94 } 94 }
95 } 95 }
96 96
97 static void 97 static void
98 w_string(char *s, int n, WFILE *p) 98 w_string(char *s, Py_ssize_t n, WFILE *p)
99 { 99 {
100 if (p->fp != NULL) { 100 if (p->fp != NULL) {
101 fwrite(s, 1, n, p->fp); 101 fwrite(s, 1, n, p->fp);
102 } 102 }
103 else { 103 else {
121 w_byte((char)( x & 0xff), p); 121 w_byte((char)( x & 0xff), p);
122 w_byte((char)((x>> 8) & 0xff), p); 122 w_byte((char)((x>> 8) & 0xff), p);
123 w_byte((char)((x>>16) & 0xff), p); 123 w_byte((char)((x>>16) & 0xff), p);
124 w_byte((char)((x>>24) & 0xff), p); 124 w_byte((char)((x>>24) & 0xff), p);
125 } 125 }
126
127 #define SIZE32_MAX 0x7FFFFFFF
128
129 #if SIZEOF_SIZE_T > 4
130 # define W_SIZE(n, p) do { \
131 if ((n) > SIZE32_MAX) { \
132 (p)->depth--; \
133 (p)->error = WFERR_UNMARSHALLABLE; \
134 return; \
135 } \
136 w_long((long)(n), p); \
137 } while(0)
138 #else
139 # define W_SIZE w_long
140 #endif
126 141
127 /* We assume that Python longs are stored internally in base some power of 142 /* We assume that Python longs are stored internally in base some power of
128 2**15; for the sake of portability we'll always read and write them in base 143 2**15; for the sake of portability we'll always read and write them in base
129 exactly 2**15. */ 144 exactly 2**15. */
130 145
155 assert(d != 0); /* a PyLong is always normalized */ 170 assert(d != 0); /* a PyLong is always normalized */
156 do { 171 do {
157 d >>= PyLong_MARSHAL_SHIFT; 172 d >>= PyLong_MARSHAL_SHIFT;
158 l++; 173 l++;
159 } while (d != 0); 174 } while (d != 0);
175 if (l > SIZE32_MAX) {
176 p->depth--;
177 p->error = WFERR_UNMARSHALLABLE;
178 return;
179 }
160 w_long((long)(Py_SIZE(ob) > 0 ? l : -l), p); 180 w_long((long)(Py_SIZE(ob) > 0 ? l : -l), p);
161 181
162 for (i=0; i < n-1; i++) { 182 for (i=0; i < n-1; i++) {
163 d = ob->ob_digit[i]; 183 d = ob->ob_digit[i];
164 for (j=0; j < PyLong_MARSHAL_RATIO; j++) { 184 for (j=0; j < PyLong_MARSHAL_RATIO; j++) {
243 return; 263 return;
244 } 264 }
245 n = strlen(buf); 265 n = strlen(buf);
246 w_byte(TYPE_FLOAT, p); 266 w_byte(TYPE_FLOAT, p);
247 w_byte((int)n, p); 267 w_byte((int)n, p);
248 w_string(buf, (int)n, p); 268 w_string(buf, n, p);
249 PyMem_Free(buf); 269 PyMem_Free(buf);
250 } 270 }
251 } 271 }
252 else if (PyComplex_CheckExact(v)) { 272 else if (PyComplex_CheckExact(v)) {
253 if (p->version > 1) { 273 if (p->version > 1) {
275 p->error = WFERR_NOMEMORY; 295 p->error = WFERR_NOMEMORY;
276 return; 296 return;
277 } 297 }
278 n = strlen(buf); 298 n = strlen(buf);
279 w_byte((int)n, p); 299 w_byte((int)n, p);
280 w_string(buf, (int)n, p); 300 w_string(buf, n, p);
281 PyMem_Free(buf); 301 PyMem_Free(buf);
282 buf = PyOS_double_to_string(PyComplex_ImagAsDouble(v), 302 buf = PyOS_double_to_string(PyComplex_ImagAsDouble(v),
283 'g', 17, 0, NULL); 303 'g', 17, 0, NULL);
284 if (!buf) { 304 if (!buf) {
285 p->error = WFERR_NOMEMORY; 305 p->error = WFERR_NOMEMORY;
286 return; 306 return;
287 } 307 }
288 n = strlen(buf); 308 n = strlen(buf);
289 w_byte((int)n, p); 309 w_byte((int)n, p);
290 w_string(buf, (int)n, p); 310 w_string(buf, n, p);
291 PyMem_Free(buf); 311 PyMem_Free(buf);
292 } 312 }
293 } 313 }
294 else if (PyBytes_CheckExact(v)) { 314 else if (PyBytes_CheckExact(v)) {
295 w_byte(TYPE_STRING, p); 315 w_byte(TYPE_STRING, p);
296 n = PyBytes_GET_SIZE(v); 316 n = PyBytes_GET_SIZE(v);
297 if (n > INT_MAX) { 317 W_SIZE(n, p);
298 /* huge strings are not supported */ 318 w_string(PyBytes_AS_STRING(v), n, p);
299 p->depth--;
300 p->error = WFERR_UNMARSHALLABLE;
301 return;
302 }
303 w_long((long)n, p);
304 w_string(PyBytes_AS_STRING(v), (int)n, p);
305 } 319 }
306 else if (PyUnicode_CheckExact(v)) { 320 else if (PyUnicode_CheckExact(v)) {
307 PyObject *utf8; 321 PyObject *utf8;
308 utf8 = PyUnicode_AsEncodedString(v, "utf8", "surrogatepass"); 322 utf8 = PyUnicode_AsEncodedString(v, "utf8", "surrogatepass");
309 if (utf8 == NULL) { 323 if (utf8 == NULL) {
311 p->error = WFERR_UNMARSHALLABLE; 325 p->error = WFERR_UNMARSHALLABLE;
312 return; 326 return;
313 } 327 }
314 w_byte(TYPE_UNICODE, p); 328 w_byte(TYPE_UNICODE, p);
315 n = PyBytes_GET_SIZE(utf8); 329 n = PyBytes_GET_SIZE(utf8);
316 if (n > INT_MAX) { 330 W_SIZE(n, p);
317 p->depth--; 331 w_string(PyBytes_AS_STRING(utf8), n, p);
318 p->error = WFERR_UNMARSHALLABLE;
319 return;
320 }
321 w_long((long)n, p);
322 w_string(PyBytes_AS_STRING(utf8), (int)n, p);
323 Py_DECREF(utf8); 332 Py_DECREF(utf8);
324 } 333 }
325 else if (PyTuple_CheckExact(v)) { 334 else if (PyTuple_CheckExact(v)) {
326 w_byte(TYPE_TUPLE, p); 335 w_byte(TYPE_TUPLE, p);
327 n = PyTuple_Size(v); 336 n = PyTuple_Size(v);
328 w_long((long)n, p); 337 W_SIZE(n, p);
329 for (i = 0; i < n; i++) { 338 for (i = 0; i < n; i++) {
330 w_object(PyTuple_GET_ITEM(v, i), p); 339 w_object(PyTuple_GET_ITEM(v, i), p);
331 } 340 }
332 } 341 }
333 else if (PyList_CheckExact(v)) { 342 else if (PyList_CheckExact(v)) {
334 w_byte(TYPE_LIST, p); 343 w_byte(TYPE_LIST, p);
335 n = PyList_GET_SIZE(v); 344 n = PyList_GET_SIZE(v);
336 w_long((long)n, p); 345 W_SIZE(n, p);
337 for (i = 0; i < n; i++) { 346 for (i = 0; i < n; i++) {
338 w_object(PyList_GET_ITEM(v, i), p); 347 w_object(PyList_GET_ITEM(v, i), p);
339 } 348 }
340 } 349 }
341 else if (PyDict_CheckExact(v)) { 350 else if (PyDict_CheckExact(v)) {
361 if (n == -1) { 370 if (n == -1) {
362 p->depth--; 371 p->depth--;
363 p->error = WFERR_UNMARSHALLABLE; 372 p->error = WFERR_UNMARSHALLABLE;
364 return; 373 return;
365 } 374 }
366 w_long((long)n, p); 375 W_SIZE(n, p);
367 it = PyObject_GetIter(v); 376 it = PyObject_GetIter(v);
368 if (it == NULL) { 377 if (it == NULL) {
369 p->depth--; 378 p->depth--;
370 p->error = WFERR_UNMARSHALLABLE; 379 p->error = WFERR_UNMARSHALLABLE;
371 return; 380 return;
411 return; 420 return;
412 } 421 }
413 w_byte(TYPE_STRING, p); 422 w_byte(TYPE_STRING, p);
414 n = view.len; 423 n = view.len;
415 s = view.buf; 424 s = view.buf;
416 if (n > INT_MAX) { 425 W_SIZE(n, p);
417 p->depth--; 426 w_string(s, n, p);
418 p->error = WFERR_UNMARSHALLABLE;
419 return;
420 }
421 w_long((long)n, p);
422 w_string(s, (int)n, p);
423 PyBuffer_Release(&view); 427 PyBuffer_Release(&view);
424 } 428 }
425 else { 429 else {
426 w_byte(TYPE_UNKNOWN, p); 430 w_byte(TYPE_UNKNOWN, p);
427 p->error = WFERR_UNMARSHALLABLE; 431 p->error = WFERR_UNMARSHALLABLE;
454 458
455 typedef WFILE RFILE; /* Same struct with different invariants */ 459 typedef WFILE RFILE; /* Same struct with different invariants */
456 460
457 #define rs_byte(p) (((p)->ptr < (p)->end) ? (unsigned char)*(p)->ptr++ : EOF) 461 #define rs_byte(p) (((p)->ptr < (p)->end) ? (unsigned char)*(p)->ptr++ : EOF)
458 462
459 static int 463 static Py_ssize_t
460 r_string(char *s, int n, RFILE *p) 464 r_string(char *s, Py_ssize_t n, RFILE *p)
461 { 465 {
462 char *ptr; 466 char *ptr;
463 int read, left; 467 Py_ssize_t read, left;
464 468
465 if (!p->readable) { 469 if (!p->readable) {
466 if (p->fp != NULL) 470 if (p->fp != NULL)
467 /* The result fits into int because it must be <=n. */ 471 /* The result fits into int because it must be <=n. */
468 read = (int) fread(s, 1, n, p->fp); 472 read = fread(s, 1, n, p->fp);
469 else { 473 else {
470 left = (int)(p->end - p->ptr); 474 left = p->end - p->ptr;
471 read = (left < n) ? left : n; 475 read = (left < n) ? left : n;
472 memcpy(s, p->ptr, read); 476 memcpy(s, p->ptr, read);
473 p->ptr += read; 477 p->ptr += read;
474 } 478 }
475 } 479 }
476 else { 480 else {
477 _Py_IDENTIFIER(read); 481 _Py_IDENTIFIER(read);
478 482
479 PyObject *data = _PyObject_CallMethodId(p->readable, &PyId_read, "i", n); 483 PyObject *data = _PyObject_CallMethodId(p->readable, &PyId_read, "n", n);
480 read = 0; 484 read = 0;
481 if (data != NULL) { 485 if (data != NULL) {
482 if (!PyBytes_Check(data)) { 486 if (!PyBytes_Check(data)) {
483 PyErr_Format(PyExc_TypeError, 487 PyErr_Format(PyExc_TypeError,
484 "f.read() returned not bytes but %.100s", 488 "f.read() returned not bytes but %.100s",
504 static int 508 static int
505 r_byte(RFILE *p) 509 r_byte(RFILE *p)
506 { 510 {
507 int c = EOF; 511 int c = EOF;
508 unsigned char ch; 512 unsigned char ch;
509 int n; 513 Py_ssize_t n;
510 514
511 if (!p->readable) 515 if (!p->readable)
512 c = p->fp ? getc(p->fp) : rs_byte(p); 516 c = p->fp ? getc(p->fp) : rs_byte(p);
513 else { 517 else {
514 n = r_string((char *) &ch, 1, p); 518 n = r_string((char *) &ch, 1, p);
588 592
589 static PyObject * 593 static PyObject *
590 r_PyLong(RFILE *p) 594 r_PyLong(RFILE *p)
591 { 595 {
592 PyLongObject *ob; 596 PyLongObject *ob;
593 int size, i, j, md, shorts_in_top_digit; 597 long n, size, i;
594 long n; 598 int j, md, shorts_in_top_digit;
595 digit d; 599 digit d;
596 600
597 n = r_long(p); 601 n = r_long(p);
598 if (PyErr_Occurred()) 602 if (PyErr_Occurred())
599 return NULL; 603 return NULL;
600 if (n == 0) 604 if (n == 0)
601 return (PyObject *)_PyLong_New(0); 605 return (PyObject *)_PyLong_New(0);
602 if (n < -INT_MAX || n > INT_MAX) { 606 if (n < -SIZE32_MAX || n > SIZE32_MAX) {
603 PyErr_SetString(PyExc_ValueError, 607 PyErr_SetString(PyExc_ValueError,
604 "bad marshal data (long size out of range)"); 608 "bad marshal data (long size out of range)");
605 return NULL; 609 return NULL;
606 } 610 }
607 611
728 { 732 {
729 char buf[256]; 733 char buf[256];
730 double dx; 734 double dx;
731 retval = NULL; 735 retval = NULL;
732 n = r_byte(p); 736 n = r_byte(p);
733 if (n == EOF || r_string(buf, (int)n, p) != n) { 737 if (n == EOF || r_string(buf, n, p) != n) {
734 PyErr_SetString(PyExc_EOFError, 738 PyErr_SetString(PyExc_EOFError,
735 "EOF read where object expected"); 739 "EOF read where object expected");
736 break; 740 break;
737 } 741 }
738 buf[n] = '\0'; 742 buf[n] = '\0';
766 { 770 {
767 char buf[256]; 771 char buf[256];
768 Py_complex c; 772 Py_complex c;
769 retval = NULL; 773 retval = NULL;
770 n = r_byte(p); 774 n = r_byte(p);
771 if (n == EOF || r_string(buf, (int)n, p) != n) { 775 if (n == EOF || r_string(buf, n, p) != n) {
772 PyErr_SetString(PyExc_EOFError, 776 PyErr_SetString(PyExc_EOFError,
773 "EOF read where object expected"); 777 "EOF read where object expected");
774 break; 778 break;
775 } 779 }
776 buf[n] = '\0'; 780 buf[n] = '\0';
777 c.real = PyOS_string_to_double(buf, NULL, NULL); 781 c.real = PyOS_string_to_double(buf, NULL, NULL);
778 if (c.real == -1.0 && PyErr_Occurred()) 782 if (c.real == -1.0 && PyErr_Occurred())
779 break; 783 break;
780 n = r_byte(p); 784 n = r_byte(p);
781 if (n == EOF || r_string(buf, (int)n, p) != n) { 785 if (n == EOF || r_string(buf, n, p) != n) {
782 PyErr_SetString(PyExc_EOFError, 786 PyErr_SetString(PyExc_EOFError,
783 "EOF read where object expected"); 787 "EOF read where object expected");
784 break; 788 break;
785 } 789 }
786 buf[n] = '\0'; 790 buf[n] = '\0';
825 n = r_long(p); 829 n = r_long(p);
826 if (PyErr_Occurred()) { 830 if (PyErr_Occurred()) {
827 retval = NULL; 831 retval = NULL;
828 break; 832 break;
829 } 833 }
830 if (n < 0 || n > INT_MAX) { 834 if (n < 0 || n > SIZE32_MAX) {
831 PyErr_SetString(PyExc_ValueError, "bad marshal data (string size out of range)"); 835 PyErr_SetString(PyExc_ValueError, "bad marshal data (string size out of range)");
832 retval = NULL; 836 retval = NULL;
833 break; 837 break;
834 } 838 }
835 v = PyBytes_FromStringAndSize((char *)NULL, n); 839 v = PyBytes_FromStringAndSize((char *)NULL, n);
836 if (v == NULL) { 840 if (v == NULL) {
837 retval = NULL; 841 retval = NULL;
838 break; 842 break;
839 } 843 }
840 if (r_string(PyBytes_AS_STRING(v), (int)n, p) != n) { 844 if (r_string(PyBytes_AS_STRING(v), n, p) != n) {
841 Py_DECREF(v); 845 Py_DECREF(v);
842 PyErr_SetString(PyExc_EOFError, 846 PyErr_SetString(PyExc_EOFError,
843 "EOF read where object expected"); 847 "EOF read where object expected");
844 retval = NULL; 848 retval = NULL;
845 break; 849 break;
854 n = r_long(p); 858 n = r_long(p);
855 if (PyErr_Occurred()) { 859 if (PyErr_Occurred()) {
856 retval = NULL; 860 retval = NULL;
857 break; 861 break;
858 } 862 }
859 if (n < 0 || n > INT_MAX) { 863 if (n < 0 || n > SIZE32_MAX) {
860 PyErr_SetString(PyExc_ValueError, "bad marshal data (unicode size out of range)"); 864 PyErr_SetString(PyExc_ValueError, "bad marshal data (unicode size out of range)");
861 retval = NULL; 865 retval = NULL;
862 break; 866 break;
863 } 867 }
864 buffer = PyMem_NEW(char, n); 868 buffer = PyMem_NEW(char, n);
865 if (buffer == NULL) { 869 if (buffer == NULL) {
866 retval = PyErr_NoMemory(); 870 retval = PyErr_NoMemory();
867 break; 871 break;
868 } 872 }
869 if (r_string(buffer, (int)n, p) != n) { 873 if (r_string(buffer, n, p) != n) {
870 PyMem_DEL(buffer); 874 PyMem_DEL(buffer);
871 PyErr_SetString(PyExc_EOFError, 875 PyErr_SetString(PyExc_EOFError,
872 "EOF read where object expected"); 876 "EOF read where object expected");
873 retval = NULL; 877 retval = NULL;
874 break; 878 break;
883 n = r_long(p); 887 n = r_long(p);
884 if (PyErr_Occurred()) { 888 if (PyErr_Occurred()) {
885 retval = NULL; 889 retval = NULL;
886 break; 890 break;
887 } 891 }
888 if (n < 0 || n > INT_MAX) { 892 if (n < 0 || n > SIZE32_MAX) {
889 PyErr_SetString(PyExc_ValueError, "bad marshal data (tuple size out of range)"); 893 PyErr_SetString(PyExc_ValueError, "bad marshal data (tuple size out of range)");
890 retval = NULL; 894 retval = NULL;
891 break; 895 break;
892 } 896 }
893 v = PyTuple_New((int)n); 897 v = PyTuple_New(n);
894 if (v == NULL) { 898 if (v == NULL) {
895 retval = NULL; 899 retval = NULL;
896 break; 900 break;
897 } 901 }
898 for (i = 0; i < n; i++) { 902 for (i = 0; i < n; i++) {
903 "NULL object in marshal data for tuple"); 907 "NULL object in marshal data for tuple");
904 Py_DECREF(v); 908 Py_DECREF(v);
905 v = NULL; 909 v = NULL;
906 break; 910 break;
907 } 911 }
908 PyTuple_SET_ITEM(v, (int)i, v2); 912 PyTuple_SET_ITEM(v, i, v2);
909 } 913 }
910 retval = v; 914 retval = v;
911 break; 915 break;
912 916
913 case TYPE_LIST: 917 case TYPE_LIST:
914 n = r_long(p); 918 n = r_long(p);
915 if (PyErr_Occurred()) { 919 if (PyErr_Occurred()) {
916 retval = NULL; 920 retval = NULL;
917 break; 921 break;
918 } 922 }
919 if (n < 0 || n > INT_MAX) { 923 if (n < 0 || n > SIZE32_MAX) {
920 PyErr_SetString(PyExc_ValueError, "bad marshal data (list size out of range)"); 924 PyErr_SetString(PyExc_ValueError, "bad marshal data (list size out of range)");
921 retval = NULL; 925 retval = NULL;
922 break; 926 break;
923 } 927 }
924 v = PyList_New((int)n); 928 v = PyList_New(n);
925 if (v == NULL) { 929 if (v == NULL) {
926 retval = NULL; 930 retval = NULL;
927 break; 931 break;
928 } 932 }
929 for (i = 0; i < n; i++) { 933 for (i = 0; i < n; i++) {
934 "NULL object in marshal data for list"); 938 "NULL object in marshal data for list");
935 Py_DECREF(v); 939 Py_DECREF(v);
936 v = NULL; 940 v = NULL;
937 break; 941 break;
938 } 942 }
939 PyList_SET_ITEM(v, (int)i, v2); 943 PyList_SET_ITEM(v, i, v2);
940 } 944 }
941 retval = v; 945 retval = v;
942 break; 946 break;
943 947
944 case TYPE_DICT: 948 case TYPE_DICT:
970 n = r_long(p); 974 n = r_long(p);
971 if (PyErr_Occurred()) { 975 if (PyErr_Occurred()) {
972 retval = NULL; 976 retval = NULL;
973 break; 977 break;
974 } 978 }
975 if (n < 0 || n > INT_MAX) { 979 if (n < 0 || n > SIZE32_MAX) {
976 PyErr_SetString(PyExc_ValueError, "bad marshal data (set size out of range)"); 980 PyErr_SetString(PyExc_ValueError, "bad marshal data (set size out of range)");
977 retval = NULL; 981 retval = NULL;
978 break; 982 break;
979 } 983 }
980 v = (type == TYPE_SET) ? PySet_New(NULL) : PyFrozenSet_New(NULL); 984 v = (type == TYPE_SET) ? PySet_New(NULL) : PyFrozenSet_New(NULL);
1178 off_t filesize; 1182 off_t filesize;
1179 filesize = getfilesize(fp); 1183 filesize = getfilesize(fp);
1180 if (filesize > 0 && filesize <= REASONABLE_FILE_LIMIT) { 1184 if (filesize > 0 && filesize <= REASONABLE_FILE_LIMIT) {
1181 char* pBuf = (char *)PyMem_MALLOC(filesize); 1185 char* pBuf = (char *)PyMem_MALLOC(filesize);
1182 if (pBuf != NULL) { 1186 if (pBuf != NULL) {
1183 PyObject* v; 1187 size_t n = fread(pBuf, 1, (size_t)filesize, fp);
1184 size_t n; 1188 PyObject* v = PyMarshal_ReadObjectFromString(pBuf, n);
1185 /* filesize must fit into an int, because it
1186 is smaller than REASONABLE_FILE_LIMIT */
1187 n = fread(pBuf, 1, (int)filesize, fp);
1188 v = PyMarshal_ReadObjectFromString(pBuf, n);
1189 PyMem_FREE(pBuf); 1189 PyMem_FREE(pBuf);
1190 return v; 1190 return v;
1191 } 1191 }
1192 1192
1193 } 1193 }