changeset 87654:d565310e7ae3

Issue #17897: Optimized unpickle prefetching.
author Serhiy Storchaka <storchaka@gmail.com>
date Sat, 30 Nov 2013 23:15:38 +0200
parents 1ceb6f84b617
children f20d966dc499
files Misc/NEWS Modules/_pickle.c
diffstat 2 files changed, 27 insertions(+), 33 deletions(-) [+]
line wrap: on
line diff
--- a/Misc/NEWS	Sat Nov 30 01:05:51 2013 -0800
+++ b/Misc/NEWS	Sat Nov 30 23:15:38 2013 +0200
@@ -18,6 +18,8 @@
 Library
 -------
 
+- Issue #17897: Optimized unpickle prefetching.
+
 - Issue #3693: Make the error message more helpful when the array.array()
   constructor is given a str. Move the array module typecode documentation to
   the docstring of the constructor.
--- a/Modules/_pickle.c	Sat Nov 30 01:05:51 2013 -0800
+++ b/Modules/_pickle.c	Sat Nov 30 23:15:38 2013 +0200
@@ -1121,7 +1121,7 @@
 _Unpickler_ReadFromFile(UnpicklerObject *self, Py_ssize_t n)
 {
     PyObject *data;
-    Py_ssize_t read_size, prefetched_size = 0;
+    Py_ssize_t read_size;
 
     assert(self->read != NULL);
 
@@ -1134,7 +1134,29 @@
         Py_DECREF(empty_tuple);
     }
     else {
-        PyObject *len = PyLong_FromSsize_t(n);
+        PyObject *len;
+        /* Prefetch some data without advancing the file pointer, if possible */
+        if (self->peek && n < PREFETCH) {
+            len = PyLong_FromSsize_t(PREFETCH);
+            if (len == NULL)
+                return -1;
+            data = _Pickle_FastCall(self->peek, len);
+            if (data == NULL) {
+                if (!PyErr_ExceptionMatches(PyExc_NotImplementedError))
+                    return -1;
+                /* peek() is probably not supported by the given file object */
+                PyErr_Clear();
+                Py_CLEAR(self->peek);
+            }
+            else {
+                read_size = _Unpickler_SetStringInput(self, data);
+                Py_DECREF(data);
+                self->prefetched_idx = 0;
+                if (n <= read_size)
+                    return n;
+            }
+        }
+        len = PyLong_FromSsize_t(n);
         if (len == NULL)
             return -1;
         data = _Pickle_FastCall(self->read, len);
@@ -1142,38 +1164,8 @@
     if (data == NULL)
         return -1;
 
-    /* Prefetch some data without advancing the file pointer, if possible */
-    if (self->peek) {
-        PyObject *len, *prefetched;
-        len = PyLong_FromSsize_t(PREFETCH);
-        if (len == NULL) {
-            Py_DECREF(data);
-            return -1;
-        }
-        prefetched = _Pickle_FastCall(self->peek, len);
-        if (prefetched == NULL) {
-            if (PyErr_ExceptionMatches(PyExc_NotImplementedError)) {
-                /* peek() is probably not supported by the given file object */
-                PyErr_Clear();
-                Py_CLEAR(self->peek);
-            }
-            else {
-                Py_DECREF(data);
-                return -1;
-            }
-        }
-        else {
-            assert(PyBytes_Check(prefetched));
-            prefetched_size = PyBytes_GET_SIZE(prefetched);
-            PyBytes_ConcatAndDel(&data, prefetched);
-            if (data == NULL)
-                return -1;
-        }
-    }
-
-    read_size = _Unpickler_SetStringInput(self, data) - prefetched_size;
+    read_size = _Unpickler_SetStringInput(self, data);
     Py_DECREF(data);
-    self->prefetched_idx = read_size;
     return read_size;
 }