diff Modules/cStringIO.c @ 82089:a025b04332fe 2.7

Issue #7358: cStringIO.StringIO now supports writing to and reading from a stream larger than 2 GiB on 64-bit systems.
author Serhiy Storchaka <storchaka@gmail.com>
date Sat, 09 Feb 2013 13:47:43 +0200
parents ba1c48f8b571
children 680959a3ae2e
line wrap: on
line diff
--- a/Modules/cStringIO.c	Sat Feb 09 12:20:18 2013 +0200
+++ b/Modules/cStringIO.c	Sat Feb 09 13:47:43 2013 +0200
@@ -170,10 +170,15 @@
         n = l;
         if (n < 0) n=0;
     }
+    if (n > INT_MAX) {
+        PyErr_SetString(PyExc_OverflowError,
+                        "length too large");
+        return -1;
+    }
 
     *output=((IOobject*)self)->buf + ((IOobject*)self)->pos;
     ((IOobject*)self)->pos += n;
-    return n;
+    return (int)n;
 }
 
 static PyObject *
@@ -192,26 +197,33 @@
 
 static int
 IO_creadline(PyObject *self, char **output) {
-    char *n, *s;
-    Py_ssize_t l;
+    char *n, *start, *end;
+    Py_ssize_t len;
 
     if (!IO__opencheck(IOOOBJECT(self))) return -1;
 
-    for (n = ((IOobject*)self)->buf + ((IOobject*)self)->pos,
-           s = ((IOobject*)self)->buf + ((IOobject*)self)->string_size;
-         n < s && *n != '\n'; n++);
+    n = start = ((IOobject*)self)->buf + ((IOobject*)self)->pos;
+    end = ((IOobject*)self)->buf + ((IOobject*)self)->string_size;
+    while (n < end && *n != '\n')
+        n++;
+
+    if (n < end) n++;
 
-    if (n < s) n++;
+    len = n - start;
+    if (len > INT_MAX) {
+        PyErr_SetString(PyExc_OverflowError,
+                        "length too large");
+        return -1;
+    }
 
-    *output=((IOobject*)self)->buf + ((IOobject*)self)->pos;
-    l = n - ((IOobject*)self)->buf - ((IOobject*)self)->pos;
+    *output=start;
 
-    assert(IOOOBJECT(self)->pos <= PY_SSIZE_T_MAX - l);
+    assert(IOOOBJECT(self)->pos <= PY_SSIZE_T_MAX - len);
     assert(IOOOBJECT(self)->pos >= 0);
     assert(IOOOBJECT(self)->string_size >= 0);
 
-    ((IOobject*)self)->pos += l;
-    return (int)l;
+    ((IOobject*)self)->pos += len;
+    return (int)len;
 }
 
 static PyObject *
@@ -239,9 +251,9 @@
     int n;
     char *output;
     PyObject *result, *line;
-    int hint = 0, length = 0;
+    Py_ssize_t hint = 0, length = 0;
 
-    if (!PyArg_ParseTuple(args, "|i:readlines", &hint)) return NULL;
+    if (!PyArg_ParseTuple(args, "|n:readlines", &hint)) return NULL;
 
     result = PyList_New(0);
     if (!result)
@@ -377,31 +389,41 @@
 
 
 static int
-O_cwrite(PyObject *self, const char *c, Py_ssize_t  l) {
-    Py_ssize_t newl;
+O_cwrite(PyObject *self, const char *c, Py_ssize_t  len) {
+    Py_ssize_t newpos;
     Oobject *oself;
     char *newbuf;
 
     if (!IO__opencheck(IOOOBJECT(self))) return -1;
     oself = (Oobject *)self;
 
-    newl = oself->pos+l;
-    if (newl >= oself->buf_size) {
-        oself->buf_size *= 2;
-        if (oself->buf_size <= newl) {
-            assert(newl + 1 < INT_MAX);
-            oself->buf_size = (int)(newl+1);
+    if (len > INT_MAX) {
+        PyErr_SetString(PyExc_OverflowError,
+                        "length too large");
+        return -1;
+    }
+    assert(len >= 0);
+    if (oself->pos >= PY_SSIZE_T_MAX - len) {
+        PyErr_SetString(PyExc_OverflowError,
+                        "new position too large");
+        return -1;
+    }
+    newpos = oself->pos + len;
+    if (newpos >= oself->buf_size) {
+        size_t newsize = oself->buf_size;
+        newsize *= 2;
+        if (newsize <= (size_t)newpos || newsize > PY_SSIZE_T_MAX) {
+            assert(newpos < PY_SSIZE_T_MAX - 1);
+            newsize = newpos + 1;
         }
-        newbuf = (char*)realloc(oself->buf, oself->buf_size);
+        newbuf = (char*)realloc(oself->buf, newsize);
         if (!newbuf) {
             PyErr_SetString(PyExc_MemoryError,"out of memory");
-            free(oself->buf);
-            oself->buf = 0;
-            oself->buf_size = oself->pos = 0;
             return -1;
-          }
+        }
+        oself->buf_size = (Py_ssize_t)newsize;
         oself->buf = newbuf;
-      }
+    }
 
     if (oself->string_size < oself->pos) {
         /* In case of overseek, pad with null bytes the buffer region between
@@ -416,16 +438,15 @@
                (oself->pos - oself->string_size) * sizeof(char));
     }
 
-    memcpy(oself->buf+oself->pos,c,l);
+    memcpy(oself->buf + oself->pos, c, len);
 
-    assert(oself->pos + l < INT_MAX);
-    oself->pos += (int)l;
+    oself->pos = newpos;
 
     if (oself->string_size < oself->pos) {
         oself->string_size = oself->pos;
     }
 
-    return (int)l;
+    return (int)len;
 }
 
 static PyObject *