Mercurial > cpython-lru_cache-weakref

diff Modules/_codecsmodule.c @ 33428:57478a13fbf6 legacy-trunk

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
SF patch #998993: The UTF-8 and the UTF-16 stateful decoders now support decoding incomplete input (when the input stream is temporarily exhausted). codecs.StreamReader now implements buffering, which enables proper readline support for the UTF-16 decoders. codecs.StreamReader.read() has a new argument chars which specifies the number of characters to return. codecs.StreamReader.readline() and codecs.StreamReader.readlines() have a new argument keepends. Trailing "\n"s will be stripped from the lines if keepends is false. Added C APIs PyUnicode_DecodeUTF8Stateful and PyUnicode_DecodeUTF16Stateful.
author Walter Dörwald <walter@livinglogic.de>
date Tue, 07 Sep 2004 20:24:22 +0000
parents ab979ff8b02a
children 0d44d9071e97
line wrap: on
line diff
--- a/Modules/_codecsmodule.c	Tue Sep 07 17:48:26 2004 +0000
+++ b/Modules/_codecsmodule.c	Tue Sep 07 20:24:22 2004 +0000
@@ -269,13 +269,20 @@
     const char *data;
     int size;
     const char *errors = NULL;
-    
-    if (!PyArg_ParseTuple(args, "t#|z:utf_8_decode",
-			  &data, &size, &errors))
+    int final = 0;
+    int consumed;
+    PyObject *decoded = NULL;
+
+    if (!PyArg_ParseTuple(args, "t#|zi:utf_8_decode",
+			  &data, &size, &errors, &final))
 	return NULL;
-
-    return codec_tuple(PyUnicode_DecodeUTF8(data, size, errors),
-		       size);
+    consumed = size;
+	
+    decoded = PyUnicode_DecodeUTF8Stateful(data, size, errors,
+					   final ? NULL : &consumed);
+    if (decoded == NULL)
+	return NULL;
+    return codec_tuple(decoded, consumed);
 }
 
 static PyObject *
@@ -286,12 +293,19 @@
     int size;
     const char *errors = NULL;
     int byteorder = 0;
-    
-    if (!PyArg_ParseTuple(args, "t#|z:utf_16_decode",
-			  &data, &size, &errors))
+    int final = 0;
+    int consumed;
+    PyObject *decoded;
+
+    if (!PyArg_ParseTuple(args, "t#|zi:utf_16_decode",
+			  &data, &size, &errors, &final))
 	return NULL;
-    return codec_tuple(PyUnicode_DecodeUTF16(data, size, errors, &byteorder),
-		       size);
+    consumed = size;
+    decoded = PyUnicode_DecodeUTF16Stateful(data, size, errors, &byteorder,
+					    final ? NULL : &consumed);
+    if (decoded == NULL)
+	return NULL;
+    return codec_tuple(decoded, consumed);
 }
 
 static PyObject *
@@ -302,12 +316,20 @@
     int size;
     const char *errors = NULL;
     int byteorder = -1;
+    int final = 0;
+    int consumed;
+    PyObject *decoded = NULL;
     
-    if (!PyArg_ParseTuple(args, "t#|z:utf_16_le_decode",
-			  &data, &size, &errors))
+    if (!PyArg_ParseTuple(args, "t#|zi:utf_16_le_decode",
+			  &data, &size, &errors, &final))
 	return NULL;
-    return codec_tuple(PyUnicode_DecodeUTF16(data, size, errors, &byteorder),
-		       size);
+    consumed = size;
+    decoded = PyUnicode_DecodeUTF16Stateful(data, size, errors,
+	&byteorder, final ? NULL : &consumed);
+    if (decoded == NULL)
+	return NULL;
+    return codec_tuple(decoded, consumed);
+
 }
 
 static PyObject *
@@ -318,12 +340,19 @@
     int size;
     const char *errors = NULL;
     int byteorder = 1;
+    int final = 0;
+    int consumed;
+    PyObject *decoded = NULL;
     
-    if (!PyArg_ParseTuple(args, "t#|z:utf_16_be_decode",
-			  &data, &size, &errors))
+    if (!PyArg_ParseTuple(args, "t#|zi:utf_16_be_decode",
+			  &data, &size, &errors, &final))
 	return NULL;
-    return codec_tuple(PyUnicode_DecodeUTF16(data, size, errors, &byteorder),
-		       size);
+    consumed = size;
+    decoded = PyUnicode_DecodeUTF16Stateful(data, size, errors,
+	&byteorder, final ? NULL : &consumed);
+    if (decoded == NULL)
+	return NULL;
+    return codec_tuple(decoded, consumed);
 }
 
 /* This non-standard version also provides access to the byteorder
@@ -343,15 +372,19 @@
     const char *errors = NULL;
     int byteorder = 0;
     PyObject *unicode, *tuple;
-    
-    if (!PyArg_ParseTuple(args, "t#|zi:utf_16_ex_decode",
-			  &data, &size, &errors, &byteorder))
+    int final = 0;
+    int consumed;
+
+    if (!PyArg_ParseTuple(args, "t#|zii:utf_16_ex_decode",
+			  &data, &size, &errors, &byteorder, &final))
 	return NULL;
 
-    unicode = PyUnicode_DecodeUTF16(data, size, errors, &byteorder);
+    consumed = size;
+    unicode = PyUnicode_DecodeUTF16Stateful(data, size, errors, &byteorder,
+					    final ? NULL : &consumed);
     if (unicode == NULL)
 	return NULL;
-    tuple = Py_BuildValue("Oii", unicode, size, byteorder);
+    tuple = Py_BuildValue("Oii", unicode, consumed, byteorder);
     Py_DECREF(unicode);
     return tuple;
 }