Mercurial > cpython-lru_cache-weakref
diff Lib/test/test_codeccallbacks.py @ 62710:9dc5f385f494 3.1
Merged revisions 82413,82468 via svnmerge from
svn+ssh://pythondev@svn.python.org/python/branches/py3k
........
r82413 | ezio.melotti | 2010-07-01 10:32:02 +0300 (Thu, 01 Jul 2010) | 13 lines
Update PyUnicode_DecodeUTF8 from RFC 2279 to RFC 3629.
1) #8271: when a byte sequence is invalid, only the start byte and all the
valid continuation bytes are now replaced by U+FFFD, instead of replacing
the number of bytes specified by the start byte.
See http://www.unicode.org/versions/Unicode5.2.0/ch03.pdf (pages 94-95);
2) 5- and 6-bytes-long UTF-8 sequences are now considered invalid (no changes
in behavior);
3) Change the error messages "unexpected code byte" to "invalid start byte"
and "invalid data" to "invalid continuation byte";
4) Add an extensive set of tests in test_unicode;
5) Fix test_codeccallbacks because it was failing after this change.
........
r82468 | ezio.melotti | 2010-07-03 07:52:19 +0300 (Sat, 03 Jul 2010) | 1 line
Update comment about surrogates.
........
| author | Ezio Melotti <ezio.melotti@gmail.com> |
|---|---|
| date | Sat, 03 Jul 2010 05:18:50 +0000 |
| parents | d06a3ebf306f |
| children | fc76e8c617dc |
line wrap: on
line diff
--- a/Lib/test/test_codeccallbacks.py Fri Jul 02 23:22:03 2010 +0000 +++ b/Lib/test/test_codeccallbacks.py Sat Jul 03 05:18:50 2010 +0000 @@ -153,28 +153,30 @@ sout += bytes("\\U%08x" % sys.maxunicode, "ascii") self.assertEqual(sin.encode("iso-8859-15", "backslashreplace"), sout) - def test_decoderelaxedutf8(self): - # This is the test for a decoding callback handler, - # that relaxes the UTF-8 minimal encoding restriction. - # A null byte that is encoded as "\xc0\x80" will be - # decoded as a null byte. All other illegal sequences - # will be handled strictly. + def test_decoding_callbacks(self): + # This is a test for a decoding callback handler + # that allows the decoding of the invalid sequence + # "\xc0\x80" and returns "\x00" instead of raising an error. + # All other illegal sequences will be handled strictly. def relaxedutf8(exc): if not isinstance(exc, UnicodeDecodeError): raise TypeError("don't know how to handle %r" % exc) - if exc.object[exc.start:exc.end].startswith(b"\xc0\x80"): + if exc.object[exc.start:exc.start+2] == b"\xc0\x80": return ("\x00", exc.start+2) # retry after two bytes else: raise exc - codecs.register_error( - "test.relaxedutf8", relaxedutf8) + codecs.register_error("test.relaxedutf8", relaxedutf8) + # all the "\xc0\x80" will be decoded to "\x00" sin = b"a\x00b\xc0\x80c\xc3\xbc\xc0\x80\xc0\x80" sout = "a\x00b\x00c\xfc\x00\x00" self.assertEqual(sin.decode("utf-8", "test.relaxedutf8"), sout) + + # "\xc0\x81" is not valid and a UnicodeDecodeError will be raised sin = b"\xc0\x80\xc0\x81" - self.assertRaises(UnicodeError, sin.decode, "utf-8", "test.relaxedutf8") + self.assertRaises(UnicodeDecodeError, sin.decode, + "utf-8", "test.relaxedutf8") def test_charmapencode(self): # For charmap encodings the replacement string will be