Mercurial > cpython-lru_cache-weakref

annotate Lib/test/test_multibytecodec.py @ 78069:a8aa77458e12 3.2

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Add c_size_t and c_ssize_t to the ctypes summary table.
author Antoine Pitrou <solipsis@pitrou.net>
date Thu, 12 Jul 2012 20:31:50 +0200
parents 7f2ab2f95a04
children 61aaec67b521
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
59522
fcfe324663a6 convert shebang lines: python -> python3
Benjamin Peterson <benjamin@python.org>
parents: 58456
diff changeset
1 #!/usr/bin/env python3
31386
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
2 #
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
3 # test_multibytecodec.py
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
4 # Unit test for multibytecodec itself
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
5 #
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
6
47201
9c930a32cebb #2621 rename test.test_support to test.support
Benjamin Peterson <benjamin@python.org>
parents: 44160
diff changeset
7 from test import support
9c930a32cebb #2621 rename test.test_support to test.support
Benjamin Peterson <benjamin@python.org>
parents: 44160
diff changeset
8 from test.support import TESTFN
41758
0ead1064ead0 Make all the multibyte codec tests pass.
Guido van Rossum <guido@python.org>
parents: 41642
diff changeset
9 import unittest, io, codecs, sys, os
48563
b51cd5d3795c Merged revisions 65012,65035,65037-65040,65048,65057,65077,65091-65095,65097-65099,65127-65128,65131,65133-65136,65139,65149-65151,65155,65158-65159,65176-65178,65183-65184,65187-65190,65192,65194 via svnmerge from
Georg Brandl <georg@python.org>
parents: 47201
diff changeset
10 import _multibytecodec
39442
654c380cf8b9 Merged revisions 46753-51188 via svnmerge from
Thomas Wouters <thomas@python.org>
parents: 38453
diff changeset
11
654c380cf8b9 Merged revisions 46753-51188 via svnmerge from
Thomas Wouters <thomas@python.org>
parents: 38453
diff changeset
12 ALL_CJKENCODINGS = [
654c380cf8b9 Merged revisions 46753-51188 via svnmerge from
Thomas Wouters <thomas@python.org>
parents: 38453
diff changeset
13 # _codecs_cn
654c380cf8b9 Merged revisions 46753-51188 via svnmerge from
Thomas Wouters <thomas@python.org>
parents: 38453
diff changeset
14 'gb2312', 'gbk', 'gb18030', 'hz',
654c380cf8b9 Merged revisions 46753-51188 via svnmerge from
Thomas Wouters <thomas@python.org>
parents: 38453
diff changeset
15 # _codecs_hk
654c380cf8b9 Merged revisions 46753-51188 via svnmerge from
Thomas Wouters <thomas@python.org>
parents: 38453
diff changeset
16 'big5hkscs',
654c380cf8b9 Merged revisions 46753-51188 via svnmerge from
Thomas Wouters <thomas@python.org>
parents: 38453
diff changeset
17 # _codecs_jp
654c380cf8b9 Merged revisions 46753-51188 via svnmerge from
Thomas Wouters <thomas@python.org>
parents: 38453
diff changeset
18 'cp932', 'shift_jis', 'euc_jp', 'euc_jisx0213', 'shift_jisx0213',
654c380cf8b9 Merged revisions 46753-51188 via svnmerge from
Thomas Wouters <thomas@python.org>
parents: 38453
diff changeset
19 'euc_jis_2004', 'shift_jis_2004',
654c380cf8b9 Merged revisions 46753-51188 via svnmerge from
Thomas Wouters <thomas@python.org>
parents: 38453
diff changeset
20 # _codecs_kr
654c380cf8b9 Merged revisions 46753-51188 via svnmerge from
Thomas Wouters <thomas@python.org>
parents: 38453
diff changeset
21 'cp949', 'euc_kr', 'johab',
654c380cf8b9 Merged revisions 46753-51188 via svnmerge from
Thomas Wouters <thomas@python.org>
parents: 38453
diff changeset
22 # _codecs_tw
654c380cf8b9 Merged revisions 46753-51188 via svnmerge from
Thomas Wouters <thomas@python.org>
parents: 38453
diff changeset
23 'big5', 'cp950',
654c380cf8b9 Merged revisions 46753-51188 via svnmerge from
Thomas Wouters <thomas@python.org>
parents: 38453
diff changeset
24 # _codecs_iso2022
654c380cf8b9 Merged revisions 46753-51188 via svnmerge from
Thomas Wouters <thomas@python.org>
parents: 38453
diff changeset
25 'iso2022_jp', 'iso2022_jp_1', 'iso2022_jp_2', 'iso2022_jp_2004',
654c380cf8b9 Merged revisions 46753-51188 via svnmerge from
Thomas Wouters <thomas@python.org>
parents: 38453
diff changeset
26 'iso2022_jp_3', 'iso2022_jp_ext', 'iso2022_kr',
654c380cf8b9 Merged revisions 46753-51188 via svnmerge from
Thomas Wouters <thomas@python.org>
parents: 38453
diff changeset
27 ]
37879
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 36960
diff changeset
28
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 36960
diff changeset
29 class Test_MultibyteCodec(unittest.TestCase):
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 36960
diff changeset
30
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 36960
diff changeset
31 def test_nullcoding(self):
39442
654c380cf8b9 Merged revisions 46753-51188 via svnmerge from
Thomas Wouters <thomas@python.org>
parents: 38453
diff changeset
32 for enc in ALL_CJKENCODINGS:
41758
0ead1064ead0 Make all the multibyte codec tests pass.
Guido van Rossum <guido@python.org>
parents: 41642
diff changeset
33 self.assertEqual(b''.decode(enc), '')
0ead1064ead0 Make all the multibyte codec tests pass.
Guido van Rossum <guido@python.org>
parents: 41642
diff changeset
34 self.assertEqual(str(b'', enc), '')
0ead1064ead0 Make all the multibyte codec tests pass.
Guido van Rossum <guido@python.org>
parents: 41642
diff changeset
35 self.assertEqual(''.encode(enc), b'')
37879
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 36960
diff changeset
36
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 36960
diff changeset
37 def test_str_decode(self):
39442
654c380cf8b9 Merged revisions 46753-51188 via svnmerge from
Thomas Wouters <thomas@python.org>
parents: 38453
diff changeset
38 for enc in ALL_CJKENCODINGS:
41758
0ead1064ead0 Make all the multibyte codec tests pass.
Guido van Rossum <guido@python.org>
parents: 41642
diff changeset
39 self.assertEqual('abcd'.encode(enc), b'abcd')
37879
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 36960
diff changeset
40
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 36960
diff changeset
41 def test_errorcallback_longindex(self):
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 36960
diff changeset
42 dec = codecs.getdecoder('euc-kr')
44160
9d152530910a Removed PyInt_GetMax and sys.maxint
Christian Heimes <christian@cheimes.de>
parents: 42918
diff changeset
43 myreplace = lambda exc: ('', sys.maxsize+1)
37879
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 36960
diff changeset
44 codecs.register_error('test.cjktest', myreplace)
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 36960
diff changeset
45 self.assertRaises(IndexError, dec,
50977
1f0a3ac8bf85 Merged revisions 68857 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 48563
diff changeset
46 b'apple\x92ham\x93spam', 'test.cjktest')
37879
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 36960
diff changeset
47
39442
654c380cf8b9 Merged revisions 46753-51188 via svnmerge from
Thomas Wouters <thomas@python.org>
parents: 38453
diff changeset
48 def test_codingspec(self):
654c380cf8b9 Merged revisions 46753-51188 via svnmerge from
Thomas Wouters <thomas@python.org>
parents: 38453
diff changeset
49 try:
654c380cf8b9 Merged revisions 46753-51188 via svnmerge from
Thomas Wouters <thomas@python.org>
parents: 38453
diff changeset
50 for enc in ALL_CJKENCODINGS:
65338
1782da440fb1 Merged revisions 85503 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 55593
diff changeset
51 code = '# coding: {}\n'.format(enc)
1782da440fb1 Merged revisions 85503 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 55593
diff changeset
52 exec(code)
39442
654c380cf8b9 Merged revisions 46753-51188 via svnmerge from
Thomas Wouters <thomas@python.org>
parents: 38453
diff changeset
53 finally:
47201
9c930a32cebb #2621 rename test.test_support to test.support
Benjamin Peterson <benjamin@python.org>
parents: 44160
diff changeset
54 support.unlink(TESTFN)
39442
654c380cf8b9 Merged revisions 46753-51188 via svnmerge from
Thomas Wouters <thomas@python.org>
parents: 38453
diff changeset
55
48563
b51cd5d3795c Merged revisions 65012,65035,65037-65040,65048,65057,65077,65091-65095,65097-65099,65127-65128,65131,65133-65136,65139,65149-65151,65155,65158-65159,65176-65178,65183-65184,65187-65190,65192,65194 via svnmerge from
Georg Brandl <georg@python.org>
parents: 47201
diff changeset
56 def test_init_segfault(self):
b51cd5d3795c Merged revisions 65012,65035,65037-65040,65048,65057,65077,65091-65095,65097-65099,65127-65128,65131,65133-65136,65139,65149-65151,65155,65158-65159,65176-65178,65183-65184,65187-65190,65192,65194 via svnmerge from
Georg Brandl <georg@python.org>
parents: 47201
diff changeset
57 # bug #3305: this used to segfault
b51cd5d3795c Merged revisions 65012,65035,65037-65040,65048,65057,65077,65091-65095,65097-65099,65127-65128,65131,65133-65136,65139,65149-65151,65155,65158-65159,65176-65178,65183-65184,65187-65190,65192,65194 via svnmerge from
Georg Brandl <georg@python.org>
parents: 47201
diff changeset
58 self.assertRaises(AttributeError,
b51cd5d3795c Merged revisions 65012,65035,65037-65040,65048,65057,65077,65091-65095,65097-65099,65127-65128,65131,65133-65136,65139,65149-65151,65155,65158-65159,65176-65178,65183-65184,65187-65190,65192,65194 via svnmerge from
Georg Brandl <georg@python.org>
parents: 47201
diff changeset
59 _multibytecodec.MultibyteStreamReader, None)
b51cd5d3795c Merged revisions 65012,65035,65037-65040,65048,65057,65077,65091-65095,65097-65099,65127-65128,65131,65133-65136,65139,65149-65151,65155,65158-65159,65176-65178,65183-65184,65187-65190,65192,65194 via svnmerge from
Georg Brandl <georg@python.org>
parents: 47201
diff changeset
60 self.assertRaises(AttributeError,
b51cd5d3795c Merged revisions 65012,65035,65037-65040,65048,65057,65077,65091-65095,65097-65099,65127-65128,65131,65133-65136,65139,65149-65151,65155,65158-65159,65176-65178,65183-65184,65187-65190,65192,65194 via svnmerge from
Georg Brandl <georg@python.org>
parents: 47201
diff changeset
61 _multibytecodec.MultibyteStreamWriter, None)
b51cd5d3795c Merged revisions 65012,65035,65037-65040,65048,65057,65077,65091-65095,65097-65099,65127-65128,65131,65133-65136,65139,65149-65151,65155,65158-65159,65176-65178,65183-65184,65187-65190,65192,65194 via svnmerge from
Georg Brandl <georg@python.org>
parents: 47201
diff changeset
62
50977
1f0a3ac8bf85 Merged revisions 68857 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 48563
diff changeset
63 def test_decode_unicode(self):
1f0a3ac8bf85 Merged revisions 68857 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 48563
diff changeset
64 # Trying to decode an unicode string should raise a TypeError
1f0a3ac8bf85 Merged revisions 68857 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 48563
diff changeset
65 for enc in ALL_CJKENCODINGS:
1f0a3ac8bf85 Merged revisions 68857 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 48563
diff changeset
66 self.assertRaises(TypeError, codecs.getdecoder(enc), "")
48563
b51cd5d3795c Merged revisions 65012,65035,65037-65040,65048,65057,65077,65091-65095,65097-65099,65127-65128,65131,65133-65136,65139,65149-65151,65155,65158-65159,65176-65178,65183-65184,65187-65190,65192,65194 via svnmerge from
Georg Brandl <georg@python.org>
parents: 47201
diff changeset
67
37879
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 36960
diff changeset
68 class Test_IncrementalEncoder(unittest.TestCase):
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 36960
diff changeset
69
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 36960
diff changeset
70 def test_stateless(self):
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 36960
diff changeset
71 # cp949 encoder isn't stateful at all.
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 36960
diff changeset
72 encoder = codecs.getincrementalencoder('cp949')()
41575
beacf40b5960 Rip out all the u"..." literals and calls to unicode().
Guido van Rossum <guido@python.org>
parents: 41554
diff changeset
73 self.assertEqual(encoder.encode('\ud30c\uc774\uc36c \ub9c8\uc744'),
41758
0ead1064ead0 Make all the multibyte codec tests pass.
Guido van Rossum <guido@python.org>
parents: 41642
diff changeset
74 b'\xc6\xc4\xc0\xcc\xbd\xe3 \xb8\xb6\xc0\xbb')
37879
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 36960
diff changeset
75 self.assertEqual(encoder.reset(), None)
41575
beacf40b5960 Rip out all the u"..." literals and calls to unicode().
Guido van Rossum <guido@python.org>
parents: 41554
diff changeset
76 self.assertEqual(encoder.encode('\u2606\u223c\u2606', True),
41758
0ead1064ead0 Make all the multibyte codec tests pass.
Guido van Rossum <guido@python.org>
parents: 41642
diff changeset
77 b'\xa1\xd9\xa1\xad\xa1\xd9')
37879
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 36960
diff changeset
78 self.assertEqual(encoder.reset(), None)
41758
0ead1064ead0 Make all the multibyte codec tests pass.
Guido van Rossum <guido@python.org>
parents: 41642
diff changeset
79 self.assertEqual(encoder.encode('', True), b'')
0ead1064ead0 Make all the multibyte codec tests pass.
Guido van Rossum <guido@python.org>
parents: 41642
diff changeset
80 self.assertEqual(encoder.encode('', False), b'')
37879
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 36960
diff changeset
81 self.assertEqual(encoder.reset(), None)
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 36960
diff changeset
82
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 36960
diff changeset
83 def test_stateful(self):
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 36960
diff changeset
84 # jisx0213 encoder is stateful for a few codepoints. eg)
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 36960
diff changeset
85 # U+00E6 => A9DC
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 36960
diff changeset
86 # U+00E6 U+0300 => ABC4
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 36960
diff changeset
87 # U+0300 => ABDC
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 36960
diff changeset
88
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 36960
diff changeset
89 encoder = codecs.getincrementalencoder('jisx0213')()
41758
0ead1064ead0 Make all the multibyte codec tests pass.
Guido van Rossum <guido@python.org>
parents: 41642
diff changeset
90 self.assertEqual(encoder.encode('\u00e6\u0300'), b'\xab\xc4')
0ead1064ead0 Make all the multibyte codec tests pass.
Guido van Rossum <guido@python.org>
parents: 41642
diff changeset
91 self.assertEqual(encoder.encode('\u00e6'), b'')
0ead1064ead0 Make all the multibyte codec tests pass.
Guido van Rossum <guido@python.org>
parents: 41642
diff changeset
92 self.assertEqual(encoder.encode('\u0300'), b'\xab\xc4')
0ead1064ead0 Make all the multibyte codec tests pass.
Guido van Rossum <guido@python.org>
parents: 41642
diff changeset
93 self.assertEqual(encoder.encode('\u00e6', True), b'\xa9\xdc')
37879
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 36960
diff changeset
94
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 36960
diff changeset
95 self.assertEqual(encoder.reset(), None)
41758
0ead1064ead0 Make all the multibyte codec tests pass.
Guido van Rossum <guido@python.org>
parents: 41642
diff changeset
96 self.assertEqual(encoder.encode('\u0300'), b'\xab\xdc')
37879
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 36960
diff changeset
97
41758
0ead1064ead0 Make all the multibyte codec tests pass.
Guido van Rossum <guido@python.org>
parents: 41642
diff changeset
98 self.assertEqual(encoder.encode('\u00e6'), b'')
0ead1064ead0 Make all the multibyte codec tests pass.
Guido van Rossum <guido@python.org>
parents: 41642
diff changeset
99 self.assertEqual(encoder.encode('', True), b'\xa9\xdc')
0ead1064ead0 Make all the multibyte codec tests pass.
Guido van Rossum <guido@python.org>
parents: 41642
diff changeset
100 self.assertEqual(encoder.encode('', True), b'')
37879
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 36960
diff changeset
101
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 36960
diff changeset
102 def test_stateful_keep_buffer(self):
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 36960
diff changeset
103 encoder = codecs.getincrementalencoder('jisx0213')()
41758
0ead1064ead0 Make all the multibyte codec tests pass.
Guido van Rossum <guido@python.org>
parents: 41642
diff changeset
104 self.assertEqual(encoder.encode('\u00e6'), b'')
41575
beacf40b5960 Rip out all the u"..." literals and calls to unicode().
Guido van Rossum <guido@python.org>
parents: 41554
diff changeset
105 self.assertRaises(UnicodeEncodeError, encoder.encode, '\u0123')
41758
0ead1064ead0 Make all the multibyte codec tests pass.
Guido van Rossum <guido@python.org>
parents: 41642
diff changeset
106 self.assertEqual(encoder.encode('\u0300\u00e6'), b'\xab\xc4')
41575
beacf40b5960 Rip out all the u"..." literals and calls to unicode().
Guido van Rossum <guido@python.org>
parents: 41554
diff changeset
107 self.assertRaises(UnicodeEncodeError, encoder.encode, '\u0123')
37879
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 36960
diff changeset
108 self.assertEqual(encoder.reset(), None)
41758
0ead1064ead0 Make all the multibyte codec tests pass.
Guido van Rossum <guido@python.org>
parents: 41642
diff changeset
109 self.assertEqual(encoder.encode('\u0300'), b'\xab\xdc')
0ead1064ead0 Make all the multibyte codec tests pass.
Guido van Rossum <guido@python.org>
parents: 41642
diff changeset
110 self.assertEqual(encoder.encode('\u00e6'), b'')
41575
beacf40b5960 Rip out all the u"..." literals and calls to unicode().
Guido van Rossum <guido@python.org>
parents: 41554
diff changeset
111 self.assertRaises(UnicodeEncodeError, encoder.encode, '\u0123')
41758
0ead1064ead0 Make all the multibyte codec tests pass.
Guido van Rossum <guido@python.org>
parents: 41642
diff changeset
112 self.assertEqual(encoder.encode('', True), b'\xa9\xdc')
37879
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 36960
diff changeset
113
52730
acdd11e39f0b Issue #5640: Fix _multibytecodec so that CJK codecs don't repeat
Hye-Shik Chang <hyeshik@gmail.com>
parents: 50977
diff changeset
114 def test_issue5640(self):
acdd11e39f0b Issue #5640: Fix _multibytecodec so that CJK codecs don't repeat
Hye-Shik Chang <hyeshik@gmail.com>
parents: 50977
diff changeset
115 encoder = codecs.getincrementalencoder('shift-jis')('backslashreplace')
acdd11e39f0b Issue #5640: Fix _multibytecodec so that CJK codecs don't repeat
Hye-Shik Chang <hyeshik@gmail.com>
parents: 50977
diff changeset
116 self.assertEqual(encoder.encode('\xff'), b'\\xff')
acdd11e39f0b Issue #5640: Fix _multibytecodec so that CJK codecs don't repeat
Hye-Shik Chang <hyeshik@gmail.com>
parents: 50977
diff changeset
117 self.assertEqual(encoder.encode('\n'), b'\n')
37879
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 36960
diff changeset
118
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 36960
diff changeset
119 class Test_IncrementalDecoder(unittest.TestCase):
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 36960
diff changeset
120
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 36960
diff changeset
121 def test_dbcs(self):
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 36960
diff changeset
122 # cp949 decoder is simple with only 1 or 2 bytes sequences.
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 36960
diff changeset
123 decoder = codecs.getincrementaldecoder('cp949')()
41758
0ead1064ead0 Make all the multibyte codec tests pass.
Guido van Rossum <guido@python.org>
parents: 41642
diff changeset
124 self.assertEqual(decoder.decode(b'\xc6\xc4\xc0\xcc\xbd'),
41575
beacf40b5960 Rip out all the u"..." literals and calls to unicode().
Guido van Rossum <guido@python.org>
parents: 41554
diff changeset
125 '\ud30c\uc774')
41758
0ead1064ead0 Make all the multibyte codec tests pass.
Guido van Rossum <guido@python.org>
parents: 41642
diff changeset
126 self.assertEqual(decoder.decode(b'\xe3 \xb8\xb6\xc0\xbb'),
41575
beacf40b5960 Rip out all the u"..." literals and calls to unicode().
Guido van Rossum <guido@python.org>
parents: 41554
diff changeset
127 '\uc36c \ub9c8\uc744')
41758
0ead1064ead0 Make all the multibyte codec tests pass.
Guido van Rossum <guido@python.org>
parents: 41642
diff changeset
128 self.assertEqual(decoder.decode(b''), '')
37879
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 36960
diff changeset
129
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 36960
diff changeset
130 def test_dbcs_keep_buffer(self):
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 36960
diff changeset
131 decoder = codecs.getincrementaldecoder('cp949')()
41758
0ead1064ead0 Make all the multibyte codec tests pass.
Guido van Rossum <guido@python.org>
parents: 41642
diff changeset
132 self.assertEqual(decoder.decode(b'\xc6\xc4\xc0'), '\ud30c')
42918
9220931e4741 More changes needed to make things work once bytes and str are truly divorced.
Guido van Rossum <guido@python.org>
parents: 42552
diff changeset
133 self.assertRaises(UnicodeDecodeError, decoder.decode, b'', True)
41758
0ead1064ead0 Make all the multibyte codec tests pass.
Guido van Rossum <guido@python.org>
parents: 41642
diff changeset
134 self.assertEqual(decoder.decode(b'\xcc'), '\uc774')
37879
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 36960
diff changeset
135
41758
0ead1064ead0 Make all the multibyte codec tests pass.
Guido van Rossum <guido@python.org>
parents: 41642
diff changeset
136 self.assertEqual(decoder.decode(b'\xc6\xc4\xc0'), '\ud30c')
42918
9220931e4741 More changes needed to make things work once bytes and str are truly divorced.
Guido van Rossum <guido@python.org>
parents: 42552
diff changeset
137 self.assertRaises(UnicodeDecodeError, decoder.decode,
9220931e4741 More changes needed to make things work once bytes and str are truly divorced.
Guido van Rossum <guido@python.org>
parents: 42552
diff changeset
138 b'\xcc\xbd', True)
41758
0ead1064ead0 Make all the multibyte codec tests pass.
Guido van Rossum <guido@python.org>
parents: 41642
diff changeset
139 self.assertEqual(decoder.decode(b'\xcc'), '\uc774')
37879
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 36960
diff changeset
140
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 36960
diff changeset
141 def test_iso2022(self):
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 36960
diff changeset
142 decoder = codecs.getincrementaldecoder('iso2022-jp')()
42918
9220931e4741 More changes needed to make things work once bytes and str are truly divorced.
Guido van Rossum <guido@python.org>
parents: 42552
diff changeset
143 ESC = b'\x1b'
9220931e4741 More changes needed to make things work once bytes and str are truly divorced.
Guido van Rossum <guido@python.org>
parents: 42552
diff changeset
144 self.assertEqual(decoder.decode(ESC + b'('), '')
9220931e4741 More changes needed to make things work once bytes and str are truly divorced.
Guido van Rossum <guido@python.org>
parents: 42552
diff changeset
145 self.assertEqual(decoder.decode(b'B', True), '')
9220931e4741 More changes needed to make things work once bytes and str are truly divorced.
Guido van Rossum <guido@python.org>
parents: 42552
diff changeset
146 self.assertEqual(decoder.decode(ESC + b'$'), '')
9220931e4741 More changes needed to make things work once bytes and str are truly divorced.
Guido van Rossum <guido@python.org>
parents: 42552
diff changeset
147 self.assertEqual(decoder.decode(b'B@$'), '\u4e16')
9220931e4741 More changes needed to make things work once bytes and str are truly divorced.
Guido van Rossum <guido@python.org>
parents: 42552
diff changeset
148 self.assertEqual(decoder.decode(b'@$@'), '\u4e16')
9220931e4741 More changes needed to make things work once bytes and str are truly divorced.
Guido van Rossum <guido@python.org>
parents: 42552
diff changeset
149 self.assertEqual(decoder.decode(b'$', True), '\u4e16')
37879
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 36960
diff changeset
150 self.assertEqual(decoder.reset(), None)
42918
9220931e4741 More changes needed to make things work once bytes and str are truly divorced.
Guido van Rossum <guido@python.org>
parents: 42552
diff changeset
151 self.assertEqual(decoder.decode(b'@$'), '@$')
9220931e4741 More changes needed to make things work once bytes and str are truly divorced.
Guido van Rossum <guido@python.org>
parents: 42552
diff changeset
152 self.assertEqual(decoder.decode(ESC + b'$'), '')
9220931e4741 More changes needed to make things work once bytes and str are truly divorced.
Guido van Rossum <guido@python.org>
parents: 42552
diff changeset
153 self.assertRaises(UnicodeDecodeError, decoder.decode, b'', True)
9220931e4741 More changes needed to make things work once bytes and str are truly divorced.
Guido van Rossum <guido@python.org>
parents: 42552
diff changeset
154 self.assertEqual(decoder.decode(b'B@$'), '\u4e16')
37879
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 36960
diff changeset
155
50977
1f0a3ac8bf85 Merged revisions 68857 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 48563
diff changeset
156 def test_decode_unicode(self):
1f0a3ac8bf85 Merged revisions 68857 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 48563
diff changeset
157 # Trying to decode an unicode string should raise a TypeError
1f0a3ac8bf85 Merged revisions 68857 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 48563
diff changeset
158 for enc in ALL_CJKENCODINGS:
1f0a3ac8bf85 Merged revisions 68857 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 48563
diff changeset
159 decoder = codecs.getincrementaldecoder(enc)()
1f0a3ac8bf85 Merged revisions 68857 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 48563
diff changeset
160 self.assertRaises(TypeError, decoder.decode, "")
1f0a3ac8bf85 Merged revisions 68857 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 48563
diff changeset
161
42084
229c28cb9afb Merged revisions 55817-55961 via svnmerge from
Guido van Rossum <guido@python.org>
parents: 41758
diff changeset
162 class Test_StreamReader(unittest.TestCase):
229c28cb9afb Merged revisions 55817-55961 via svnmerge from
Guido van Rossum <guido@python.org>
parents: 41758
diff changeset
163 def test_bug1728403(self):
229c28cb9afb Merged revisions 55817-55961 via svnmerge from
Guido van Rossum <guido@python.org>
parents: 41758
diff changeset
164 try:
42160
bf222ebff0e5 Fix the last remaining problem with test_multibytecodec.py;
Guido van Rossum <guido@python.org>
parents: 42084
diff changeset
165 f = open(TESTFN, 'wb')
bf222ebff0e5 Fix the last remaining problem with test_multibytecodec.py;
Guido van Rossum <guido@python.org>
parents: 42084
diff changeset
166 try:
bf222ebff0e5 Fix the last remaining problem with test_multibytecodec.py;
Guido van Rossum <guido@python.org>
parents: 42084
diff changeset
167 f.write(b'\xa1')
bf222ebff0e5 Fix the last remaining problem with test_multibytecodec.py;
Guido van Rossum <guido@python.org>
parents: 42084
diff changeset
168 finally:
bf222ebff0e5 Fix the last remaining problem with test_multibytecodec.py;
Guido van Rossum <guido@python.org>
parents: 42084
diff changeset
169 f.close()
42084
229c28cb9afb Merged revisions 55817-55961 via svnmerge from
Guido van Rossum <guido@python.org>
parents: 41758
diff changeset
170 f = codecs.open(TESTFN, encoding='cp949')
42160
bf222ebff0e5 Fix the last remaining problem with test_multibytecodec.py;
Guido van Rossum <guido@python.org>
parents: 42084
diff changeset
171 try:
bf222ebff0e5 Fix the last remaining problem with test_multibytecodec.py;
Guido van Rossum <guido@python.org>
parents: 42084
diff changeset
172 self.assertRaises(UnicodeDecodeError, f.read, 2)
bf222ebff0e5 Fix the last remaining problem with test_multibytecodec.py;
Guido van Rossum <guido@python.org>
parents: 42084
diff changeset
173 finally:
bf222ebff0e5 Fix the last remaining problem with test_multibytecodec.py;
Guido van Rossum <guido@python.org>
parents: 42084
diff changeset
174 f.close()
42084
229c28cb9afb Merged revisions 55817-55961 via svnmerge from
Guido van Rossum <guido@python.org>
parents: 41758
diff changeset
175 finally:
47201
9c930a32cebb #2621 rename test.test_support to test.support
Benjamin Peterson <benjamin@python.org>
parents: 44160
diff changeset
176 support.unlink(TESTFN)
31386
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
177
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
178 class Test_StreamWriter(unittest.TestCase):
41575
beacf40b5960 Rip out all the u"..." literals and calls to unicode().
Guido van Rossum <guido@python.org>
parents: 41554
diff changeset
179 if len('\U00012345') == 2: # UCS2
31386
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
180 def test_gb18030(self):
41758
0ead1064ead0 Make all the multibyte codec tests pass.
Guido van Rossum <guido@python.org>
parents: 41642
diff changeset
181 s= io.BytesIO()
37879
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 36960
diff changeset
182 c = codecs.getwriter('gb18030')(s)
41575
beacf40b5960 Rip out all the u"..." literals and calls to unicode().
Guido van Rossum <guido@python.org>
parents: 41554
diff changeset
183 c.write('123')
41758
0ead1064ead0 Make all the multibyte codec tests pass.
Guido van Rossum <guido@python.org>
parents: 41642
diff changeset
184 self.assertEqual(s.getvalue(), b'123')
41575
beacf40b5960 Rip out all the u"..." literals and calls to unicode().
Guido van Rossum <guido@python.org>
parents: 41554
diff changeset
185 c.write('\U00012345')
41758
0ead1064ead0 Make all the multibyte codec tests pass.
Guido van Rossum <guido@python.org>
parents: 41642
diff changeset
186 self.assertEqual(s.getvalue(), b'123\x907\x959')
41575
beacf40b5960 Rip out all the u"..." literals and calls to unicode().
Guido van Rossum <guido@python.org>
parents: 41554
diff changeset
187 c.write('\U00012345'[0])
41758
0ead1064ead0 Make all the multibyte codec tests pass.
Guido van Rossum <guido@python.org>
parents: 41642
diff changeset
188 self.assertEqual(s.getvalue(), b'123\x907\x959')
41575
beacf40b5960 Rip out all the u"..." literals and calls to unicode().
Guido van Rossum <guido@python.org>
parents: 41554
diff changeset
189 c.write('\U00012345'[1] + '\U00012345' + '\uac00\u00ac')
31386
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
190 self.assertEqual(s.getvalue(),
41758
0ead1064ead0 Make all the multibyte codec tests pass.
Guido van Rossum <guido@python.org>
parents: 41642
diff changeset
191 b'123\x907\x959\x907\x959\x907\x959\x827\xcf5\x810\x851')
41575
beacf40b5960 Rip out all the u"..." literals and calls to unicode().
Guido van Rossum <guido@python.org>
parents: 41554
diff changeset
192 c.write('\U00012345'[0])
31386
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
193 self.assertEqual(s.getvalue(),
41758
0ead1064ead0 Make all the multibyte codec tests pass.
Guido van Rossum <guido@python.org>
parents: 41642
diff changeset
194 b'123\x907\x959\x907\x959\x907\x959\x827\xcf5\x810\x851')
31386
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
195 self.assertRaises(UnicodeError, c.reset)
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
196 self.assertEqual(s.getvalue(),
41758
0ead1064ead0 Make all the multibyte codec tests pass.
Guido van Rossum <guido@python.org>
parents: 41642
diff changeset
197 b'123\x907\x959\x907\x959\x907\x959\x827\xcf5\x810\x851')
31386
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
198
37879
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 36960
diff changeset
199 def test_utf_8(self):
41758
0ead1064ead0 Make all the multibyte codec tests pass.
Guido van Rossum <guido@python.org>
parents: 41642
diff changeset
200 s= io.BytesIO()
37879
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 36960
diff changeset
201 c = codecs.getwriter('utf-8')(s)
41575
beacf40b5960 Rip out all the u"..." literals and calls to unicode().
Guido van Rossum <guido@python.org>
parents: 41554
diff changeset
202 c.write('123')
41758
0ead1064ead0 Make all the multibyte codec tests pass.
Guido van Rossum <guido@python.org>
parents: 41642
diff changeset
203 self.assertEqual(s.getvalue(), b'123')
41575
beacf40b5960 Rip out all the u"..." literals and calls to unicode().
Guido van Rossum <guido@python.org>
parents: 41554
diff changeset
204 c.write('\U00012345')
41758
0ead1064ead0 Make all the multibyte codec tests pass.
Guido van Rossum <guido@python.org>
parents: 41642
diff changeset
205 self.assertEqual(s.getvalue(), b'123\xf0\x92\x8d\x85')
37879
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 36960
diff changeset
206
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 36960
diff changeset
207 # Python utf-8 codec can't buffer surrogate pairs yet.
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 36960
diff changeset
208 if 0:
41575
beacf40b5960 Rip out all the u"..." literals and calls to unicode().
Guido van Rossum <guido@python.org>
parents: 41554
diff changeset
209 c.write('\U00012345'[0])
41758
0ead1064ead0 Make all the multibyte codec tests pass.
Guido van Rossum <guido@python.org>
parents: 41642
diff changeset
210 self.assertEqual(s.getvalue(), b'123\xf0\x92\x8d\x85')
41575
beacf40b5960 Rip out all the u"..." literals and calls to unicode().
Guido van Rossum <guido@python.org>
parents: 41554
diff changeset
211 c.write('\U00012345'[1] + '\U00012345' + '\uac00\u00ac')
31386
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
212 self.assertEqual(s.getvalue(),
41758
0ead1064ead0 Make all the multibyte codec tests pass.
Guido van Rossum <guido@python.org>
parents: 41642
diff changeset
213 b'123\xf0\x92\x8d\x85\xf0\x92\x8d\x85\xf0\x92\x8d\x85'
0ead1064ead0 Make all the multibyte codec tests pass.
Guido van Rossum <guido@python.org>
parents: 41642
diff changeset
214 b'\xea\xb0\x80\xc2\xac')
41575
beacf40b5960 Rip out all the u"..." literals and calls to unicode().
Guido van Rossum <guido@python.org>
parents: 41554
diff changeset
215 c.write('\U00012345'[0])
31386
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
216 self.assertEqual(s.getvalue(),
41758
0ead1064ead0 Make all the multibyte codec tests pass.
Guido van Rossum <guido@python.org>
parents: 41642
diff changeset
217 b'123\xf0\x92\x8d\x85\xf0\x92\x8d\x85\xf0\x92\x8d\x85'
0ead1064ead0 Make all the multibyte codec tests pass.
Guido van Rossum <guido@python.org>
parents: 41642
diff changeset
218 b'\xea\xb0\x80\xc2\xac')
31386
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
219 c.reset()
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
220 self.assertEqual(s.getvalue(),
41758
0ead1064ead0 Make all the multibyte codec tests pass.
Guido van Rossum <guido@python.org>
parents: 41642
diff changeset
221 b'123\xf0\x92\x8d\x85\xf0\x92\x8d\x85\xf0\x92\x8d\x85'
0ead1064ead0 Make all the multibyte codec tests pass.
Guido van Rossum <guido@python.org>
parents: 41642
diff changeset
222 b'\xea\xb0\x80\xc2\xac\xed\xa0\x88')
41575
beacf40b5960 Rip out all the u"..." literals and calls to unicode().
Guido van Rossum <guido@python.org>
parents: 41554
diff changeset
223 c.write('\U00012345'[1])
31386
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
224 self.assertEqual(s.getvalue(),
41758
0ead1064ead0 Make all the multibyte codec tests pass.
Guido van Rossum <guido@python.org>
parents: 41642
diff changeset
225 b'123\xf0\x92\x8d\x85\xf0\x92\x8d\x85\xf0\x92\x8d\x85'
0ead1064ead0 Make all the multibyte codec tests pass.
Guido van Rossum <guido@python.org>
parents: 41642
diff changeset
226 b'\xea\xb0\x80\xc2\xac\xed\xa0\x88\xed\xbd\x85')
31386
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
227
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
228 else: # UCS4
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
229 pass
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
230
32727
69dadc2ca14d Bring CJKCodecs 1.1 into trunk. This completely reorganizes source
Hye-Shik Chang <hyeshik@gmail.com>
parents: 31393
diff changeset
231 def test_streamwriter_strwrite(self):
41758
0ead1064ead0 Make all the multibyte codec tests pass.
Guido van Rossum <guido@python.org>
parents: 41642
diff changeset
232 s = io.BytesIO()
32727
69dadc2ca14d Bring CJKCodecs 1.1 into trunk. This completely reorganizes source
Hye-Shik Chang <hyeshik@gmail.com>
parents: 31393
diff changeset
233 wr = codecs.getwriter('gb18030')(s)
69dadc2ca14d Bring CJKCodecs 1.1 into trunk. This completely reorganizes source
Hye-Shik Chang <hyeshik@gmail.com>
parents: 31393
diff changeset
234 wr.write('abcd')
41758
0ead1064ead0 Make all the multibyte codec tests pass.
Guido van Rossum <guido@python.org>
parents: 41642
diff changeset
235 self.assertEqual(s.getvalue(), b'abcd')
32727
69dadc2ca14d Bring CJKCodecs 1.1 into trunk. This completely reorganizes source
Hye-Shik Chang <hyeshik@gmail.com>
parents: 31393
diff changeset
236
36960
3ad42f99a6db Bug #1448490: Fix a bug that ISO-2022 codecs could not handle
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32727
diff changeset
237 class Test_ISO2022(unittest.TestCase):
3ad42f99a6db Bug #1448490: Fix a bug that ISO-2022 codecs could not handle
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32727
diff changeset
238 def test_g2(self):
42918
9220931e4741 More changes needed to make things work once bytes and str are truly divorced.
Guido van Rossum <guido@python.org>
parents: 42552
diff changeset
239 iso2022jp2 = b'\x1b(B:hu4:unit\x1b.A\x1bNi de famille'
41575
beacf40b5960 Rip out all the u"..." literals and calls to unicode().
Guido van Rossum <guido@python.org>
parents: 41554
diff changeset
240 uni = ':hu4:unit\xe9 de famille'
36960
3ad42f99a6db Bug #1448490: Fix a bug that ISO-2022 codecs could not handle
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32727
diff changeset
241 self.assertEqual(iso2022jp2.decode('iso2022-jp-2'), uni)
3ad42f99a6db Bug #1448490: Fix a bug that ISO-2022 codecs could not handle
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32727
diff changeset
242
40450
cc992d75d5b3 Four months of trunk changes (including a few releases...)
Thomas Wouters <thomas@python.org>
parents: 39782
diff changeset
243 def test_iso2022_jp_g0(self):
58456
f0f221e1a7c0 use assert[Not]In where appropriate
Ezio Melotti <ezio.melotti@gmail.com>
parents: 55593
diff changeset
244 self.assertNotIn(b'\x0e', '\N{SOFT HYPHEN}'.encode('iso-2022-jp-2'))
40450
cc992d75d5b3 Four months of trunk changes (including a few releases...)
Thomas Wouters <thomas@python.org>
parents: 39782
diff changeset
245 for encoding in ('iso-2022-jp-2004', 'iso-2022-jp-3'):
41575
beacf40b5960 Rip out all the u"..." literals and calls to unicode().
Guido van Rossum <guido@python.org>
parents: 41554
diff changeset
246 e = '\u3406'.encode(encoding)
55593
0bb232078b8c Merged revisions 73715 via svnmerge from
Georg Brandl <georg@python.org>
parents: 52730
diff changeset
247 self.assertFalse(any(x > 0x80 for x in e))
40450
cc992d75d5b3 Four months of trunk changes (including a few releases...)
Thomas Wouters <thomas@python.org>
parents: 39782
diff changeset
248
cc992d75d5b3 Four months of trunk changes (including a few releases...)
Thomas Wouters <thomas@python.org>
parents: 39782
diff changeset
249 def test_bug1572832(self):
cc992d75d5b3 Four months of trunk changes (including a few releases...)
Thomas Wouters <thomas@python.org>
parents: 39782
diff changeset
250 if sys.maxunicode >= 0x10000:
41585
825d966338ea Rename 'unicode' to 'str' in its tp_name field. Rename 'str' to 'str8'.
Guido van Rossum <guido@python.org>
parents: 41575
diff changeset
251 myunichr = chr
40450
cc992d75d5b3 Four months of trunk changes (including a few releases...)
Thomas Wouters <thomas@python.org>
parents: 39782
diff changeset
252 else:
41585
825d966338ea Rename 'unicode' to 'str' in its tp_name field. Rename 'str' to 'str8'.
Guido van Rossum <guido@python.org>
parents: 41575
diff changeset
253 myunichr = lambda x: chr(0xD7C0+(x>>10)) + chr(0xDC00+(x&0x3FF))
40450
cc992d75d5b3 Four months of trunk changes (including a few releases...)
Thomas Wouters <thomas@python.org>
parents: 39782
diff changeset
254
41642
a6eb6acfe04a Merged revisions 55007-55179 via svnmerge from
Guido van Rossum <guido@python.org>
parents: 41585
diff changeset
255 for x in range(0x10000, 0x110000):
40450
cc992d75d5b3 Four months of trunk changes (including a few releases...)
Thomas Wouters <thomas@python.org>
parents: 39782
diff changeset
256 # Any ISO 2022 codec will cause the segfault
cc992d75d5b3 Four months of trunk changes (including a few releases...)
Thomas Wouters <thomas@python.org>
parents: 39782
diff changeset
257 myunichr(x).encode('iso_2022_jp', 'ignore')
cc992d75d5b3 Four months of trunk changes (including a few releases...)
Thomas Wouters <thomas@python.org>
parents: 39782
diff changeset
258
70347
bd17396895fb Issue #12100: Don't reset incremental encoders of CJK codecs at each call to
Victor Stinner <victor.stinner@haypocalc.com>
parents: 65338
diff changeset
259 class TestStateful(unittest.TestCase):
bd17396895fb Issue #12100: Don't reset incremental encoders of CJK codecs at each call to
Victor Stinner <victor.stinner@haypocalc.com>
parents: 65338
diff changeset
260 text = '\u4E16\u4E16'
bd17396895fb Issue #12100: Don't reset incremental encoders of CJK codecs at each call to
Victor Stinner <victor.stinner@haypocalc.com>
parents: 65338
diff changeset
261 encoding = 'iso-2022-jp'
bd17396895fb Issue #12100: Don't reset incremental encoders of CJK codecs at each call to
Victor Stinner <victor.stinner@haypocalc.com>
parents: 65338
diff changeset
262 expected = b'\x1b$B@$@$'
bd17396895fb Issue #12100: Don't reset incremental encoders of CJK codecs at each call to
Victor Stinner <victor.stinner@haypocalc.com>
parents: 65338
diff changeset
263 expected_reset = b'\x1b$B@$@$\x1b(B'
bd17396895fb Issue #12100: Don't reset incremental encoders of CJK codecs at each call to
Victor Stinner <victor.stinner@haypocalc.com>
parents: 65338
diff changeset
264
bd17396895fb Issue #12100: Don't reset incremental encoders of CJK codecs at each call to
Victor Stinner <victor.stinner@haypocalc.com>
parents: 65338
diff changeset
265 def test_encode(self):
bd17396895fb Issue #12100: Don't reset incremental encoders of CJK codecs at each call to
Victor Stinner <victor.stinner@haypocalc.com>
parents: 65338
diff changeset
266 self.assertEqual(self.text.encode(self.encoding), self.expected_reset)
bd17396895fb Issue #12100: Don't reset incremental encoders of CJK codecs at each call to
Victor Stinner <victor.stinner@haypocalc.com>
parents: 65338
diff changeset
267
bd17396895fb Issue #12100: Don't reset incremental encoders of CJK codecs at each call to
Victor Stinner <victor.stinner@haypocalc.com>
parents: 65338
diff changeset
268 def test_incrementalencoder(self):
bd17396895fb Issue #12100: Don't reset incremental encoders of CJK codecs at each call to
Victor Stinner <victor.stinner@haypocalc.com>
parents: 65338
diff changeset
269 encoder = codecs.getincrementalencoder(self.encoding)()
bd17396895fb Issue #12100: Don't reset incremental encoders of CJK codecs at each call to
Victor Stinner <victor.stinner@haypocalc.com>
parents: 65338
diff changeset
270 output = b''.join(
bd17396895fb Issue #12100: Don't reset incremental encoders of CJK codecs at each call to
Victor Stinner <victor.stinner@haypocalc.com>
parents: 65338
diff changeset
271 encoder.encode(char)
bd17396895fb Issue #12100: Don't reset incremental encoders of CJK codecs at each call to
Victor Stinner <victor.stinner@haypocalc.com>
parents: 65338
diff changeset
272 for char in self.text)
bd17396895fb Issue #12100: Don't reset incremental encoders of CJK codecs at each call to
Victor Stinner <victor.stinner@haypocalc.com>
parents: 65338
diff changeset
273 self.assertEqual(output, self.expected)
bd17396895fb Issue #12100: Don't reset incremental encoders of CJK codecs at each call to
Victor Stinner <victor.stinner@haypocalc.com>
parents: 65338
diff changeset
274
bd17396895fb Issue #12100: Don't reset incremental encoders of CJK codecs at each call to
Victor Stinner <victor.stinner@haypocalc.com>
parents: 65338
diff changeset
275 def test_incrementalencoder_final(self):
bd17396895fb Issue #12100: Don't reset incremental encoders of CJK codecs at each call to
Victor Stinner <victor.stinner@haypocalc.com>
parents: 65338
diff changeset
276 encoder = codecs.getincrementalencoder(self.encoding)()
bd17396895fb Issue #12100: Don't reset incremental encoders of CJK codecs at each call to
Victor Stinner <victor.stinner@haypocalc.com>
parents: 65338
diff changeset
277 last_index = len(self.text) - 1
bd17396895fb Issue #12100: Don't reset incremental encoders of CJK codecs at each call to
Victor Stinner <victor.stinner@haypocalc.com>
parents: 65338
diff changeset
278 output = b''.join(
bd17396895fb Issue #12100: Don't reset incremental encoders of CJK codecs at each call to
Victor Stinner <victor.stinner@haypocalc.com>
parents: 65338
diff changeset
279 encoder.encode(char, index == last_index)
bd17396895fb Issue #12100: Don't reset incremental encoders of CJK codecs at each call to
Victor Stinner <victor.stinner@haypocalc.com>
parents: 65338
diff changeset
280 for index, char in enumerate(self.text))
bd17396895fb Issue #12100: Don't reset incremental encoders of CJK codecs at each call to
Victor Stinner <victor.stinner@haypocalc.com>
parents: 65338
diff changeset
281 self.assertEqual(output, self.expected_reset)
bd17396895fb Issue #12100: Don't reset incremental encoders of CJK codecs at each call to
Victor Stinner <victor.stinner@haypocalc.com>
parents: 65338
diff changeset
282
bd17396895fb Issue #12100: Don't reset incremental encoders of CJK codecs at each call to
Victor Stinner <victor.stinner@haypocalc.com>
parents: 65338
diff changeset
283 class TestHZStateful(TestStateful):
bd17396895fb Issue #12100: Don't reset incremental encoders of CJK codecs at each call to
Victor Stinner <victor.stinner@haypocalc.com>
parents: 65338
diff changeset
284 text = '\u804a\u804a'
bd17396895fb Issue #12100: Don't reset incremental encoders of CJK codecs at each call to
Victor Stinner <victor.stinner@haypocalc.com>
parents: 65338
diff changeset
285 encoding = 'hz'
bd17396895fb Issue #12100: Don't reset incremental encoders of CJK codecs at each call to
Victor Stinner <victor.stinner@haypocalc.com>
parents: 65338
diff changeset
286 expected = b'~{ADAD'
bd17396895fb Issue #12100: Don't reset incremental encoders of CJK codecs at each call to
Victor Stinner <victor.stinner@haypocalc.com>
parents: 65338
diff changeset
287 expected_reset = b'~{ADAD~}'
bd17396895fb Issue #12100: Don't reset incremental encoders of CJK codecs at each call to
Victor Stinner <victor.stinner@haypocalc.com>
parents: 65338
diff changeset
288
31386
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
289 def test_main():
47201
9c930a32cebb #2621 rename test.test_support to test.support
Benjamin Peterson <benjamin@python.org>
parents: 44160
diff changeset
290 support.run_unittest(__name__)
31386
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
291
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
292 if __name__ == "__main__":
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
293 test_main()