Mercurial > cpython-lru_cache-weakref

annotate Lib/test/multibytecodec_support.py @ 79691:05c8928362ee

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Merge __next__ method link fixes with 3.3.
author Ezio Melotti <ezio.melotti@gmail.com>
date Fri, 12 Oct 2012 13:46:37 +0300
parents 46a620c87db8
children 72623edae4df
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
59522
fcfe324663a6 convert shebang lines: python -> python3
Benjamin Peterson <benjamin@python.org>
parents: 57467
diff changeset
1 #!/usr/bin/env python3
31386
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
2 #
76178
46a620c87db8 Rename test_ file that is really a support file to remove test_ prefix.
R David Murray <rdmurray@bitdance.com>
parents: 72516
diff changeset
3 # multibytecodec_support.py
31386
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
4 # Common Unittest Routines for CJK codecs
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
5 #
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
6
70155
16503022c4b8 Issue #12057: Convert CJK encoding testcase BLOB into multiple text files
Victor Stinner <victor.stinner@haypocalc.com>
parents: 65338
diff changeset
7 import codecs
16503022c4b8 Issue #12057: Convert CJK encoding testcase BLOB into multiple text files
Victor Stinner <victor.stinner@haypocalc.com>
parents: 65338
diff changeset
8 import os
16503022c4b8 Issue #12057: Convert CJK encoding testcase BLOB into multiple text files
Victor Stinner <victor.stinner@haypocalc.com>
parents: 65338
diff changeset
9 import re
16503022c4b8 Issue #12057: Convert CJK encoding testcase BLOB into multiple text files
Victor Stinner <victor.stinner@haypocalc.com>
parents: 65338
diff changeset
10 import sys
16503022c4b8 Issue #12057: Convert CJK encoding testcase BLOB into multiple text files
Victor Stinner <victor.stinner@haypocalc.com>
parents: 65338
diff changeset
11 import unittest
57467
74230e5ab373 Merged revisions 76535 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 57026
diff changeset
12 from http.client import HTTPException
47201
9c930a32cebb #2621 rename test.test_support to test.support
Benjamin Peterson <benjamin@python.org>
parents: 47147
diff changeset
13 from test import support
41758
0ead1064ead0 Make all the multibyte codec tests pass.
Guido van Rossum <guido@python.org>
parents: 41642
diff changeset
14 from io import BytesIO
31386
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
15
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
16 class TestBase:
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
17 encoding = '' # codec name
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
18 codec = None # codec tuple (with 4 elements)
41758
0ead1064ead0 Make all the multibyte codec tests pass.
Guido van Rossum <guido@python.org>
parents: 41642
diff changeset
19 tstring = None # must set. 2 strings to test StreamReader
31386
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
20
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
21 codectests = None # must set. codec test tuple
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
22 roundtriptest = 1 # set if roundtrip is possible with unicode
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
23 has_iso10646 = 0 # set if this encoding contains whole iso10646 map
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
24 xmlcharnametest = None # string to test xmlcharrefreplace
41575
beacf40b5960 Rip out all the u"..." literals and calls to unicode().
Guido van Rossum <guido@python.org>
parents: 40644
diff changeset
25 unmappedunicode = '\udeee' # a unicode codepoint that is not mapped.
31386
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
26
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
27 def setUp(self):
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
28 if self.codec is None:
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
29 self.codec = codecs.lookup(self.encoding)
37879
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
30 self.encode = self.codec.encode
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
31 self.decode = self.codec.decode
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
32 self.reader = self.codec.streamreader
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
33 self.writer = self.codec.streamwriter
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
34 self.incrementalencoder = self.codec.incrementalencoder
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
35 self.incrementaldecoder = self.codec.incrementaldecoder
31386
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
36
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
37 def test_chunkcoding(self):
43516
8d40c5348ca9 Don't depend on str8.splitlines() in test_chunkcoding().
Guido van Rossum <guido@python.org>
parents: 42466
diff changeset
38 tstring_lines = []
8d40c5348ca9 Don't depend on str8.splitlines() in test_chunkcoding().
Guido van Rossum <guido@python.org>
parents: 42466
diff changeset
39 for b in self.tstring:
8d40c5348ca9 Don't depend on str8.splitlines() in test_chunkcoding().
Guido van Rossum <guido@python.org>
parents: 42466
diff changeset
40 lines = b.split(b"\n")
8d40c5348ca9 Don't depend on str8.splitlines() in test_chunkcoding().
Guido van Rossum <guido@python.org>
parents: 42466
diff changeset
41 last = lines.pop()
8d40c5348ca9 Don't depend on str8.splitlines() in test_chunkcoding().
Guido van Rossum <guido@python.org>
parents: 42466
diff changeset
42 assert last == b""
8d40c5348ca9 Don't depend on str8.splitlines() in test_chunkcoding().
Guido van Rossum <guido@python.org>
parents: 42466
diff changeset
43 lines = [line + b"\n" for line in lines]
8d40c5348ca9 Don't depend on str8.splitlines() in test_chunkcoding().
Guido van Rossum <guido@python.org>
parents: 42466
diff changeset
44 tstring_lines.append(lines)
8d40c5348ca9 Don't depend on str8.splitlines() in test_chunkcoding().
Guido van Rossum <guido@python.org>
parents: 42466
diff changeset
45 for native, utf8 in zip(*tstring_lines):
31386
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
46 u = self.decode(native)[0]
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
47 self.assertEqual(u, utf8.decode('utf-8'))
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
48 if self.roundtriptest:
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
49 self.assertEqual(native, self.encode(u)[0])
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
50
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
51 def test_errorhandle(self):
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
52 for source, scheme, expected in self.codectests:
41758
0ead1064ead0 Make all the multibyte codec tests pass.
Guido van Rossum <guido@python.org>
parents: 41642
diff changeset
53 if isinstance(source, bytes):
31386
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
54 func = self.decode
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
55 else:
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
56 func = self.encode
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
57 if expected:
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
58 result = func(source, scheme)[0]
43777
0d462d789b18 Merging the py3k-pep3137 branch back into the py3k branch.
Guido van Rossum <guido@python.org>
parents: 43516
diff changeset
59 if func is self.decode:
55593
0bb232078b8c Merged revisions 73715 via svnmerge from
Georg Brandl <georg@python.org>
parents: 47201
diff changeset
60 self.assertTrue(type(result) is str, type(result))
70628
8572bf1b56ec Issue #12016: Add test_errorhandle() to TestBase_Mapping of
Victor Stinner <victor.stinner@haypocalc.com>
parents: 70541
diff changeset
61 self.assertEqual(result, expected,
8572bf1b56ec Issue #12016: Add test_errorhandle() to TestBase_Mapping of
Victor Stinner <victor.stinner@haypocalc.com>
parents: 70541
diff changeset
62 '%a.decode(%r, %r)=%a != %a'
8572bf1b56ec Issue #12016: Add test_errorhandle() to TestBase_Mapping of
Victor Stinner <victor.stinner@haypocalc.com>
parents: 70541
diff changeset
63 % (source, self.encoding, scheme, result,
8572bf1b56ec Issue #12016: Add test_errorhandle() to TestBase_Mapping of
Victor Stinner <victor.stinner@haypocalc.com>
parents: 70541
diff changeset
64 expected))
43777
0d462d789b18 Merging the py3k-pep3137 branch back into the py3k branch.
Guido van Rossum <guido@python.org>
parents: 43516
diff changeset
65 else:
55593
0bb232078b8c Merged revisions 73715 via svnmerge from
Georg Brandl <georg@python.org>
parents: 47201
diff changeset
66 self.assertTrue(type(result) is bytes, type(result))
70628
8572bf1b56ec Issue #12016: Add test_errorhandle() to TestBase_Mapping of
Victor Stinner <victor.stinner@haypocalc.com>
parents: 70541
diff changeset
67 self.assertEqual(result, expected,
8572bf1b56ec Issue #12016: Add test_errorhandle() to TestBase_Mapping of
Victor Stinner <victor.stinner@haypocalc.com>
parents: 70541
diff changeset
68 '%a.encode(%r, %r)=%a != %a'
8572bf1b56ec Issue #12016: Add test_errorhandle() to TestBase_Mapping of
Victor Stinner <victor.stinner@haypocalc.com>
parents: 70541
diff changeset
69 % (source, self.encoding, scheme, result,
8572bf1b56ec Issue #12016: Add test_errorhandle() to TestBase_Mapping of
Victor Stinner <victor.stinner@haypocalc.com>
parents: 70541
diff changeset
70 expected))
31386
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
71 else:
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
72 self.assertRaises(UnicodeError, func, source, scheme)
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
73
37879
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
74 def test_xmlcharrefreplace(self):
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
75 if self.has_iso10646:
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
76 return
31386
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
77
41575
beacf40b5960 Rip out all the u"..." literals and calls to unicode().
Guido van Rossum <guido@python.org>
parents: 40644
diff changeset
78 s = "\u0b13\u0b23\u0b60 nd eggs"
37879
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
79 self.assertEqual(
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
80 self.encode(s, "xmlcharrefreplace")[0],
41758
0ead1064ead0 Make all the multibyte codec tests pass.
Guido van Rossum <guido@python.org>
parents: 41642
diff changeset
81 b"&#2835;&#2851;&#2912; nd eggs"
37879
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
82 )
31386
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
83
37879
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
84 def test_customreplace_encode(self):
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
85 if self.has_iso10646:
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
86 return
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
87
47147
7296a2fb4786 rename HTMLParser to html.parser and htmlentitydefs to html.entities;
Fred Drake <fdrake@acm.org>
parents: 45714
diff changeset
88 from html.entities import codepoint2name
31386
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
89
37879
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
90 def xmlcharnamereplace(exc):
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
91 if not isinstance(exc, UnicodeEncodeError):
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
92 raise TypeError("don't know how to handle %r" % exc)
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
93 l = []
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
94 for c in exc.object[exc.start:exc.end]:
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
95 if ord(c) in codepoint2name:
41575
beacf40b5960 Rip out all the u"..." literals and calls to unicode().
Guido van Rossum <guido@python.org>
parents: 40644
diff changeset
96 l.append("&%s;" % codepoint2name[ord(c)])
31386
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
97 else:
41575
beacf40b5960 Rip out all the u"..." literals and calls to unicode().
Guido van Rossum <guido@python.org>
parents: 40644
diff changeset
98 l.append("&#%d;" % ord(c))
beacf40b5960 Rip out all the u"..." literals and calls to unicode().
Guido van Rossum <guido@python.org>
parents: 40644
diff changeset
99 return ("".join(l), exc.end)
37879
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
100
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
101 codecs.register_error("test.xmlcharnamereplace", xmlcharnamereplace)
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
102
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
103 if self.xmlcharnametest:
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
104 sin, sout = self.xmlcharnametest
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
105 else:
41575
beacf40b5960 Rip out all the u"..." literals and calls to unicode().
Guido van Rossum <guido@python.org>
parents: 40644
diff changeset
106 sin = "\xab\u211c\xbb = \u2329\u1234\u232a"
41758
0ead1064ead0 Make all the multibyte codec tests pass.
Guido van Rossum <guido@python.org>
parents: 41642
diff changeset
107 sout = b"&laquo;&real;&raquo; = &lang;&#4660;&rang;"
37879
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
108 self.assertEqual(self.encode(sin,
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
109 "test.xmlcharnamereplace")[0], sout)
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
110
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
111 def test_callback_wrong_objects(self):
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
112 def myreplace(exc):
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
113 return (ret, exc.end)
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
114 codecs.register_error("test.cjktest", myreplace)
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
115
41758
0ead1064ead0 Make all the multibyte codec tests pass.
Guido van Rossum <guido@python.org>
parents: 41642
diff changeset
116 for ret in ([1, 2, 3], [], None, object(), b'string', b''):
37879
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
117 self.assertRaises(TypeError, self.encode, self.unmappedunicode,
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
118 'test.cjktest')
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
119
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
120 def test_callback_long_index(self):
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
121 def myreplace(exc):
41575
beacf40b5960 Rip out all the u"..." literals and calls to unicode().
Guido van Rossum <guido@python.org>
parents: 40644
diff changeset
122 return ('x', int(exc.end))
37879
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
123 codecs.register_error("test.cjktest", myreplace)
41575
beacf40b5960 Rip out all the u"..." literals and calls to unicode().
Guido van Rossum <guido@python.org>
parents: 40644
diff changeset
124 self.assertEqual(self.encode('abcd' + self.unmappedunicode + 'efgh',
41758
0ead1064ead0 Make all the multibyte codec tests pass.
Guido van Rossum <guido@python.org>
parents: 41642
diff changeset
125 'test.cjktest'), (b'abcdxefgh', 9))
37879
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
126
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
127 def myreplace(exc):
44160
9d152530910a Removed PyInt_GetMax and sys.maxint
Christian Heimes <christian@cheimes.de>
parents: 43777
diff changeset
128 return ('x', sys.maxsize + 1)
37879
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
129 codecs.register_error("test.cjktest", myreplace)
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
130 self.assertRaises(IndexError, self.encode, self.unmappedunicode,
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
131 'test.cjktest')
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
132
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
133 def test_callback_None_index(self):
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
134 def myreplace(exc):
41575
beacf40b5960 Rip out all the u"..." literals and calls to unicode().
Guido van Rossum <guido@python.org>
parents: 40644
diff changeset
135 return ('x', None)
37879
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
136 codecs.register_error("test.cjktest", myreplace)
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
137 self.assertRaises(TypeError, self.encode, self.unmappedunicode,
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
138 'test.cjktest')
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
139
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
140 def test_callback_backward_index(self):
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
141 def myreplace(exc):
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
142 if myreplace.limit > 0:
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
143 myreplace.limit -= 1
41575
beacf40b5960 Rip out all the u"..." literals and calls to unicode().
Guido van Rossum <guido@python.org>
parents: 40644
diff changeset
144 return ('REPLACED', 0)
37879
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
145 else:
41575
beacf40b5960 Rip out all the u"..." literals and calls to unicode().
Guido van Rossum <guido@python.org>
parents: 40644
diff changeset
146 return ('TERMINAL', exc.end)
37879
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
147 myreplace.limit = 3
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
148 codecs.register_error("test.cjktest", myreplace)
41575
beacf40b5960 Rip out all the u"..." literals and calls to unicode().
Guido van Rossum <guido@python.org>
parents: 40644
diff changeset
149 self.assertEqual(self.encode('abcd' + self.unmappedunicode + 'efgh',
37879
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
150 'test.cjktest'),
41758
0ead1064ead0 Make all the multibyte codec tests pass.
Guido van Rossum <guido@python.org>
parents: 41642
diff changeset
151 (b'abcdREPLACEDabcdREPLACEDabcdREPLACEDabcdTERMINALefgh', 9))
37879
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
152
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
153 def test_callback_forward_index(self):
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
154 def myreplace(exc):
41575
beacf40b5960 Rip out all the u"..." literals and calls to unicode().
Guido van Rossum <guido@python.org>
parents: 40644
diff changeset
155 return ('REPLACED', exc.end + 2)
37879
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
156 codecs.register_error("test.cjktest", myreplace)
41575
beacf40b5960 Rip out all the u"..." literals and calls to unicode().
Guido van Rossum <guido@python.org>
parents: 40644
diff changeset
157 self.assertEqual(self.encode('abcd' + self.unmappedunicode + 'efgh',
41758
0ead1064ead0 Make all the multibyte codec tests pass.
Guido van Rossum <guido@python.org>
parents: 41642
diff changeset
158 'test.cjktest'), (b'abcdREPLACEDgh', 9))
31386
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
159
37879
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
160 def test_callback_index_outofbound(self):
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
161 def myreplace(exc):
41575
beacf40b5960 Rip out all the u"..." literals and calls to unicode().
Guido van Rossum <guido@python.org>
parents: 40644
diff changeset
162 return ('TERM', 100)
37879
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
163 codecs.register_error("test.cjktest", myreplace)
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
164 self.assertRaises(IndexError, self.encode, self.unmappedunicode,
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
165 'test.cjktest')
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
166
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
167 def test_incrementalencoder(self):
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
168 UTF8Reader = codecs.getreader('utf-8')
41642
a6eb6acfe04a Merged revisions 55007-55179 via svnmerge from
Guido van Rossum <guido@python.org>
parents: 41585
diff changeset
169 for sizehint in [None] + list(range(1, 33)) + \
37879
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
170 [64, 128, 256, 512, 1024]:
41758
0ead1064ead0 Make all the multibyte codec tests pass.
Guido van Rossum <guido@python.org>
parents: 41642
diff changeset
171 istream = UTF8Reader(BytesIO(self.tstring[1]))
0ead1064ead0 Make all the multibyte codec tests pass.
Guido van Rossum <guido@python.org>
parents: 41642
diff changeset
172 ostream = BytesIO()
37879
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
173 encoder = self.incrementalencoder()
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
174 while 1:
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
175 if sizehint is not None:
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
176 data = istream.read(sizehint)
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
177 else:
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
178 data = istream.read()
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
179
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
180 if not data:
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
181 break
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
182 e = encoder.encode(data)
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
183 ostream.write(e)
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
184
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
185 self.assertEqual(ostream.getvalue(), self.tstring[0])
31386
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
186
37879
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
187 def test_incrementaldecoder(self):
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
188 UTF8Writer = codecs.getwriter('utf-8')
41642
a6eb6acfe04a Merged revisions 55007-55179 via svnmerge from
Guido van Rossum <guido@python.org>
parents: 41585
diff changeset
189 for sizehint in [None, -1] + list(range(1, 33)) + \
37879
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
190 [64, 128, 256, 512, 1024]:
41758
0ead1064ead0 Make all the multibyte codec tests pass.
Guido van Rossum <guido@python.org>
parents: 41642
diff changeset
191 istream = BytesIO(self.tstring[0])
0ead1064ead0 Make all the multibyte codec tests pass.
Guido van Rossum <guido@python.org>
parents: 41642
diff changeset
192 ostream = UTF8Writer(BytesIO())
37879
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
193 decoder = self.incrementaldecoder()
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
194 while 1:
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
195 data = istream.read(sizehint)
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
196 if not data:
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
197 break
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
198 else:
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
199 u = decoder.decode(data)
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
200 ostream.write(u)
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
201
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
202 self.assertEqual(ostream.getvalue(), self.tstring[1])
31386
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
203
37879
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
204 def test_incrementalencoder_error_callback(self):
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
205 inv = self.unmappedunicode
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
206
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
207 e = self.incrementalencoder()
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
208 self.assertRaises(UnicodeEncodeError, e.encode, inv, True)
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
209
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
210 e.errors = 'ignore'
41758
0ead1064ead0 Make all the multibyte codec tests pass.
Guido van Rossum <guido@python.org>
parents: 41642
diff changeset
211 self.assertEqual(e.encode(inv, True), b'')
37879
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
212
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
213 e.reset()
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
214 def tempreplace(exc):
41575
beacf40b5960 Rip out all the u"..." literals and calls to unicode().
Guido van Rossum <guido@python.org>
parents: 40644
diff changeset
215 return ('called', exc.end)
37879
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
216 codecs.register_error('test.incremental_error_callback', tempreplace)
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
217 e.errors = 'test.incremental_error_callback'
41758
0ead1064ead0 Make all the multibyte codec tests pass.
Guido van Rossum <guido@python.org>
parents: 41642
diff changeset
218 self.assertEqual(e.encode(inv, True), b'called')
37879
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
219
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
220 # again
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
221 e.errors = 'ignore'
41758
0ead1064ead0 Make all the multibyte codec tests pass.
Guido van Rossum <guido@python.org>
parents: 41642
diff changeset
222 self.assertEqual(e.encode(inv, True), b'')
31386
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
223
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
224 def test_streamreader(self):
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
225 UTF8Writer = codecs.getwriter('utf-8')
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
226 for name in ["read", "readline", "readlines"]:
41642
a6eb6acfe04a Merged revisions 55007-55179 via svnmerge from
Guido van Rossum <guido@python.org>
parents: 41585
diff changeset
227 for sizehint in [None, -1] + list(range(1, 33)) + \
31386
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
228 [64, 128, 256, 512, 1024]:
41758
0ead1064ead0 Make all the multibyte codec tests pass.
Guido van Rossum <guido@python.org>
parents: 41642
diff changeset
229 istream = self.reader(BytesIO(self.tstring[0]))
0ead1064ead0 Make all the multibyte codec tests pass.
Guido van Rossum <guido@python.org>
parents: 41642
diff changeset
230 ostream = UTF8Writer(BytesIO())
31386
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
231 func = getattr(istream, name)
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
232 while 1:
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
233 data = func(sizehint)
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
234 if not data:
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
235 break
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
236 if name == "readlines":
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
237 ostream.writelines(data)
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
238 else:
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
239 ostream.write(data)
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
240
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
241 self.assertEqual(ostream.getvalue(), self.tstring[1])
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
242
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
243 def test_streamwriter(self):
37879
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 35998
diff changeset
244 readfuncs = ('read', 'readline', 'readlines')
31386
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
245 UTF8Reader = codecs.getreader('utf-8')
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
246 for name in readfuncs:
41642
a6eb6acfe04a Merged revisions 55007-55179 via svnmerge from
Guido van Rossum <guido@python.org>
parents: 41585
diff changeset
247 for sizehint in [None] + list(range(1, 33)) + \
31386
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
248 [64, 128, 256, 512, 1024]:
41758
0ead1064ead0 Make all the multibyte codec tests pass.
Guido van Rossum <guido@python.org>
parents: 41642
diff changeset
249 istream = UTF8Reader(BytesIO(self.tstring[1]))
0ead1064ead0 Make all the multibyte codec tests pass.
Guido van Rossum <guido@python.org>
parents: 41642
diff changeset
250 ostream = self.writer(BytesIO())
31386
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
251 func = getattr(istream, name)
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
252 while 1:
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
253 if sizehint is not None:
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
254 data = func(sizehint)
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
255 else:
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
256 data = func()
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
257
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
258 if not data:
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
259 break
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
260 if name == "readlines":
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
261 ostream.writelines(data)
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
262 else:
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
263 ostream.write(data)
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
264
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
265 self.assertEqual(ostream.getvalue(), self.tstring[0])
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
266
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
267
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
268 class TestBase_Mapping(unittest.TestCase):
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
269 pass_enctest = []
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
270 pass_dectest = []
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
271 supmaps = []
70628
8572bf1b56ec Issue #12016: Add test_errorhandle() to TestBase_Mapping of
Victor Stinner <victor.stinner@haypocalc.com>
parents: 70541
diff changeset
272 codectests = []
31386
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
273
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
274 def __init__(self, *args, **kw):
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
275 unittest.TestCase.__init__(self, *args, **kw)
45714
c830616bd2fc Skip tests if the file mapping file cannot be fetched.
Martin v. Löwis <martin@v.loewis.de>
parents: 45471
diff changeset
276 try:
65338
1782da440fb1 Merged revisions 85503 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 57026
diff changeset
277 self.open_mapping_file().close() # test it to report the error early
57467
74230e5ab373 Merged revisions 76535 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 57026
diff changeset
278 except (IOError, HTTPException):
57026
b0e4efb83d60 Merged revisions 76010 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 55593
diff changeset
279 self.skipTest("Could not retrieve "+self.mapfileurl)
35998
96d1fa36964d Patch #1276356: Implement new resource "urlfetch" for regrtest.
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32727
diff changeset
280
96d1fa36964d Patch #1276356: Implement new resource "urlfetch" for regrtest.
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32727
diff changeset
281 def open_mapping_file(self):
47201
9c930a32cebb #2621 rename test.test_support to test.support
Benjamin Peterson <benjamin@python.org>
parents: 47147
diff changeset
282 return support.open_urlresource(self.mapfileurl)
31386
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
283
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
284 def test_mapping_file(self):
42466
3fc3cedc49ca Merged revisions 56753-56781 via svnmerge from
Guido van Rossum <guido@python.org>
parents: 42352
diff changeset
285 if self.mapfileurl.endswith('.xml'):
3fc3cedc49ca Merged revisions 56753-56781 via svnmerge from
Guido van Rossum <guido@python.org>
parents: 42352
diff changeset
286 self._test_mapping_file_ucm()
3fc3cedc49ca Merged revisions 56753-56781 via svnmerge from
Guido van Rossum <guido@python.org>
parents: 42352
diff changeset
287 else:
3fc3cedc49ca Merged revisions 56753-56781 via svnmerge from
Guido van Rossum <guido@python.org>
parents: 42352
diff changeset
288 self._test_mapping_file_plain()
3fc3cedc49ca Merged revisions 56753-56781 via svnmerge from
Guido van Rossum <guido@python.org>
parents: 42352
diff changeset
289
3fc3cedc49ca Merged revisions 56753-56781 via svnmerge from
Guido van Rossum <guido@python.org>
parents: 42352
diff changeset
290 def _test_mapping_file_plain(self):
41585
825d966338ea Rename 'unicode' to 'str' in its tp_name field. Rename 'str' to 'str8'.
Guido van Rossum <guido@python.org>
parents: 41575
diff changeset
291 unichrs = lambda s: ''.join(map(chr, map(eval, s.split('+'))))
31386
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
292 urt_wa = {}
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
293
65338
1782da440fb1 Merged revisions 85503 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 57026
diff changeset
294 with self.open_mapping_file() as f:
1782da440fb1 Merged revisions 85503 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 57026
diff changeset
295 for line in f:
1782da440fb1 Merged revisions 85503 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 57026
diff changeset
296 if not line:
1782da440fb1 Merged revisions 85503 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 57026
diff changeset
297 break
1782da440fb1 Merged revisions 85503 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 57026
diff changeset
298 data = line.split('#')[0].strip().split()
1782da440fb1 Merged revisions 85503 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 57026
diff changeset
299 if len(data) != 2:
1782da440fb1 Merged revisions 85503 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 57026
diff changeset
300 continue
31386
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
301
65338
1782da440fb1 Merged revisions 85503 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 57026
diff changeset
302 csetval = eval(data[0])
1782da440fb1 Merged revisions 85503 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 57026
diff changeset
303 if csetval <= 0x7F:
1782da440fb1 Merged revisions 85503 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 57026
diff changeset
304 csetch = bytes([csetval & 0xff])
1782da440fb1 Merged revisions 85503 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 57026
diff changeset
305 elif csetval >= 0x1000000:
1782da440fb1 Merged revisions 85503 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 57026
diff changeset
306 csetch = bytes([(csetval >> 24), ((csetval >> 16) & 0xff),
1782da440fb1 Merged revisions 85503 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 57026
diff changeset
307 ((csetval >> 8) & 0xff), (csetval & 0xff)])
1782da440fb1 Merged revisions 85503 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 57026
diff changeset
308 elif csetval >= 0x10000:
1782da440fb1 Merged revisions 85503 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 57026
diff changeset
309 csetch = bytes([(csetval >> 16), ((csetval >> 8) & 0xff),
1782da440fb1 Merged revisions 85503 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 57026
diff changeset
310 (csetval & 0xff)])
1782da440fb1 Merged revisions 85503 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 57026
diff changeset
311 elif csetval >= 0x100:
1782da440fb1 Merged revisions 85503 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 57026
diff changeset
312 csetch = bytes([(csetval >> 8), (csetval & 0xff)])
1782da440fb1 Merged revisions 85503 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 57026
diff changeset
313 else:
1782da440fb1 Merged revisions 85503 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 57026
diff changeset
314 continue
31386
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
315
65338
1782da440fb1 Merged revisions 85503 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 57026
diff changeset
316 unich = unichrs(data[1])
1782da440fb1 Merged revisions 85503 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 57026
diff changeset
317 if ord(unich) == 0xfffd or unich in urt_wa:
1782da440fb1 Merged revisions 85503 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 57026
diff changeset
318 continue
1782da440fb1 Merged revisions 85503 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 57026
diff changeset
319 urt_wa[unich] = csetch
31386
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
320
65338
1782da440fb1 Merged revisions 85503 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 57026
diff changeset
321 self._testpoint(csetch, unich)
31386
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
322
42466
3fc3cedc49ca Merged revisions 56753-56781 via svnmerge from
Guido van Rossum <guido@python.org>
parents: 42352
diff changeset
323 def _test_mapping_file_ucm(self):
65338
1782da440fb1 Merged revisions 85503 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 57026
diff changeset
324 with self.open_mapping_file() as f:
1782da440fb1 Merged revisions 85503 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 57026
diff changeset
325 ucmdata = f.read()
42466
3fc3cedc49ca Merged revisions 56753-56781 via svnmerge from
Guido van Rossum <guido@python.org>
parents: 42352
diff changeset
326 uc = re.findall('<a u="([A-F0-9]{4})" b="([0-9A-F ]+)"/>', ucmdata)
3fc3cedc49ca Merged revisions 56753-56781 via svnmerge from
Guido van Rossum <guido@python.org>
parents: 42352
diff changeset
327 for uni, coded in uc:
3fc3cedc49ca Merged revisions 56753-56781 via svnmerge from
Guido van Rossum <guido@python.org>
parents: 42352
diff changeset
328 unich = chr(int(uni, 16))
3fc3cedc49ca Merged revisions 56753-56781 via svnmerge from
Guido van Rossum <guido@python.org>
parents: 42352
diff changeset
329 codech = bytes(int(c, 16) for c in coded.split())
3fc3cedc49ca Merged revisions 56753-56781 via svnmerge from
Guido van Rossum <guido@python.org>
parents: 42352
diff changeset
330 self._testpoint(codech, unich)
3fc3cedc49ca Merged revisions 56753-56781 via svnmerge from
Guido van Rossum <guido@python.org>
parents: 42352
diff changeset
331
31386
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
332 def test_mapping_supplemental(self):
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
333 for mapping in self.supmaps:
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
334 self._testpoint(*mapping)
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
335
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
336 def _testpoint(self, csetch, unich):
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
337 if (csetch, unich) not in self.pass_enctest:
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
338 self.assertEqual(unich.encode(self.encoding), csetch)
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
339 if (csetch, unich) not in self.pass_dectest:
41575
beacf40b5960 Rip out all the u"..." literals and calls to unicode().
Guido van Rossum <guido@python.org>
parents: 40644
diff changeset
340 self.assertEqual(str(csetch, self.encoding), unich)
31386
887ce39f95f2 Add CJK codecs support as discussed on python-dev. (SF #873597)
Hye-Shik Chang <hyeshik@gmail.com>
parents:
diff changeset
341
70628
8572bf1b56ec Issue #12016: Add test_errorhandle() to TestBase_Mapping of
Victor Stinner <victor.stinner@haypocalc.com>
parents: 70541
diff changeset
342 def test_errorhandle(self):
8572bf1b56ec Issue #12016: Add test_errorhandle() to TestBase_Mapping of
Victor Stinner <victor.stinner@haypocalc.com>
parents: 70541
diff changeset
343 for source, scheme, expected in self.codectests:
8572bf1b56ec Issue #12016: Add test_errorhandle() to TestBase_Mapping of
Victor Stinner <victor.stinner@haypocalc.com>
parents: 70541
diff changeset
344 if isinstance(source, bytes):
8572bf1b56ec Issue #12016: Add test_errorhandle() to TestBase_Mapping of
Victor Stinner <victor.stinner@haypocalc.com>
parents: 70541
diff changeset
345 func = source.decode
8572bf1b56ec Issue #12016: Add test_errorhandle() to TestBase_Mapping of
Victor Stinner <victor.stinner@haypocalc.com>
parents: 70541
diff changeset
346 else:
8572bf1b56ec Issue #12016: Add test_errorhandle() to TestBase_Mapping of
Victor Stinner <victor.stinner@haypocalc.com>
parents: 70541
diff changeset
347 func = source.encode
8572bf1b56ec Issue #12016: Add test_errorhandle() to TestBase_Mapping of
Victor Stinner <victor.stinner@haypocalc.com>
parents: 70541
diff changeset
348 if expected:
8572bf1b56ec Issue #12016: Add test_errorhandle() to TestBase_Mapping of
Victor Stinner <victor.stinner@haypocalc.com>
parents: 70541
diff changeset
349 if isinstance(source, bytes):
8572bf1b56ec Issue #12016: Add test_errorhandle() to TestBase_Mapping of
Victor Stinner <victor.stinner@haypocalc.com>
parents: 70541
diff changeset
350 result = func(self.encoding, scheme)
8572bf1b56ec Issue #12016: Add test_errorhandle() to TestBase_Mapping of
Victor Stinner <victor.stinner@haypocalc.com>
parents: 70541
diff changeset
351 self.assertTrue(type(result) is str, type(result))
8572bf1b56ec Issue #12016: Add test_errorhandle() to TestBase_Mapping of
Victor Stinner <victor.stinner@haypocalc.com>
parents: 70541
diff changeset
352 self.assertEqual(result, expected,
8572bf1b56ec Issue #12016: Add test_errorhandle() to TestBase_Mapping of
Victor Stinner <victor.stinner@haypocalc.com>
parents: 70541
diff changeset
353 '%a.decode(%r, %r)=%a != %a'
8572bf1b56ec Issue #12016: Add test_errorhandle() to TestBase_Mapping of
Victor Stinner <victor.stinner@haypocalc.com>
parents: 70541
diff changeset
354 % (source, self.encoding, scheme, result,
8572bf1b56ec Issue #12016: Add test_errorhandle() to TestBase_Mapping of
Victor Stinner <victor.stinner@haypocalc.com>
parents: 70541
diff changeset
355 expected))
8572bf1b56ec Issue #12016: Add test_errorhandle() to TestBase_Mapping of
Victor Stinner <victor.stinner@haypocalc.com>
parents: 70541
diff changeset
356 else:
8572bf1b56ec Issue #12016: Add test_errorhandle() to TestBase_Mapping of
Victor Stinner <victor.stinner@haypocalc.com>
parents: 70541
diff changeset
357 result = func(self.encoding, scheme)
8572bf1b56ec Issue #12016: Add test_errorhandle() to TestBase_Mapping of
Victor Stinner <victor.stinner@haypocalc.com>
parents: 70541
diff changeset
358 self.assertTrue(type(result) is bytes, type(result))
8572bf1b56ec Issue #12016: Add test_errorhandle() to TestBase_Mapping of
Victor Stinner <victor.stinner@haypocalc.com>
parents: 70541
diff changeset
359 self.assertEqual(result, expected,
8572bf1b56ec Issue #12016: Add test_errorhandle() to TestBase_Mapping of
Victor Stinner <victor.stinner@haypocalc.com>
parents: 70541
diff changeset
360 '%a.encode(%r, %r)=%a != %a'
8572bf1b56ec Issue #12016: Add test_errorhandle() to TestBase_Mapping of
Victor Stinner <victor.stinner@haypocalc.com>
parents: 70541
diff changeset
361 % (source, self.encoding, scheme, result,
8572bf1b56ec Issue #12016: Add test_errorhandle() to TestBase_Mapping of
Victor Stinner <victor.stinner@haypocalc.com>
parents: 70541
diff changeset
362 expected))
8572bf1b56ec Issue #12016: Add test_errorhandle() to TestBase_Mapping of
Victor Stinner <victor.stinner@haypocalc.com>
parents: 70541
diff changeset
363 else:
8572bf1b56ec Issue #12016: Add test_errorhandle() to TestBase_Mapping of
Victor Stinner <victor.stinner@haypocalc.com>
parents: 70541
diff changeset
364 self.assertRaises(UnicodeError, func, self.encoding, scheme)
8572bf1b56ec Issue #12016: Add test_errorhandle() to TestBase_Mapping of
Victor Stinner <victor.stinner@haypocalc.com>
parents: 70541
diff changeset
365
70155
16503022c4b8 Issue #12057: Convert CJK encoding testcase BLOB into multiple text files
Victor Stinner <victor.stinner@haypocalc.com>
parents: 65338
diff changeset
366 def load_teststring(name):
16503022c4b8 Issue #12057: Convert CJK encoding testcase BLOB into multiple text files
Victor Stinner <victor.stinner@haypocalc.com>
parents: 65338
diff changeset
367 dir = os.path.join(os.path.dirname(__file__), 'cjkencodings')
16503022c4b8 Issue #12057: Convert CJK encoding testcase BLOB into multiple text files
Victor Stinner <victor.stinner@haypocalc.com>
parents: 65338
diff changeset
368 with open(os.path.join(dir, name + '.txt'), 'rb') as f:
16503022c4b8 Issue #12057: Convert CJK encoding testcase BLOB into multiple text files
Victor Stinner <victor.stinner@haypocalc.com>
parents: 65338
diff changeset
369 encoded = f.read()
16503022c4b8 Issue #12057: Convert CJK encoding testcase BLOB into multiple text files
Victor Stinner <victor.stinner@haypocalc.com>
parents: 65338
diff changeset
370 with open(os.path.join(dir, name + '-utf8.txt'), 'rb') as f:
16503022c4b8 Issue #12057: Convert CJK encoding testcase BLOB into multiple text files
Victor Stinner <victor.stinner@haypocalc.com>
parents: 65338
diff changeset
371 utf8 = f.read()
16503022c4b8 Issue #12057: Convert CJK encoding testcase BLOB into multiple text files
Victor Stinner <victor.stinner@haypocalc.com>
parents: 65338
diff changeset
372 return encoded, utf8