Mercurial > cpython-lru_cache-weakref

annotate Lib/test/test_normalization.py @ 39042:c02fa187e0f4 legacy-trunk

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Part of SF patch #1484695. This removes dead code. The chksum was already verified in .frombuf() on the lines above. If there was a problem an exception is raised, so there was no way this condition could have been true.
author Neal Norwitz <nnorwitz@gmail.com>
date Mon, 10 Jul 2006 00:23:17 +0000
parents 760ffb0fba37
children ea6fbf404c34 4cdf21a0dbbc
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
35998
96d1fa36964d Patch #1276356: Implement new resource "urlfetch" for regrtest.
Hye-Shik Chang <hyeshik@gmail.com>
parents: 30867
diff changeset
1 from test.test_support import (verbose, TestFailed, TestSkipped, verify,
96d1fa36964d Patch #1276356: Implement new resource "urlfetch" for regrtest.
Hye-Shik Chang <hyeshik@gmail.com>
parents: 30867
diff changeset
2 open_urlresource)
26379
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
3 import sys
26391
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
4 import os
26379
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
5 from unicodedata import normalize
26391
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
6
36929
fab44c5c8df4 Update test data to 4.1; disable PRI #29 for now.
Martin v. Löwis <martin@v.loewis.de>
parents: 35998
diff changeset
7 TESTDATAFILE = "NormalizationTest" + os.extsep + "txt"
fab44c5c8df4 Update test data to 4.1; disable PRI #29 for now.
Martin v. Löwis <martin@v.loewis.de>
parents: 35998
diff changeset
8 TESTDATAURL = "http://www.unicode.org/Public/4.1.0/ucd/" + TESTDATAFILE
26379
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
9
37188
760ffb0fba37 Exceptions should inherit from Exception now.
Neal Norwitz <nnorwitz@gmail.com>
parents: 36937
diff changeset
10 class RangeError(Exception):
26379
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
11 pass
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
12
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
13 def NFC(str):
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
14 return normalize("NFC", str)
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
15
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
16 def NFKC(str):
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
17 return normalize("NFKC", str)
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
18
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
19 def NFD(str):
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
20 return normalize("NFD", str)
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
21
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
22 def NFKD(str):
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
23 return normalize("NFKD", str)
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
24
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
25 def unistr(data):
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
26 data = [int(x, 16) for x in data.split(" ")]
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
27 for x in data:
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
28 if x > sys.maxunicode:
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
29 raise RangeError
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
30 return u"".join([unichr(x) for x in data])
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
31
26391
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
32 def test_main():
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
33 part1_data = {}
35998
96d1fa36964d Patch #1276356: Implement new resource "urlfetch" for regrtest.
Hye-Shik Chang <hyeshik@gmail.com>
parents: 30867
diff changeset
34 for line in open_urlresource(TESTDATAURL):
26391
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
35 if '#' in line:
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
36 line = line.split('#')[0]
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
37 line = line.strip()
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
38 if not line:
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
39 continue
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
40 if line.startswith("@Part"):
36929
fab44c5c8df4 Update test data to 4.1; disable PRI #29 for now.
Martin v. Löwis <martin@v.loewis.de>
parents: 35998
diff changeset
41 part = line.split()[0]
fab44c5c8df4 Update test data to 4.1; disable PRI #29 for now.
Martin v. Löwis <martin@v.loewis.de>
parents: 35998
diff changeset
42 continue
fab44c5c8df4 Update test data to 4.1; disable PRI #29 for now.
Martin v. Löwis <martin@v.loewis.de>
parents: 35998
diff changeset
43 if part == "@Part3":
fab44c5c8df4 Update test data to 4.1; disable PRI #29 for now.
Martin v. Löwis <martin@v.loewis.de>
parents: 35998
diff changeset
44 # XXX we don't support PRI #29 yet, so skip these tests for now
26391
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
45 continue
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
46 try:
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
47 c1,c2,c3,c4,c5 = [unistr(x) for x in line.split(';')[:-1]]
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
48 except RangeError:
36937
df43c04d2554 Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 36929
diff changeset
49 # Skip unsupported characters;
36929
fab44c5c8df4 Update test data to 4.1; disable PRI #29 for now.
Martin v. Löwis <martin@v.loewis.de>
parents: 35998
diff changeset
50 # try atleast adding c1 if we are in part1
fab44c5c8df4 Update test data to 4.1; disable PRI #29 for now.
Martin v. Löwis <martin@v.loewis.de>
parents: 35998
diff changeset
51 if part == "@Part1":
36937
df43c04d2554 Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 36929
diff changeset
52 try:
36929
fab44c5c8df4 Update test data to 4.1; disable PRI #29 for now.
Martin v. Löwis <martin@v.loewis.de>
parents: 35998
diff changeset
53 c1=unistr(line.split(';')[0])
fab44c5c8df4 Update test data to 4.1; disable PRI #29 for now.
Martin v. Löwis <martin@v.loewis.de>
parents: 35998
diff changeset
54 except RangeError:
fab44c5c8df4 Update test data to 4.1; disable PRI #29 for now.
Martin v. Löwis <martin@v.loewis.de>
parents: 35998
diff changeset
55 pass
fab44c5c8df4 Update test data to 4.1; disable PRI #29 for now.
Martin v. Löwis <martin@v.loewis.de>
parents: 35998
diff changeset
56 else:
fab44c5c8df4 Update test data to 4.1; disable PRI #29 for now.
Martin v. Löwis <martin@v.loewis.de>
parents: 35998
diff changeset
57 part1_data[c1] = 1
26391
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
58 continue
26387
fcf9734e1021 Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 26379
diff changeset
59
26391
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
60 if verbose:
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
61 print line
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
62
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
63 # Perform tests
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
64 verify(c2 == NFC(c1) == NFC(c2) == NFC(c3), line)
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
65 verify(c4 == NFC(c4) == NFC(c5), line)
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
66 verify(c3 == NFD(c1) == NFD(c2) == NFD(c3), line)
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
67 verify(c5 == NFD(c4) == NFD(c5), line)
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
68 verify(c4 == NFKC(c1) == NFKC(c2) == NFKC(c3) == NFKC(c4) == NFKC(c5),
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
69 line)
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
70 verify(c5 == NFKD(c1) == NFKD(c2) == NFKD(c3) == NFKD(c4) == NFKD(c5),
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
71 line)
26379
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
72
26391
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
73 # Record part 1 data
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
74 if part == "@Part1":
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
75 part1_data[c1] = 1
26379
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
76
26391
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
77 # Perform tests for all other data
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
78 for c in range(sys.maxunicode+1):
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
79 X = unichr(c)
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
80 if X in part1_data:
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
81 continue
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
82 assert X == NFC(X) == NFD(X) == NFKC(X) == NFKD(X), c
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
83
30867
04ca4d1f751f Overallocate target buffer for normalization more early. Fixes #834676.
Martin v. Löwis <martin@v.loewis.de>
parents: 30056
diff changeset
84 # Check for bug 834676
04ca4d1f751f Overallocate target buffer for normalization more early. Fixes #834676.
Martin v. Löwis <martin@v.loewis.de>
parents: 30056
diff changeset
85 normalize('NFC',u'\ud55c\uae00')
04ca4d1f751f Overallocate target buffer for normalization more early. Fixes #834676.
Martin v. Löwis <martin@v.loewis.de>
parents: 30056
diff changeset
86
26391
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
87 if __name__ == "__main__":
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
88 test_main()