Mercurial > cpython-lru_cache-weakref

annotate Lib/test/test_normalization.py @ 32527:0da660e05e69 legacy-trunk

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
"#if WITH_THREAD" is incorrect; must be #ifdef instead; WITH_THREAD isn't always set to an integer value when it's defined.
author Tim Peters <tim.peters@gmail.com>
date Wed, 07 Jul 2004 20:42:07 +0000
parents 04ca4d1f751f
children 96d1fa36964d
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
26379
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
1 from test.test_support import verbose, TestFailed, TestSkipped, verify
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
2 import sys
26391
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
3 import os
26379
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
4 from unicodedata import normalize
26391
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
5
30056
e477b599924d Correct URL of normalization file. Fixes #781065. Will backport to 2.3.
Martin v. Löwis <martin@v.loewis.de>
parents: 29066
diff changeset
6 TESTDATAFILE = "NormalizationTest-3.2.0" + os.extsep + "txt"
28729
31cbd46f0c7d Do a little more searching for the data file for the test: this allows
Fred Drake <fdrake@acm.org>
parents: 26392
diff changeset
7
31cbd46f0c7d Do a little more searching for the data file for the test: this allows
Fred Drake <fdrake@acm.org>
parents: 26392
diff changeset
8 # This search allows using a build directory just inside the source
31cbd46f0c7d Do a little more searching for the data file for the test: this allows
Fred Drake <fdrake@acm.org>
parents: 26392
diff changeset
9 # directory, and saving just one copy of the test data in the source
31cbd46f0c7d Do a little more searching for the data file for the test: this allows
Fred Drake <fdrake@acm.org>
parents: 26392
diff changeset
10 # tree, rather than having a copy in each build directory.
31cbd46f0c7d Do a little more searching for the data file for the test: this allows
Fred Drake <fdrake@acm.org>
parents: 26392
diff changeset
11 # There might be a better way to do this.
31cbd46f0c7d Do a little more searching for the data file for the test: this allows
Fred Drake <fdrake@acm.org>
parents: 26392
diff changeset
12
31cbd46f0c7d Do a little more searching for the data file for the test: this allows
Fred Drake <fdrake@acm.org>
parents: 26392
diff changeset
13 for path in [os.path.curdir, os.path.pardir]:
31cbd46f0c7d Do a little more searching for the data file for the test: this allows
Fred Drake <fdrake@acm.org>
parents: 26392
diff changeset
14 fn = os.path.join(path, TESTDATAFILE)
31cbd46f0c7d Do a little more searching for the data file for the test: this allows
Fred Drake <fdrake@acm.org>
parents: 26392
diff changeset
15 skip_expected = not os.path.exists(fn)
31cbd46f0c7d Do a little more searching for the data file for the test: this allows
Fred Drake <fdrake@acm.org>
parents: 26392
diff changeset
16 if not skip_expected:
31cbd46f0c7d Do a little more searching for the data file for the test: this allows
Fred Drake <fdrake@acm.org>
parents: 26392
diff changeset
17 TESTDATAFILE = fn
31cbd46f0c7d Do a little more searching for the data file for the test: this allows
Fred Drake <fdrake@acm.org>
parents: 26392
diff changeset
18 break
26379
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
19
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
20 class RangeError:
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
21 pass
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
22
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
23 def NFC(str):
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
24 return normalize("NFC", str)
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
25
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
26 def NFKC(str):
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
27 return normalize("NFKC", str)
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
28
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
29 def NFD(str):
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
30 return normalize("NFD", str)
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
31
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
32 def NFKD(str):
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
33 return normalize("NFKD", str)
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
34
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
35 def unistr(data):
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
36 data = [int(x, 16) for x in data.split(" ")]
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
37 for x in data:
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
38 if x > sys.maxunicode:
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
39 raise RangeError
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
40 return u"".join([unichr(x) for x in data])
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
41
26391
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
42 def test_main():
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
43 if skip_expected:
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
44 raise TestSkipped(TESTDATAFILE + " not found, download from " +
30056
e477b599924d Correct URL of normalization file. Fixes #781065. Will backport to 2.3.
Martin v. Löwis <martin@v.loewis.de>
parents: 29066
diff changeset
45 "http://www.unicode.org/Public/3.2-Update/" + TESTDATAFILE)
26391
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
46
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
47 part1_data = {}
26392
482ae055bbd4 Reduced memory burden by iterating over the normalization test input
Tim Peters <tim.peters@gmail.com>
parents: 26391
diff changeset
48 for line in open(TESTDATAFILE):
26391
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
49 if '#' in line:
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
50 line = line.split('#')[0]
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
51 line = line.strip()
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
52 if not line:
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
53 continue
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
54 if line.startswith("@Part"):
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
55 part = line
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
56 continue
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
57 try:
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
58 c1,c2,c3,c4,c5 = [unistr(x) for x in line.split(';')[:-1]]
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
59 except RangeError:
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
60 # Skip unsupported characters
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
61 continue
26387
fcf9734e1021 Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 26379
diff changeset
62
26391
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
63 if verbose:
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
64 print line
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
65
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
66 # Perform tests
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
67 verify(c2 == NFC(c1) == NFC(c2) == NFC(c3), line)
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
68 verify(c4 == NFC(c4) == NFC(c5), line)
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
69 verify(c3 == NFD(c1) == NFD(c2) == NFD(c3), line)
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
70 verify(c5 == NFD(c4) == NFD(c5), line)
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
71 verify(c4 == NFKC(c1) == NFKC(c2) == NFKC(c3) == NFKC(c4) == NFKC(c5),
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
72 line)
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
73 verify(c5 == NFKD(c1) == NFKD(c2) == NFKD(c3) == NFKD(c4) == NFKD(c5),
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
74 line)
26379
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
75
26391
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
76 # Record part 1 data
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
77 if part == "@Part1":
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
78 part1_data[c1] = 1
26379
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
79
26391
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
80 # Perform tests for all other data
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
81 for c in range(sys.maxunicode+1):
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
82 X = unichr(c)
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
83 if X in part1_data:
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
84 continue
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
85 assert X == NFC(X) == NFD(X) == NFKC(X) == NFKD(X), c
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
86
30867
04ca4d1f751f Overallocate target buffer for normalization more early. Fixes #834676.
Martin v. Löwis <martin@v.loewis.de>
parents: 30056
diff changeset
87 # Check for bug 834676
04ca4d1f751f Overallocate target buffer for normalization more early. Fixes #834676.
Martin v. Löwis <martin@v.loewis.de>
parents: 30056
diff changeset
88 normalize('NFC',u'\ud55c\uae00')
04ca4d1f751f Overallocate target buffer for normalization more early. Fixes #834676.
Martin v. Löwis <martin@v.loewis.de>
parents: 30056
diff changeset
89
26391
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
90 if __name__ == "__main__":
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
91 test_main()