Mercurial > cpython-lru_cache-weakref

annotate Lib/test/test_normalization.py @ 28646:193b8aec00ce legacy-trunk

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Changes from Jonathan Riehl to allow his pgen extension (PEP 269) to work. This includes some more code that used to be part of pgen in the main parser; I'm okay with that. I'll see if the Windows build needs work next.
author Guido van Rossum <guido@python.org>
date Thu, 17 Apr 2003 14:55:42 +0000
parents 482ae055bbd4
children 31cbd46f0c7d
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
26379
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
1 from test.test_support import verbose, TestFailed, TestSkipped, verify
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
2 import sys
26391
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
3 import os
26379
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
4 from unicodedata import normalize
26391
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
5
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
6 TESTDATAFILE = "NormalizationTest.txt"
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
7 skip_expected = not os.path.exists(TESTDATAFILE)
26379
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
8
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
9 class RangeError:
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
10 pass
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
11
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
12 def NFC(str):
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
13 return normalize("NFC", str)
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
14
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
15 def NFKC(str):
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
16 return normalize("NFKC", str)
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
17
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
18 def NFD(str):
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
19 return normalize("NFD", str)
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
20
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
21 def NFKD(str):
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
22 return normalize("NFKD", str)
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
23
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
24 def unistr(data):
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
25 data = [int(x, 16) for x in data.split(" ")]
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
26 for x in data:
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
27 if x > sys.maxunicode:
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
28 raise RangeError
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
29 return u"".join([unichr(x) for x in data])
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
30
26391
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
31 def test_main():
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
32 if skip_expected:
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
33 raise TestSkipped(TESTDATAFILE + " not found, download from " +
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
34 "http://www.unicode.org/Public/UNIDATA/" + TESTDATAFILE)
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
35
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
36 part1_data = {}
26392
482ae055bbd4 Reduced memory burden by iterating over the normalization test input
Tim Peters <tim.peters@gmail.com>
parents: 26391
diff changeset
37 for line in open(TESTDATAFILE):
26391
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
38 if '#' in line:
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
39 line = line.split('#')[0]
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
40 line = line.strip()
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
41 if not line:
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
42 continue
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
43 if line.startswith("@Part"):
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
44 part = line
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
45 continue
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
46 try:
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
47 c1,c2,c3,c4,c5 = [unistr(x) for x in line.split(';')[:-1]]
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
48 except RangeError:
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
49 # Skip unsupported characters
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
50 continue
26387
fcf9734e1021 Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 26379
diff changeset
51
26391
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
52 if verbose:
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
53 print line
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
54
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
55 # Perform tests
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
56 verify(c2 == NFC(c1) == NFC(c2) == NFC(c3), line)
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
57 verify(c4 == NFC(c4) == NFC(c5), line)
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
58 verify(c3 == NFD(c1) == NFD(c2) == NFD(c3), line)
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
59 verify(c5 == NFD(c4) == NFD(c5), line)
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
60 verify(c4 == NFKC(c1) == NFKC(c2) == NFKC(c3) == NFKC(c4) == NFKC(c5),
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
61 line)
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
62 verify(c5 == NFKD(c1) == NFKD(c2) == NFKD(c3) == NFKD(c4) == NFKD(c5),
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
63 line)
26379
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
64
26391
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
65 # Record part 1 data
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
66 if part == "@Part1":
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
67 part1_data[c1] = 1
26379
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
68
26391
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
69 # Perform tests for all other data
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
70 for c in range(sys.maxunicode+1):
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
71 X = unichr(c)
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
72 if X in part1_data:
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
73 continue
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
74 assert X == NFC(X) == NFD(X) == NFKC(X) == NFKD(X), c
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
75
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
76 if __name__ == "__main__":
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
77 test_main()