Mercurial > cpython-lru_cache-weakref

annotate Lib/test/test_normalization.py @ 87690:66c1c9f32567 default tip

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Add use_weakref option to functools.lru_cache The use_weakref option makes lru_cache create weak references to the function arguments when creating the cache key, which prevents them from staying in the memory forever even after all other references to them are gone. This is a proof of concept for http://bugs.python.org/issue19859
author Radomir Dopieralski <devel@sheep.art.pl>
date Tue, 03 Dec 2013 16:02:48 +0100
parents 4cc308acd26d
children
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
47201
9c930a32cebb #2621 rename test.test_support to test.support
Benjamin Peterson <benjamin@python.org>
parents: 43089
diff changeset
1 from test.support import run_unittest, open_urlresource
41554
06145fbc7ab9 Merged revisions 53952-54987 via svnmerge from
Guido van Rossum <guido@python.org>
parents: 40775
diff changeset
2 import unittest
06145fbc7ab9 Merged revisions 53952-54987 via svnmerge from
Guido van Rossum <guido@python.org>
parents: 40775
diff changeset
3
57467
74230e5ab373 Merged revisions 76535 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 57051
diff changeset
4 from http.client import HTTPException
26379
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
5 import sys
26391
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
6 import os
49133
f97779262bac Merged revisions 66369 via svnmerge from
Martin v. Löwis <martin@v.loewis.de>
parents: 49130
diff changeset
7 from unicodedata import normalize, unidata_version
26391
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
8
42622
4d2aa4bfb2d6 Remove RISCOS support
Skip Montanaro <skip@pobox.com>
parents: 41585
diff changeset
9 TESTDATAFILE = "NormalizationTest.txt"
49133
f97779262bac Merged revisions 66369 via svnmerge from
Martin v. Löwis <martin@v.loewis.de>
parents: 49130
diff changeset
10 TESTDATAURL = "http://www.unicode.org/Public/" + unidata_version + "/ucd/" + TESTDATAFILE
f97779262bac Merged revisions 66369 via svnmerge from
Martin v. Löwis <martin@v.loewis.de>
parents: 49130
diff changeset
11
59720
8a28be45c242 Merged revisions 78982,78986 via svnmerge from
Florent Xicluna <florent.xicluna@gmail.com>
parents: 57467
diff changeset
12 def check_version(testfile):
8a28be45c242 Merged revisions 78982,78986 via svnmerge from
Florent Xicluna <florent.xicluna@gmail.com>
parents: 57467
diff changeset
13 hdr = testfile.readline()
8a28be45c242 Merged revisions 78982,78986 via svnmerge from
Florent Xicluna <florent.xicluna@gmail.com>
parents: 57467
diff changeset
14 return unidata_version in hdr
26379
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
15
37188
760ffb0fba37 Exceptions should inherit from Exception now.
Neal Norwitz <nnorwitz@gmail.com>
parents: 36937
diff changeset
16 class RangeError(Exception):
26379
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
17 pass
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
18
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
19 def NFC(str):
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
20 return normalize("NFC", str)
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
21
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
22 def NFKC(str):
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
23 return normalize("NFKC", str)
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
24
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
25 def NFD(str):
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
26 return normalize("NFD", str)
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
27
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
28 def NFKD(str):
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
29 return normalize("NFKD", str)
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
30
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
31 def unistr(data):
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
32 data = [int(x, 16) for x in data.split(" ")]
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
33 for x in data:
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
34 if x > sys.maxunicode:
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
35 raise RangeError
41585
825d966338ea Rename 'unicode' to 'str' in its tp_name field. Rename 'str' to 'str8'.
Guido van Rossum <guido@python.org>
parents: 41575
diff changeset
36 return "".join([chr(x) for x in data])
26379
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
37
41554
06145fbc7ab9 Merged revisions 53952-54987 via svnmerge from
Guido van Rossum <guido@python.org>
parents: 40775
diff changeset
38 class NormalizationTest(unittest.TestCase):
06145fbc7ab9 Merged revisions 53952-54987 via svnmerge from
Guido van Rossum <guido@python.org>
parents: 40775
diff changeset
39 def test_main(self):
59720
8a28be45c242 Merged revisions 78982,78986 via svnmerge from
Florent Xicluna <florent.xicluna@gmail.com>
parents: 57467
diff changeset
40 part = None
41554
06145fbc7ab9 Merged revisions 53952-54987 via svnmerge from
Guido van Rossum <guido@python.org>
parents: 40775
diff changeset
41 part1_data = {}
57054
4580fe0719e7 Merged revisions 76036 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 55593
diff changeset
42 # Hit the exception early
4580fe0719e7 Merged revisions 76036 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 55593
diff changeset
43 try:
59720
8a28be45c242 Merged revisions 78982,78986 via svnmerge from
Florent Xicluna <florent.xicluna@gmail.com>
parents: 57467
diff changeset
44 testdata = open_urlresource(TESTDATAURL, encoding="utf-8",
8a28be45c242 Merged revisions 78982,78986 via svnmerge from
Florent Xicluna <florent.xicluna@gmail.com>
parents: 57467
diff changeset
45 check=check_version)
81039
7d69d04522e3 Replace IOError with OSError (#16715)
Andrew Svetlov <andrew.svetlov@gmail.com>
parents: 68257
diff changeset
46 except (OSError, HTTPException):
57054
4580fe0719e7 Merged revisions 76036 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 55593
diff changeset
47 self.skipTest("Could not retrieve " + TESTDATAURL)
67087
89a416fd7c29 Fix ResourceWarning in test_normalization
Antoine Pitrou <solipsis@pitrou.net>
parents: 59720
diff changeset
48 self.addCleanup(testdata.close)
59720
8a28be45c242 Merged revisions 78982,78986 via svnmerge from
Florent Xicluna <florent.xicluna@gmail.com>
parents: 57467
diff changeset
49 for line in testdata:
41554
06145fbc7ab9 Merged revisions 53952-54987 via svnmerge from
Guido van Rossum <guido@python.org>
parents: 40775
diff changeset
50 if '#' in line:
06145fbc7ab9 Merged revisions 53952-54987 via svnmerge from
Guido van Rossum <guido@python.org>
parents: 40775
diff changeset
51 line = line.split('#')[0]
06145fbc7ab9 Merged revisions 53952-54987 via svnmerge from
Guido van Rossum <guido@python.org>
parents: 40775
diff changeset
52 line = line.strip()
06145fbc7ab9 Merged revisions 53952-54987 via svnmerge from
Guido van Rossum <guido@python.org>
parents: 40775
diff changeset
53 if not line:
06145fbc7ab9 Merged revisions 53952-54987 via svnmerge from
Guido van Rossum <guido@python.org>
parents: 40775
diff changeset
54 continue
06145fbc7ab9 Merged revisions 53952-54987 via svnmerge from
Guido van Rossum <guido@python.org>
parents: 40775
diff changeset
55 if line.startswith("@Part"):
06145fbc7ab9 Merged revisions 53952-54987 via svnmerge from
Guido van Rossum <guido@python.org>
parents: 40775
diff changeset
56 part = line.split()[0]
06145fbc7ab9 Merged revisions 53952-54987 via svnmerge from
Guido van Rossum <guido@python.org>
parents: 40775
diff changeset
57 continue
06145fbc7ab9 Merged revisions 53952-54987 via svnmerge from
Guido van Rossum <guido@python.org>
parents: 40775
diff changeset
58 try:
06145fbc7ab9 Merged revisions 53952-54987 via svnmerge from
Guido van Rossum <guido@python.org>
parents: 40775
diff changeset
59 c1,c2,c3,c4,c5 = [unistr(x) for x in line.split(';')[:-1]]
06145fbc7ab9 Merged revisions 53952-54987 via svnmerge from
Guido van Rossum <guido@python.org>
parents: 40775
diff changeset
60 except RangeError:
06145fbc7ab9 Merged revisions 53952-54987 via svnmerge from
Guido van Rossum <guido@python.org>
parents: 40775
diff changeset
61 # Skip unsupported characters;
85231
61227b4c169f #18466: fix more typos. Patch by Févry Thibault.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 68257
diff changeset
62 # try at least adding c1 if we are in part1
41554
06145fbc7ab9 Merged revisions 53952-54987 via svnmerge from
Guido van Rossum <guido@python.org>
parents: 40775
diff changeset
63 if part == "@Part1":
06145fbc7ab9 Merged revisions 53952-54987 via svnmerge from
Guido van Rossum <guido@python.org>
parents: 40775
diff changeset
64 try:
06145fbc7ab9 Merged revisions 53952-54987 via svnmerge from
Guido van Rossum <guido@python.org>
parents: 40775
diff changeset
65 c1 = unistr(line.split(';')[0])
06145fbc7ab9 Merged revisions 53952-54987 via svnmerge from
Guido van Rossum <guido@python.org>
parents: 40775
diff changeset
66 except RangeError:
06145fbc7ab9 Merged revisions 53952-54987 via svnmerge from
Guido van Rossum <guido@python.org>
parents: 40775
diff changeset
67 pass
06145fbc7ab9 Merged revisions 53952-54987 via svnmerge from
Guido van Rossum <guido@python.org>
parents: 40775
diff changeset
68 else:
06145fbc7ab9 Merged revisions 53952-54987 via svnmerge from
Guido van Rossum <guido@python.org>
parents: 40775
diff changeset
69 part1_data[c1] = 1
06145fbc7ab9 Merged revisions 53952-54987 via svnmerge from
Guido van Rossum <guido@python.org>
parents: 40775
diff changeset
70 continue
06145fbc7ab9 Merged revisions 53952-54987 via svnmerge from
Guido van Rossum <guido@python.org>
parents: 40775
diff changeset
71
06145fbc7ab9 Merged revisions 53952-54987 via svnmerge from
Guido van Rossum <guido@python.org>
parents: 40775
diff changeset
72 # Perform tests
55593
0bb232078b8c Merged revisions 73715 via svnmerge from
Georg Brandl <georg@python.org>
parents: 50203
diff changeset
73 self.assertTrue(c2 == NFC(c1) == NFC(c2) == NFC(c3), line)
0bb232078b8c Merged revisions 73715 via svnmerge from
Georg Brandl <georg@python.org>
parents: 50203
diff changeset
74 self.assertTrue(c4 == NFC(c4) == NFC(c5), line)
0bb232078b8c Merged revisions 73715 via svnmerge from
Georg Brandl <georg@python.org>
parents: 50203
diff changeset
75 self.assertTrue(c3 == NFD(c1) == NFD(c2) == NFD(c3), line)
0bb232078b8c Merged revisions 73715 via svnmerge from
Georg Brandl <georg@python.org>
parents: 50203
diff changeset
76 self.assertTrue(c5 == NFD(c4) == NFD(c5), line)
0bb232078b8c Merged revisions 73715 via svnmerge from
Georg Brandl <georg@python.org>
parents: 50203
diff changeset
77 self.assertTrue(c4 == NFKC(c1) == NFKC(c2) == \
41554
06145fbc7ab9 Merged revisions 53952-54987 via svnmerge from
Guido van Rossum <guido@python.org>
parents: 40775
diff changeset
78 NFKC(c3) == NFKC(c4) == NFKC(c5),
06145fbc7ab9 Merged revisions 53952-54987 via svnmerge from
Guido van Rossum <guido@python.org>
parents: 40775
diff changeset
79 line)
55593
0bb232078b8c Merged revisions 73715 via svnmerge from
Georg Brandl <georg@python.org>
parents: 50203
diff changeset
80 self.assertTrue(c5 == NFKD(c1) == NFKD(c2) == \
41554
06145fbc7ab9 Merged revisions 53952-54987 via svnmerge from
Guido van Rossum <guido@python.org>
parents: 40775
diff changeset
81 NFKD(c3) == NFKD(c4) == NFKD(c5),
06145fbc7ab9 Merged revisions 53952-54987 via svnmerge from
Guido van Rossum <guido@python.org>
parents: 40775
diff changeset
82 line)
06145fbc7ab9 Merged revisions 53952-54987 via svnmerge from
Guido van Rossum <guido@python.org>
parents: 40775
diff changeset
83
06145fbc7ab9 Merged revisions 53952-54987 via svnmerge from
Guido van Rossum <guido@python.org>
parents: 40775
diff changeset
84 # Record part 1 data
36929
fab44c5c8df4 Update test data to 4.1; disable PRI #29 for now.
Martin v. Löwis <martin@v.loewis.de>
parents: 35998
diff changeset
85 if part == "@Part1":
41554
06145fbc7ab9 Merged revisions 53952-54987 via svnmerge from
Guido van Rossum <guido@python.org>
parents: 40775
diff changeset
86 part1_data[c1] = 1
26387
fcf9734e1021 Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 26379
diff changeset
87
41554
06145fbc7ab9 Merged revisions 53952-54987 via svnmerge from
Guido van Rossum <guido@python.org>
parents: 40775
diff changeset
88 # Perform tests for all other data
06145fbc7ab9 Merged revisions 53952-54987 via svnmerge from
Guido van Rossum <guido@python.org>
parents: 40775
diff changeset
89 for c in range(sys.maxunicode+1):
41585
825d966338ea Rename 'unicode' to 'str' in its tp_name field. Rename 'str' to 'str8'.
Guido van Rossum <guido@python.org>
parents: 41575
diff changeset
90 X = chr(c)
41554
06145fbc7ab9 Merged revisions 53952-54987 via svnmerge from
Guido van Rossum <guido@python.org>
parents: 40775
diff changeset
91 if X in part1_data:
06145fbc7ab9 Merged revisions 53952-54987 via svnmerge from
Guido van Rossum <guido@python.org>
parents: 40775
diff changeset
92 continue
55593
0bb232078b8c Merged revisions 73715 via svnmerge from
Georg Brandl <georg@python.org>
parents: 50203
diff changeset
93 self.assertTrue(X == NFC(X) == NFD(X) == NFKC(X) == NFKD(X), c)
26391
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
94
41554
06145fbc7ab9 Merged revisions 53952-54987 via svnmerge from
Guido van Rossum <guido@python.org>
parents: 40775
diff changeset
95 def test_bug_834676(self):
06145fbc7ab9 Merged revisions 53952-54987 via svnmerge from
Guido van Rossum <guido@python.org>
parents: 40775
diff changeset
96 # Check for bug 834676
41575
beacf40b5960 Rip out all the u"..." literals and calls to unicode().
Guido van Rossum <guido@python.org>
parents: 41554
diff changeset
97 normalize('NFC', '\ud55c\uae00')
41554
06145fbc7ab9 Merged revisions 53952-54987 via svnmerge from
Guido van Rossum <guido@python.org>
parents: 40775
diff changeset
98
26379
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
99
41554
06145fbc7ab9 Merged revisions 53952-54987 via svnmerge from
Guido van Rossum <guido@python.org>
parents: 40775
diff changeset
100 def test_main():
06145fbc7ab9 Merged revisions 53952-54987 via svnmerge from
Guido van Rossum <guido@python.org>
parents: 40775
diff changeset
101 run_unittest(NormalizationTest)
30867
04ca4d1f751f Overallocate target buffer for normalization more early. Fixes #834676.
Martin v. Löwis <martin@v.loewis.de>
parents: 30056
diff changeset
102
26391
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
103 if __name__ == "__main__":
d207d1d78c5d Fiddled things so that test_normalization is expected to be skipped if
Tim Peters <tim.peters@gmail.com>
parents: 26388
diff changeset
104 test_main()