Mercurial > cpython-lru_cache-weakref

annotate Tools/unicode/mkstringprep.py @ 58765:2ba7677bfa95 legacy-trunk

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
testCopysign was defined twice in test_math; combine the definitions
author Mark Dickinson <dickinsm@gmail.com>
date Sat, 06 Feb 2010 23:11:25 +0000
parents c3a096bfbd6b
children c51929233e19 8aa5530e8407 a6eb6acfe04a
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
32125
b9edd3418a31 Applying SF patch #949329 on behalf of Raymond Hettinger.
Armin Rigo <arigo@tunes.org>
parents: 28671
diff changeset
1 import re, unicodedata, sys
28671
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
2
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
3 if sys.maxunicode == 65535:
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
4 raise RuntimeError, "need UCS-4 Python"
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
5
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
6 def gen_category(cats):
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
7 for i in range(0, 0x110000):
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
8 if unicodedata.category(unichr(i)) in cats:
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
9 yield(i)
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
10
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
11 def gen_bidirectional(cats):
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
12 for i in range(0, 0x110000):
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
13 if unicodedata.bidirectional(unichr(i)) in cats:
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
14 yield(i)
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
15
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
16 def compact_set(l):
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
17 single = []
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
18 tuple = []
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
19 prev = None
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
20 span = 0
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
21 for e in l:
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
22 if prev is None:
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
23 prev = e
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
24 span = 0
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
25 continue
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
26 if prev+span+1 != e:
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
27 if span > 2:
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
28 tuple.append((prev,prev+span+1))
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
29 else:
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
30 for i in range(prev, prev+span+1):
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
31 single.append(i)
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
32 prev = e
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
33 span = 0
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
34 else:
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
35 span += 1
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
36 if span:
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
37 tuple.append((prev,prev+span+1))
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
38 else:
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
39 single.append(prev)
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
40 tuple = " + ".join(["range(%d,%d)" % t for t in tuple])
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
41 if not single:
32125
b9edd3418a31 Applying SF patch #949329 on behalf of Raymond Hettinger.
Armin Rigo <arigo@tunes.org>
parents: 28671
diff changeset
42 return "set(%s)" % tuple
28671
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
43 if not tuple:
32125
b9edd3418a31 Applying SF patch #949329 on behalf of Raymond Hettinger.
Armin Rigo <arigo@tunes.org>
parents: 28671
diff changeset
44 return "set(%s)" % repr(single)
b9edd3418a31 Applying SF patch #949329 on behalf of Raymond Hettinger.
Armin Rigo <arigo@tunes.org>
parents: 28671
diff changeset
45 return "set(%s + %s)" % (repr(single),tuple)
28671
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
46
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
47 ############## Read the tables in the RFC #######################
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
48
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
49 data = open("rfc3454.txt").readlines()
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
50
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
51 tables = []
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
52 curname = None
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
53 for l in data:
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
54 l = l.strip()
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
55 if not l:
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
56 continue
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
57 # Skip RFC page breaks
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
58 if l.startswith("Hoffman & Blanchet") or\
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
59 l.startswith("RFC 3454"):
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
60 continue
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
61 # Find start/end lines
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
62 m = re.match("----- (Start|End) Table ([A-Z](.[0-9])+) -----", l)
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
63 if m:
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
64 if m.group(1) == "Start":
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
65 if curname:
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
66 raise "Double Start",(curname, l)
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
67 curname = m.group(2)
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
68 table = {}
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
69 tables.append((curname, table))
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
70 continue
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
71 else:
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
72 if not curname:
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
73 raise "End without start", l
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
74 curname = None
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
75 continue
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
76 if not curname:
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
77 continue
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
78 # Now we are in a table
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
79 fields = l.split(";")
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
80 if len(fields) > 1:
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
81 # Drop comment field
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
82 fields = fields[:-1]
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
83 if len(fields) == 1:
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
84 fields = fields[0].split("-")
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
85 if len(fields) > 1:
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
86 # range
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
87 try:
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
88 start, end = fields
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
89 except ValueError:
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
90 raise "Unpacking problem", l
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
91 else:
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
92 start = end = fields[0]
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
93 start = int(start, 16)
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
94 end = int(end, 16)
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
95 for i in range(start, end+1):
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
96 table[i] = i
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
97 else:
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
98 code, value = fields
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
99 value = value.strip()
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
100 if value:
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
101 value = [int(v, 16) for v in value.split(" ")]
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
102 else:
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
103 # table B.1
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
104 value = None
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
105 table[int(code, 16)] = value
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
106
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
107 ########### Generate compact Python versions of the tables #############
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
108
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
109 print """# This file is generated by mkstringprep.py. DO NOT EDIT.
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
110 \"\"\"Library that exposes various tables found in the StringPrep RFC 3454.
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
111
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
112 There are two kinds of tables: sets, for which a member test is provided,
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
113 and mappings, for which a mapping function is provided.
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
114 \"\"\"
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
115
32125
b9edd3418a31 Applying SF patch #949329 on behalf of Raymond Hettinger.
Armin Rigo <arigo@tunes.org>
parents: 28671
diff changeset
116 import unicodedata
28671
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
117 """
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
118
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
119 print "assert unicodedata.unidata_version == %s" % repr(unicodedata.unidata_version)
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
120
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
121 # A.1 is the table of unassigned characters
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
122 # XXX Plane 15 PUA is listed as unassigned in Python.
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
123 name, table = tables[0]
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
124 del tables[0]
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
125 assert name == "A.1"
32125
b9edd3418a31 Applying SF patch #949329 on behalf of Raymond Hettinger.
Armin Rigo <arigo@tunes.org>
parents: 28671
diff changeset
126 table = set(table.keys())
b9edd3418a31 Applying SF patch #949329 on behalf of Raymond Hettinger.
Armin Rigo <arigo@tunes.org>
parents: 28671
diff changeset
127 Cn = set(gen_category(["Cn"]))
28671
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
128
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
129 # FDD0..FDEF are process internal codes
32125
b9edd3418a31 Applying SF patch #949329 on behalf of Raymond Hettinger.
Armin Rigo <arigo@tunes.org>
parents: 28671
diff changeset
130 Cn -= set(range(0xFDD0, 0xFDF0))
28671
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
131 # not a character
32125
b9edd3418a31 Applying SF patch #949329 on behalf of Raymond Hettinger.
Armin Rigo <arigo@tunes.org>
parents: 28671
diff changeset
132 Cn -= set(range(0xFFFE, 0x110000, 0x10000))
b9edd3418a31 Applying SF patch #949329 on behalf of Raymond Hettinger.
Armin Rigo <arigo@tunes.org>
parents: 28671
diff changeset
133 Cn -= set(range(0xFFFF, 0x110000, 0x10000))
28671
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
134
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
135 # assert table == Cn
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
136
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
137 print """
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
138 def in_table_a1(code):
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
139 if unicodedata.category(code) != 'Cn': return False
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
140 c = ord(code)
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
141 if 0xFDD0 <= c < 0xFDF0: return False
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
142 return (c & 0xFFFF) not in (0xFFFE, 0xFFFF)
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
143 """
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
144
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
145 # B.1 cannot easily be derived
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
146 name, table = tables[0]
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
147 del tables[0]
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
148 assert name == "B.1"
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
149 table = table.keys()
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
150 table.sort()
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
151 print """
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
152 b1_set = """ + compact_set(table) + """
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
153 def in_table_b1(code):
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
154 return ord(code) in b1_set
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
155 """
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
156
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
157 # B.2 and B.3 is case folding.
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
158 # It takes CaseFolding.txt into account, which is
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
159 # not available in the Python database. Since
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
160 # B.2 is derived from B.3, we process B.3 first.
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
161 # B.3 supposedly *is* CaseFolding-3.2.0.txt.
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
162
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
163 name, table_b2 = tables[0]
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
164 del tables[0]
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
165 assert name == "B.2"
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
166
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
167 name, table_b3 = tables[0]
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
168 del tables[0]
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
169 assert name == "B.3"
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
170
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
171 # B.3 is mostly Python's .lower, except for a number
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
172 # of special cases, e.g. considering canonical forms.
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
173
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
174 b3_exceptions = {}
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
175
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
176 for k,v in table_b2.items():
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
177 if map(ord, unichr(k).lower()) != v:
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
178 b3_exceptions[k] = u"".join(map(unichr,v))
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
179
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
180 b3 = b3_exceptions.items()
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
181 b3.sort()
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
182
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
183 print """
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
184 b3_exceptions = {"""
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
185 for i,(k,v) in enumerate(b3):
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
186 print "0x%x:%s," % (k, repr(v)),
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
187 if i % 4 == 3:
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
188 print
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
189 print "}"
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
190
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
191 print """
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
192 def map_table_b3(code):
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
193 r = b3_exceptions.get(ord(code))
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
194 if r is not None: return r
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
195 return code.lower()
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
196 """
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
197
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
198 def map_table_b3(code):
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
199 r = b3_exceptions.get(ord(code))
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
200 if r is not None: return r
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
201 return code.lower()
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
202
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
203 # B.2 is case folding for NFKC. This is the same as B.3,
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
204 # except where NormalizeWithKC(Fold(a)) !=
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
205 # NormalizeWithKC(Fold(NormalizeWithKC(Fold(a))))
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
206
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
207 def map_table_b2(a):
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
208 al = map_table_b3(a)
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
209 b = unicodedata.normalize("NFKC", al)
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
210 bl = u"".join([map_table_b3(ch) for ch in b])
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
211 c = unicodedata.normalize("NFKC", bl)
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
212 if b != c:
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
213 return c
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
214 else:
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
215 return al
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
216
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
217 specials = {}
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
218 for k,v in table_b2.items():
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
219 if map(ord, map_table_b2(unichr(k))) != v:
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
220 specials[k] = v
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
221
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
222 # B.3 should not add any additional special cases
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
223 assert specials == {}
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
224
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
225 print """
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
226 def map_table_b2(a):
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
227 al = map_table_b3(a)
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
228 b = unicodedata.normalize("NFKC", al)
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
229 bl = u"".join([map_table_b3(ch) for ch in b])
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
230 c = unicodedata.normalize("NFKC", bl)
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
231 if b != c:
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
232 return c
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
233 else:
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
234 return al
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
235 """
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
236
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
237 # C.1.1 is a table with a single character
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
238 name, table = tables[0]
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
239 del tables[0]
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
240 assert name == "C.1.1"
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
241 assert table == {0x20:0x20}
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
242
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
243 print """
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
244 def in_table_c11(code):
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
245 return code == u" "
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
246 """
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
247
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
248 # C.1.2 is the rest of all space characters
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
249 name, table = tables[0]
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
250 del tables[0]
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
251 assert name == "C.1.2"
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
252
32125
b9edd3418a31 Applying SF patch #949329 on behalf of Raymond Hettinger.
Armin Rigo <arigo@tunes.org>
parents: 28671
diff changeset
253 # table = set(table.keys())
b9edd3418a31 Applying SF patch #949329 on behalf of Raymond Hettinger.
Armin Rigo <arigo@tunes.org>
parents: 28671
diff changeset
254 # Zs = set(gen_category(["Zs"])) - set([0x20])
28671
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
255 # assert Zs == table
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
256
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
257 print """
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
258 def in_table_c12(code):
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
259 return unicodedata.category(code) == "Zs" and code != u" "
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
260
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
261 def in_table_c11_c12(code):
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
262 return unicodedata.category(code) == "Zs"
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
263 """
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
264
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
265 # C.2.1 ASCII control characters
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
266 name, table_c21 = tables[0]
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
267 del tables[0]
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
268 assert name == "C.2.1"
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
269
32125
b9edd3418a31 Applying SF patch #949329 on behalf of Raymond Hettinger.
Armin Rigo <arigo@tunes.org>
parents: 28671
diff changeset
270 Cc = set(gen_category(["Cc"]))
b9edd3418a31 Applying SF patch #949329 on behalf of Raymond Hettinger.
Armin Rigo <arigo@tunes.org>
parents: 28671
diff changeset
271 Cc_ascii = Cc & set(range(128))
b9edd3418a31 Applying SF patch #949329 on behalf of Raymond Hettinger.
Armin Rigo <arigo@tunes.org>
parents: 28671
diff changeset
272 table_c21 = set(table_c21.keys())
28671
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
273 assert Cc_ascii == table_c21
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
274
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
275 print """
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
276 def in_table_c21(code):
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
277 return ord(code) < 128 and unicodedata.category(code) == "Cc"
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
278 """
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
279
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
280 # C.2.2 Non-ASCII control characters. It also includes
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
281 # a number of characters in category Cf.
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
282 name, table_c22 = tables[0]
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
283 del tables[0]
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
284 assert name == "C.2.2"
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
285
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
286 Cc_nonascii = Cc - Cc_ascii
32125
b9edd3418a31 Applying SF patch #949329 on behalf of Raymond Hettinger.
Armin Rigo <arigo@tunes.org>
parents: 28671
diff changeset
287 table_c22 = set(table_c22.keys())
28671
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
288 assert len(Cc_nonascii - table_c22) == 0
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
289
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
290 specials = list(table_c22 - Cc_nonascii)
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
291 specials.sort()
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
292
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
293 print """c22_specials = """ + compact_set(specials) + """
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
294 def in_table_c22(code):
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
295 c = ord(code)
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
296 if c < 128: return False
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
297 if unicodedata.category(code) == "Cc": return True
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
298 return c in c22_specials
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
299
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
300 def in_table_c21_c22(code):
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
301 return unicodedata.category(code) == "Cc" or \\
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
302 ord(code) in c22_specials
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
303 """
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
304
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
305 # C.3 Private use
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
306 name, table = tables[0]
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
307 del tables[0]
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
308 assert name == "C.3"
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
309
32125
b9edd3418a31 Applying SF patch #949329 on behalf of Raymond Hettinger.
Armin Rigo <arigo@tunes.org>
parents: 28671
diff changeset
310 Co = set(gen_category(["Co"]))
b9edd3418a31 Applying SF patch #949329 on behalf of Raymond Hettinger.
Armin Rigo <arigo@tunes.org>
parents: 28671
diff changeset
311 assert set(table.keys()) == Co
28671
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
312
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
313 print """
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
314 def in_table_c3(code):
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
315 return unicodedata.category(code) == "Co"
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
316 """
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
317
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
318 # C.4 Non-character code points, xFFFE, xFFFF
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
319 # plus process internal codes
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
320 name, table = tables[0]
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
321 del tables[0]
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
322 assert name == "C.4"
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
323
32125
b9edd3418a31 Applying SF patch #949329 on behalf of Raymond Hettinger.
Armin Rigo <arigo@tunes.org>
parents: 28671
diff changeset
324 nonchar = set(range(0xFDD0,0xFDF0) +
28671
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
325 range(0xFFFE,0x110000,0x10000) +
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
326 range(0xFFFF,0x110000,0x10000))
32125
b9edd3418a31 Applying SF patch #949329 on behalf of Raymond Hettinger.
Armin Rigo <arigo@tunes.org>
parents: 28671
diff changeset
327 table = set(table.keys())
28671
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
328 assert table == nonchar
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
329
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
330 print """
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
331 def in_table_c4(code):
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
332 c = ord(code)
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
333 if c < 0xFDD0: return False
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
334 if c < 0xFDF0: return True
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
335 return (ord(code) & 0xFFFF) in (0xFFFE, 0xFFFF)
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
336 """
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
337
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
338 # C.5 Surrogate codes
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
339 name, table = tables[0]
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
340 del tables[0]
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
341 assert name == "C.5"
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
342
32125
b9edd3418a31 Applying SF patch #949329 on behalf of Raymond Hettinger.
Armin Rigo <arigo@tunes.org>
parents: 28671
diff changeset
343 Cs = set(gen_category(["Cs"]))
b9edd3418a31 Applying SF patch #949329 on behalf of Raymond Hettinger.
Armin Rigo <arigo@tunes.org>
parents: 28671
diff changeset
344 assert set(table.keys()) == Cs
28671
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
345
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
346 print """
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
347 def in_table_c5(code):
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
348 return unicodedata.category(code) == "Cs"
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
349 """
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
350
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
351 # C.6 Inappropriate for plain text
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
352 name, table = tables[0]
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
353 del tables[0]
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
354 assert name == "C.6"
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
355
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
356 table = table.keys()
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
357 table.sort()
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
358
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
359 print """
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
360 c6_set = """ + compact_set(table) + """
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
361 def in_table_c6(code):
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
362 return ord(code) in c6_set
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
363 """
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
364
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
365 # C.7 Inappropriate for canonical representation
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
366 name, table = tables[0]
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
367 del tables[0]
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
368 assert name == "C.7"
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
369
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
370 table = table.keys()
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
371 table.sort()
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
372
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
373 print """
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
374 c7_set = """ + compact_set(table) + """
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
375 def in_table_c7(code):
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
376 return ord(code) in c7_set
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
377 """
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
378
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
379 # C.8 Change display properties or are deprecated
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
380 name, table = tables[0]
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
381 del tables[0]
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
382 assert name == "C.8"
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
383
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
384 table = table.keys()
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
385 table.sort()
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
386
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
387 print """
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
388 c8_set = """ + compact_set(table) + """
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
389 def in_table_c8(code):
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
390 return ord(code) in c8_set
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
391 """
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
392
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
393 # C.9 Tagging characters
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
394 name, table = tables[0]
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
395 del tables[0]
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
396 assert name == "C.9"
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
397
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
398 table = table.keys()
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
399 table.sort()
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
400
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
401 print """
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
402 c9_set = """ + compact_set(table) + """
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
403 def in_table_c9(code):
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
404 return ord(code) in c9_set
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
405 """
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
406
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
407 # D.1 Characters with bidirectional property "R" or "AL"
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
408 name, table = tables[0]
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
409 del tables[0]
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
410 assert name == "D.1"
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
411
32125
b9edd3418a31 Applying SF patch #949329 on behalf of Raymond Hettinger.
Armin Rigo <arigo@tunes.org>
parents: 28671
diff changeset
412 RandAL = set(gen_bidirectional(["R","AL"]))
b9edd3418a31 Applying SF patch #949329 on behalf of Raymond Hettinger.
Armin Rigo <arigo@tunes.org>
parents: 28671
diff changeset
413 assert set(table.keys()) == RandAL
28671
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
414
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
415 print """
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
416 def in_table_d1(code):
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
417 return unicodedata.bidirectional(code) in ("R","AL")
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
418 """
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
419
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
420 # D.2 Characters with bidirectional property "L"
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
421 name, table = tables[0]
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
422 del tables[0]
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
423 assert name == "D.2"
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
424
32125
b9edd3418a31 Applying SF patch #949329 on behalf of Raymond Hettinger.
Armin Rigo <arigo@tunes.org>
parents: 28671
diff changeset
425 L = set(gen_bidirectional(["L"]))
b9edd3418a31 Applying SF patch #949329 on behalf of Raymond Hettinger.
Armin Rigo <arigo@tunes.org>
parents: 28671
diff changeset
426 assert set(table.keys()) == L
28671
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
427
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
428 print """
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
429 def in_table_d2(code):
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
430 return unicodedata.bidirectional(code) == "L"
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
431 """