Mercurial > cpython-lru_cache-weakref

annotate Tools/unicode/mkstringprep.py @ 33091:48100a01ac7c 2.3

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Ignore the locale's encoding if it is an empty string.
author Martin v. Löwis <martin@v.loewis.de>
date Thu, 12 Aug 2004 13:14:20 +0000
parents d41d8de9768f
children b9edd3418a31
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
28671
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
1 import re, unicodedata, sys, sets
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
2 from sets import Set
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
3
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
4 if sys.maxunicode == 65535:
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
5 raise RuntimeError, "need UCS-4 Python"
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
6
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
7 def gen_category(cats):
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
8 for i in range(0, 0x110000):
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
9 if unicodedata.category(unichr(i)) in cats:
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
10 yield(i)
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
11
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
12 def gen_bidirectional(cats):
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
13 for i in range(0, 0x110000):
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
14 if unicodedata.bidirectional(unichr(i)) in cats:
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
15 yield(i)
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
16
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
17 def compact_set(l):
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
18 single = []
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
19 tuple = []
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
20 prev = None
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
21 span = 0
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
22 for e in l:
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
23 if prev is None:
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
24 prev = e
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
25 span = 0
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
26 continue
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
27 if prev+span+1 != e:
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
28 if span > 2:
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
29 tuple.append((prev,prev+span+1))
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
30 else:
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
31 for i in range(prev, prev+span+1):
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
32 single.append(i)
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
33 prev = e
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
34 span = 0
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
35 else:
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
36 span += 1
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
37 if span:
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
38 tuple.append((prev,prev+span+1))
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
39 else:
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
40 single.append(prev)
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
41 tuple = " + ".join(["range(%d,%d)" % t for t in tuple])
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
42 if not single:
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
43 return "sets.Set(%s)" % tuple
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
44 if not tuple:
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
45 return "sets.Set(%s)" % repr(single)
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
46 return "sets.Set(%s + %s)" % (repr(single),tuple)
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
47
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
48 ############## Read the tables in the RFC #######################
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
49
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
50 data = open("rfc3454.txt").readlines()
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
51
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
52 tables = []
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
53 curname = None
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
54 for l in data:
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
55 l = l.strip()
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
56 if not l:
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
57 continue
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
58 # Skip RFC page breaks
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
59 if l.startswith("Hoffman & Blanchet") or\
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
60 l.startswith("RFC 3454"):
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
61 continue
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
62 # Find start/end lines
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
63 m = re.match("----- (Start|End) Table ([A-Z](.[0-9])+) -----", l)
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
64 if m:
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
65 if m.group(1) == "Start":
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
66 if curname:
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
67 raise "Double Start",(curname, l)
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
68 curname = m.group(2)
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
69 table = {}
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
70 tables.append((curname, table))
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
71 continue
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
72 else:
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
73 if not curname:
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
74 raise "End without start", l
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
75 curname = None
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
76 continue
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
77 if not curname:
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
78 continue
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
79 # Now we are in a table
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
80 fields = l.split(";")
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
81 if len(fields) > 1:
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
82 # Drop comment field
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
83 fields = fields[:-1]
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
84 if len(fields) == 1:
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
85 fields = fields[0].split("-")
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
86 if len(fields) > 1:
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
87 # range
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
88 try:
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
89 start, end = fields
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
90 except ValueError:
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
91 raise "Unpacking problem", l
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
92 else:
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
93 start = end = fields[0]
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
94 start = int(start, 16)
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
95 end = int(end, 16)
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
96 for i in range(start, end+1):
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
97 table[i] = i
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
98 else:
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
99 code, value = fields
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
100 value = value.strip()
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
101 if value:
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
102 value = [int(v, 16) for v in value.split(" ")]
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
103 else:
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
104 # table B.1
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
105 value = None
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
106 table[int(code, 16)] = value
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
107
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
108 ########### Generate compact Python versions of the tables #############
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
109
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
110 print """# This file is generated by mkstringprep.py. DO NOT EDIT.
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
111 \"\"\"Library that exposes various tables found in the StringPrep RFC 3454.
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
112
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
113 There are two kinds of tables: sets, for which a member test is provided,
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
114 and mappings, for which a mapping function is provided.
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
115 \"\"\"
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
116
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
117 import unicodedata, sets
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
118 """
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
119
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
120 print "assert unicodedata.unidata_version == %s" % repr(unicodedata.unidata_version)
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
121
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
122 # A.1 is the table of unassigned characters
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
123 # XXX Plane 15 PUA is listed as unassigned in Python.
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
124 name, table = tables[0]
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
125 del tables[0]
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
126 assert name == "A.1"
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
127 table = Set(table.keys())
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
128 Cn = Set(gen_category(["Cn"]))
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
129
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
130 # FDD0..FDEF are process internal codes
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
131 Cn -= Set(range(0xFDD0, 0xFDF0))
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
132 # not a character
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
133 Cn -= Set(range(0xFFFE, 0x110000, 0x10000))
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
134 Cn -= Set(range(0xFFFF, 0x110000, 0x10000))
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
135
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
136 # assert table == Cn
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
137
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
138 print """
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
139 def in_table_a1(code):
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
140 if unicodedata.category(code) != 'Cn': return False
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
141 c = ord(code)
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
142 if 0xFDD0 <= c < 0xFDF0: return False
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
143 return (c & 0xFFFF) not in (0xFFFE, 0xFFFF)
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
144 """
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
145
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
146 # B.1 cannot easily be derived
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
147 name, table = tables[0]
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
148 del tables[0]
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
149 assert name == "B.1"
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
150 table = table.keys()
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
151 table.sort()
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
152 print """
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
153 b1_set = """ + compact_set(table) + """
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
154 def in_table_b1(code):
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
155 return ord(code) in b1_set
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
156 """
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
157
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
158 # B.2 and B.3 is case folding.
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
159 # It takes CaseFolding.txt into account, which is
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
160 # not available in the Python database. Since
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
161 # B.2 is derived from B.3, we process B.3 first.
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
162 # B.3 supposedly *is* CaseFolding-3.2.0.txt.
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
163
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
164 name, table_b2 = tables[0]
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
165 del tables[0]
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
166 assert name == "B.2"
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
167
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
168 name, table_b3 = tables[0]
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
169 del tables[0]
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
170 assert name == "B.3"
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
171
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
172 # B.3 is mostly Python's .lower, except for a number
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
173 # of special cases, e.g. considering canonical forms.
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
174
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
175 b3_exceptions = {}
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
176
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
177 for k,v in table_b2.items():
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
178 if map(ord, unichr(k).lower()) != v:
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
179 b3_exceptions[k] = u"".join(map(unichr,v))
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
180
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
181 b3 = b3_exceptions.items()
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
182 b3.sort()
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
183
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
184 print """
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
185 b3_exceptions = {"""
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
186 for i,(k,v) in enumerate(b3):
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
187 print "0x%x:%s," % (k, repr(v)),
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
188 if i % 4 == 3:
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
189 print
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
190 print "}"
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
191
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
192 print """
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
193 def map_table_b3(code):
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
194 r = b3_exceptions.get(ord(code))
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
195 if r is not None: return r
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
196 return code.lower()
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
197 """
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
198
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
199 def map_table_b3(code):
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
200 r = b3_exceptions.get(ord(code))
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
201 if r is not None: return r
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
202 return code.lower()
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
203
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
204 # B.2 is case folding for NFKC. This is the same as B.3,
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
205 # except where NormalizeWithKC(Fold(a)) !=
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
206 # NormalizeWithKC(Fold(NormalizeWithKC(Fold(a))))
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
207
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
208 def map_table_b2(a):
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
209 al = map_table_b3(a)
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
210 b = unicodedata.normalize("NFKC", al)
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
211 bl = u"".join([map_table_b3(ch) for ch in b])
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
212 c = unicodedata.normalize("NFKC", bl)
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
213 if b != c:
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
214 return c
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
215 else:
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
216 return al
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
217
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
218 specials = {}
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
219 for k,v in table_b2.items():
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
220 if map(ord, map_table_b2(unichr(k))) != v:
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
221 specials[k] = v
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
222
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
223 # B.3 should not add any additional special cases
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
224 assert specials == {}
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
225
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
226 print """
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
227 def map_table_b2(a):
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
228 al = map_table_b3(a)
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
229 b = unicodedata.normalize("NFKC", al)
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
230 bl = u"".join([map_table_b3(ch) for ch in b])
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
231 c = unicodedata.normalize("NFKC", bl)
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
232 if b != c:
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
233 return c
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
234 else:
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
235 return al
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
236 """
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
237
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
238 # C.1.1 is a table with a single character
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
239 name, table = tables[0]
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
240 del tables[0]
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
241 assert name == "C.1.1"
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
242 assert table == {0x20:0x20}
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
243
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
244 print """
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
245 def in_table_c11(code):
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
246 return code == u" "
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
247 """
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
248
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
249 # C.1.2 is the rest of all space characters
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
250 name, table = tables[0]
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
251 del tables[0]
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
252 assert name == "C.1.2"
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
253
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
254 # table = Set(table.keys())
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
255 # Zs = Set(gen_category(["Zs"])) - Set([0x20])
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
256 # assert Zs == table
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
257
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
258 print """
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
259 def in_table_c12(code):
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
260 return unicodedata.category(code) == "Zs" and code != u" "
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
261
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
262 def in_table_c11_c12(code):
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
263 return unicodedata.category(code) == "Zs"
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
264 """
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
265
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
266 # C.2.1 ASCII control characters
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
267 name, table_c21 = tables[0]
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
268 del tables[0]
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
269 assert name == "C.2.1"
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
270
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
271 Cc = Set(gen_category(["Cc"]))
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
272 Cc_ascii = Cc & Set(range(128))
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
273 table_c21 = Set(table_c21.keys())
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
274 assert Cc_ascii == table_c21
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
275
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
276 print """
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
277 def in_table_c21(code):
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
278 return ord(code) < 128 and unicodedata.category(code) == "Cc"
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
279 """
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
280
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
281 # C.2.2 Non-ASCII control characters. It also includes
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
282 # a number of characters in category Cf.
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
283 name, table_c22 = tables[0]
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
284 del tables[0]
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
285 assert name == "C.2.2"
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
286
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
287 Cc_nonascii = Cc - Cc_ascii
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
288 table_c22 = Set(table_c22.keys())
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
289 assert len(Cc_nonascii - table_c22) == 0
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
290
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
291 specials = list(table_c22 - Cc_nonascii)
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
292 specials.sort()
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
293
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
294 print """c22_specials = """ + compact_set(specials) + """
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
295 def in_table_c22(code):
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
296 c = ord(code)
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
297 if c < 128: return False
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
298 if unicodedata.category(code) == "Cc": return True
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
299 return c in c22_specials
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
300
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
301 def in_table_c21_c22(code):
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
302 return unicodedata.category(code) == "Cc" or \\
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
303 ord(code) in c22_specials
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
304 """
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
305
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
306 # C.3 Private use
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
307 name, table = tables[0]
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
308 del tables[0]
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
309 assert name == "C.3"
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
310
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
311 Co = Set(gen_category(["Co"]))
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
312 assert Set(table.keys()) == Co
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
313
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
314 print """
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
315 def in_table_c3(code):
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
316 return unicodedata.category(code) == "Co"
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
317 """
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
318
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
319 # C.4 Non-character code points, xFFFE, xFFFF
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
320 # plus process internal codes
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
321 name, table = tables[0]
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
322 del tables[0]
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
323 assert name == "C.4"
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
324
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
325 nonchar = Set(range(0xFDD0,0xFDF0) +
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
326 range(0xFFFE,0x110000,0x10000) +
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
327 range(0xFFFF,0x110000,0x10000))
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
328 table = Set(table.keys())
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
329 assert table == nonchar
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
330
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
331 print """
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
332 def in_table_c4(code):
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
333 c = ord(code)
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
334 if c < 0xFDD0: return False
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
335 if c < 0xFDF0: return True
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
336 return (ord(code) & 0xFFFF) in (0xFFFE, 0xFFFF)
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
337 """
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
338
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
339 # C.5 Surrogate codes
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
340 name, table = tables[0]
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
341 del tables[0]
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
342 assert name == "C.5"
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
343
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
344 Cs = Set(gen_category(["Cs"]))
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
345 assert Set(table.keys()) == Cs
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
346
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
347 print """
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
348 def in_table_c5(code):
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
349 return unicodedata.category(code) == "Cs"
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
350 """
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
351
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
352 # C.6 Inappropriate for plain text
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
353 name, table = tables[0]
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
354 del tables[0]
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
355 assert name == "C.6"
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
356
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
357 table = table.keys()
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
358 table.sort()
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
359
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
360 print """
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
361 c6_set = """ + compact_set(table) + """
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
362 def in_table_c6(code):
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
363 return ord(code) in c6_set
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
364 """
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
365
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
366 # C.7 Inappropriate for canonical representation
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
367 name, table = tables[0]
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
368 del tables[0]
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
369 assert name == "C.7"
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
370
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
371 table = table.keys()
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
372 table.sort()
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
373
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
374 print """
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
375 c7_set = """ + compact_set(table) + """
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
376 def in_table_c7(code):
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
377 return ord(code) in c7_set
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
378 """
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
379
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
380 # C.8 Change display properties or are deprecated
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
381 name, table = tables[0]
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
382 del tables[0]
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
383 assert name == "C.8"
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
384
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
385 table = table.keys()
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
386 table.sort()
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
387
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
388 print """
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
389 c8_set = """ + compact_set(table) + """
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
390 def in_table_c8(code):
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
391 return ord(code) in c8_set
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
392 """
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
393
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
394 # C.9 Tagging characters
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
395 name, table = tables[0]
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
396 del tables[0]
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
397 assert name == "C.9"
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
398
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
399 table = table.keys()
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
400 table.sort()
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
401
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
402 print """
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
403 c9_set = """ + compact_set(table) + """
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
404 def in_table_c9(code):
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
405 return ord(code) in c9_set
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
406 """
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
407
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
408 # D.1 Characters with bidirectional property "R" or "AL"
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
409 name, table = tables[0]
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
410 del tables[0]
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
411 assert name == "D.1"
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
412
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
413 RandAL = Set(gen_bidirectional(["R","AL"]))
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
414 assert Set(table.keys()) == RandAL
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
415
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
416 print """
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
417 def in_table_d1(code):
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
418 return unicodedata.bidirectional(code) in ("R","AL")
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
419 """
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
420
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
421 # D.2 Characters with bidirectional property "L"
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
422 name, table = tables[0]
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
423 del tables[0]
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
424 assert name == "D.2"
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
425
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
426 L = Set(gen_bidirectional(["L"]))
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
427 assert Set(table.keys()) == L
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
428
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
429 print """
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
430 def in_table_d2(code):
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
431 return unicodedata.bidirectional(code) == "L"
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
432 """
d41d8de9768f Implement IDNA (Internationalized Domain Names in Applications).
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
433