Mercurial > cpython-lru_cache-weakref

annotate Tools/unicode/makeunicodedata.py @ 22360:c8a6ea00a9e1 legacy-trunk

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
restructure a bit to not rely on test case execution ordering add test case for bug #482460
author Skip Montanaro <skip@pobox.com>
date Sun, 17 Mar 2002 23:03:42 +0000
parents 45c065668ea5
children 8a7a2f1f88f0
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
14825
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
1 #
14849
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
2 # (re)generate unicode property and type databases
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
3 #
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
4 # this script converts a unicode 3.0 database file to
16080
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
5 # Modules/unicodedata_db.h, Modules/unicodename_db.h,
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
6 # and Objects/unicodetype_db.h
14835
96d31a143857 unicode database compression, step 2:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14834
diff changeset
7 #
96d31a143857 unicode database compression, step 2:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14834
diff changeset
8 # history:
96d31a143857 unicode database compression, step 2:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14834
diff changeset
9 # 2000-09-24 fl created (based on bits and pieces from unidb)
96d31a143857 unicode database compression, step 2:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14834
diff changeset
10 # 2000-09-25 fl merged tim's splitbin fixes, separate decomposition table
14849
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
11 # 2000-09-25 fl added character type table
16080
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
12 # 2000-09-26 fl added LINEBREAK, DECIMAL, and DIGIT flags/fields (2.0)
15376
493ca9e4260e Added 38,642 missing characters to the Unicode database (first-last
Fredrik Lundh <fredrik@pythonware.com>
parents: 15339
diff changeset
13 # 2000-11-03 fl expand first/last ranges
16080
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
14 # 2001-01-19 fl added character name tables (2.1)
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 16080
diff changeset
15 # 2001-01-21 fl added decomp compression; dynamic phrasebook threshold
14835
96d31a143857 unicode database compression, step 2:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14834
diff changeset
16 #
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 16080
diff changeset
17 # written by Fredrik Lundh (fredrik@pythonware.com)
14825
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
18 #
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
19
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
20 import sys
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
21
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
22 SCRIPT = sys.argv[0]
16080
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
23 VERSION = "2.1"
14825
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
24
14849
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
25 UNICODE_DATA = "UnicodeData-Latest.txt"
14825
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
26
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
27 CATEGORY_NAMES = [ "Cn", "Lu", "Ll", "Lt", "Mn", "Mc", "Me", "Nd",
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
28 "Nl", "No", "Zs", "Zl", "Zp", "Cc", "Cf", "Cs", "Co", "Cn", "Lm",
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
29 "Lo", "Pc", "Pd", "Ps", "Pe", "Pi", "Pf", "Po", "Sm", "Sc", "Sk",
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
30 "So" ]
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
31
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
32 BIDIRECTIONAL_NAMES = [ "", "L", "LRE", "LRO", "R", "AL", "RLE", "RLO",
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
33 "PDF", "EN", "ES", "ET", "AN", "CS", "NSM", "BN", "B", "S", "WS",
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
34 "ON" ]
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
35
14852
463faf01685e unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14849
diff changeset
36 # note: should match definitions in Objects/unicodectype.c
14849
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
37 ALPHA_MASK = 0x01
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
38 DECIMAL_MASK = 0x02
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
39 DIGIT_MASK = 0x04
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
40 LOWER_MASK = 0x08
14852
463faf01685e unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14849
diff changeset
41 LINEBREAK_MASK = 0x10
14849
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
42 SPACE_MASK = 0x20
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
43 TITLE_MASK = 0x40
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
44 UPPER_MASK = 0x80
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
45
15376
493ca9e4260e Added 38,642 missing characters to the Unicode database (first-last
Fredrik Lundh <fredrik@pythonware.com>
parents: 15339
diff changeset
46 def maketables(trace=0):
14825
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
47
16080
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
48 print "--- Reading", UNICODE_DATA, "..."
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
49
14825
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
50 unicode = UnicodeData(UNICODE_DATA)
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
51
15376
493ca9e4260e Added 38,642 missing characters to the Unicode database (first-last
Fredrik Lundh <fredrik@pythonware.com>
parents: 15339
diff changeset
52 print len(filter(None, unicode.table)), "characters"
493ca9e4260e Added 38,642 missing characters to the Unicode database (first-last
Fredrik Lundh <fredrik@pythonware.com>
parents: 15339
diff changeset
53
16091
45c065668ea5 Unicode nits: Don't include unicodedatabase.h no more. And make sure
Fredrik Lundh <fredrik@pythonware.com>
parents: 16090
diff changeset
54 makeunicodename(unicode, trace)
16080
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
55 makeunicodedata(unicode, trace)
16091
45c065668ea5 Unicode nits: Don't include unicodedatabase.h no more. And make sure
Fredrik Lundh <fredrik@pythonware.com>
parents: 16090
diff changeset
56 makeunicodetype(unicode, trace)
16080
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
57
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
58 # --------------------------------------------------------------------
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
59 # unicode character properties
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
60
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
61 def makeunicodedata(unicode, trace):
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
62
14835
96d31a143857 unicode database compression, step 2:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14834
diff changeset
63 dummy = (0, 0, 0, 0)
14825
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
64 table = [dummy]
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
65 cache = {0: dummy}
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
66 index = [0] * len(unicode.chars)
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
67
16080
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
68 FILE = "Modules/unicodedata_db.h"
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
69
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
70 print "--- Preparing", FILE, "..."
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
71
14835
96d31a143857 unicode database compression, step 2:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14834
diff changeset
72 # 1) database properties
16080
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
73
14825
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
74 for char in unicode.chars:
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
75 record = unicode.table[char]
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
76 if record:
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
77 # extract database properties
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
78 category = CATEGORY_NAMES.index(record[2])
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
79 combining = int(record[3])
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
80 bidirectional = BIDIRECTIONAL_NAMES.index(record[4])
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
81 mirrored = record[9] == "Y"
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
82 item = (
14835
96d31a143857 unicode database compression, step 2:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14834
diff changeset
83 category, combining, bidirectional, mirrored
14825
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
84 )
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
85 # add entry to index and item tables
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
86 i = cache.get(item)
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
87 if i is None:
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
88 cache[item] = i = len(table)
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
89 table.append(item)
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
90 index[char] = i
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
91
14835
96d31a143857 unicode database compression, step 2:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14834
diff changeset
92 # 2) decomposition data
96d31a143857 unicode database compression, step 2:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14834
diff changeset
93
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 16080
diff changeset
94 decomp_data = [0]
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 16080
diff changeset
95 decomp_prefix = [""]
14835
96d31a143857 unicode database compression, step 2:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14834
diff changeset
96 decomp_index = [0] * len(unicode.chars)
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 16080
diff changeset
97 decomp_size = 0
14835
96d31a143857 unicode database compression, step 2:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14834
diff changeset
98
96d31a143857 unicode database compression, step 2:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14834
diff changeset
99 for char in unicode.chars:
96d31a143857 unicode database compression, step 2:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14834
diff changeset
100 record = unicode.table[char]
96d31a143857 unicode database compression, step 2:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14834
diff changeset
101 if record:
96d31a143857 unicode database compression, step 2:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14834
diff changeset
102 if record[5]:
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 16080
diff changeset
103 decomp = string.split(record[5])
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 16080
diff changeset
104 # prefix
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 16080
diff changeset
105 if decomp[0][0] == "<":
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 16080
diff changeset
106 prefix = decomp.pop(0)
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 16080
diff changeset
107 else:
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 16080
diff changeset
108 prefix = ""
14835
96d31a143857 unicode database compression, step 2:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14834
diff changeset
109 try:
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 16080
diff changeset
110 i = decomp_prefix.index(prefix)
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 16080
diff changeset
111 except ValueError:
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 16080
diff changeset
112 i = len(decomp_prefix)
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 16080
diff changeset
113 decomp_prefix.append(prefix)
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 16080
diff changeset
114 prefix = i
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 16080
diff changeset
115 assert prefix < 256
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 16080
diff changeset
116 # content
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 16080
diff changeset
117 decomp = [prefix + (len(decomp)<<8)] +\
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 16080
diff changeset
118 map(lambda s: int(s, 16), decomp)
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 16080
diff changeset
119 try:
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 16080
diff changeset
120 i = decomp_data.index(decomp)
14835
96d31a143857 unicode database compression, step 2:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14834
diff changeset
121 except ValueError:
96d31a143857 unicode database compression, step 2:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14834
diff changeset
122 i = len(decomp_data)
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 16080
diff changeset
123 decomp_data.extend(decomp)
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 16080
diff changeset
124 decomp_size = decomp_size + len(decomp) * 2
14835
96d31a143857 unicode database compression, step 2:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14834
diff changeset
125 else:
96d31a143857 unicode database compression, step 2:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14834
diff changeset
126 i = 0
96d31a143857 unicode database compression, step 2:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14834
diff changeset
127 decomp_index[char] = i
14825
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
128
16080
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
129 print len(table), "unique properties"
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 16080
diff changeset
130 print len(decomp_prefix), "unique decomposition prefixes"
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 16080
diff changeset
131 print len(decomp_data), "unique decomposition entries:",
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 16080
diff changeset
132 print decomp_size, "bytes"
14825
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
133
15376
493ca9e4260e Added 38,642 missing characters to the Unicode database (first-last
Fredrik Lundh <fredrik@pythonware.com>
parents: 15339
diff changeset
134 print "--- Writing", FILE, "..."
493ca9e4260e Added 38,642 missing characters to the Unicode database (first-last
Fredrik Lundh <fredrik@pythonware.com>
parents: 15339
diff changeset
135
15339
c2d7451b42f6 Remove bogus stdout redirection and use of sys.__stdout__; use
Fred Drake <fdrake@acm.org>
parents: 14856
diff changeset
136 fp = open(FILE, "w")
c2d7451b42f6 Remove bogus stdout redirection and use of sys.__stdout__; use
Fred Drake <fdrake@acm.org>
parents: 14856
diff changeset
137 print >>fp, "/* this file was generated by %s %s */" % (SCRIPT, VERSION)
c2d7451b42f6 Remove bogus stdout redirection and use of sys.__stdout__; use
Fred Drake <fdrake@acm.org>
parents: 14856
diff changeset
138 print >>fp
c2d7451b42f6 Remove bogus stdout redirection and use of sys.__stdout__; use
Fred Drake <fdrake@acm.org>
parents: 14856
diff changeset
139 print >>fp, "/* a list of unique database records */"
c2d7451b42f6 Remove bogus stdout redirection and use of sys.__stdout__; use
Fred Drake <fdrake@acm.org>
parents: 14856
diff changeset
140 print >>fp, \
c2d7451b42f6 Remove bogus stdout redirection and use of sys.__stdout__; use
Fred Drake <fdrake@acm.org>
parents: 14856
diff changeset
141 "const _PyUnicode_DatabaseRecord _PyUnicode_Database_Records[] = {"
14825
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
142 for item in table:
15339
c2d7451b42f6 Remove bogus stdout redirection and use of sys.__stdout__; use
Fred Drake <fdrake@acm.org>
parents: 14856
diff changeset
143 print >>fp, " {%d, %d, %d, %d}," % item
c2d7451b42f6 Remove bogus stdout redirection and use of sys.__stdout__; use
Fred Drake <fdrake@acm.org>
parents: 14856
diff changeset
144 print >>fp, "};"
c2d7451b42f6 Remove bogus stdout redirection and use of sys.__stdout__; use
Fred Drake <fdrake@acm.org>
parents: 14856
diff changeset
145 print >>fp
14825
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
146
16080
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
147 # FIXME: <fl> the following tables could be made static, and
14849
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
148 # the support code moved into unicodedatabase.c
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
149
15339
c2d7451b42f6 Remove bogus stdout redirection and use of sys.__stdout__; use
Fred Drake <fdrake@acm.org>
parents: 14856
diff changeset
150 print >>fp, "/* string literals */"
c2d7451b42f6 Remove bogus stdout redirection and use of sys.__stdout__; use
Fred Drake <fdrake@acm.org>
parents: 14856
diff changeset
151 print >>fp, "const char *_PyUnicode_CategoryNames[] = {"
14825
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
152 for name in CATEGORY_NAMES:
15339
c2d7451b42f6 Remove bogus stdout redirection and use of sys.__stdout__; use
Fred Drake <fdrake@acm.org>
parents: 14856
diff changeset
153 print >>fp, " \"%s\"," % name
c2d7451b42f6 Remove bogus stdout redirection and use of sys.__stdout__; use
Fred Drake <fdrake@acm.org>
parents: 14856
diff changeset
154 print >>fp, " NULL"
c2d7451b42f6 Remove bogus stdout redirection and use of sys.__stdout__; use
Fred Drake <fdrake@acm.org>
parents: 14856
diff changeset
155 print >>fp, "};"
14825
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
156
15339
c2d7451b42f6 Remove bogus stdout redirection and use of sys.__stdout__; use
Fred Drake <fdrake@acm.org>
parents: 14856
diff changeset
157 print >>fp, "const char *_PyUnicode_BidirectionalNames[] = {"
14825
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
158 for name in BIDIRECTIONAL_NAMES:
15339
c2d7451b42f6 Remove bogus stdout redirection and use of sys.__stdout__; use
Fred Drake <fdrake@acm.org>
parents: 14856
diff changeset
159 print >>fp, " \"%s\"," % name
c2d7451b42f6 Remove bogus stdout redirection and use of sys.__stdout__; use
Fred Drake <fdrake@acm.org>
parents: 14856
diff changeset
160 print >>fp, " NULL"
c2d7451b42f6 Remove bogus stdout redirection and use of sys.__stdout__; use
Fred Drake <fdrake@acm.org>
parents: 14856
diff changeset
161 print >>fp, "};"
14825
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
162
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 16080
diff changeset
163 print >>fp, "static const char *decomp_prefix[] = {"
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 16080
diff changeset
164 for name in decomp_prefix:
15339
c2d7451b42f6 Remove bogus stdout redirection and use of sys.__stdout__; use
Fred Drake <fdrake@acm.org>
parents: 14856
diff changeset
165 print >>fp, " \"%s\"," % name
c2d7451b42f6 Remove bogus stdout redirection and use of sys.__stdout__; use
Fred Drake <fdrake@acm.org>
parents: 14856
diff changeset
166 print >>fp, " NULL"
c2d7451b42f6 Remove bogus stdout redirection and use of sys.__stdout__; use
Fred Drake <fdrake@acm.org>
parents: 14856
diff changeset
167 print >>fp, "};"
14835
96d31a143857 unicode database compression, step 2:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14834
diff changeset
168
14849
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
169 # split record index table
15376
493ca9e4260e Added 38,642 missing characters to the Unicode database (first-last
Fredrik Lundh <fredrik@pythonware.com>
parents: 15339
diff changeset
170 index1, index2, shift = splitbins(index, trace)
14825
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
171
15339
c2d7451b42f6 Remove bogus stdout redirection and use of sys.__stdout__; use
Fred Drake <fdrake@acm.org>
parents: 14856
diff changeset
172 print >>fp, "/* index tables for the database records */"
c2d7451b42f6 Remove bogus stdout redirection and use of sys.__stdout__; use
Fred Drake <fdrake@acm.org>
parents: 14856
diff changeset
173 print >>fp, "#define SHIFT", shift
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 16080
diff changeset
174 Array("index1", index1).dump(fp, trace)
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 16080
diff changeset
175 Array("index2", index2).dump(fp, trace)
14825
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
176
14849
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
177 # split decomposition index table
15376
493ca9e4260e Added 38,642 missing characters to the Unicode database (first-last
Fredrik Lundh <fredrik@pythonware.com>
parents: 15339
diff changeset
178 index1, index2, shift = splitbins(decomp_index, trace)
14835
96d31a143857 unicode database compression, step 2:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14834
diff changeset
179
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 16080
diff changeset
180 print >>fp, "/* decomposition data */"
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 16080
diff changeset
181 Array("decomp_data", decomp_data).dump(fp, trace)
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 16080
diff changeset
182
15339
c2d7451b42f6 Remove bogus stdout redirection and use of sys.__stdout__; use
Fred Drake <fdrake@acm.org>
parents: 14856
diff changeset
183 print >>fp, "/* index tables for the decomposition data */"
c2d7451b42f6 Remove bogus stdout redirection and use of sys.__stdout__; use
Fred Drake <fdrake@acm.org>
parents: 14856
diff changeset
184 print >>fp, "#define DECOMP_SHIFT", shift
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 16080
diff changeset
185 Array("decomp_index1", index1).dump(fp, trace)
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 16080
diff changeset
186 Array("decomp_index2", index2).dump(fp, trace)
14825
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
187
16080
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
188 fp.close()
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
189
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
190 # --------------------------------------------------------------------
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
191 # unicode character type tables
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
192
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
193 def makeunicodetype(unicode, trace):
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
194
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
195 FILE = "Objects/unicodetype_db.h"
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
196
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
197 print "--- Preparing", FILE, "..."
14849
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
198
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
199 # extract unicode types
14852
463faf01685e unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14849
diff changeset
200 dummy = (0, 0, 0, 0, 0, 0)
14849
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
201 table = [dummy]
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
202 cache = {0: dummy}
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
203 index = [0] * len(unicode.chars)
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
204
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
205 for char in unicode.chars:
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
206 record = unicode.table[char]
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
207 if record:
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
208 # extract database properties
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
209 category = record[2]
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
210 bidirectional = record[4]
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
211 flags = 0
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
212 if category in ["Lm", "Lt", "Lu", "Ll", "Lo"]:
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
213 flags |= ALPHA_MASK
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
214 if category == "Ll":
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
215 flags |= LOWER_MASK
14852
463faf01685e unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14849
diff changeset
216 if category == "Zl" or bidirectional == "B":
463faf01685e unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14849
diff changeset
217 flags |= LINEBREAK_MASK
14849
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
218 if category == "Zs" or bidirectional in ("WS", "B", "S"):
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
219 flags |= SPACE_MASK
14856
83aff1ea5b70 - don't set the titlecase flag for uppercase letters (sorry, tim)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14852
diff changeset
220 if category == "Lt":
14849
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
221 flags |= TITLE_MASK
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
222 if category == "Lu":
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
223 flags |= UPPER_MASK
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
224 # use delta predictor for upper/lower/title
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
225 if record[12]:
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
226 upper = (int(record[12], 16) - char) & 0xffff
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
227 else:
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
228 upper = 0
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
229 if record[13]:
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
230 lower = (int(record[13], 16) - char) & 0xffff
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
231 else:
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
232 lower = 0
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
233 if record[14]:
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
234 title = (int(record[14], 16) - char) & 0xffff
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
235 else:
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
236 title = 0
14852
463faf01685e unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14849
diff changeset
237 # decimal digit, integer digit
463faf01685e unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14849
diff changeset
238 decimal = 0
463faf01685e unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14849
diff changeset
239 if record[6]:
463faf01685e unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14849
diff changeset
240 flags |= DECIMAL_MASK
463faf01685e unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14849
diff changeset
241 decimal = int(record[6])
463faf01685e unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14849
diff changeset
242 digit = 0
463faf01685e unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14849
diff changeset
243 if record[7]:
463faf01685e unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14849
diff changeset
244 flags |= DIGIT_MASK
463faf01685e unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14849
diff changeset
245 digit = int(record[7])
14849
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
246 item = (
14852
463faf01685e unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14849
diff changeset
247 flags, upper, lower, title, decimal, digit
14849
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
248 )
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
249 # add entry to index and item tables
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
250 i = cache.get(item)
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
251 if i is None:
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
252 cache[item] = i = len(table)
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
253 table.append(item)
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
254 index[char] = i
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
255
16080
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
256 print len(table), "unique character type entries"
14849
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
257
15376
493ca9e4260e Added 38,642 missing characters to the Unicode database (first-last
Fredrik Lundh <fredrik@pythonware.com>
parents: 15339
diff changeset
258 print "--- Writing", FILE, "..."
493ca9e4260e Added 38,642 missing characters to the Unicode database (first-last
Fredrik Lundh <fredrik@pythonware.com>
parents: 15339
diff changeset
259
16080
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
260 fp = open(FILE, "w")
15339
c2d7451b42f6 Remove bogus stdout redirection and use of sys.__stdout__; use
Fred Drake <fdrake@acm.org>
parents: 14856
diff changeset
261 print >>fp, "/* this file was generated by %s %s */" % (SCRIPT, VERSION)
c2d7451b42f6 Remove bogus stdout redirection and use of sys.__stdout__; use
Fred Drake <fdrake@acm.org>
parents: 14856
diff changeset
262 print >>fp
c2d7451b42f6 Remove bogus stdout redirection and use of sys.__stdout__; use
Fred Drake <fdrake@acm.org>
parents: 14856
diff changeset
263 print >>fp, "/* a list of unique character type descriptors */"
c2d7451b42f6 Remove bogus stdout redirection and use of sys.__stdout__; use
Fred Drake <fdrake@acm.org>
parents: 14856
diff changeset
264 print >>fp, "const _PyUnicode_TypeRecord _PyUnicode_TypeRecords[] = {"
14849
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
265 for item in table:
15339
c2d7451b42f6 Remove bogus stdout redirection and use of sys.__stdout__; use
Fred Drake <fdrake@acm.org>
parents: 14856
diff changeset
266 print >>fp, " {%d, %d, %d, %d, %d, %d}," % item
c2d7451b42f6 Remove bogus stdout redirection and use of sys.__stdout__; use
Fred Drake <fdrake@acm.org>
parents: 14856
diff changeset
267 print >>fp, "};"
c2d7451b42f6 Remove bogus stdout redirection and use of sys.__stdout__; use
Fred Drake <fdrake@acm.org>
parents: 14856
diff changeset
268 print >>fp
14849
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
269
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
270 # split decomposition index table
15376
493ca9e4260e Added 38,642 missing characters to the Unicode database (first-last
Fredrik Lundh <fredrik@pythonware.com>
parents: 15339
diff changeset
271 index1, index2, shift = splitbins(index, trace)
14849
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
272
15339
c2d7451b42f6 Remove bogus stdout redirection and use of sys.__stdout__; use
Fred Drake <fdrake@acm.org>
parents: 14856
diff changeset
273 print >>fp, "/* type indexes */"
c2d7451b42f6 Remove bogus stdout redirection and use of sys.__stdout__; use
Fred Drake <fdrake@acm.org>
parents: 14856
diff changeset
274 print >>fp, "#define SHIFT", shift
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 16080
diff changeset
275 Array("index1", index1).dump(fp, trace)
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 16080
diff changeset
276 Array("index2", index2).dump(fp, trace)
14849
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
277
16080
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
278 fp.close()
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
279
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
280 # --------------------------------------------------------------------
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
281 # unicode name database
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
282
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
283 def makeunicodename(unicode, trace):
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
284
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
285 FILE = "Modules/unicodename_db.h"
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
286
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
287 print "--- Preparing", FILE, "..."
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
288
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
289 # collect names
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
290 names = [None] * len(unicode.chars)
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
291
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
292 for char in unicode.chars:
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
293 record = unicode.table[char]
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
294 if record:
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
295 name = record[1].strip()
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
296 if name and name[0] != "<":
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
297 names[char] = name + chr(0)
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
298
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
299 print len(filter(lambda n: n is not None, names)), "distinct names"
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
300
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
301 # collect unique words from names (note that we differ between
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
302 # words inside a sentence, and words ending a sentence. the
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
303 # latter includes the trailing null byte.
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
304
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
305 words = {}
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
306 n = b = 0
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
307 for char in unicode.chars:
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
308 name = names[char]
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
309 if name:
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
310 w = name.split()
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
311 b = b + len(name)
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
312 n = n + len(w)
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
313 for w in w:
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
314 l = words.get(w)
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
315 if l:
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
316 l.append(None)
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
317 else:
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
318 words[w] = [len(words)]
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
319
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
320 print n, "words in text;", b, "bytes"
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
321
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
322 wordlist = words.items()
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
323
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
324 # sort on falling frequency
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
325 wordlist.sort(lambda a, b: len(b[1])-len(a[1]))
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
326
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 16080
diff changeset
327 # figure out how many phrasebook escapes we need
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 16080
diff changeset
328 escapes = 0
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 16080
diff changeset
329 while escapes * 256 < len(wordlist):
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 16080
diff changeset
330 escapes = escapes + 1
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 16080
diff changeset
331 print escapes, "escapes"
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 16080
diff changeset
332
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 16080
diff changeset
333 short = 256 - escapes
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 16080
diff changeset
334
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 16080
diff changeset
335 assert short > 0
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 16080
diff changeset
336
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 16080
diff changeset
337 print short, "short indexes in lexicon"
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 16080
diff changeset
338
16080
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
339 # statistics
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
340 n = 0
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 16080
diff changeset
341 for i in range(short):
16080
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
342 n = n + len(wordlist[i][1])
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 16080
diff changeset
343 print n, "short indexes in phrasebook"
16080
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
344
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 16080
diff changeset
345 # pick the most commonly used words, and sort the rest on falling
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 16080
diff changeset
346 # length (to maximize overlap)
16080
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
347
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 16080
diff changeset
348 wordlist, wordtail = wordlist[:short], wordlist[short:]
16080
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
349 wordtail.sort(lambda a, b: len(b[0])-len(a[0]))
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
350 wordlist.extend(wordtail)
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
351
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
352 # generate lexicon from words
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
353
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
354 lexicon_offset = [0]
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
355 lexicon = ""
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
356 words = {}
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
357
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
358 # build a lexicon string
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
359 offset = 0
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
360 for w, x in wordlist:
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
361 # encoding: bit 7 indicates last character in word (chr(128)
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
362 # indicates the last character in an entire string)
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
363 ww = w[:-1] + chr(ord(w[-1])+128)
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
364 # reuse string tails, when possible
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
365 o = string.find(lexicon, ww)
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
366 if o < 0:
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
367 o = offset
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
368 lexicon = lexicon + ww
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
369 offset = offset + len(w)
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
370 words[w] = len(lexicon_offset)
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 16080
diff changeset
371 lexicon_offset.append(o)
16080
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
372
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
373 lexicon = map(ord, lexicon)
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
374
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
375 # generate phrasebook from names and lexicon
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
376 phrasebook = [0]
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
377 phrasebook_offset = [0] * len(unicode.chars)
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
378 for char in unicode.chars:
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
379 name = names[char]
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
380 if name:
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
381 w = name.split()
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
382 phrasebook_offset[char] = len(phrasebook)
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
383 for w in w:
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
384 i = words[w]
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 16080
diff changeset
385 if i < short:
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 16080
diff changeset
386 phrasebook.append(i)
16080
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
387 else:
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 16080
diff changeset
388 # store as two bytes
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 16080
diff changeset
389 phrasebook.append((i>>8) + short)
16080
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
390 phrasebook.append(i&255)
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
391
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 16080
diff changeset
392 assert getsize(phrasebook) == 1
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 16080
diff changeset
393
16080
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
394 #
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
395 # unicode name hash table
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
396
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
397 # extract names
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
398 data = []
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
399 for char in unicode.chars:
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
400 record = unicode.table[char]
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
401 if record:
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
402 name = record[1].strip()
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
403 if name and name[0] != "<":
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
404 data.append((name, char))
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
405
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
406 # the magic number 47 was chosen to minimize the number of
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
407 # collisions on the current data set. if you like, change it
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
408 # and see what happens...
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
409
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
410 codehash = Hash("code", data, 47)
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
411
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
412 print "--- Writing", FILE, "..."
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
413
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
414 fp = open(FILE, "w")
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
415 print >>fp, "/* this file was generated by %s %s */" % (SCRIPT, VERSION)
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
416 print >>fp
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
417 print >>fp, "#define NAME_MAXLEN", 256
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
418 print >>fp
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
419 print >>fp, "/* lexicon */"
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 16080
diff changeset
420 Array("lexicon", lexicon).dump(fp, trace)
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 16080
diff changeset
421 Array("lexicon_offset", lexicon_offset).dump(fp, trace)
16080
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
422
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
423 # split decomposition index table
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
424 offset1, offset2, shift = splitbins(phrasebook_offset, trace)
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
425
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
426 print >>fp, "/* code->name phrasebook */"
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
427 print >>fp, "#define phrasebook_shift", shift
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 16080
diff changeset
428 print >>fp, "#define phrasebook_short", short
16080
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
429
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 16080
diff changeset
430 Array("phrasebook", phrasebook).dump(fp, trace)
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 16080
diff changeset
431 Array("phrasebook_offset1", offset1).dump(fp, trace)
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 16080
diff changeset
432 Array("phrasebook_offset2", offset2).dump(fp, trace)
16080
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
433
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
434 print >>fp, "/* name->code dictionary */"
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 16080
diff changeset
435 codehash.dump(fp, trace)
16080
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
436
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
437 fp.close()
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
438
14825
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
439 # --------------------------------------------------------------------
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
440 # the following support code is taken from the unidb utilities
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
441 # Copyright (c) 1999-2000 by Secret Labs AB
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
442
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
443 # load a unicode-data file from disk
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
444
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
445 import string, sys
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
446
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
447 class UnicodeData:
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
448
15376
493ca9e4260e Added 38,642 missing characters to the Unicode database (first-last
Fredrik Lundh <fredrik@pythonware.com>
parents: 15339
diff changeset
449 def __init__(self, filename, expand=1):
14825
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
450 file = open(filename)
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
451 table = [None] * 65536
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
452 while 1:
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
453 s = file.readline()
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
454 if not s:
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
455 break
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
456 s = string.split(string.strip(s), ";")
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
457 char = string.atoi(s[0], 16)
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
458 table[char] = s
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
459
15376
493ca9e4260e Added 38,642 missing characters to the Unicode database (first-last
Fredrik Lundh <fredrik@pythonware.com>
parents: 15339
diff changeset
460 # expand first-last ranges (ignore surrogates and private use)
493ca9e4260e Added 38,642 missing characters to the Unicode database (first-last
Fredrik Lundh <fredrik@pythonware.com>
parents: 15339
diff changeset
461 if expand:
493ca9e4260e Added 38,642 missing characters to the Unicode database (first-last
Fredrik Lundh <fredrik@pythonware.com>
parents: 15339
diff changeset
462 field = None
493ca9e4260e Added 38,642 missing characters to the Unicode database (first-last
Fredrik Lundh <fredrik@pythonware.com>
parents: 15339
diff changeset
463 for i in range(0, 0xD800):
493ca9e4260e Added 38,642 missing characters to the Unicode database (first-last
Fredrik Lundh <fredrik@pythonware.com>
parents: 15339
diff changeset
464 s = table[i]
493ca9e4260e Added 38,642 missing characters to the Unicode database (first-last
Fredrik Lundh <fredrik@pythonware.com>
parents: 15339
diff changeset
465 if s:
493ca9e4260e Added 38,642 missing characters to the Unicode database (first-last
Fredrik Lundh <fredrik@pythonware.com>
parents: 15339
diff changeset
466 if s[1][-6:] == "First>":
493ca9e4260e Added 38,642 missing characters to the Unicode database (first-last
Fredrik Lundh <fredrik@pythonware.com>
parents: 15339
diff changeset
467 s[1] = ""
493ca9e4260e Added 38,642 missing characters to the Unicode database (first-last
Fredrik Lundh <fredrik@pythonware.com>
parents: 15339
diff changeset
468 field = s[:]
493ca9e4260e Added 38,642 missing characters to the Unicode database (first-last
Fredrik Lundh <fredrik@pythonware.com>
parents: 15339
diff changeset
469 elif s[1][-5:] == "Last>":
493ca9e4260e Added 38,642 missing characters to the Unicode database (first-last
Fredrik Lundh <fredrik@pythonware.com>
parents: 15339
diff changeset
470 s[1] = ""
493ca9e4260e Added 38,642 missing characters to the Unicode database (first-last
Fredrik Lundh <fredrik@pythonware.com>
parents: 15339
diff changeset
471 field = None
493ca9e4260e Added 38,642 missing characters to the Unicode database (first-last
Fredrik Lundh <fredrik@pythonware.com>
parents: 15339
diff changeset
472 elif field:
493ca9e4260e Added 38,642 missing characters to the Unicode database (first-last
Fredrik Lundh <fredrik@pythonware.com>
parents: 15339
diff changeset
473 field[0] = hex(i)
493ca9e4260e Added 38,642 missing characters to the Unicode database (first-last
Fredrik Lundh <fredrik@pythonware.com>
parents: 15339
diff changeset
474 table[i] = field
493ca9e4260e Added 38,642 missing characters to the Unicode database (first-last
Fredrik Lundh <fredrik@pythonware.com>
parents: 15339
diff changeset
475
14825
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
476 # public attributes
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
477 self.filename = filename
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
478 self.table = table
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
479 self.chars = range(65536) # unicode
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
480
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
481 def uselatin1(self):
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
482 # restrict character range to ISO Latin 1
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
483 self.chars = range(256)
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
484
16080
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
485 # hash table tools
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
486
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
487 # this is a straight-forward reimplementation of Python's built-in
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
488 # dictionary type, using a static data structure, and a custom string
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
489 # hash algorithm.
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
490
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
491 def myhash(s, magic):
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
492 h = 0
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
493 for c in map(ord, string.upper(s)):
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
494 h = (h * magic) + c
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
495 ix = h & 0xff000000
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
496 if ix:
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
497 h = (h ^ ((ix>>24) & 0xff)) & 0x00ffffff
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
498 return h
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
499
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
500 SIZES = [
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
501 (4,3), (8,3), (16,3), (32,5), (64,3), (128,3), (256,29), (512,17),
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
502 (1024,9), (2048,5), (4096,83), (8192,27), (16384,43), (32768,3),
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
503 (65536,45), (131072,9), (262144,39), (524288,39), (1048576,9),
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
504 (2097152,5), (4194304,3), (8388608,33), (16777216,27)
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
505 ]
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
506
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
507 class Hash:
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
508 def __init__(self, name, data, magic):
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
509 # turn a (key, value) list into a static hash table structure
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
510
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
511 # determine table size
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
512 for size, poly in SIZES:
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
513 if size > len(data):
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
514 poly = size + poly
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
515 break
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
516 else:
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
517 raise AssertionError, "ran out of polynominals"
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
518
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
519 print size, "slots in hash table"
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
520
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
521 table = [None] * size
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
522
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
523 mask = size-1
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
524
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
525 n = 0
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
526
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
527 hash = myhash
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
528
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
529 # initialize hash table
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
530 for key, value in data:
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
531 h = hash(key, magic)
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
532 i = (~h) & mask
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
533 v = table[i]
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
534 if v is None:
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
535 table[i] = value
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
536 continue
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
537 incr = (h ^ (h >> 3)) & mask;
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
538 if not incr:
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
539 incr = mask
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
540 while 1:
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
541 n = n + 1
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
542 i = (i + incr) & mask
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
543 v = table[i]
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
544 if v is None:
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
545 table[i] = value
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
546 break
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
547 incr = incr << 1
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
548 if incr > mask:
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
549 incr = incr ^ poly
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
550
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
551 print n, "collisions"
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
552 self.collisions = n
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
553
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
554 for i in range(len(table)):
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
555 if table[i] is None:
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
556 table[i] = 0
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
557
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
558 self.data = Array(name + "_hash", table)
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
559 self.magic = magic
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
560 self.name = name
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
561 self.size = size
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
562 self.poly = poly
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
563
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 16080
diff changeset
564 def dump(self, file, trace):
16080
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
565 # write data to file, as a C array
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 16080
diff changeset
566 self.data.dump(file, trace)
16080
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
567 file.write("#define %s_magic %d\n" % (self.name, self.magic))
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
568 file.write("#define %s_size %d\n" % (self.name, self.size))
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
569 file.write("#define %s_poly %d\n" % (self.name, self.poly))
93f836432c2b forgot to check in the new makeunicodedata.py script
Fredrik Lundh <fredrik@pythonware.com>
parents: 15376
diff changeset
570
14825
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
571 # stuff to deal with arrays of unsigned integers
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
572
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
573 class Array:
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
574
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
575 def __init__(self, name, data):
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
576 self.name = name
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
577 self.data = data
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
578
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 16080
diff changeset
579 def dump(self, file, trace=0):
14825
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
580 # write data to file, as a C array
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
581 size = getsize(self.data)
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 16080
diff changeset
582 if trace:
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 16080
diff changeset
583 print >>sys.stderr, self.name+":", size*len(self.data), "bytes"
14825
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
584 file.write("static ")
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
585 if size == 1:
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
586 file.write("unsigned char")
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
587 elif size == 2:
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
588 file.write("unsigned short")
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
589 else:
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
590 file.write("unsigned int")
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
591 file.write(" " + self.name + "[] = {\n")
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
592 if self.data:
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
593 s = " "
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
594 for item in self.data:
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
595 i = str(item) + ", "
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
596 if len(s) + len(i) > 78:
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
597 file.write(s + "\n")
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
598 s = " " + i
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
599 else:
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
600 s = s + i
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
601 if string.strip(s):
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
602 file.write(s + "\n")
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
603 file.write("};\n\n")
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
604
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
605 def getsize(data):
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
606 # return smallest possible integer size for the given array
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
607 maxdata = max(data)
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
608 if maxdata < 256:
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
609 return 1
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
610 elif maxdata < 65536:
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
611 return 2
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
612 else:
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
613 return 4
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
614
14834
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
615 def splitbins(t, trace=0):
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
616 """t, trace=0 -> (t1, t2, shift). Split a table to save space.
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
617
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
618 t is a sequence of ints. This function can be useful to save space if
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
619 many of the ints are the same. t1 and t2 are lists of ints, and shift
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
620 is an int, chosen to minimize the combined size of t1 and t2 (in C
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
621 code), and where for each i in range(len(t)),
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
622 t[i] == t2[(t1[i >> shift] << shift) + (i & mask)]
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
623 where mask is a bitmask isolating the last "shift" bits.
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
624
15376
493ca9e4260e Added 38,642 missing characters to the Unicode database (first-last
Fredrik Lundh <fredrik@pythonware.com>
parents: 15339
diff changeset
625 If optional arg trace is non-zero (default zero), progress info
493ca9e4260e Added 38,642 missing characters to the Unicode database (first-last
Fredrik Lundh <fredrik@pythonware.com>
parents: 15339
diff changeset
626 is printed to sys.stderr. The higher the value, the more info
493ca9e4260e Added 38,642 missing characters to the Unicode database (first-last
Fredrik Lundh <fredrik@pythonware.com>
parents: 15339
diff changeset
627 you'll get.
14834
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
628 """
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
629
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
630 import sys
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
631 if trace:
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
632 def dump(t1, t2, shift, bytes):
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
633 print >>sys.stderr, "%d+%d bins at shift %d; %d bytes" % (
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
634 len(t1), len(t2), shift, bytes)
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
635 print >>sys.stderr, "Size of original table:", len(t)*getsize(t), \
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
636 "bytes"
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
637 n = len(t)-1 # last valid index
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
638 maxshift = 0 # the most we can shift n and still have something left
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
639 if n > 0:
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
640 while n >> 1:
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
641 n >>= 1
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
642 maxshift += 1
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
643 del n
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
644 bytes = sys.maxint # smallest total size so far
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
645 t = tuple(t) # so slices can be dict keys
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
646 for shift in range(maxshift + 1):
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
647 t1 = []
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
648 t2 = []
14825
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
649 size = 2**shift
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
650 bincache = {}
14834
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
651 for i in range(0, len(t), size):
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
652 bin = t[i:i+size]
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
653 index = bincache.get(bin)
14825
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
654 if index is None:
14834
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
655 index = len(t2)
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
656 bincache[bin] = index
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
657 t2.extend(bin)
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
658 t1.append(index >> shift)
14825
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
659 # determine memory size
14834
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
660 b = len(t1)*getsize(t1) + len(t2)*getsize(t2)
15376
493ca9e4260e Added 38,642 missing characters to the Unicode database (first-last
Fredrik Lundh <fredrik@pythonware.com>
parents: 15339
diff changeset
661 if trace > 1:
14834
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
662 dump(t1, t2, shift, b)
14825
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
663 if b < bytes:
14834
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
664 best = t1, t2, shift
14825
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
665 bytes = b
14834
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
666 t1, t2, shift = best
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
667 if trace:
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
668 print >>sys.stderr, "Best:",
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
669 dump(t1, t2, shift, bytes)
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
670 if __debug__:
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
671 # exhaustively verify that the decomposition is correct
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
672 mask = ~((~0) << shift) # i.e., low-bit mask of shift bits
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
673 for i in xrange(len(t)):
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
674 assert t[i] == t2[(t1[i >> shift] << shift) + (i & mask)]
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
675 return best
14825
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
676
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
677 if __name__ == "__main__":
15376
493ca9e4260e Added 38,642 missing characters to the Unicode database (first-last
Fredrik Lundh <fredrik@pythonware.com>
parents: 15339
diff changeset
678 maketables(1)