Mercurial > cpython-lru_cache-weakref

annotate Tools/unicode/makeunicodedata.py @ 15842:f6cd36498a50 legacy-trunk

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Document extensions to .pth files.
author Martin v. Löwis <martin@v.loewis.de>
date Sat, 13 Jan 2001 09:54:41 +0000
parents 493ca9e4260e
children 93f836432c2b
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
14825
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
1 #
14849
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
2 # (re)generate unicode property and type databases
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
3 #
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
4 # this script converts a unicode 3.0 database file to
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
5 # Modules/unicodedata_db.h and Objects/unicodetype_db.h
14835
96d31a143857 unicode database compression, step 2:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14834
diff changeset
6 #
96d31a143857 unicode database compression, step 2:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14834
diff changeset
7 # history:
96d31a143857 unicode database compression, step 2:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14834
diff changeset
8 # 2000-09-24 fl created (based on bits and pieces from unidb)
96d31a143857 unicode database compression, step 2:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14834
diff changeset
9 # 2000-09-25 fl merged tim's splitbin fixes, separate decomposition table
14849
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
10 # 2000-09-25 fl added character type table
14856
83aff1ea5b70 - don't set the titlecase flag for uppercase letters (sorry, tim)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14852
diff changeset
11 # 2000-09-26 fl added LINEBREAK, DECIMAL, and DIGIT flags/fields
15376
493ca9e4260e Added 38,642 missing characters to the Unicode database (first-last
Fredrik Lundh <fredrik@pythonware.com>
parents: 15339
diff changeset
12 # 2000-11-03 fl expand first/last ranges
14835
96d31a143857 unicode database compression, step 2:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14834
diff changeset
13 #
96d31a143857 unicode database compression, step 2:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14834
diff changeset
14 # written by Fredrik Lundh (fredrik@pythonware.com), September 2000
14825
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
15 #
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
16
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
17 import sys
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
18
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
19 SCRIPT = sys.argv[0]
14835
96d31a143857 unicode database compression, step 2:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14834
diff changeset
20 VERSION = "1.1"
14825
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
21
14849
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
22 UNICODE_DATA = "UnicodeData-Latest.txt"
14825
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
23
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
24 CATEGORY_NAMES = [ "Cn", "Lu", "Ll", "Lt", "Mn", "Mc", "Me", "Nd",
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
25 "Nl", "No", "Zs", "Zl", "Zp", "Cc", "Cf", "Cs", "Co", "Cn", "Lm",
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
26 "Lo", "Pc", "Pd", "Ps", "Pe", "Pi", "Pf", "Po", "Sm", "Sc", "Sk",
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
27 "So" ]
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
28
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
29 BIDIRECTIONAL_NAMES = [ "", "L", "LRE", "LRO", "R", "AL", "RLE", "RLO",
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
30 "PDF", "EN", "ES", "ET", "AN", "CS", "NSM", "BN", "B", "S", "WS",
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
31 "ON" ]
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
32
14852
463faf01685e unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14849
diff changeset
33 # note: should match definitions in Objects/unicodectype.c
14849
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
34 ALPHA_MASK = 0x01
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
35 DECIMAL_MASK = 0x02
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
36 DIGIT_MASK = 0x04
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
37 LOWER_MASK = 0x08
14852
463faf01685e unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14849
diff changeset
38 LINEBREAK_MASK = 0x10
14849
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
39 SPACE_MASK = 0x20
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
40 TITLE_MASK = 0x40
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
41 UPPER_MASK = 0x80
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
42
15376
493ca9e4260e Added 38,642 missing characters to the Unicode database (first-last
Fredrik Lundh <fredrik@pythonware.com>
parents: 15339
diff changeset
43 def maketables(trace=0):
14825
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
44
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
45 unicode = UnicodeData(UNICODE_DATA)
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
46
15376
493ca9e4260e Added 38,642 missing characters to the Unicode database (first-last
Fredrik Lundh <fredrik@pythonware.com>
parents: 15339
diff changeset
47 print "--- Processing", UNICODE_DATA, "..."
493ca9e4260e Added 38,642 missing characters to the Unicode database (first-last
Fredrik Lundh <fredrik@pythonware.com>
parents: 15339
diff changeset
48 print len(filter(None, unicode.table)), "characters"
493ca9e4260e Added 38,642 missing characters to the Unicode database (first-last
Fredrik Lundh <fredrik@pythonware.com>
parents: 15339
diff changeset
49
14825
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
50 # extract unicode properties
14835
96d31a143857 unicode database compression, step 2:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14834
diff changeset
51 dummy = (0, 0, 0, 0)
14825
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
52 table = [dummy]
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
53 cache = {0: dummy}
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
54 index = [0] * len(unicode.chars)
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
55
14835
96d31a143857 unicode database compression, step 2:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14834
diff changeset
56 # 1) database properties
14825
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
57 for char in unicode.chars:
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
58 record = unicode.table[char]
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
59 if record:
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
60 # extract database properties
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
61 category = CATEGORY_NAMES.index(record[2])
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
62 combining = int(record[3])
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
63 bidirectional = BIDIRECTIONAL_NAMES.index(record[4])
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
64 mirrored = record[9] == "Y"
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
65 item = (
14835
96d31a143857 unicode database compression, step 2:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14834
diff changeset
66 category, combining, bidirectional, mirrored
14825
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
67 )
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
68 # add entry to index and item tables
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
69 i = cache.get(item)
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
70 if i is None:
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
71 cache[item] = i = len(table)
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
72 table.append(item)
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
73 index[char] = i
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
74
14835
96d31a143857 unicode database compression, step 2:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14834
diff changeset
75 # 2) decomposition data
96d31a143857 unicode database compression, step 2:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14834
diff changeset
76
96d31a143857 unicode database compression, step 2:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14834
diff changeset
77 # FIXME: <fl> using the encoding stuff from unidb would save
96d31a143857 unicode database compression, step 2:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14834
diff changeset
78 # another 50k or so, but I'll leave that for 2.1...
96d31a143857 unicode database compression, step 2:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14834
diff changeset
79
96d31a143857 unicode database compression, step 2:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14834
diff changeset
80 decomp_data = [""]
96d31a143857 unicode database compression, step 2:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14834
diff changeset
81 decomp_index = [0] * len(unicode.chars)
96d31a143857 unicode database compression, step 2:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14834
diff changeset
82
96d31a143857 unicode database compression, step 2:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14834
diff changeset
83 for char in unicode.chars:
96d31a143857 unicode database compression, step 2:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14834
diff changeset
84 record = unicode.table[char]
96d31a143857 unicode database compression, step 2:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14834
diff changeset
85 if record:
96d31a143857 unicode database compression, step 2:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14834
diff changeset
86 if record[5]:
96d31a143857 unicode database compression, step 2:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14834
diff changeset
87 try:
96d31a143857 unicode database compression, step 2:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14834
diff changeset
88 i = decomp_data.index(record[5])
96d31a143857 unicode database compression, step 2:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14834
diff changeset
89 except ValueError:
96d31a143857 unicode database compression, step 2:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14834
diff changeset
90 i = len(decomp_data)
96d31a143857 unicode database compression, step 2:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14834
diff changeset
91 decomp_data.append(record[5])
96d31a143857 unicode database compression, step 2:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14834
diff changeset
92 else:
96d31a143857 unicode database compression, step 2:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14834
diff changeset
93 i = 0
96d31a143857 unicode database compression, step 2:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14834
diff changeset
94 decomp_index[char] = i
14825
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
95
14849
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
96 FILE = "Modules/unicodedata_db.h"
14825
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
97
15376
493ca9e4260e Added 38,642 missing characters to the Unicode database (first-last
Fredrik Lundh <fredrik@pythonware.com>
parents: 15339
diff changeset
98 print "--- Writing", FILE, "..."
493ca9e4260e Added 38,642 missing characters to the Unicode database (first-last
Fredrik Lundh <fredrik@pythonware.com>
parents: 15339
diff changeset
99
493ca9e4260e Added 38,642 missing characters to the Unicode database (first-last
Fredrik Lundh <fredrik@pythonware.com>
parents: 15339
diff changeset
100 print len(table), "unique properties"
493ca9e4260e Added 38,642 missing characters to the Unicode database (first-last
Fredrik Lundh <fredrik@pythonware.com>
parents: 15339
diff changeset
101 print len(decomp_data), "unique decomposition entries"
493ca9e4260e Added 38,642 missing characters to the Unicode database (first-last
Fredrik Lundh <fredrik@pythonware.com>
parents: 15339
diff changeset
102
15339
c2d7451b42f6 Remove bogus stdout redirection and use of sys.__stdout__; use
Fred Drake <fdrake@acm.org>
parents: 14856
diff changeset
103 fp = open(FILE, "w")
c2d7451b42f6 Remove bogus stdout redirection and use of sys.__stdout__; use
Fred Drake <fdrake@acm.org>
parents: 14856
diff changeset
104 print >>fp, "/* this file was generated by %s %s */" % (SCRIPT, VERSION)
c2d7451b42f6 Remove bogus stdout redirection and use of sys.__stdout__; use
Fred Drake <fdrake@acm.org>
parents: 14856
diff changeset
105 print >>fp
c2d7451b42f6 Remove bogus stdout redirection and use of sys.__stdout__; use
Fred Drake <fdrake@acm.org>
parents: 14856
diff changeset
106 print >>fp, "/* a list of unique database records */"
c2d7451b42f6 Remove bogus stdout redirection and use of sys.__stdout__; use
Fred Drake <fdrake@acm.org>
parents: 14856
diff changeset
107 print >>fp, \
c2d7451b42f6 Remove bogus stdout redirection and use of sys.__stdout__; use
Fred Drake <fdrake@acm.org>
parents: 14856
diff changeset
108 "const _PyUnicode_DatabaseRecord _PyUnicode_Database_Records[] = {"
14825
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
109 for item in table:
15339
c2d7451b42f6 Remove bogus stdout redirection and use of sys.__stdout__; use
Fred Drake <fdrake@acm.org>
parents: 14856
diff changeset
110 print >>fp, " {%d, %d, %d, %d}," % item
c2d7451b42f6 Remove bogus stdout redirection and use of sys.__stdout__; use
Fred Drake <fdrake@acm.org>
parents: 14856
diff changeset
111 print >>fp, "};"
c2d7451b42f6 Remove bogus stdout redirection and use of sys.__stdout__; use
Fred Drake <fdrake@acm.org>
parents: 14856
diff changeset
112 print >>fp
14825
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
113
14849
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
114 # FIXME: the following tables should be made static, and
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
115 # the support code moved into unicodedatabase.c
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
116
15339
c2d7451b42f6 Remove bogus stdout redirection and use of sys.__stdout__; use
Fred Drake <fdrake@acm.org>
parents: 14856
diff changeset
117 print >>fp, "/* string literals */"
c2d7451b42f6 Remove bogus stdout redirection and use of sys.__stdout__; use
Fred Drake <fdrake@acm.org>
parents: 14856
diff changeset
118 print >>fp, "const char *_PyUnicode_CategoryNames[] = {"
14825
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
119 for name in CATEGORY_NAMES:
15339
c2d7451b42f6 Remove bogus stdout redirection and use of sys.__stdout__; use
Fred Drake <fdrake@acm.org>
parents: 14856
diff changeset
120 print >>fp, " \"%s\"," % name
c2d7451b42f6 Remove bogus stdout redirection and use of sys.__stdout__; use
Fred Drake <fdrake@acm.org>
parents: 14856
diff changeset
121 print >>fp, " NULL"
c2d7451b42f6 Remove bogus stdout redirection and use of sys.__stdout__; use
Fred Drake <fdrake@acm.org>
parents: 14856
diff changeset
122 print >>fp, "};"
14825
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
123
15339
c2d7451b42f6 Remove bogus stdout redirection and use of sys.__stdout__; use
Fred Drake <fdrake@acm.org>
parents: 14856
diff changeset
124 print >>fp, "const char *_PyUnicode_BidirectionalNames[] = {"
14825
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
125 for name in BIDIRECTIONAL_NAMES:
15339
c2d7451b42f6 Remove bogus stdout redirection and use of sys.__stdout__; use
Fred Drake <fdrake@acm.org>
parents: 14856
diff changeset
126 print >>fp, " \"%s\"," % name
c2d7451b42f6 Remove bogus stdout redirection and use of sys.__stdout__; use
Fred Drake <fdrake@acm.org>
parents: 14856
diff changeset
127 print >>fp, " NULL"
c2d7451b42f6 Remove bogus stdout redirection and use of sys.__stdout__; use
Fred Drake <fdrake@acm.org>
parents: 14856
diff changeset
128 print >>fp, "};"
14825
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
129
15339
c2d7451b42f6 Remove bogus stdout redirection and use of sys.__stdout__; use
Fred Drake <fdrake@acm.org>
parents: 14856
diff changeset
130 print >>fp, "static const char *decomp_data[] = {"
14835
96d31a143857 unicode database compression, step 2:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14834
diff changeset
131 for name in decomp_data:
15339
c2d7451b42f6 Remove bogus stdout redirection and use of sys.__stdout__; use
Fred Drake <fdrake@acm.org>
parents: 14856
diff changeset
132 print >>fp, " \"%s\"," % name
c2d7451b42f6 Remove bogus stdout redirection and use of sys.__stdout__; use
Fred Drake <fdrake@acm.org>
parents: 14856
diff changeset
133 print >>fp, " NULL"
c2d7451b42f6 Remove bogus stdout redirection and use of sys.__stdout__; use
Fred Drake <fdrake@acm.org>
parents: 14856
diff changeset
134 print >>fp, "};"
14835
96d31a143857 unicode database compression, step 2:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14834
diff changeset
135
14849
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
136 # split record index table
15376
493ca9e4260e Added 38,642 missing characters to the Unicode database (first-last
Fredrik Lundh <fredrik@pythonware.com>
parents: 15339
diff changeset
137 index1, index2, shift = splitbins(index, trace)
14825
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
138
15339
c2d7451b42f6 Remove bogus stdout redirection and use of sys.__stdout__; use
Fred Drake <fdrake@acm.org>
parents: 14856
diff changeset
139 print >>fp, "/* index tables for the database records */"
c2d7451b42f6 Remove bogus stdout redirection and use of sys.__stdout__; use
Fred Drake <fdrake@acm.org>
parents: 14856
diff changeset
140 print >>fp, "#define SHIFT", shift
c2d7451b42f6 Remove bogus stdout redirection and use of sys.__stdout__; use
Fred Drake <fdrake@acm.org>
parents: 14856
diff changeset
141 Array("index1", index1).dump(fp)
c2d7451b42f6 Remove bogus stdout redirection and use of sys.__stdout__; use
Fred Drake <fdrake@acm.org>
parents: 14856
diff changeset
142 Array("index2", index2).dump(fp)
14825
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
143
14849
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
144 # split decomposition index table
15376
493ca9e4260e Added 38,642 missing characters to the Unicode database (first-last
Fredrik Lundh <fredrik@pythonware.com>
parents: 15339
diff changeset
145 index1, index2, shift = splitbins(decomp_index, trace)
14835
96d31a143857 unicode database compression, step 2:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14834
diff changeset
146
15339
c2d7451b42f6 Remove bogus stdout redirection and use of sys.__stdout__; use
Fred Drake <fdrake@acm.org>
parents: 14856
diff changeset
147 print >>fp, "/* index tables for the decomposition data */"
c2d7451b42f6 Remove bogus stdout redirection and use of sys.__stdout__; use
Fred Drake <fdrake@acm.org>
parents: 14856
diff changeset
148 print >>fp, "#define DECOMP_SHIFT", shift
c2d7451b42f6 Remove bogus stdout redirection and use of sys.__stdout__; use
Fred Drake <fdrake@acm.org>
parents: 14856
diff changeset
149 Array("decomp_index1", index1).dump(fp)
c2d7451b42f6 Remove bogus stdout redirection and use of sys.__stdout__; use
Fred Drake <fdrake@acm.org>
parents: 14856
diff changeset
150 Array("decomp_index2", index2).dump(fp)
14825
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
151
14849
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
152 #
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
153 # 3) unicode type data
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
154
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
155 # extract unicode types
14852
463faf01685e unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14849
diff changeset
156 dummy = (0, 0, 0, 0, 0, 0)
14849
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
157 table = [dummy]
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
158 cache = {0: dummy}
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
159 index = [0] * len(unicode.chars)
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
160
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
161 for char in unicode.chars:
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
162 record = unicode.table[char]
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
163 if record:
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
164 # extract database properties
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
165 category = record[2]
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
166 bidirectional = record[4]
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
167 flags = 0
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
168 if category in ["Lm", "Lt", "Lu", "Ll", "Lo"]:
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
169 flags |= ALPHA_MASK
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
170 if category == "Ll":
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
171 flags |= LOWER_MASK
14852
463faf01685e unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14849
diff changeset
172 if category == "Zl" or bidirectional == "B":
463faf01685e unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14849
diff changeset
173 flags |= LINEBREAK_MASK
14849
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
174 if category == "Zs" or bidirectional in ("WS", "B", "S"):
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
175 flags |= SPACE_MASK
14856
83aff1ea5b70 - don't set the titlecase flag for uppercase letters (sorry, tim)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14852
diff changeset
176 if category == "Lt":
14849
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
177 flags |= TITLE_MASK
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
178 if category == "Lu":
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
179 flags |= UPPER_MASK
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
180 # use delta predictor for upper/lower/title
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
181 if record[12]:
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
182 upper = (int(record[12], 16) - char) & 0xffff
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
183 else:
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
184 upper = 0
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
185 if record[13]:
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
186 lower = (int(record[13], 16) - char) & 0xffff
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
187 else:
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
188 lower = 0
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
189 if record[14]:
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
190 title = (int(record[14], 16) - char) & 0xffff
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
191 else:
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
192 title = 0
14852
463faf01685e unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14849
diff changeset
193 # decimal digit, integer digit
463faf01685e unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14849
diff changeset
194 decimal = 0
463faf01685e unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14849
diff changeset
195 if record[6]:
463faf01685e unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14849
diff changeset
196 flags |= DECIMAL_MASK
463faf01685e unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14849
diff changeset
197 decimal = int(record[6])
463faf01685e unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14849
diff changeset
198 digit = 0
463faf01685e unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14849
diff changeset
199 if record[7]:
463faf01685e unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14849
diff changeset
200 flags |= DIGIT_MASK
463faf01685e unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14849
diff changeset
201 digit = int(record[7])
14849
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
202 item = (
14852
463faf01685e unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14849
diff changeset
203 flags, upper, lower, title, decimal, digit
14849
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
204 )
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
205 # add entry to index and item tables
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
206 i = cache.get(item)
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
207 if i is None:
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
208 cache[item] = i = len(table)
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
209 table.append(item)
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
210 index[char] = i
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
211
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
212 FILE = "Objects/unicodetype_db.h"
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
213
15339
c2d7451b42f6 Remove bogus stdout redirection and use of sys.__stdout__; use
Fred Drake <fdrake@acm.org>
parents: 14856
diff changeset
214 fp = open(FILE, "w")
14849
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
215
15376
493ca9e4260e Added 38,642 missing characters to the Unicode database (first-last
Fredrik Lundh <fredrik@pythonware.com>
parents: 15339
diff changeset
216 print "--- Writing", FILE, "..."
493ca9e4260e Added 38,642 missing characters to the Unicode database (first-last
Fredrik Lundh <fredrik@pythonware.com>
parents: 15339
diff changeset
217
493ca9e4260e Added 38,642 missing characters to the Unicode database (first-last
Fredrik Lundh <fredrik@pythonware.com>
parents: 15339
diff changeset
218 print len(table), "unique character type entries"
493ca9e4260e Added 38,642 missing characters to the Unicode database (first-last
Fredrik Lundh <fredrik@pythonware.com>
parents: 15339
diff changeset
219
15339
c2d7451b42f6 Remove bogus stdout redirection and use of sys.__stdout__; use
Fred Drake <fdrake@acm.org>
parents: 14856
diff changeset
220 print >>fp, "/* this file was generated by %s %s */" % (SCRIPT, VERSION)
c2d7451b42f6 Remove bogus stdout redirection and use of sys.__stdout__; use
Fred Drake <fdrake@acm.org>
parents: 14856
diff changeset
221 print >>fp
c2d7451b42f6 Remove bogus stdout redirection and use of sys.__stdout__; use
Fred Drake <fdrake@acm.org>
parents: 14856
diff changeset
222 print >>fp, "/* a list of unique character type descriptors */"
c2d7451b42f6 Remove bogus stdout redirection and use of sys.__stdout__; use
Fred Drake <fdrake@acm.org>
parents: 14856
diff changeset
223 print >>fp, "const _PyUnicode_TypeRecord _PyUnicode_TypeRecords[] = {"
14849
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
224 for item in table:
15339
c2d7451b42f6 Remove bogus stdout redirection and use of sys.__stdout__; use
Fred Drake <fdrake@acm.org>
parents: 14856
diff changeset
225 print >>fp, " {%d, %d, %d, %d, %d, %d}," % item
c2d7451b42f6 Remove bogus stdout redirection and use of sys.__stdout__; use
Fred Drake <fdrake@acm.org>
parents: 14856
diff changeset
226 print >>fp, "};"
c2d7451b42f6 Remove bogus stdout redirection and use of sys.__stdout__; use
Fred Drake <fdrake@acm.org>
parents: 14856
diff changeset
227 print >>fp
14849
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
228
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
229 # split decomposition index table
15376
493ca9e4260e Added 38,642 missing characters to the Unicode database (first-last
Fredrik Lundh <fredrik@pythonware.com>
parents: 15339
diff changeset
230 index1, index2, shift = splitbins(index, trace)
14849
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
231
15339
c2d7451b42f6 Remove bogus stdout redirection and use of sys.__stdout__; use
Fred Drake <fdrake@acm.org>
parents: 14856
diff changeset
232 print >>fp, "/* type indexes */"
c2d7451b42f6 Remove bogus stdout redirection and use of sys.__stdout__; use
Fred Drake <fdrake@acm.org>
parents: 14856
diff changeset
233 print >>fp, "#define SHIFT", shift
c2d7451b42f6 Remove bogus stdout redirection and use of sys.__stdout__; use
Fred Drake <fdrake@acm.org>
parents: 14856
diff changeset
234 Array("index1", index1).dump(fp)
c2d7451b42f6 Remove bogus stdout redirection and use of sys.__stdout__; use
Fred Drake <fdrake@acm.org>
parents: 14856
diff changeset
235 Array("index2", index2).dump(fp)
14849
92cdbc0f55d5 unicode database compression, step 3:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
236
14825
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
237 # --------------------------------------------------------------------
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
238 # the following support code is taken from the unidb utilities
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
239 # Copyright (c) 1999-2000 by Secret Labs AB
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
240
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
241 # load a unicode-data file from disk
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
242
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
243 import string, sys
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
244
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
245 class UnicodeData:
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
246
15376
493ca9e4260e Added 38,642 missing characters to the Unicode database (first-last
Fredrik Lundh <fredrik@pythonware.com>
parents: 15339
diff changeset
247 def __init__(self, filename, expand=1):
14825
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
248 file = open(filename)
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
249 table = [None] * 65536
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
250 while 1:
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
251 s = file.readline()
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
252 if not s:
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
253 break
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
254 s = string.split(string.strip(s), ";")
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
255 char = string.atoi(s[0], 16)
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
256 table[char] = s
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
257
15376
493ca9e4260e Added 38,642 missing characters to the Unicode database (first-last
Fredrik Lundh <fredrik@pythonware.com>
parents: 15339
diff changeset
258 # expand first-last ranges (ignore surrogates and private use)
493ca9e4260e Added 38,642 missing characters to the Unicode database (first-last
Fredrik Lundh <fredrik@pythonware.com>
parents: 15339
diff changeset
259 if expand:
493ca9e4260e Added 38,642 missing characters to the Unicode database (first-last
Fredrik Lundh <fredrik@pythonware.com>
parents: 15339
diff changeset
260 field = None
493ca9e4260e Added 38,642 missing characters to the Unicode database (first-last
Fredrik Lundh <fredrik@pythonware.com>
parents: 15339
diff changeset
261 for i in range(0, 0xD800):
493ca9e4260e Added 38,642 missing characters to the Unicode database (first-last
Fredrik Lundh <fredrik@pythonware.com>
parents: 15339
diff changeset
262 s = table[i]
493ca9e4260e Added 38,642 missing characters to the Unicode database (first-last
Fredrik Lundh <fredrik@pythonware.com>
parents: 15339
diff changeset
263 if s:
493ca9e4260e Added 38,642 missing characters to the Unicode database (first-last
Fredrik Lundh <fredrik@pythonware.com>
parents: 15339
diff changeset
264 if s[1][-6:] == "First>":
493ca9e4260e Added 38,642 missing characters to the Unicode database (first-last
Fredrik Lundh <fredrik@pythonware.com>
parents: 15339
diff changeset
265 s[1] = ""
493ca9e4260e Added 38,642 missing characters to the Unicode database (first-last
Fredrik Lundh <fredrik@pythonware.com>
parents: 15339
diff changeset
266 field = s[:]
493ca9e4260e Added 38,642 missing characters to the Unicode database (first-last
Fredrik Lundh <fredrik@pythonware.com>
parents: 15339
diff changeset
267 elif s[1][-5:] == "Last>":
493ca9e4260e Added 38,642 missing characters to the Unicode database (first-last
Fredrik Lundh <fredrik@pythonware.com>
parents: 15339
diff changeset
268 s[1] = ""
493ca9e4260e Added 38,642 missing characters to the Unicode database (first-last
Fredrik Lundh <fredrik@pythonware.com>
parents: 15339
diff changeset
269 field = None
493ca9e4260e Added 38,642 missing characters to the Unicode database (first-last
Fredrik Lundh <fredrik@pythonware.com>
parents: 15339
diff changeset
270 elif field:
493ca9e4260e Added 38,642 missing characters to the Unicode database (first-last
Fredrik Lundh <fredrik@pythonware.com>
parents: 15339
diff changeset
271 field[0] = hex(i)
493ca9e4260e Added 38,642 missing characters to the Unicode database (first-last
Fredrik Lundh <fredrik@pythonware.com>
parents: 15339
diff changeset
272 table[i] = field
493ca9e4260e Added 38,642 missing characters to the Unicode database (first-last
Fredrik Lundh <fredrik@pythonware.com>
parents: 15339
diff changeset
273
14825
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
274 # public attributes
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
275 self.filename = filename
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
276 self.table = table
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
277 self.chars = range(65536) # unicode
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
278
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
279 def uselatin1(self):
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
280 # restrict character range to ISO Latin 1
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
281 self.chars = range(256)
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
282
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
283 # stuff to deal with arrays of unsigned integers
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
284
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
285 class Array:
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
286
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
287 def __init__(self, name, data):
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
288 self.name = name
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
289 self.data = data
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
290
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
291 def dump(self, file):
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
292 # write data to file, as a C array
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
293 size = getsize(self.data)
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
294 # print >>sys.stderr, self.name+":", size*len(self.data), "bytes"
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
295 file.write("static ")
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
296 if size == 1:
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
297 file.write("unsigned char")
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
298 elif size == 2:
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
299 file.write("unsigned short")
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
300 else:
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
301 file.write("unsigned int")
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
302 file.write(" " + self.name + "[] = {\n")
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
303 if self.data:
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
304 s = " "
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
305 for item in self.data:
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
306 i = str(item) + ", "
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
307 if len(s) + len(i) > 78:
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
308 file.write(s + "\n")
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
309 s = " " + i
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
310 else:
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
311 s = s + i
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
312 if string.strip(s):
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
313 file.write(s + "\n")
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
314 file.write("};\n\n")
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
315
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
316 def getsize(data):
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
317 # return smallest possible integer size for the given array
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
318 maxdata = max(data)
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
319 if maxdata < 256:
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
320 return 1
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
321 elif maxdata < 65536:
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
322 return 2
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
323 else:
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
324 return 4
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
325
14834
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
326 def splitbins(t, trace=0):
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
327 """t, trace=0 -> (t1, t2, shift). Split a table to save space.
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
328
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
329 t is a sequence of ints. This function can be useful to save space if
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
330 many of the ints are the same. t1 and t2 are lists of ints, and shift
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
331 is an int, chosen to minimize the combined size of t1 and t2 (in C
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
332 code), and where for each i in range(len(t)),
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
333 t[i] == t2[(t1[i >> shift] << shift) + (i & mask)]
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
334 where mask is a bitmask isolating the last "shift" bits.
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
335
15376
493ca9e4260e Added 38,642 missing characters to the Unicode database (first-last
Fredrik Lundh <fredrik@pythonware.com>
parents: 15339
diff changeset
336 If optional arg trace is non-zero (default zero), progress info
493ca9e4260e Added 38,642 missing characters to the Unicode database (first-last
Fredrik Lundh <fredrik@pythonware.com>
parents: 15339
diff changeset
337 is printed to sys.stderr. The higher the value, the more info
493ca9e4260e Added 38,642 missing characters to the Unicode database (first-last
Fredrik Lundh <fredrik@pythonware.com>
parents: 15339
diff changeset
338 you'll get.
14834
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
339 """
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
340
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
341 import sys
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
342 if trace:
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
343 def dump(t1, t2, shift, bytes):
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
344 print >>sys.stderr, "%d+%d bins at shift %d; %d bytes" % (
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
345 len(t1), len(t2), shift, bytes)
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
346 print >>sys.stderr, "Size of original table:", len(t)*getsize(t), \
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
347 "bytes"
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
348 n = len(t)-1 # last valid index
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
349 maxshift = 0 # the most we can shift n and still have something left
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
350 if n > 0:
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
351 while n >> 1:
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
352 n >>= 1
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
353 maxshift += 1
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
354 del n
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
355 bytes = sys.maxint # smallest total size so far
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
356 t = tuple(t) # so slices can be dict keys
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
357 for shift in range(maxshift + 1):
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
358 t1 = []
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
359 t2 = []
14825
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
360 size = 2**shift
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
361 bincache = {}
14834
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
362 for i in range(0, len(t), size):
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
363 bin = t[i:i+size]
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
364 index = bincache.get(bin)
14825
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
365 if index is None:
14834
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
366 index = len(t2)
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
367 bincache[bin] = index
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
368 t2.extend(bin)
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
369 t1.append(index >> shift)
14825
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
370 # determine memory size
14834
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
371 b = len(t1)*getsize(t1) + len(t2)*getsize(t2)
15376
493ca9e4260e Added 38,642 missing characters to the Unicode database (first-last
Fredrik Lundh <fredrik@pythonware.com>
parents: 15339
diff changeset
372 if trace > 1:
14834
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
373 dump(t1, t2, shift, b)
14825
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
374 if b < bytes:
14834
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
375 best = t1, t2, shift
14825
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
376 bytes = b
14834
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
377 t1, t2, shift = best
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
378 if trace:
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
379 print >>sys.stderr, "Best:",
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
380 dump(t1, t2, shift, bytes)
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
381 if __debug__:
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
382 # exhaustively verify that the decomposition is correct
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
383 mask = ~((~0) << shift) # i.e., low-bit mask of shift bits
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
384 for i in xrange(len(t)):
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
385 assert t[i] == t2[(t1[i >> shift] << shift) + (i & mask)]
1d627242e29a Fiddled w/ /F's cool new splitbins function: documented it, generalized it
Tim Peters <tim.peters@gmail.com>
parents: 14825
diff changeset
386 return best
14825
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
387
9fd030af2fe0 unicode database compression, step 1:
Fredrik Lundh <fredrik@pythonware.com>
parents:
diff changeset
388 if __name__ == "__main__":
15376
493ca9e4260e Added 38,642 missing characters to the Unicode database (first-last
Fredrik Lundh <fredrik@pythonware.com>
parents: 15339
diff changeset
389 maketables(1)