Mercurial > cpython-lru_cache-weakref
annotate Tools/scripts/gencodec.py @ 36392:2012c548ce69 2.4
Correct misspellings.
| author | Georg Brandl <georg@python.org> |
|---|---|
| date | Mon, 23 Jan 2006 22:00:32 +0000 |
| parents | 9267aeb72985 |
| children |
| rev | line source |
|---|---|
| 11966 | 1 """ Unicode Mapping Parser and Codec Generator. |
| 2 | |
| 3 This script parses Unicode mapping files as available from the Unicode | |
|
15706
bf9bb0e3dd34
This patch changes the default behaviour of the builtin charmap
Marc-André Lemburg <mal@egenix.com>
parents:
12014
diff
changeset
|
4 site (ftp://ftp.unicode.org/Public/MAPPINGS/) and creates Python codec |
|
bf9bb0e3dd34
This patch changes the default behaviour of the builtin charmap
Marc-André Lemburg <mal@egenix.com>
parents:
12014
diff
changeset
|
5 modules from them. The codecs use the standard character mapping codec |
|
bf9bb0e3dd34
This patch changes the default behaviour of the builtin charmap
Marc-André Lemburg <mal@egenix.com>
parents:
12014
diff
changeset
|
6 to actually apply the mapping. |
| 11966 | 7 |
| 8 Synopsis: gencodec.py dir codec_prefix | |
| 9 | |
| 10 All files in dir are scanned and those producing non-empty mappings | |
| 11 will be written to <codec_prefix><mapname>.py with <mapname> being the | |
| 12 first part of the map's filename ('a' in a.b.c.txt) converted to | |
| 13 lowercase with hyphens replaced by underscores. | |
| 14 | |
|
12014
ba80c9064491
Of course, I forget how to type long ago...
Fred Drake <fdrake@acm.org>
parents:
12012
diff
changeset
|
15 The tool also writes marshalled versions of the mapping tables to the |
| 11966 | 16 same location (with .mapping extension). |
| 17 | |
| 18 Written by Marc-Andre Lemburg (mal@lemburg.com). | |
| 19 | |
| 20 (c) Copyright CNRI, All Rights Reserved. NO WARRANTY. | |
|
15706
bf9bb0e3dd34
This patch changes the default behaviour of the builtin charmap
Marc-André Lemburg <mal@egenix.com>
parents:
12014
diff
changeset
|
21 (c) Copyright Guido van Rossum, 2000. |
| 11966 | 22 |
| 23 """#" | |
| 24 | |
|
25219
8a7a2f1f88f0
Apply diff2.txt from SF patch http://www.python.org/sf/572113
Walter Dörwald <walter@livinglogic.de>
parents:
17843
diff
changeset
|
25 import re,os,time,marshal |
| 11966 | 26 |
| 27 # Create numeric tables or character based ones ? | |
| 28 numeric = 1 | |
| 29 | |
| 30 mapRE = re.compile('((?:0x[0-9a-fA-F]+\+?)+)' | |
| 31 '\s+' | |
| 32 '((?:(?:0x[0-9a-fA-Z]+|<[A-Za-z]+>)\+?)*)' | |
| 33 '\s*' | |
| 34 '(#.+)?') | |
| 35 | |
| 36 def parsecodes(codes, | |
|
25219
8a7a2f1f88f0
Apply diff2.txt from SF patch http://www.python.org/sf/572113
Walter Dörwald <walter@livinglogic.de>
parents:
17843
diff
changeset
|
37 len=len, filter=filter,range=range): |
| 11966 | 38 |
| 39 """ Converts code combinations to either a single code integer | |
| 40 or a tuple of integers. | |
| 41 | |
| 42 meta-codes (in angular brackets, e.g. <LR> and <RL>) are | |
| 43 ignored. | |
| 44 | |
| 45 Empty codes or illegal ones are returned as None. | |
| 46 | |
| 47 """ | |
| 48 if not codes: | |
| 49 return None | |
|
25219
8a7a2f1f88f0
Apply diff2.txt from SF patch http://www.python.org/sf/572113
Walter Dörwald <walter@livinglogic.de>
parents:
17843
diff
changeset
|
50 l = codes.split('+') |
| 11966 | 51 if len(l) == 1: |
|
25219
8a7a2f1f88f0
Apply diff2.txt from SF patch http://www.python.org/sf/572113
Walter Dörwald <walter@livinglogic.de>
parents:
17843
diff
changeset
|
52 return int(l[0],16) |
| 11966 | 53 for i in range(len(l)): |
| 54 try: | |
|
25219
8a7a2f1f88f0
Apply diff2.txt from SF patch http://www.python.org/sf/572113
Walter Dörwald <walter@livinglogic.de>
parents:
17843
diff
changeset
|
55 l[i] = int(l[i],16) |
| 11966 | 56 except ValueError: |
| 57 l[i] = None | |
| 58 l = filter(lambda x: x is not None, l) | |
| 59 if len(l) == 1: | |
| 60 return l[0] | |
| 61 else: | |
| 62 return tuple(l) | |
| 63 | |
|
25219
8a7a2f1f88f0
Apply diff2.txt from SF patch http://www.python.org/sf/572113
Walter Dörwald <walter@livinglogic.de>
parents:
17843
diff
changeset
|
64 def readmap(filename): |
| 11966 | 65 |
| 66 f = open(filename,'r') | |
| 67 lines = f.readlines() | |
| 68 f.close() | |
| 69 enc2uni = {} | |
|
15706
bf9bb0e3dd34
This patch changes the default behaviour of the builtin charmap
Marc-André Lemburg <mal@egenix.com>
parents:
12014
diff
changeset
|
70 identity = [] |
|
bf9bb0e3dd34
This patch changes the default behaviour of the builtin charmap
Marc-André Lemburg <mal@egenix.com>
parents:
12014
diff
changeset
|
71 unmapped = range(256) |
|
bf9bb0e3dd34
This patch changes the default behaviour of the builtin charmap
Marc-André Lemburg <mal@egenix.com>
parents:
12014
diff
changeset
|
72 for i in range(256): |
|
bf9bb0e3dd34
This patch changes the default behaviour of the builtin charmap
Marc-André Lemburg <mal@egenix.com>
parents:
12014
diff
changeset
|
73 unmapped[i] = i |
| 11966 | 74 for line in lines: |
|
25219
8a7a2f1f88f0
Apply diff2.txt from SF patch http://www.python.org/sf/572113
Walter Dörwald <walter@livinglogic.de>
parents:
17843
diff
changeset
|
75 line = line.strip() |
| 11966 | 76 if not line or line[0] == '#': |
| 77 continue | |
| 78 m = mapRE.match(line) | |
| 79 if not m: | |
| 80 #print '* not matched: %s' % repr(line) | |
| 81 continue | |
| 82 enc,uni,comment = m.groups() | |
| 83 enc = parsecodes(enc) | |
| 84 uni = parsecodes(uni) | |
| 85 if not comment: | |
| 86 comment = '' | |
| 87 else: | |
| 88 comment = comment[1:] | |
|
15706
bf9bb0e3dd34
This patch changes the default behaviour of the builtin charmap
Marc-André Lemburg <mal@egenix.com>
parents:
12014
diff
changeset
|
89 if enc < 256: |
|
bf9bb0e3dd34
This patch changes the default behaviour of the builtin charmap
Marc-André Lemburg <mal@egenix.com>
parents:
12014
diff
changeset
|
90 unmapped.remove(enc) |
|
bf9bb0e3dd34
This patch changes the default behaviour of the builtin charmap
Marc-André Lemburg <mal@egenix.com>
parents:
12014
diff
changeset
|
91 if enc == uni: |
|
bf9bb0e3dd34
This patch changes the default behaviour of the builtin charmap
Marc-André Lemburg <mal@egenix.com>
parents:
12014
diff
changeset
|
92 identity.append(enc) |
|
bf9bb0e3dd34
This patch changes the default behaviour of the builtin charmap
Marc-André Lemburg <mal@egenix.com>
parents:
12014
diff
changeset
|
93 else: |
|
bf9bb0e3dd34
This patch changes the default behaviour of the builtin charmap
Marc-André Lemburg <mal@egenix.com>
parents:
12014
diff
changeset
|
94 enc2uni[enc] = (uni,comment) |
|
bf9bb0e3dd34
This patch changes the default behaviour of the builtin charmap
Marc-André Lemburg <mal@egenix.com>
parents:
12014
diff
changeset
|
95 else: |
| 11966 | 96 enc2uni[enc] = (uni,comment) |
|
15706
bf9bb0e3dd34
This patch changes the default behaviour of the builtin charmap
Marc-André Lemburg <mal@egenix.com>
parents:
12014
diff
changeset
|
97 # If there are more identity-mapped entries than unmapped entries, |
| 27620 | 98 # it pays to generate an identity dictionary first, and add explicit |
|
15706
bf9bb0e3dd34
This patch changes the default behaviour of the builtin charmap
Marc-André Lemburg <mal@egenix.com>
parents:
12014
diff
changeset
|
99 # mappings to None for the rest |
|
bf9bb0e3dd34
This patch changes the default behaviour of the builtin charmap
Marc-André Lemburg <mal@egenix.com>
parents:
12014
diff
changeset
|
100 if len(identity)>=len(unmapped): |
|
bf9bb0e3dd34
This patch changes the default behaviour of the builtin charmap
Marc-André Lemburg <mal@egenix.com>
parents:
12014
diff
changeset
|
101 for enc in unmapped: |
|
bf9bb0e3dd34
This patch changes the default behaviour of the builtin charmap
Marc-André Lemburg <mal@egenix.com>
parents:
12014
diff
changeset
|
102 enc2uni[enc] = (None, "") |
|
bf9bb0e3dd34
This patch changes the default behaviour of the builtin charmap
Marc-André Lemburg <mal@egenix.com>
parents:
12014
diff
changeset
|
103 enc2uni['IDENTITY'] = 256 |
|
bf9bb0e3dd34
This patch changes the default behaviour of the builtin charmap
Marc-André Lemburg <mal@egenix.com>
parents:
12014
diff
changeset
|
104 |
| 11966 | 105 return enc2uni |
| 106 | |
|
25219
8a7a2f1f88f0
Apply diff2.txt from SF patch http://www.python.org/sf/572113
Walter Dörwald <walter@livinglogic.de>
parents:
17843
diff
changeset
|
107 def hexrepr(t): |
| 11966 | 108 |
| 109 if t is None: | |
| 110 return 'None' | |
| 111 try: | |
| 112 len(t) | |
| 113 except: | |
| 114 return '0x%04x' % t | |
|
25219
8a7a2f1f88f0
Apply diff2.txt from SF patch http://www.python.org/sf/572113
Walter Dörwald <walter@livinglogic.de>
parents:
17843
diff
changeset
|
115 return '(' + ', '.join(map(lambda t: '0x%04x' % t, t)) + ')' |
| 11966 | 116 |
|
25219
8a7a2f1f88f0
Apply diff2.txt from SF patch http://www.python.org/sf/572113
Walter Dörwald <walter@livinglogic.de>
parents:
17843
diff
changeset
|
117 def unicoderepr(t): |
| 11966 | 118 |
| 119 if t is None: | |
| 120 return 'None' | |
| 121 if numeric: | |
| 122 return hexrepr(t) | |
| 123 else: | |
| 124 try: | |
| 125 len(t) | |
| 126 except: | |
| 127 return repr(unichr(t)) | |
|
25219
8a7a2f1f88f0
Apply diff2.txt from SF patch http://www.python.org/sf/572113
Walter Dörwald <walter@livinglogic.de>
parents:
17843
diff
changeset
|
128 return repr(''.join(map(unichr, t))) |
| 11966 | 129 |
|
25219
8a7a2f1f88f0
Apply diff2.txt from SF patch http://www.python.org/sf/572113
Walter Dörwald <walter@livinglogic.de>
parents:
17843
diff
changeset
|
130 def keyrepr(t): |
| 11966 | 131 |
| 132 if t is None: | |
| 133 return 'None' | |
| 134 if numeric: | |
| 135 return hexrepr(t) | |
| 136 else: | |
| 137 try: | |
| 138 len(t) | |
| 139 except: | |
| 140 if t < 256: | |
| 141 return repr(chr(t)) | |
| 142 else: | |
| 143 return repr(unichr(t)) | |
|
25219
8a7a2f1f88f0
Apply diff2.txt from SF patch http://www.python.org/sf/572113
Walter Dörwald <walter@livinglogic.de>
parents:
17843
diff
changeset
|
144 return repr(''.join(map(chr, t))) |
| 11966 | 145 |
| 146 def codegen(name,map,comments=1): | |
| 147 | |
| 148 """ Returns Python source for the given map. | |
| 149 | |
| 150 Comments are included in the source, if comments is true (default). | |
| 151 | |
| 152 """ | |
| 153 l = [ | |
| 154 '''\ | |
|
15706
bf9bb0e3dd34
This patch changes the default behaviour of the builtin charmap
Marc-André Lemburg <mal@egenix.com>
parents:
12014
diff
changeset
|
155 """ Python Character Mapping Codec generated from '%s' with gencodec.py. |
| 11966 | 156 |
| 157 """#" | |
| 158 | |
| 159 import codecs | |
| 160 | |
| 161 ### Codec APIs | |
| 162 | |
| 163 class Codec(codecs.Codec): | |
| 164 | |
| 165 def encode(self,input,errors='strict'): | |
| 166 | |
| 167 return codecs.charmap_encode(input,errors,encoding_map) | |
|
15930
14a27acc755a
Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents:
15706
diff
changeset
|
168 |
| 11966 | 169 def decode(self,input,errors='strict'): |
| 170 | |
| 171 return codecs.charmap_decode(input,errors,decoding_map) | |
| 172 | |
| 173 class StreamWriter(Codec,codecs.StreamWriter): | |
| 174 pass | |
|
15930
14a27acc755a
Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents:
15706
diff
changeset
|
175 |
| 11966 | 176 class StreamReader(Codec,codecs.StreamReader): |
| 177 pass | |
| 178 | |
| 179 ### encodings module API | |
| 180 | |
| 181 def getregentry(): | |
| 182 | |
| 183 return (Codec().encode,Codec().decode,StreamReader,StreamWriter) | |
| 184 | |
| 185 ### Decoding Map | |
| 186 ''' % name, | |
| 187 ] | |
|
15706
bf9bb0e3dd34
This patch changes the default behaviour of the builtin charmap
Marc-André Lemburg <mal@egenix.com>
parents:
12014
diff
changeset
|
188 |
|
bf9bb0e3dd34
This patch changes the default behaviour of the builtin charmap
Marc-André Lemburg <mal@egenix.com>
parents:
12014
diff
changeset
|
189 if map.has_key("IDENTITY"): |
|
bf9bb0e3dd34
This patch changes the default behaviour of the builtin charmap
Marc-André Lemburg <mal@egenix.com>
parents:
12014
diff
changeset
|
190 l.append("decoding_map = codecs.make_identity_dict(range(%d))" |
|
bf9bb0e3dd34
This patch changes the default behaviour of the builtin charmap
Marc-André Lemburg <mal@egenix.com>
parents:
12014
diff
changeset
|
191 % map["IDENTITY"]) |
|
bf9bb0e3dd34
This patch changes the default behaviour of the builtin charmap
Marc-André Lemburg <mal@egenix.com>
parents:
12014
diff
changeset
|
192 l.append("decoding_map.update({") |
|
bf9bb0e3dd34
This patch changes the default behaviour of the builtin charmap
Marc-André Lemburg <mal@egenix.com>
parents:
12014
diff
changeset
|
193 splits = 1 |
|
bf9bb0e3dd34
This patch changes the default behaviour of the builtin charmap
Marc-André Lemburg <mal@egenix.com>
parents:
12014
diff
changeset
|
194 del map["IDENTITY"] |
|
bf9bb0e3dd34
This patch changes the default behaviour of the builtin charmap
Marc-André Lemburg <mal@egenix.com>
parents:
12014
diff
changeset
|
195 else: |
|
bf9bb0e3dd34
This patch changes the default behaviour of the builtin charmap
Marc-André Lemburg <mal@egenix.com>
parents:
12014
diff
changeset
|
196 l.append("decoding_map = {") |
|
bf9bb0e3dd34
This patch changes the default behaviour of the builtin charmap
Marc-André Lemburg <mal@egenix.com>
parents:
12014
diff
changeset
|
197 splits = 0 |
|
15930
14a27acc755a
Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents:
15706
diff
changeset
|
198 |
| 11966 | 199 mappings = map.items() |
| 200 mappings.sort() | |
| 201 append = l.append | |
| 202 i = 0 | |
| 203 for e,value in mappings: | |
| 204 try: | |
| 205 (u,c) = value | |
| 206 except TypeError: | |
| 207 u = value | |
| 208 c = '' | |
| 209 key = keyrepr(e) | |
| 210 if c and comments: | |
| 211 append('\t%s: %s,\t# %s' % (key,unicoderepr(u),c)) | |
| 212 else: | |
| 213 append('\t%s: %s,' % (key,unicoderepr(u))) | |
|
15706
bf9bb0e3dd34
This patch changes the default behaviour of the builtin charmap
Marc-André Lemburg <mal@egenix.com>
parents:
12014
diff
changeset
|
214 i += 1 |
| 11966 | 215 if i == 4096: |
| 216 # Split the definition into parts to that the Python | |
| 217 # parser doesn't dump core | |
| 218 if splits == 0: | |
| 219 append('}') | |
| 220 else: | |
| 221 append('})') | |
|
15706
bf9bb0e3dd34
This patch changes the default behaviour of the builtin charmap
Marc-André Lemburg <mal@egenix.com>
parents:
12014
diff
changeset
|
222 append('decoding_map.update({') |
| 11966 | 223 i = 0 |
| 224 splits = splits + 1 | |
| 225 if splits == 0: | |
| 226 append('}') | |
| 227 else: | |
| 228 append('})') | |
| 229 append(''' | |
| 230 ### Encoding Map | |
| 231 | |
|
17843
3e256f9d99ad
Moved the encoding map building logic from the individual mapping
Marc-André Lemburg <mal@egenix.com>
parents:
15930
diff
changeset
|
232 encoding_map = codecs.make_encoding_map(decoding_map) |
| 11966 | 233 ''') |
|
25219
8a7a2f1f88f0
Apply diff2.txt from SF patch http://www.python.org/sf/572113
Walter Dörwald <walter@livinglogic.de>
parents:
17843
diff
changeset
|
234 return '\n'.join(l) |
| 11966 | 235 |
| 236 def pymap(name,map,pyfile,comments=1): | |
| 237 | |
| 238 code = codegen(name,map,comments) | |
| 239 f = open(pyfile,'w') | |
| 240 f.write(code) | |
| 241 f.close() | |
| 242 | |
| 243 def marshalmap(name,map,marshalfile): | |
| 244 | |
| 245 d = {} | |
| 246 for e,(u,c) in map.items(): | |
| 247 d[e] = (u,c) | |
| 248 f = open(marshalfile,'wb') | |
| 249 marshal.dump(d,f) | |
| 250 f.close() | |
| 251 | |
| 252 def convertdir(dir,prefix='',comments=1): | |
| 253 | |
| 254 mapnames = os.listdir(dir) | |
| 255 for mapname in mapnames: | |
| 256 name = os.path.split(mapname)[1] | |
|
25219
8a7a2f1f88f0
Apply diff2.txt from SF patch http://www.python.org/sf/572113
Walter Dörwald <walter@livinglogic.de>
parents:
17843
diff
changeset
|
257 name = name.replace('-','_') |
|
8a7a2f1f88f0
Apply diff2.txt from SF patch http://www.python.org/sf/572113
Walter Dörwald <walter@livinglogic.de>
parents:
17843
diff
changeset
|
258 name = name.split('.')[0] |
|
8a7a2f1f88f0
Apply diff2.txt from SF patch http://www.python.org/sf/572113
Walter Dörwald <walter@livinglogic.de>
parents:
17843
diff
changeset
|
259 name = name.lower() |
| 11966 | 260 codefile = name + '.py' |
| 261 marshalfile = name + '.mapping' | |
| 262 print 'converting %s to %s and %s' % (mapname, | |
| 263 prefix + codefile, | |
| 264 prefix + marshalfile) | |
| 265 try: | |
| 266 map = readmap(os.path.join(dir,mapname)) | |
| 267 if not map: | |
| 268 print '* map is empty; skipping' | |
| 269 else: | |
| 270 pymap(mapname, map, prefix + codefile,comments) | |
| 271 marshalmap(mapname, map, prefix + marshalfile) | |
| 272 except ValueError: | |
| 273 print '* conversion failed' | |
| 274 | |
| 275 def rewritepythondir(dir,prefix='',comments=1): | |
|
15930
14a27acc755a
Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents:
15706
diff
changeset
|
276 |
| 11966 | 277 mapnames = os.listdir(dir) |
| 278 for mapname in mapnames: | |
|
15706
bf9bb0e3dd34
This patch changes the default behaviour of the builtin charmap
Marc-André Lemburg <mal@egenix.com>
parents:
12014
diff
changeset
|
279 if not mapname.endswith('.mapping'): |
| 11966 | 280 continue |
| 281 codefile = mapname[:-len('.mapping')] + '.py' | |
| 282 print 'converting %s to %s' % (mapname, | |
| 283 prefix + codefile) | |
| 284 try: | |
| 285 map = marshal.load(open(os.path.join(dir,mapname), | |
| 286 'rb')) | |
| 287 if not map: | |
| 288 print '* map is empty; skipping' | |
| 289 else: | |
| 290 pymap(mapname, map, prefix + codefile,comments) | |
| 291 except ValueError, why: | |
| 292 print '* conversion failed: %s' % why | |
| 293 | |
| 294 if __name__ == '__main__': | |
| 295 | |
| 296 import sys | |
| 297 if 1: | |
| 298 apply(convertdir,tuple(sys.argv[1:])) | |
| 299 else: | |
| 300 apply(rewritepythondir,tuple(sys.argv[1:])) |