Mercurial > cpython-lru_cache-weakref
annotate Tools/scripts/gencodec.py @ 27620:bbd8abaa0f33 legacy-trunk
Fix comment typo.
| author | Walter Dörwald <walter@livinglogic.de> |
|---|---|
| date | Sun, 02 Feb 2003 23:39:45 +0000 |
| parents | 8a7a2f1f88f0 |
| children | 9267aeb72985 |
| rev | line source |
|---|---|
| 11966 | 1 """ Unicode Mapping Parser and Codec Generator. |
| 2 | |
| 3 This script parses Unicode mapping files as available from the Unicode | |
|
15706
bf9bb0e3dd34
This patch changes the default behaviour of the builtin charmap
Marc-André Lemburg <mal@egenix.com>
parents:
12014
diff
changeset
|
4 site (ftp://ftp.unicode.org/Public/MAPPINGS/) and creates Python codec |
|
bf9bb0e3dd34
This patch changes the default behaviour of the builtin charmap
Marc-André Lemburg <mal@egenix.com>
parents:
12014
diff
changeset
|
5 modules from them. The codecs use the standard character mapping codec |
|
bf9bb0e3dd34
This patch changes the default behaviour of the builtin charmap
Marc-André Lemburg <mal@egenix.com>
parents:
12014
diff
changeset
|
6 to actually apply the mapping. |
| 11966 | 7 |
| 8 Synopsis: gencodec.py dir codec_prefix | |
| 9 | |
| 10 All files in dir are scanned and those producing non-empty mappings | |
| 11 will be written to <codec_prefix><mapname>.py with <mapname> being the | |
| 12 first part of the map's filename ('a' in a.b.c.txt) converted to | |
| 13 lowercase with hyphens replaced by underscores. | |
| 14 | |
|
12014
ba80c9064491
Of course, I forget how to type long ago...
Fred Drake <fdrake@acm.org>
parents:
12012
diff
changeset
|
15 The tool also writes marshalled versions of the mapping tables to the |
| 11966 | 16 same location (with .mapping extension). |
| 17 | |
| 18 Written by Marc-Andre Lemburg (mal@lemburg.com). | |
| 19 | |
| 20 (c) Copyright CNRI, All Rights Reserved. NO WARRANTY. | |
|
15706
bf9bb0e3dd34
This patch changes the default behaviour of the builtin charmap
Marc-André Lemburg <mal@egenix.com>
parents:
12014
diff
changeset
|
21 (c) Copyright Guido van Rossum, 2000. |
| 11966 | 22 |
| 23 """#" | |
| 24 | |
|
25219
8a7a2f1f88f0
Apply diff2.txt from SF patch http://www.python.org/sf/572113
Walter Dörwald <walter@livinglogic.de>
parents:
17843
diff
changeset
|
25 import re,os,time,marshal |
| 11966 | 26 |
| 27 # Create numeric tables or character based ones ? | |
| 28 numeric = 1 | |
| 29 | |
| 30 mapRE = re.compile('((?:0x[0-9a-fA-F]+\+?)+)' | |
| 31 '\s+' | |
| 32 '((?:(?:0x[0-9a-fA-Z]+|<[A-Za-z]+>)\+?)*)' | |
| 33 '\s*' | |
| 34 '(#.+)?') | |
| 35 | |
| 36 def parsecodes(codes, | |
|
25219
8a7a2f1f88f0
Apply diff2.txt from SF patch http://www.python.org/sf/572113
Walter Dörwald <walter@livinglogic.de>
parents:
17843
diff
changeset
|
37 len=len, filter=filter,range=range): |
| 11966 | 38 |
| 39 """ Converts code combinations to either a single code integer | |
| 40 or a tuple of integers. | |
| 41 | |
| 42 meta-codes (in angular brackets, e.g. <LR> and <RL>) are | |
| 43 ignored. | |
| 44 | |
| 45 Empty codes or illegal ones are returned as None. | |
| 46 | |
| 47 """ | |
| 48 if not codes: | |
| 49 return None | |
|
25219
8a7a2f1f88f0
Apply diff2.txt from SF patch http://www.python.org/sf/572113
Walter Dörwald <walter@livinglogic.de>
parents:
17843
diff
changeset
|
50 l = codes.split('+') |
| 11966 | 51 if len(l) == 1: |
|
25219
8a7a2f1f88f0
Apply diff2.txt from SF patch http://www.python.org/sf/572113
Walter Dörwald <walter@livinglogic.de>
parents:
17843
diff
changeset
|
52 return int(l[0],16) |
| 11966 | 53 for i in range(len(l)): |
| 54 try: | |
|
25219
8a7a2f1f88f0
Apply diff2.txt from SF patch http://www.python.org/sf/572113
Walter Dörwald <walter@livinglogic.de>
parents:
17843
diff
changeset
|
55 l[i] = int(l[i],16) |
| 11966 | 56 except ValueError: |
| 57 l[i] = None | |
| 58 l = filter(lambda x: x is not None, l) | |
| 59 if len(l) == 1: | |
| 60 return l[0] | |
| 61 else: | |
| 62 return tuple(l) | |
| 63 | |
|
25219
8a7a2f1f88f0
Apply diff2.txt from SF patch http://www.python.org/sf/572113
Walter Dörwald <walter@livinglogic.de>
parents:
17843
diff
changeset
|
64 def readmap(filename): |
| 11966 | 65 |
| 66 f = open(filename,'r') | |
| 67 lines = f.readlines() | |
| 68 f.close() | |
| 69 enc2uni = {} | |
|
15706
bf9bb0e3dd34
This patch changes the default behaviour of the builtin charmap
Marc-André Lemburg <mal@egenix.com>
parents:
12014
diff
changeset
|
70 identity = [] |
|
bf9bb0e3dd34
This patch changes the default behaviour of the builtin charmap
Marc-André Lemburg <mal@egenix.com>
parents:
12014
diff
changeset
|
71 unmapped = range(256) |
|
bf9bb0e3dd34
This patch changes the default behaviour of the builtin charmap
Marc-André Lemburg <mal@egenix.com>
parents:
12014
diff
changeset
|
72 for i in range(256): |
|
bf9bb0e3dd34
This patch changes the default behaviour of the builtin charmap
Marc-André Lemburg <mal@egenix.com>
parents:
12014
diff
changeset
|
73 unmapped[i] = i |
| 11966 | 74 for line in lines: |
|
25219
8a7a2f1f88f0
Apply diff2.txt from SF patch http://www.python.org/sf/572113
Walter Dörwald <walter@livinglogic.de>
parents:
17843
diff
changeset
|
75 line = line.strip() |
| 11966 | 76 if not line or line[0] == '#': |
| 77 continue | |
| 78 m = mapRE.match(line) | |
| 79 if not m: | |
| 80 #print '* not matched: %s' % repr(line) | |
| 81 continue | |
| 82 enc,uni,comment = m.groups() | |
| 83 enc = parsecodes(enc) | |
| 84 uni = parsecodes(uni) | |
| 85 if not comment: | |
| 86 comment = '' | |
| 87 else: | |
| 88 comment = comment[1:] | |
|
15706
bf9bb0e3dd34
This patch changes the default behaviour of the builtin charmap
Marc-André Lemburg <mal@egenix.com>
parents:
12014
diff
changeset
|
89 if enc < 256: |
|
bf9bb0e3dd34
This patch changes the default behaviour of the builtin charmap
Marc-André Lemburg <mal@egenix.com>
parents:
12014
diff
changeset
|
90 unmapped.remove(enc) |
|
bf9bb0e3dd34
This patch changes the default behaviour of the builtin charmap
Marc-André Lemburg <mal@egenix.com>
parents:
12014
diff
changeset
|
91 if enc == uni: |
|
bf9bb0e3dd34
This patch changes the default behaviour of the builtin charmap
Marc-André Lemburg <mal@egenix.com>
parents:
12014
diff
changeset
|
92 identity.append(enc) |
|
bf9bb0e3dd34
This patch changes the default behaviour of the builtin charmap
Marc-André Lemburg <mal@egenix.com>
parents:
12014
diff
changeset
|
93 else: |
|
bf9bb0e3dd34
This patch changes the default behaviour of the builtin charmap
Marc-André Lemburg <mal@egenix.com>
parents:
12014
diff
changeset
|
94 enc2uni[enc] = (uni,comment) |
|
bf9bb0e3dd34
This patch changes the default behaviour of the builtin charmap
Marc-André Lemburg <mal@egenix.com>
parents:
12014
diff
changeset
|
95 else: |
| 11966 | 96 enc2uni[enc] = (uni,comment) |
|
15706
bf9bb0e3dd34
This patch changes the default behaviour of the builtin charmap
Marc-André Lemburg <mal@egenix.com>
parents:
12014
diff
changeset
|
97 # If there are more identity-mapped entries than unmapped entries, |
| 27620 | 98 # it pays to generate an identity dictionary first, and add explicit |
|
15706
bf9bb0e3dd34
This patch changes the default behaviour of the builtin charmap
Marc-André Lemburg <mal@egenix.com>
parents:
12014
diff
changeset
|
99 # mappings to None for the rest |
|
bf9bb0e3dd34
This patch changes the default behaviour of the builtin charmap
Marc-André Lemburg <mal@egenix.com>
parents:
12014
diff
changeset
|
100 if len(identity)>=len(unmapped): |
|
bf9bb0e3dd34
This patch changes the default behaviour of the builtin charmap
Marc-André Lemburg <mal@egenix.com>
parents:
12014
diff
changeset
|
101 for enc in unmapped: |
|
bf9bb0e3dd34
This patch changes the default behaviour of the builtin charmap
Marc-André Lemburg <mal@egenix.com>
parents:
12014
diff
changeset
|
102 enc2uni[enc] = (None, "") |
|
bf9bb0e3dd34
This patch changes the default behaviour of the builtin charmap
Marc-André Lemburg <mal@egenix.com>
parents:
12014
diff
changeset
|
103 enc2uni['IDENTITY'] = 256 |
|
bf9bb0e3dd34
This patch changes the default behaviour of the builtin charmap
Marc-André Lemburg <mal@egenix.com>
parents:
12014
diff
changeset
|
104 |
| 11966 | 105 return enc2uni |
| 106 | |
|
25219
8a7a2f1f88f0
Apply diff2.txt from SF patch http://www.python.org/sf/572113
Walter Dörwald <walter@livinglogic.de>
parents:
17843
diff
changeset
|
107 def hexrepr(t): |
| 11966 | 108 |
| 109 if t is None: | |
| 110 return 'None' | |
| 111 try: | |
| 112 len(t) | |
| 113 except: | |
| 114 return '0x%04x' % t | |
|
25219
8a7a2f1f88f0
Apply diff2.txt from SF patch http://www.python.org/sf/572113
Walter Dörwald <walter@livinglogic.de>
parents:
17843
diff
changeset
|
115 return '(' + ', '.join(map(lambda t: '0x%04x' % t, t)) + ')' |
| 11966 | 116 |
|
25219
8a7a2f1f88f0
Apply diff2.txt from SF patch http://www.python.org/sf/572113
Walter Dörwald <walter@livinglogic.de>
parents:
17843
diff
changeset
|
117 def unicoderepr(t): |
| 11966 | 118 |
| 119 if t is None: | |
| 120 return 'None' | |
| 121 if numeric: | |
| 122 return hexrepr(t) | |
| 123 else: | |
| 124 try: | |
| 125 len(t) | |
| 126 except: | |
| 127 return repr(unichr(t)) | |
|
25219
8a7a2f1f88f0
Apply diff2.txt from SF patch http://www.python.org/sf/572113
Walter Dörwald <walter@livinglogic.de>
parents:
17843
diff
changeset
|
128 return repr(''.join(map(unichr, t))) |
| 11966 | 129 |
|
25219
8a7a2f1f88f0
Apply diff2.txt from SF patch http://www.python.org/sf/572113
Walter Dörwald <walter@livinglogic.de>
parents:
17843
diff
changeset
|
130 def keyrepr(t): |
| 11966 | 131 |
| 132 if t is None: | |
| 133 return 'None' | |
| 134 if numeric: | |
| 135 return hexrepr(t) | |
| 136 else: | |
| 137 try: | |
| 138 len(t) | |
| 139 except: | |
| 140 if t < 256: | |
| 141 return repr(chr(t)) | |
| 142 else: | |
| 143 return repr(unichr(t)) | |
|
25219
8a7a2f1f88f0
Apply diff2.txt from SF patch http://www.python.org/sf/572113
Walter Dörwald <walter@livinglogic.de>
parents:
17843
diff
changeset
|
144 return repr(''.join(map(chr, t))) |
| 11966 | 145 |
| 146 def codegen(name,map,comments=1): | |
| 147 | |
| 148 """ Returns Python source for the given map. | |
| 149 | |
| 150 Comments are included in the source, if comments is true (default). | |
| 151 | |
| 152 """ | |
| 153 l = [ | |
| 154 '''\ | |
|
15706
bf9bb0e3dd34
This patch changes the default behaviour of the builtin charmap
Marc-André Lemburg <mal@egenix.com>
parents:
12014
diff
changeset
|
155 """ Python Character Mapping Codec generated from '%s' with gencodec.py. |
| 11966 | 156 |
| 157 Written by Marc-Andre Lemburg (mal@lemburg.com). | |
| 158 | |
| 159 (c) Copyright CNRI, All Rights Reserved. NO WARRANTY. | |
|
15706
bf9bb0e3dd34
This patch changes the default behaviour of the builtin charmap
Marc-André Lemburg <mal@egenix.com>
parents:
12014
diff
changeset
|
160 (c) Copyright 2000 Guido van Rossum. |
| 11966 | 161 |
| 162 """#" | |
| 163 | |
| 164 import codecs | |
| 165 | |
| 166 ### Codec APIs | |
| 167 | |
| 168 class Codec(codecs.Codec): | |
| 169 | |
| 170 def encode(self,input,errors='strict'): | |
| 171 | |
| 172 return codecs.charmap_encode(input,errors,encoding_map) | |
|
15930
14a27acc755a
Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents:
15706
diff
changeset
|
173 |
| 11966 | 174 def decode(self,input,errors='strict'): |
| 175 | |
| 176 return codecs.charmap_decode(input,errors,decoding_map) | |
| 177 | |
| 178 class StreamWriter(Codec,codecs.StreamWriter): | |
| 179 pass | |
|
15930
14a27acc755a
Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents:
15706
diff
changeset
|
180 |
| 11966 | 181 class StreamReader(Codec,codecs.StreamReader): |
| 182 pass | |
| 183 | |
| 184 ### encodings module API | |
| 185 | |
| 186 def getregentry(): | |
| 187 | |
| 188 return (Codec().encode,Codec().decode,StreamReader,StreamWriter) | |
| 189 | |
| 190 ### Decoding Map | |
| 191 ''' % name, | |
| 192 ] | |
|
15706
bf9bb0e3dd34
This patch changes the default behaviour of the builtin charmap
Marc-André Lemburg <mal@egenix.com>
parents:
12014
diff
changeset
|
193 |
|
bf9bb0e3dd34
This patch changes the default behaviour of the builtin charmap
Marc-André Lemburg <mal@egenix.com>
parents:
12014
diff
changeset
|
194 if map.has_key("IDENTITY"): |
|
bf9bb0e3dd34
This patch changes the default behaviour of the builtin charmap
Marc-André Lemburg <mal@egenix.com>
parents:
12014
diff
changeset
|
195 l.append("decoding_map = codecs.make_identity_dict(range(%d))" |
|
bf9bb0e3dd34
This patch changes the default behaviour of the builtin charmap
Marc-André Lemburg <mal@egenix.com>
parents:
12014
diff
changeset
|
196 % map["IDENTITY"]) |
|
bf9bb0e3dd34
This patch changes the default behaviour of the builtin charmap
Marc-André Lemburg <mal@egenix.com>
parents:
12014
diff
changeset
|
197 l.append("decoding_map.update({") |
|
bf9bb0e3dd34
This patch changes the default behaviour of the builtin charmap
Marc-André Lemburg <mal@egenix.com>
parents:
12014
diff
changeset
|
198 splits = 1 |
|
bf9bb0e3dd34
This patch changes the default behaviour of the builtin charmap
Marc-André Lemburg <mal@egenix.com>
parents:
12014
diff
changeset
|
199 del map["IDENTITY"] |
|
bf9bb0e3dd34
This patch changes the default behaviour of the builtin charmap
Marc-André Lemburg <mal@egenix.com>
parents:
12014
diff
changeset
|
200 else: |
|
bf9bb0e3dd34
This patch changes the default behaviour of the builtin charmap
Marc-André Lemburg <mal@egenix.com>
parents:
12014
diff
changeset
|
201 l.append("decoding_map = {") |
|
bf9bb0e3dd34
This patch changes the default behaviour of the builtin charmap
Marc-André Lemburg <mal@egenix.com>
parents:
12014
diff
changeset
|
202 splits = 0 |
|
15930
14a27acc755a
Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents:
15706
diff
changeset
|
203 |
| 11966 | 204 mappings = map.items() |
| 205 mappings.sort() | |
| 206 append = l.append | |
| 207 i = 0 | |
| 208 for e,value in mappings: | |
| 209 try: | |
| 210 (u,c) = value | |
| 211 except TypeError: | |
| 212 u = value | |
| 213 c = '' | |
| 214 key = keyrepr(e) | |
| 215 if c and comments: | |
| 216 append('\t%s: %s,\t# %s' % (key,unicoderepr(u),c)) | |
| 217 else: | |
| 218 append('\t%s: %s,' % (key,unicoderepr(u))) | |
|
15706
bf9bb0e3dd34
This patch changes the default behaviour of the builtin charmap
Marc-André Lemburg <mal@egenix.com>
parents:
12014
diff
changeset
|
219 i += 1 |
| 11966 | 220 if i == 4096: |
| 221 # Split the definition into parts to that the Python | |
| 222 # parser doesn't dump core | |
| 223 if splits == 0: | |
| 224 append('}') | |
| 225 else: | |
| 226 append('})') | |
|
15706
bf9bb0e3dd34
This patch changes the default behaviour of the builtin charmap
Marc-André Lemburg <mal@egenix.com>
parents:
12014
diff
changeset
|
227 append('decoding_map.update({') |
| 11966 | 228 i = 0 |
| 229 splits = splits + 1 | |
| 230 if splits == 0: | |
| 231 append('}') | |
| 232 else: | |
| 233 append('})') | |
| 234 append(''' | |
| 235 ### Encoding Map | |
| 236 | |
|
17843
3e256f9d99ad
Moved the encoding map building logic from the individual mapping
Marc-André Lemburg <mal@egenix.com>
parents:
15930
diff
changeset
|
237 encoding_map = codecs.make_encoding_map(decoding_map) |
| 11966 | 238 ''') |
|
25219
8a7a2f1f88f0
Apply diff2.txt from SF patch http://www.python.org/sf/572113
Walter Dörwald <walter@livinglogic.de>
parents:
17843
diff
changeset
|
239 return '\n'.join(l) |
| 11966 | 240 |
| 241 def pymap(name,map,pyfile,comments=1): | |
| 242 | |
| 243 code = codegen(name,map,comments) | |
| 244 f = open(pyfile,'w') | |
| 245 f.write(code) | |
| 246 f.close() | |
| 247 | |
| 248 def marshalmap(name,map,marshalfile): | |
| 249 | |
| 250 d = {} | |
| 251 for e,(u,c) in map.items(): | |
| 252 d[e] = (u,c) | |
| 253 f = open(marshalfile,'wb') | |
| 254 marshal.dump(d,f) | |
| 255 f.close() | |
| 256 | |
| 257 def convertdir(dir,prefix='',comments=1): | |
| 258 | |
| 259 mapnames = os.listdir(dir) | |
| 260 for mapname in mapnames: | |
| 261 name = os.path.split(mapname)[1] | |
|
25219
8a7a2f1f88f0
Apply diff2.txt from SF patch http://www.python.org/sf/572113
Walter Dörwald <walter@livinglogic.de>
parents:
17843
diff
changeset
|
262 name = name.replace('-','_') |
|
8a7a2f1f88f0
Apply diff2.txt from SF patch http://www.python.org/sf/572113
Walter Dörwald <walter@livinglogic.de>
parents:
17843
diff
changeset
|
263 name = name.split('.')[0] |
|
8a7a2f1f88f0
Apply diff2.txt from SF patch http://www.python.org/sf/572113
Walter Dörwald <walter@livinglogic.de>
parents:
17843
diff
changeset
|
264 name = name.lower() |
| 11966 | 265 codefile = name + '.py' |
| 266 marshalfile = name + '.mapping' | |
| 267 print 'converting %s to %s and %s' % (mapname, | |
| 268 prefix + codefile, | |
| 269 prefix + marshalfile) | |
| 270 try: | |
| 271 map = readmap(os.path.join(dir,mapname)) | |
| 272 if not map: | |
| 273 print '* map is empty; skipping' | |
| 274 else: | |
| 275 pymap(mapname, map, prefix + codefile,comments) | |
| 276 marshalmap(mapname, map, prefix + marshalfile) | |
| 277 except ValueError: | |
| 278 print '* conversion failed' | |
| 279 | |
| 280 def rewritepythondir(dir,prefix='',comments=1): | |
|
15930
14a27acc755a
Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents:
15706
diff
changeset
|
281 |
| 11966 | 282 mapnames = os.listdir(dir) |
| 283 for mapname in mapnames: | |
|
15706
bf9bb0e3dd34
This patch changes the default behaviour of the builtin charmap
Marc-André Lemburg <mal@egenix.com>
parents:
12014
diff
changeset
|
284 if not mapname.endswith('.mapping'): |
| 11966 | 285 continue |
| 286 codefile = mapname[:-len('.mapping')] + '.py' | |
| 287 print 'converting %s to %s' % (mapname, | |
| 288 prefix + codefile) | |
| 289 try: | |
| 290 map = marshal.load(open(os.path.join(dir,mapname), | |
| 291 'rb')) | |
| 292 if not map: | |
| 293 print '* map is empty; skipping' | |
| 294 else: | |
| 295 pymap(mapname, map, prefix + codefile,comments) | |
| 296 except ValueError, why: | |
| 297 print '* conversion failed: %s' % why | |
| 298 | |
| 299 if __name__ == '__main__': | |
| 300 | |
| 301 import sys | |
| 302 if 1: | |
| 303 apply(convertdir,tuple(sys.argv[1:])) | |
| 304 else: | |
| 305 apply(rewritepythondir,tuple(sys.argv[1:])) |