Mercurial > cpython-lru_cache-weakref
annotate Tools/scripts/gencodec.py @ 15164:8930ec3bc00a legacy-trunk
Added additional test cases for pulldom modifications.
| author | Lars Gustäbel <lars@gustaebel.de> |
|---|---|
| date | Wed, 11 Oct 2000 22:36:00 +0000 |
| parents | ba80c9064491 |
| children | bf9bb0e3dd34 |
| rev | line source |
|---|---|
| 11966 | 1 """ Unicode Mapping Parser and Codec Generator. |
| 2 | |
| 3 This script parses Unicode mapping files as available from the Unicode | |
| 4 site (ftp.unicode.org) and creates Python codec modules from them. The | |
| 5 codecs use the standard character mapping codec to actually apply the | |
| 6 mapping. | |
| 7 | |
| 8 Synopsis: gencodec.py dir codec_prefix | |
| 9 | |
| 10 All files in dir are scanned and those producing non-empty mappings | |
| 11 will be written to <codec_prefix><mapname>.py with <mapname> being the | |
| 12 first part of the map's filename ('a' in a.b.c.txt) converted to | |
| 13 lowercase with hyphens replaced by underscores. | |
| 14 | |
|
12014
ba80c9064491
Of course, I forget how to type long ago...
Fred Drake <fdrake@acm.org>
parents:
12012
diff
changeset
|
15 The tool also writes marshalled versions of the mapping tables to the |
| 11966 | 16 same location (with .mapping extension). |
| 17 | |
| 18 Written by Marc-Andre Lemburg (mal@lemburg.com). | |
| 19 | |
| 20 (c) Copyright CNRI, All Rights Reserved. NO WARRANTY. | |
| 21 | |
| 22 """#" | |
| 23 | |
| 24 import string,re,os,time,marshal | |
| 25 | |
| 26 # Create numeric tables or character based ones ? | |
| 27 numeric = 1 | |
| 28 | |
| 29 mapRE = re.compile('((?:0x[0-9a-fA-F]+\+?)+)' | |
| 30 '\s+' | |
| 31 '((?:(?:0x[0-9a-fA-Z]+|<[A-Za-z]+>)\+?)*)' | |
| 32 '\s*' | |
| 33 '(#.+)?') | |
| 34 | |
| 35 def parsecodes(codes, | |
| 36 | |
| 37 split=string.split,atoi=string.atoi,len=len, | |
| 38 filter=filter,range=range): | |
| 39 | |
| 40 """ Converts code combinations to either a single code integer | |
| 41 or a tuple of integers. | |
| 42 | |
| 43 meta-codes (in angular brackets, e.g. <LR> and <RL>) are | |
| 44 ignored. | |
| 45 | |
| 46 Empty codes or illegal ones are returned as None. | |
| 47 | |
| 48 """ | |
| 49 if not codes: | |
| 50 return None | |
| 51 l = split(codes,'+') | |
| 52 if len(l) == 1: | |
| 53 return atoi(l[0],16) | |
| 54 for i in range(len(l)): | |
| 55 try: | |
| 56 l[i] = atoi(l[i],16) | |
| 57 except ValueError: | |
| 58 l[i] = None | |
| 59 l = filter(lambda x: x is not None, l) | |
| 60 if len(l) == 1: | |
| 61 return l[0] | |
| 62 else: | |
| 63 return tuple(l) | |
| 64 | |
| 65 def readmap(filename, | |
| 66 | |
| 67 strip=string.strip): | |
| 68 | |
| 69 f = open(filename,'r') | |
| 70 lines = f.readlines() | |
| 71 f.close() | |
| 72 enc2uni = {} | |
| 73 for line in lines: | |
| 74 line = strip(line) | |
| 75 if not line or line[0] == '#': | |
| 76 continue | |
| 77 m = mapRE.match(line) | |
| 78 if not m: | |
| 79 #print '* not matched: %s' % repr(line) | |
| 80 continue | |
| 81 enc,uni,comment = m.groups() | |
| 82 enc = parsecodes(enc) | |
| 83 uni = parsecodes(uni) | |
| 84 if not comment: | |
| 85 comment = '' | |
| 86 else: | |
| 87 comment = comment[1:] | |
| 88 if enc != uni: | |
| 89 enc2uni[enc] = (uni,comment) | |
| 90 return enc2uni | |
| 91 | |
| 92 def hexrepr(t, | |
| 93 | |
| 94 join=string.join): | |
| 95 | |
| 96 if t is None: | |
| 97 return 'None' | |
| 98 try: | |
| 99 len(t) | |
| 100 except: | |
| 101 return '0x%04x' % t | |
| 102 return '(' + join(map(lambda t: '0x%04x' % t, t),', ') + ')' | |
| 103 | |
| 104 def unicoderepr(t, | |
| 105 | |
| 106 join=string.join): | |
| 107 | |
| 108 if t is None: | |
| 109 return 'None' | |
| 110 if numeric: | |
| 111 return hexrepr(t) | |
| 112 else: | |
| 113 try: | |
| 114 len(t) | |
| 115 except: | |
| 116 return repr(unichr(t)) | |
| 117 return repr(join(map(unichr, t),'')) | |
| 118 | |
| 119 def keyrepr(t, | |
| 120 | |
| 121 join=string.join): | |
| 122 | |
| 123 if t is None: | |
| 124 return 'None' | |
| 125 if numeric: | |
| 126 return hexrepr(t) | |
| 127 else: | |
| 128 try: | |
| 129 len(t) | |
| 130 except: | |
| 131 if t < 256: | |
| 132 return repr(chr(t)) | |
| 133 else: | |
| 134 return repr(unichr(t)) | |
| 135 return repr(join(map(chr, t),'')) | |
| 136 | |
| 137 def codegen(name,map,comments=1): | |
| 138 | |
| 139 """ Returns Python source for the given map. | |
| 140 | |
| 141 Comments are included in the source, if comments is true (default). | |
| 142 | |
| 143 """ | |
| 144 l = [ | |
| 145 '''\ | |
| 146 """ Python Character Mapping Codec generated from '%s'. | |
| 147 | |
| 148 Written by Marc-Andre Lemburg (mal@lemburg.com). | |
| 149 | |
| 150 (c) Copyright CNRI, All Rights Reserved. NO WARRANTY. | |
| 151 | |
| 152 """#" | |
| 153 | |
| 154 import codecs | |
| 155 | |
| 156 ### Codec APIs | |
| 157 | |
| 158 class Codec(codecs.Codec): | |
| 159 | |
| 160 def encode(self,input,errors='strict'): | |
| 161 | |
| 162 return codecs.charmap_encode(input,errors,encoding_map) | |
| 163 | |
| 164 def decode(self,input,errors='strict'): | |
| 165 | |
| 166 return codecs.charmap_decode(input,errors,decoding_map) | |
| 167 | |
| 168 class StreamWriter(Codec,codecs.StreamWriter): | |
| 169 pass | |
| 170 | |
| 171 class StreamReader(Codec,codecs.StreamReader): | |
| 172 pass | |
| 173 | |
| 174 ### encodings module API | |
| 175 | |
| 176 def getregentry(): | |
| 177 | |
| 178 return (Codec().encode,Codec().decode,StreamReader,StreamWriter) | |
| 179 | |
| 180 ### Decoding Map | |
| 181 | |
| 182 decoding_map = { | |
| 183 ''' % name, | |
| 184 ] | |
| 185 mappings = map.items() | |
| 186 mappings.sort() | |
| 187 append = l.append | |
| 188 i = 0 | |
| 189 splits = 0 | |
| 190 for e,value in mappings: | |
| 191 try: | |
| 192 (u,c) = value | |
| 193 except TypeError: | |
| 194 u = value | |
| 195 c = '' | |
| 196 key = keyrepr(e) | |
| 197 if c and comments: | |
| 198 append('\t%s: %s,\t# %s' % (key,unicoderepr(u),c)) | |
| 199 else: | |
| 200 append('\t%s: %s,' % (key,unicoderepr(u))) | |
| 201 i = i + 1 | |
| 202 if i == 4096: | |
| 203 # Split the definition into parts to that the Python | |
| 204 # parser doesn't dump core | |
| 205 if splits == 0: | |
| 206 append('}') | |
| 207 else: | |
| 208 append('})') | |
| 209 append('map.update({') | |
| 210 i = 0 | |
| 211 splits = splits + 1 | |
| 212 if splits == 0: | |
| 213 append('}') | |
| 214 else: | |
| 215 append('})') | |
| 216 append(''' | |
| 217 ### Encoding Map | |
| 218 | |
| 219 encoding_map = {} | |
| 220 for k,v in decoding_map.items(): | |
| 221 encoding_map[v] = k | |
| 222 ''') | |
| 223 return string.join(l,'\n') | |
| 224 | |
| 225 def pymap(name,map,pyfile,comments=1): | |
| 226 | |
| 227 code = codegen(name,map,comments) | |
| 228 f = open(pyfile,'w') | |
| 229 f.write(code) | |
| 230 f.close() | |
| 231 | |
| 232 def marshalmap(name,map,marshalfile): | |
| 233 | |
| 234 d = {} | |
| 235 for e,(u,c) in map.items(): | |
| 236 d[e] = (u,c) | |
| 237 f = open(marshalfile,'wb') | |
| 238 marshal.dump(d,f) | |
| 239 f.close() | |
| 240 | |
| 241 def convertdir(dir,prefix='',comments=1): | |
| 242 | |
| 243 mapnames = os.listdir(dir) | |
| 244 for mapname in mapnames: | |
| 245 name = os.path.split(mapname)[1] | |
| 246 name = string.replace(name,'-','_') | |
| 247 name = string.split(name, '.')[0] | |
| 248 name = string.lower(name) | |
| 249 codefile = name + '.py' | |
| 250 marshalfile = name + '.mapping' | |
| 251 print 'converting %s to %s and %s' % (mapname, | |
| 252 prefix + codefile, | |
| 253 prefix + marshalfile) | |
| 254 try: | |
| 255 map = readmap(os.path.join(dir,mapname)) | |
| 256 if not map: | |
| 257 print '* map is empty; skipping' | |
| 258 else: | |
| 259 pymap(mapname, map, prefix + codefile,comments) | |
| 260 marshalmap(mapname, map, prefix + marshalfile) | |
| 261 except ValueError: | |
| 262 print '* conversion failed' | |
| 263 | |
| 264 def rewritepythondir(dir,prefix='',comments=1): | |
| 265 | |
| 266 mapnames = os.listdir(dir) | |
| 267 for mapname in mapnames: | |
| 268 if mapname[-len('.mapping'):] != '.mapping': | |
| 269 continue | |
| 270 codefile = mapname[:-len('.mapping')] + '.py' | |
| 271 print 'converting %s to %s' % (mapname, | |
| 272 prefix + codefile) | |
| 273 try: | |
| 274 map = marshal.load(open(os.path.join(dir,mapname), | |
| 275 'rb')) | |
| 276 if not map: | |
| 277 print '* map is empty; skipping' | |
| 278 else: | |
| 279 pymap(mapname, map, prefix + codefile,comments) | |
| 280 except ValueError, why: | |
| 281 print '* conversion failed: %s' % why | |
| 282 | |
| 283 if __name__ == '__main__': | |
| 284 | |
| 285 import sys | |
| 286 if 1: | |
| 287 apply(convertdir,tuple(sys.argv[1:])) | |
| 288 else: | |
| 289 apply(rewritepythondir,tuple(sys.argv[1:])) |