Mercurial > cpython-lru_cache-weakref
annotate Parser/asdl.py @ 79675:c0a2294de787 2.7
Issue #16199: Backport 2to3 typo fix from 3.2.
| author | Chris Jerdonek <chris.jerdonek@gmail.com> |
|---|---|
| date | Thu, 11 Oct 2012 16:15:46 -0700 |
| parents | b36cb4602e21 |
| children |
| rev | line source |
|---|---|
| 35742 | 1 """An implementation of the Zephyr Abstract Syntax Definition Language. |
| 2 | |
| 3 See http://asdl.sourceforge.net/ and | |
|
61689
80dc570177c6
Merged revisions 81286 via svnmerge from
Senthil Kumaran <orsenthil@gmail.com>
parents:
50522
diff
changeset
|
4 http://www.cs.princeton.edu/research/techreps/TR-554-97 |
| 35742 | 5 |
| 6 Only supports top level module decl, not view. I'm guessing that view | |
| 7 is intended to support the browser and I'm not interested in the | |
| 8 browser. | |
|
36703
9f3e58dbea63
Add support for version field on Modules
Martin v. Löwis <martin@v.loewis.de>
parents:
36121
diff
changeset
|
9 |
|
9f3e58dbea63
Add support for version field on Modules
Martin v. Löwis <martin@v.loewis.de>
parents:
36121
diff
changeset
|
10 Changes for Python: Add support for module versions |
| 35742 | 11 """ |
| 12 | |
| 13 import os | |
| 14 import traceback | |
| 15 | |
| 16 import spark | |
| 17 | |
|
54611
d945e6de5227
always inherit from an appropiate base class
Benjamin Peterson <benjamin@python.org>
parents:
54610
diff
changeset
|
18 class Token(object): |
| 35742 | 19 # spark seems to dispatch in the parser based on a token's |
| 20 # type attribute | |
| 21 def __init__(self, type, lineno): | |
| 22 self.type = type | |
| 23 self.lineno = lineno | |
| 24 | |
| 25 def __str__(self): | |
| 26 return self.type | |
| 27 | |
| 28 def __repr__(self): | |
| 29 return str(self) | |
| 30 | |
| 31 class Id(Token): | |
| 32 def __init__(self, value, lineno): | |
| 33 self.type = 'Id' | |
| 34 self.value = value | |
| 35 self.lineno = lineno | |
| 36 | |
| 37 def __str__(self): | |
| 38 return self.value | |
|
36721
f7b4726ada6c
Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents:
36703
diff
changeset
|
39 |
|
36703
9f3e58dbea63
Add support for version field on Modules
Martin v. Löwis <martin@v.loewis.de>
parents:
36121
diff
changeset
|
40 class String(Token): |
|
9f3e58dbea63
Add support for version field on Modules
Martin v. Löwis <martin@v.loewis.de>
parents:
36121
diff
changeset
|
41 def __init__(self, value, lineno): |
|
9f3e58dbea63
Add support for version field on Modules
Martin v. Löwis <martin@v.loewis.de>
parents:
36121
diff
changeset
|
42 self.type = 'String' |
|
9f3e58dbea63
Add support for version field on Modules
Martin v. Löwis <martin@v.loewis.de>
parents:
36121
diff
changeset
|
43 self.value = value |
|
9f3e58dbea63
Add support for version field on Modules
Martin v. Löwis <martin@v.loewis.de>
parents:
36121
diff
changeset
|
44 self.lineno = lineno |
| 35742 | 45 |
|
54611
d945e6de5227
always inherit from an appropiate base class
Benjamin Peterson <benjamin@python.org>
parents:
54610
diff
changeset
|
46 class ASDLSyntaxError(Exception): |
| 35742 | 47 |
| 48 def __init__(self, lineno, token=None, msg=None): | |
| 49 self.lineno = lineno | |
| 50 self.token = token | |
| 51 self.msg = msg | |
| 52 | |
| 53 def __str__(self): | |
| 54 if self.msg is None: | |
| 55 return "Error at '%s', line %d" % (self.token, self.lineno) | |
| 56 else: | |
| 57 return "%s, line %d" % (self.msg, self.lineno) | |
| 58 | |
| 59 class ASDLScanner(spark.GenericScanner, object): | |
| 60 | |
| 61 def tokenize(self, input): | |
| 62 self.rv = [] | |
| 63 self.lineno = 1 | |
| 64 super(ASDLScanner, self).tokenize(input) | |
| 65 return self.rv | |
| 66 | |
| 67 def t_id(self, s): | |
| 68 r"[\w\.]+" | |
| 69 # XXX doesn't distinguish upper vs. lower, which is | |
| 70 # significant for ASDL. | |
| 71 self.rv.append(Id(s, self.lineno)) | |
|
36721
f7b4726ada6c
Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents:
36703
diff
changeset
|
72 |
|
36703
9f3e58dbea63
Add support for version field on Modules
Martin v. Löwis <martin@v.loewis.de>
parents:
36121
diff
changeset
|
73 def t_string(self, s): |
|
9f3e58dbea63
Add support for version field on Modules
Martin v. Löwis <martin@v.loewis.de>
parents:
36121
diff
changeset
|
74 r'"[^"]*"' |
|
9f3e58dbea63
Add support for version field on Modules
Martin v. Löwis <martin@v.loewis.de>
parents:
36121
diff
changeset
|
75 self.rv.append(String(s, self.lineno)) |
| 35742 | 76 |
| 77 def t_xxx(self, s): # not sure what this production means | |
| 78 r"<=" | |
| 79 self.rv.append(Token(s, self.lineno)) | |
| 80 | |
| 81 def t_punctuation(self, s): | |
| 82 r"[\{\}\*\=\|\(\)\,\?\:]" | |
| 83 self.rv.append(Token(s, self.lineno)) | |
| 84 | |
| 85 def t_comment(self, s): | |
| 86 r"\-\-[^\n]*" | |
| 87 pass | |
| 88 | |
| 89 def t_newline(self, s): | |
| 90 r"\n" | |
| 91 self.lineno += 1 | |
| 92 | |
| 93 def t_whitespace(self, s): | |
| 94 r"[ \t]+" | |
| 95 pass | |
| 96 | |
| 97 def t_default(self, s): | |
| 98 r" . +" | |
| 99 raise ValueError, "unmatched input: %s" % `s` | |
| 100 | |
| 101 class ASDLParser(spark.GenericParser, object): | |
| 102 def __init__(self): | |
| 103 super(ASDLParser, self).__init__("module") | |
| 104 | |
| 105 def typestring(self, tok): | |
| 106 return tok.type | |
| 107 | |
| 108 def error(self, tok): | |
| 109 raise ASDLSyntaxError(tok.lineno, tok) | |
| 110 | |
|
36703
9f3e58dbea63
Add support for version field on Modules
Martin v. Löwis <martin@v.loewis.de>
parents:
36121
diff
changeset
|
111 def p_module_0(self, (module, name, version, _0, _1)): |
|
9f3e58dbea63
Add support for version field on Modules
Martin v. Löwis <martin@v.loewis.de>
parents:
36121
diff
changeset
|
112 " module ::= Id Id version { } " |
| 35742 | 113 if module.value != "module": |
| 114 raise ASDLSyntaxError(module.lineno, | |
| 115 msg="expected 'module', found %s" % module) | |
|
36703
9f3e58dbea63
Add support for version field on Modules
Martin v. Löwis <martin@v.loewis.de>
parents:
36121
diff
changeset
|
116 return Module(name, None, version) |
| 35742 | 117 |
|
36703
9f3e58dbea63
Add support for version field on Modules
Martin v. Löwis <martin@v.loewis.de>
parents:
36121
diff
changeset
|
118 def p_module(self, (module, name, version, _0, definitions, _1)): |
|
9f3e58dbea63
Add support for version field on Modules
Martin v. Löwis <martin@v.loewis.de>
parents:
36121
diff
changeset
|
119 " module ::= Id Id version { definitions } " |
| 35742 | 120 if module.value != "module": |
| 121 raise ASDLSyntaxError(module.lineno, | |
| 122 msg="expected 'module', found %s" % module) | |
|
36703
9f3e58dbea63
Add support for version field on Modules
Martin v. Löwis <martin@v.loewis.de>
parents:
36121
diff
changeset
|
123 return Module(name, definitions, version) |
|
36721
f7b4726ada6c
Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents:
36703
diff
changeset
|
124 |
|
36703
9f3e58dbea63
Add support for version field on Modules
Martin v. Löwis <martin@v.loewis.de>
parents:
36121
diff
changeset
|
125 def p_version(self, (version, V)): |
|
9f3e58dbea63
Add support for version field on Modules
Martin v. Löwis <martin@v.loewis.de>
parents:
36121
diff
changeset
|
126 "version ::= Id String" |
|
9f3e58dbea63
Add support for version field on Modules
Martin v. Löwis <martin@v.loewis.de>
parents:
36121
diff
changeset
|
127 if version.value != "version": |
|
9f3e58dbea63
Add support for version field on Modules
Martin v. Löwis <martin@v.loewis.de>
parents:
36121
diff
changeset
|
128 raise ASDLSyntaxError(version.lineno, |
|
9f3e58dbea63
Add support for version field on Modules
Martin v. Löwis <martin@v.loewis.de>
parents:
36121
diff
changeset
|
129 msg="expected 'version', found %" % version) |
|
9f3e58dbea63
Add support for version field on Modules
Martin v. Löwis <martin@v.loewis.de>
parents:
36121
diff
changeset
|
130 return V |
| 35742 | 131 |
| 132 def p_definition_0(self, (definition,)): | |
| 133 " definitions ::= definition " | |
| 134 return definition | |
| 135 | |
| 136 def p_definition_1(self, (definitions, definition)): | |
| 137 " definitions ::= definition definitions " | |
| 138 return definitions + definition | |
| 139 | |
| 140 def p_definition(self, (id, _, type)): | |
| 141 " definition ::= Id = type " | |
| 142 return [Type(id, type)] | |
| 143 | |
| 144 def p_type_0(self, (product,)): | |
| 145 " type ::= product " | |
| 146 return product | |
| 147 | |
| 148 def p_type_1(self, (sum,)): | |
| 149 " type ::= sum " | |
| 150 return Sum(sum) | |
| 151 | |
| 152 def p_type_2(self, (sum, id, _0, attributes, _1)): | |
| 153 " type ::= sum Id ( fields ) " | |
| 154 if id.value != "attributes": | |
| 155 raise ASDLSyntaxError(id.lineno, | |
| 156 msg="expected attributes, found %s" % id) | |
|
36786
57322cb2ca11
Patch #1440601: Add col_offset attribute to AST nodes.
Martin v. Löwis <martin@v.loewis.de>
parents:
36721
diff
changeset
|
157 if attributes: |
|
57322cb2ca11
Patch #1440601: Add col_offset attribute to AST nodes.
Martin v. Löwis <martin@v.loewis.de>
parents:
36721
diff
changeset
|
158 attributes.reverse() |
| 35742 | 159 return Sum(sum, attributes) |
| 160 | |
| 161 def p_product(self, (_0, fields, _1)): | |
| 162 " product ::= ( fields ) " | |
| 163 # XXX can't I just construct things in the right order? | |
|
36121
686306a67e9b
Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents:
35742
diff
changeset
|
164 fields.reverse() |
| 35742 | 165 return Product(fields) |
| 166 | |
| 167 def p_sum_0(self, (constructor,)): | |
|
50522
ee9ced92843a
Merged revisions 68133-68134,68141-68142,68145-68146,68148-68149,68159-68162,68166,68171-68174,68179,68195-68196,68210,68214-68215,68217-68222 via svnmerge from
Georg Brandl <georg@python.org>
parents:
36786
diff
changeset
|
168 " sum ::= constructor " |
| 35742 | 169 return [constructor] |
| 170 | |
| 171 def p_sum_1(self, (constructor, _, sum)): | |
| 172 " sum ::= constructor | sum " | |
| 173 return [constructor] + sum | |
| 174 | |
| 175 def p_sum_2(self, (constructor, _, sum)): | |
| 176 " sum ::= constructor | sum " | |
| 177 return [constructor] + sum | |
| 178 | |
| 179 def p_constructor_0(self, (id,)): | |
| 180 " constructor ::= Id " | |
| 181 return Constructor(id) | |
| 182 | |
| 183 def p_constructor_1(self, (id, _0, fields, _1)): | |
| 184 " constructor ::= Id ( fields ) " | |
| 185 # XXX can't I just construct things in the right order? | |
|
36121
686306a67e9b
Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents:
35742
diff
changeset
|
186 fields.reverse() |
| 35742 | 187 return Constructor(id, fields) |
| 188 | |
| 189 def p_fields_0(self, (field,)): | |
| 190 " fields ::= field " | |
| 191 return [field] | |
| 192 | |
| 193 def p_fields_1(self, (field, _, fields)): | |
| 194 " fields ::= field , fields " | |
| 195 return fields + [field] | |
| 196 | |
| 197 def p_field_0(self, (type,)): | |
| 198 " field ::= Id " | |
| 199 return Field(type) | |
| 200 | |
| 201 def p_field_1(self, (type, name)): | |
| 202 " field ::= Id Id " | |
| 203 return Field(type, name) | |
| 204 | |
| 205 def p_field_2(self, (type, _, name)): | |
| 206 " field ::= Id * Id " | |
|
54612
d79fde1041bd
use booleans for flags
Benjamin Peterson <benjamin@python.org>
parents:
54611
diff
changeset
|
207 return Field(type, name, seq=True) |
| 35742 | 208 |
| 209 def p_field_3(self, (type, _, name)): | |
| 210 " field ::= Id ? Id " | |
|
54612
d79fde1041bd
use booleans for flags
Benjamin Peterson <benjamin@python.org>
parents:
54611
diff
changeset
|
211 return Field(type, name, opt=True) |
| 35742 | 212 |
| 213 def p_field_4(self, (type, _)): | |
| 214 " field ::= Id * " | |
|
54612
d79fde1041bd
use booleans for flags
Benjamin Peterson <benjamin@python.org>
parents:
54611
diff
changeset
|
215 return Field(type, seq=True) |
| 35742 | 216 |
| 217 def p_field_5(self, (type, _)): | |
| 218 " field ::= Id ? " | |
|
54612
d79fde1041bd
use booleans for flags
Benjamin Peterson <benjamin@python.org>
parents:
54611
diff
changeset
|
219 return Field(type, opt=True) |
| 35742 | 220 |
| 221 builtin_types = ("identifier", "string", "int", "bool", "object") | |
| 222 | |
| 223 # below is a collection of classes to capture the AST of an AST :-) | |
| 224 # not sure if any of the methods are useful yet, but I'm adding them | |
| 225 # piecemeal as they seem helpful | |
| 226 | |
|
54610
851636b62ad9
inherit from object
Benjamin Peterson <benjamin@python.org>
parents:
50522
diff
changeset
|
227 class AST(object): |
| 35742 | 228 pass # a marker class |
| 229 | |
| 230 class Module(AST): | |
|
36703
9f3e58dbea63
Add support for version field on Modules
Martin v. Löwis <martin@v.loewis.de>
parents:
36121
diff
changeset
|
231 def __init__(self, name, dfns, version): |
| 35742 | 232 self.name = name |
| 233 self.dfns = dfns | |
|
36703
9f3e58dbea63
Add support for version field on Modules
Martin v. Löwis <martin@v.loewis.de>
parents:
36121
diff
changeset
|
234 self.version = version |
| 35742 | 235 self.types = {} # maps type name to value (from dfns) |
| 236 for type in dfns: | |
| 237 self.types[type.name.value] = type.value | |
| 238 | |
| 239 def __repr__(self): | |
| 240 return "Module(%s, %s)" % (self.name, self.dfns) | |
| 241 | |
| 242 class Type(AST): | |
| 243 def __init__(self, name, value): | |
| 244 self.name = name | |
| 245 self.value = value | |
| 246 | |
| 247 def __repr__(self): | |
| 248 return "Type(%s, %s)" % (self.name, self.value) | |
| 249 | |
| 250 class Constructor(AST): | |
| 251 def __init__(self, name, fields=None): | |
| 252 self.name = name | |
| 253 self.fields = fields or [] | |
| 254 | |
| 255 def __repr__(self): | |
| 256 return "Constructor(%s, %s)" % (self.name, self.fields) | |
| 257 | |
| 258 class Field(AST): | |
|
54612
d79fde1041bd
use booleans for flags
Benjamin Peterson <benjamin@python.org>
parents:
54611
diff
changeset
|
259 def __init__(self, type, name=None, seq=False, opt=False): |
| 35742 | 260 self.type = type |
| 261 self.name = name | |
| 262 self.seq = seq | |
| 263 self.opt = opt | |
| 264 | |
| 265 def __repr__(self): | |
| 266 if self.seq: | |
|
54612
d79fde1041bd
use booleans for flags
Benjamin Peterson <benjamin@python.org>
parents:
54611
diff
changeset
|
267 extra = ", seq=True" |
| 35742 | 268 elif self.opt: |
|
54612
d79fde1041bd
use booleans for flags
Benjamin Peterson <benjamin@python.org>
parents:
54611
diff
changeset
|
269 extra = ", opt=True" |
| 35742 | 270 else: |
| 271 extra = "" | |
| 272 if self.name is None: | |
| 273 return "Field(%s%s)" % (self.type, extra) | |
| 274 else: | |
| 275 return "Field(%s, %s%s)" % (self.type, self.name, extra) | |
| 276 | |
| 277 class Sum(AST): | |
| 278 def __init__(self, types, attributes=None): | |
| 279 self.types = types | |
| 280 self.attributes = attributes or [] | |
| 281 | |
| 282 def __repr__(self): | |
| 283 if self.attributes is None: | |
| 284 return "Sum(%s)" % self.types | |
| 285 else: | |
| 286 return "Sum(%s, %s)" % (self.types, self.attributes) | |
| 287 | |
| 288 class Product(AST): | |
| 289 def __init__(self, fields): | |
| 290 self.fields = fields | |
| 291 | |
| 292 def __repr__(self): | |
| 293 return "Product(%s)" % self.fields | |
| 294 | |
| 295 class VisitorBase(object): | |
| 296 | |
|
54612
d79fde1041bd
use booleans for flags
Benjamin Peterson <benjamin@python.org>
parents:
54611
diff
changeset
|
297 def __init__(self, skip=False): |
| 35742 | 298 self.cache = {} |
| 299 self.skip = skip | |
| 300 | |
| 301 def visit(self, object, *args): | |
| 302 meth = self._dispatch(object) | |
| 303 if meth is None: | |
| 304 return | |
| 305 try: | |
| 306 meth(object, *args) | |
| 307 except Exception, err: | |
| 308 print "Error visiting", repr(object) | |
| 309 print err | |
| 310 traceback.print_exc() | |
| 311 # XXX hack | |
| 312 if hasattr(self, 'file'): | |
| 313 self.file.flush() | |
| 314 os._exit(1) | |
| 315 | |
| 316 def _dispatch(self, object): | |
| 317 assert isinstance(object, AST), repr(object) | |
| 318 klass = object.__class__ | |
| 319 meth = self.cache.get(klass) | |
| 320 if meth is None: | |
| 321 methname = "visit" + klass.__name__ | |
| 322 if self.skip: | |
| 323 meth = getattr(self, methname, None) | |
| 324 else: | |
| 325 meth = getattr(self, methname) | |
| 326 self.cache[klass] = meth | |
| 327 return meth | |
| 328 | |
| 329 class Check(VisitorBase): | |
| 330 | |
| 331 def __init__(self): | |
|
54612
d79fde1041bd
use booleans for flags
Benjamin Peterson <benjamin@python.org>
parents:
54611
diff
changeset
|
332 super(Check, self).__init__(skip=True) |
| 35742 | 333 self.cons = {} |
| 334 self.errors = 0 | |
| 335 self.types = {} | |
| 336 | |
| 337 def visitModule(self, mod): | |
| 338 for dfn in mod.dfns: | |
| 339 self.visit(dfn) | |
| 340 | |
| 341 def visitType(self, type): | |
| 342 self.visit(type.value, str(type.name)) | |
| 343 | |
| 344 def visitSum(self, sum, name): | |
| 345 for t in sum.types: | |
| 346 self.visit(t, name) | |
| 347 | |
| 348 def visitConstructor(self, cons, name): | |
| 349 key = str(cons.name) | |
| 350 conflict = self.cons.get(key) | |
| 351 if conflict is None: | |
| 352 self.cons[key] = name | |
| 353 else: | |
| 354 print "Redefinition of constructor %s" % key | |
| 355 print "Defined in %s and %s" % (conflict, name) | |
| 356 self.errors += 1 | |
| 357 for f in cons.fields: | |
| 358 self.visit(f, key) | |
| 359 | |
| 360 def visitField(self, field, name): | |
| 361 key = str(field.type) | |
| 362 l = self.types.setdefault(key, []) | |
| 363 l.append(name) | |
| 364 | |
| 365 def visitProduct(self, prod, name): | |
| 366 for f in prod.fields: | |
| 367 self.visit(f, name) | |
| 368 | |
| 369 def check(mod): | |
| 370 v = Check() | |
| 371 v.visit(mod) | |
| 372 | |
| 373 for t in v.types: | |
| 54613 | 374 if t not in mod.types and not t in builtin_types: |
| 35742 | 375 v.errors += 1 |
| 376 uses = ", ".join(v.types[t]) | |
| 377 print "Undefined type %s, used in %s" % (t, uses) | |
|
36121
686306a67e9b
Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents:
35742
diff
changeset
|
378 |
| 35742 | 379 return not v.errors |
| 380 | |
| 381 def parse(file): | |
| 382 scanner = ASDLScanner() | |
| 383 parser = ASDLParser() | |
| 384 | |
| 385 buf = open(file).read() | |
| 386 tokens = scanner.tokenize(buf) | |
| 387 try: | |
| 388 return parser.parse(tokens) | |
| 389 except ASDLSyntaxError, err: | |
| 390 print err | |
| 391 lines = buf.split("\n") | |
| 392 print lines[err.lineno - 1] # lines starts at 0, files at 1 | |
| 393 | |
| 394 if __name__ == "__main__": | |
| 395 import glob | |
| 396 import sys | |
| 397 | |
| 398 if len(sys.argv) > 1: | |
| 399 files = sys.argv[1:] | |
| 400 else: | |
| 401 testdir = "tests" | |
| 402 files = glob.glob(testdir + "/*.asdl") | |
|
36121
686306a67e9b
Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents:
35742
diff
changeset
|
403 |
| 35742 | 404 for file in files: |
| 405 print file | |
| 406 mod = parse(file) | |
| 407 print "module", mod.name | |
| 408 print len(mod.dfns), "definitions" | |
| 409 if not check(mod): | |
| 410 print "Check failed" | |
| 411 else: | |
| 412 for dfn in mod.dfns: | |
| 413 print dfn.type |