Mercurial > cpython-lru_cache-weakref
annotate Parser/parser.c @ 73323:dad02a080bbc
Improve byte-compilation in packaging to be independent of -O or -B.
The code I fixed to comply with PEP 3147 still had one bug: When run
under python -O, some paths for pyc files would be pyo, because I called
imp.cache_from_source without explicit debug_override argument in some
places, and under -O that would return .pyo (this is well explained in
the imp docs). Now all code (util.byte_compile, build_py, install_lib)
can create .pyo files according to options given by users,
without interference from the calling Python’s own optimize mode.
On a related topic, I also removed the code that prevented byte
compilation under python -B. The rationale is that packaging gives
control over the creation of pyc files to the user with its own explicit
option, and the behavior should not be changed if the calling Python
happens to run with -B for whatever reason. I will argue that this is a
bug fix and ask to be allowed to backport this change to distutils.
Finally, I moved one nugget of information about the --compile and
--optimize options from the source into the doc. It clears up a
misunderstanding that I (and maybe other people) had.
| author | Éric Araujo <merwok@netwok.org> |
|---|---|
| date | Thu, 03 Nov 2011 05:08:28 +0100 |
| parents | cab14be0ada1 |
| children | e7f6cef7a4cc |
| rev | line source |
|---|---|
| 145 | 1 |
| 6 | 2 /* Parser implementation */ |
| 3 | |
| 4 /* For a description, see the comments at end of this file */ | |
| 5 | |
| 6 /* XXX To do: error recovery */ | |
| 7 | |
|
21181
944d6e02f359
The parser doesn't need its own implementation of assert, and having its
Tim Peters <tim.peters@gmail.com>
parents:
18739
diff
changeset
|
8 #include "Python.h" |
| 86 | 9 #include "pgenheaders.h" |
| 6 | 10 #include "token.h" |
| 11 #include "grammar.h" | |
| 12 #include "node.h" | |
| 13 #include "parser.h" | |
| 14 #include "errcode.h" | |
| 15 | |
| 16 | |
|
4839
4567e158d20c
Rename DEBUG macro to Py_DEBUG
Guido van Rossum <guido@python.org>
parents:
4503
diff
changeset
|
17 #ifdef Py_DEBUG |
|
5379
1c9a84e0981d
Another directory quickly renamed.
Guido van Rossum <guido@python.org>
parents:
4839
diff
changeset
|
18 extern int Py_DebugFlag; |
|
1c9a84e0981d
Another directory quickly renamed.
Guido van Rossum <guido@python.org>
parents:
4839
diff
changeset
|
19 #define D(x) if (!Py_DebugFlag); else x |
| 6 | 20 #else |
| 21 #define D(x) | |
| 22 #endif | |
| 23 | |
| 24 | |
| 25 /* STACK DATA TYPE */ | |
| 26 | |
|
13447
a89f7f268de4
Nuke all remaining occurrences of Py_PROTO and Py_FPROTO.
Tim Peters <tim.peters@gmail.com>
parents:
13256
diff
changeset
|
27 static void s_reset(stack *); |
| 6 | 28 |
| 29 static void | |
| 13659 | 30 s_reset(stack *s) |
| 6 | 31 { |
|
61466
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
32 s->s_top = &s->s_base[MAXSTACK]; |
| 6 | 33 } |
| 34 | |
| 35 #define s_empty(s) ((s)->s_top == &(s)->s_base[MAXSTACK]) | |
| 36 | |
| 37 static int | |
| 13659 | 38 s_push(register stack *s, dfa *d, node *parent) |
| 6 | 39 { |
|
61466
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
40 register stackentry *top; |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
41 if (s->s_top == s->s_base) { |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
42 fprintf(stderr, "s_push: parser stack overflow\n"); |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
43 return E_NOMEM; |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
44 } |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
45 top = --s->s_top; |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
46 top->s_dfa = d; |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
47 top->s_parent = parent; |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
48 top->s_state = 0; |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
49 return 0; |
| 6 | 50 } |
| 51 | |
|
4839
4567e158d20c
Rename DEBUG macro to Py_DEBUG
Guido van Rossum <guido@python.org>
parents:
4503
diff
changeset
|
52 #ifdef Py_DEBUG |
| 6 | 53 |
| 54 static void | |
| 13659 | 55 s_pop(register stack *s) |
| 6 | 56 { |
|
61466
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
57 if (s_empty(s)) |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
58 Py_FatalError("s_pop: parser stack underflow -- FATAL"); |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
59 s->s_top++; |
| 6 | 60 } |
| 61 | |
|
4839
4567e158d20c
Rename DEBUG macro to Py_DEBUG
Guido van Rossum <guido@python.org>
parents:
4503
diff
changeset
|
62 #else /* !Py_DEBUG */ |
| 6 | 63 |
| 64 #define s_pop(s) (s)->s_top++ | |
| 65 | |
| 66 #endif | |
| 67 | |
| 68 | |
| 69 /* PARSER CREATION */ | |
| 70 | |
| 71 parser_state * | |
| 13659 | 72 PyParser_New(grammar *g, int start) |
| 6 | 73 { |
|
61466
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
74 parser_state *ps; |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
75 |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
76 if (!g->g_accel) |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
77 PyGrammar_AddAccelerators(g); |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
78 ps = (parser_state *)PyMem_MALLOC(sizeof(parser_state)); |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
79 if (ps == NULL) |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
80 return NULL; |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
81 ps->p_grammar = g; |
|
36726
bbf16fd024df
from __future__ import with_statement addon for 'with', mostly written by
Thomas Wouters <thomas@python.org>
parents:
22466
diff
changeset
|
82 #ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD |
|
61466
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
83 ps->p_flags = 0; |
|
22466
1db1510abe21
Disable the parser hacks that allowed the "yield" keyword to be enabled
Neil Schemenauer <nascheme@enme.ucalgary.ca>
parents:
21181
diff
changeset
|
84 #endif |
|
61466
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
85 ps->p_tree = PyNode_New(start); |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
86 if (ps->p_tree == NULL) { |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
87 PyMem_FREE(ps); |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
88 return NULL; |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
89 } |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
90 s_reset(&ps->p_stack); |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
91 (void) s_push(&ps->p_stack, PyGrammar_FindDFA(g, start), ps->p_tree); |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
92 return ps; |
| 6 | 93 } |
| 94 | |
| 95 void | |
| 13659 | 96 PyParser_Delete(parser_state *ps) |
| 6 | 97 { |
|
61466
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
98 /* NB If you want to save the parse tree, |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
99 you must set p_tree to NULL before calling delparser! */ |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
100 PyNode_Free(ps->p_tree); |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
101 PyMem_FREE(ps); |
| 6 | 102 } |
| 103 | |
| 104 | |
| 105 /* PARSER STACK OPERATIONS */ | |
| 106 | |
| 107 static int | |
|
36786
57322cb2ca11
Patch #1440601: Add col_offset attribute to AST nodes.
Martin v. Löwis <martin@v.loewis.de>
parents:
36742
diff
changeset
|
108 shift(register stack *s, int type, char *str, int newstate, int lineno, int col_offset) |
| 6 | 109 { |
|
61466
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
110 int err; |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
111 assert(!s_empty(s)); |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
112 err = PyNode_AddChild(s->s_top->s_parent, type, str, lineno, col_offset); |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
113 if (err) |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
114 return err; |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
115 s->s_top->s_state = newstate; |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
116 return 0; |
| 6 | 117 } |
| 118 | |
| 119 static int | |
|
36786
57322cb2ca11
Patch #1440601: Add col_offset attribute to AST nodes.
Martin v. Löwis <martin@v.loewis.de>
parents:
36742
diff
changeset
|
120 push(register stack *s, int type, dfa *d, int newstate, int lineno, int col_offset) |
| 6 | 121 { |
|
61466
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
122 int err; |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
123 register node *n; |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
124 n = s->s_top->s_parent; |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
125 assert(!s_empty(s)); |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
126 err = PyNode_AddChild(n, type, (char *)NULL, lineno, col_offset); |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
127 if (err) |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
128 return err; |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
129 s->s_top->s_state = newstate; |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
130 return s_push(s, d, CHILD(n, NCH(n)-1)); |
| 6 | 131 } |
| 132 | |
| 133 | |
| 134 /* PARSER PROPER */ | |
| 135 | |
| 136 static int | |
|
18609
59446d0f8341
Preliminary support for "from __future__ import generators" to enable
Guido van Rossum <guido@python.org>
parents:
14968
diff
changeset
|
137 classify(parser_state *ps, int type, char *str) |
| 6 | 138 { |
|
61466
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
139 grammar *g = ps->p_grammar; |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
140 register int n = g->g_ll.ll_nlabels; |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
141 |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
142 if (type == NAME) { |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
143 register char *s = str; |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
144 register label *l = g->g_ll.ll_label; |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
145 register int i; |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
146 for (i = n; i > 0; i--, l++) { |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
147 if (l->lb_type != NAME || l->lb_str == NULL || |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
148 l->lb_str[0] != s[0] || |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
149 strcmp(l->lb_str, s) != 0) |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
150 continue; |
|
36726
bbf16fd024df
from __future__ import with_statement addon for 'with', mostly written by
Thomas Wouters <thomas@python.org>
parents:
22466
diff
changeset
|
151 #ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD |
|
52641
0530aadff696
The BDFL has retired! Long live the FLUFL (Friendly Language Uncle For Life)!
Brett Cannon <bcannon@gmail.com>
parents:
49594
diff
changeset
|
152 #if 0 |
|
61466
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
153 /* Leaving this in as an example */ |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
154 if (!(ps->p_flags & CO_FUTURE_WITH_STATEMENT)) { |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
155 if (s[0] == 'w' && strcmp(s, "with") == 0) |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
156 break; /* not a keyword yet */ |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
157 else if (s[0] == 'a' && strcmp(s, "as") == 0) |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
158 break; /* not a keyword yet */ |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
159 } |
|
22466
1db1510abe21
Disable the parser hacks that allowed the "yield" keyword to be enabled
Neil Schemenauer <nascheme@enme.ucalgary.ca>
parents:
21181
diff
changeset
|
160 #endif |
|
52641
0530aadff696
The BDFL has retired! Long live the FLUFL (Friendly Language Uncle For Life)!
Brett Cannon <bcannon@gmail.com>
parents:
49594
diff
changeset
|
161 #endif |
|
61466
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
162 D(printf("It's a keyword\n")); |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
163 return n - i; |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
164 } |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
165 } |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
166 |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
167 { |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
168 register label *l = g->g_ll.ll_label; |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
169 register int i; |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
170 for (i = n; i > 0; i--, l++) { |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
171 if (l->lb_type == type && l->lb_str == NULL) { |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
172 D(printf("It's a token we know\n")); |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
173 return n - i; |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
174 } |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
175 } |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
176 } |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
177 |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
178 D(printf("Illegal token\n")); |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
179 return -1; |
| 6 | 180 } |
| 181 | |
|
36726
bbf16fd024df
from __future__ import with_statement addon for 'with', mostly written by
Thomas Wouters <thomas@python.org>
parents:
22466
diff
changeset
|
182 #ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD |
|
52641
0530aadff696
The BDFL has retired! Long live the FLUFL (Friendly Language Uncle For Life)!
Brett Cannon <bcannon@gmail.com>
parents:
49594
diff
changeset
|
183 #if 0 |
|
36988
49c6eb688906
Checkpoint. 218 tests are okay; 53 are failing. Done so far:
Guido van Rossum <guido@python.org>
parents:
36786
diff
changeset
|
184 /* Leaving this in as an example */ |
|
18609
59446d0f8341
Preliminary support for "from __future__ import generators" to enable
Guido van Rossum <guido@python.org>
parents:
14968
diff
changeset
|
185 static void |
|
59446d0f8341
Preliminary support for "from __future__ import generators" to enable
Guido van Rossum <guido@python.org>
parents:
14968
diff
changeset
|
186 future_hack(parser_state *ps) |
|
59446d0f8341
Preliminary support for "from __future__ import generators" to enable
Guido van Rossum <guido@python.org>
parents:
14968
diff
changeset
|
187 { |
|
61466
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
188 node *n = ps->p_stack.s_top->s_parent; |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
189 node *ch, *cch; |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
190 int i; |
|
18609
59446d0f8341
Preliminary support for "from __future__ import generators" to enable
Guido van Rossum <guido@python.org>
parents:
14968
diff
changeset
|
191 |
|
61466
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
192 /* from __future__ import ..., must have at least 4 children */ |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
193 n = CHILD(n, 0); |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
194 if (NCH(n) < 4) |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
195 return; |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
196 ch = CHILD(n, 0); |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
197 if (STR(ch) == NULL || strcmp(STR(ch), "from") != 0) |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
198 return; |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
199 ch = CHILD(n, 1); |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
200 if (NCH(ch) == 1 && STR(CHILD(ch, 0)) && |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
201 strcmp(STR(CHILD(ch, 0)), "__future__") != 0) |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
202 return; |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
203 ch = CHILD(n, 3); |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
204 /* ch can be a star, a parenthesis or import_as_names */ |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
205 if (TYPE(ch) == STAR) |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
206 return; |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
207 if (TYPE(ch) == LPAR) |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
208 ch = CHILD(n, 4); |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
209 |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
210 for (i = 0; i < NCH(ch); i += 2) { |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
211 cch = CHILD(ch, i); |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
212 if (NCH(cch) >= 1 && TYPE(CHILD(cch, 0)) == NAME) { |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
213 char *str_ch = STR(CHILD(cch, 0)); |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
214 if (strcmp(str_ch, FUTURE_WITH_STATEMENT) == 0) { |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
215 ps->p_flags |= CO_FUTURE_WITH_STATEMENT; |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
216 } else if (strcmp(str_ch, FUTURE_PRINT_FUNCTION) == 0) { |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
217 ps->p_flags |= CO_FUTURE_PRINT_FUNCTION; |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
218 } else if (strcmp(str_ch, FUTURE_UNICODE_LITERALS) == 0) { |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
219 ps->p_flags |= CO_FUTURE_UNICODE_LITERALS; |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
220 } |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
221 } |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
222 } |
|
18609
59446d0f8341
Preliminary support for "from __future__ import generators" to enable
Guido van Rossum <guido@python.org>
parents:
14968
diff
changeset
|
223 } |
|
52641
0530aadff696
The BDFL has retired! Long live the FLUFL (Friendly Language Uncle For Life)!
Brett Cannon <bcannon@gmail.com>
parents:
49594
diff
changeset
|
224 #endif |
|
22466
1db1510abe21
Disable the parser hacks that allowed the "yield" keyword to be enabled
Neil Schemenauer <nascheme@enme.ucalgary.ca>
parents:
21181
diff
changeset
|
225 #endif /* future keyword */ |
|
18609
59446d0f8341
Preliminary support for "from __future__ import generators" to enable
Guido van Rossum <guido@python.org>
parents:
14968
diff
changeset
|
226 |
| 6 | 227 int |
| 13659 | 228 PyParser_AddToken(register parser_state *ps, register int type, char *str, |
|
61466
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
229 int lineno, int col_offset, int *expected_ret) |
| 6 | 230 { |
|
61466
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
231 register int ilabel; |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
232 int err; |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
233 |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
234 D(printf("Token %s/'%s' ... ", _PyParser_TokenNames[type], str)); |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
235 |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
236 /* Find out which label this token is */ |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
237 ilabel = classify(ps, type, str); |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
238 if (ilabel < 0) |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
239 return E_SYNTAX; |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
240 |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
241 /* Loop until the token is shifted or an error occurred */ |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
242 for (;;) { |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
243 /* Fetch the current dfa and state */ |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
244 register dfa *d = ps->p_stack.s_top->s_dfa; |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
245 register state *s = &d->d_state[ps->p_stack.s_top->s_state]; |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
246 |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
247 D(printf(" DFA '%s', state %d:", |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
248 d->d_name, ps->p_stack.s_top->s_state)); |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
249 |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
250 /* Check accelerator */ |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
251 if (s->s_lower <= ilabel && ilabel < s->s_upper) { |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
252 register int x = s->s_accel[ilabel - s->s_lower]; |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
253 if (x != -1) { |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
254 if (x & (1<<7)) { |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
255 /* Push non-terminal */ |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
256 int nt = (x >> 8) + NT_OFFSET; |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
257 int arrow = x & ((1<<7)-1); |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
258 dfa *d1 = PyGrammar_FindDFA( |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
259 ps->p_grammar, nt); |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
260 if ((err = push(&ps->p_stack, nt, d1, |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
261 arrow, lineno, col_offset)) > 0) { |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
262 D(printf(" MemError: push\n")); |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
263 return err; |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
264 } |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
265 D(printf(" Push ...\n")); |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
266 continue; |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
267 } |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
268 |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
269 /* Shift the token */ |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
270 if ((err = shift(&ps->p_stack, type, str, |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
271 x, lineno, col_offset)) > 0) { |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
272 D(printf(" MemError: shift.\n")); |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
273 return err; |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
274 } |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
275 D(printf(" Shift.\n")); |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
276 /* Pop while we are in an accept-only state */ |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
277 while (s = &d->d_state |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
278 [ps->p_stack.s_top->s_state], |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
279 s->s_accept && s->s_narcs == 1) { |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
280 D(printf(" DFA '%s', state %d: " |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
281 "Direct pop.\n", |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
282 d->d_name, |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
283 ps->p_stack.s_top->s_state)); |
|
36726
bbf16fd024df
from __future__ import with_statement addon for 'with', mostly written by
Thomas Wouters <thomas@python.org>
parents:
22466
diff
changeset
|
284 #ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD |
|
52641
0530aadff696
The BDFL has retired! Long live the FLUFL (Friendly Language Uncle For Life)!
Brett Cannon <bcannon@gmail.com>
parents:
49594
diff
changeset
|
285 #if 0 |
|
61466
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
286 if (d->d_name[0] == 'i' && |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
287 strcmp(d->d_name, |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
288 "import_stmt") == 0) |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
289 future_hack(ps); |
|
22466
1db1510abe21
Disable the parser hacks that allowed the "yield" keyword to be enabled
Neil Schemenauer <nascheme@enme.ucalgary.ca>
parents:
21181
diff
changeset
|
290 #endif |
|
52641
0530aadff696
The BDFL has retired! Long live the FLUFL (Friendly Language Uncle For Life)!
Brett Cannon <bcannon@gmail.com>
parents:
49594
diff
changeset
|
291 #endif |
|
61466
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
292 s_pop(&ps->p_stack); |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
293 if (s_empty(&ps->p_stack)) { |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
294 D(printf(" ACCEPT.\n")); |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
295 return E_DONE; |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
296 } |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
297 d = ps->p_stack.s_top->s_dfa; |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
298 } |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
299 return E_OK; |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
300 } |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
301 } |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
302 |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
303 if (s->s_accept) { |
|
36726
bbf16fd024df
from __future__ import with_statement addon for 'with', mostly written by
Thomas Wouters <thomas@python.org>
parents:
22466
diff
changeset
|
304 #ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD |
|
52641
0530aadff696
The BDFL has retired! Long live the FLUFL (Friendly Language Uncle For Life)!
Brett Cannon <bcannon@gmail.com>
parents:
49594
diff
changeset
|
305 #if 0 |
|
61466
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
306 if (d->d_name[0] == 'i' && |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
307 strcmp(d->d_name, "import_stmt") == 0) |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
308 future_hack(ps); |
|
22466
1db1510abe21
Disable the parser hacks that allowed the "yield" keyword to be enabled
Neil Schemenauer <nascheme@enme.ucalgary.ca>
parents:
21181
diff
changeset
|
309 #endif |
|
52641
0530aadff696
The BDFL has retired! Long live the FLUFL (Friendly Language Uncle For Life)!
Brett Cannon <bcannon@gmail.com>
parents:
49594
diff
changeset
|
310 #endif |
|
61466
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
311 /* Pop this dfa and try again */ |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
312 s_pop(&ps->p_stack); |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
313 D(printf(" Pop ...\n")); |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
314 if (s_empty(&ps->p_stack)) { |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
315 D(printf(" Error: bottom of stack.\n")); |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
316 return E_SYNTAX; |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
317 } |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
318 continue; |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
319 } |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
320 |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
321 /* Stuck, report syntax error */ |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
322 D(printf(" Error.\n")); |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
323 if (expected_ret) { |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
324 if (s->s_lower == s->s_upper - 1) { |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
325 /* Only one possible expected token */ |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
326 *expected_ret = ps->p_grammar-> |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
327 g_ll.ll_label[s->s_lower].lb_type; |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
328 } |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
329 else |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
330 *expected_ret = -1; |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
331 } |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
332 return E_SYNTAX; |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
333 } |
| 6 | 334 } |
| 335 | |
| 336 | |
|
4839
4567e158d20c
Rename DEBUG macro to Py_DEBUG
Guido van Rossum <guido@python.org>
parents:
4503
diff
changeset
|
337 #ifdef Py_DEBUG |
| 6 | 338 |
| 339 /* DEBUG OUTPUT */ | |
| 340 | |
| 341 void | |
| 13659 | 342 dumptree(grammar *g, node *n) |
| 6 | 343 { |
|
61466
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
344 int i; |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
345 |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
346 if (n == NULL) |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
347 printf("NIL"); |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
348 else { |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
349 label l; |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
350 l.lb_type = TYPE(n); |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
351 l.lb_str = STR(n); |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
352 printf("%s", PyGrammar_LabelRepr(&l)); |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
353 if (ISNONTERMINAL(TYPE(n))) { |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
354 printf("("); |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
355 for (i = 0; i < NCH(n); i++) { |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
356 if (i > 0) |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
357 printf(","); |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
358 dumptree(g, CHILD(n, i)); |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
359 } |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
360 printf(")"); |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
361 } |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
362 } |
| 6 | 363 } |
| 364 | |
| 365 void | |
| 13659 | 366 showtree(grammar *g, node *n) |
| 6 | 367 { |
|
61466
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
368 int i; |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
369 |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
370 if (n == NULL) |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
371 return; |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
372 if (ISNONTERMINAL(TYPE(n))) { |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
373 for (i = 0; i < NCH(n); i++) |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
374 showtree(g, CHILD(n, i)); |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
375 } |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
376 else if (ISTERMINAL(TYPE(n))) { |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
377 printf("%s", _PyParser_TokenNames[TYPE(n)]); |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
378 if (TYPE(n) == NUMBER || TYPE(n) == NAME) |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
379 printf("(%s)", STR(n)); |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
380 printf(" "); |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
381 } |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
382 else |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
383 printf("? "); |
| 6 | 384 } |
| 385 | |
| 386 void | |
| 13659 | 387 printtree(parser_state *ps) |
| 6 | 388 { |
|
61466
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
389 if (Py_DebugFlag) { |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
390 printf("Parse tree:\n"); |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
391 dumptree(ps->p_grammar, ps->p_tree); |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
392 printf("\n"); |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
393 printf("Tokens:\n"); |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
394 showtree(ps->p_grammar, ps->p_tree); |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
395 printf("\n"); |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
396 } |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
397 printf("Listing:\n"); |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
398 PyNode_ListTree(ps->p_tree); |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
399 printf("\n"); |
| 6 | 400 } |
| 401 | |
|
4839
4567e158d20c
Rename DEBUG macro to Py_DEBUG
Guido van Rossum <guido@python.org>
parents:
4503
diff
changeset
|
402 #endif /* Py_DEBUG */ |
| 6 | 403 |
| 404 /* | |
| 405 | |
| 406 Description | |
| 407 ----------- | |
| 408 | |
| 409 The parser's interface is different than usual: the function addtoken() | |
| 410 must be called for each token in the input. This makes it possible to | |
| 411 turn it into an incremental parsing system later. The parsing system | |
| 412 constructs a parse tree as it goes. | |
| 413 | |
| 414 A parsing rule is represented as a Deterministic Finite-state Automaton | |
| 415 (DFA). A node in a DFA represents a state of the parser; an arc represents | |
| 416 a transition. Transitions are either labeled with terminal symbols or | |
| 417 with non-terminals. When the parser decides to follow an arc labeled | |
| 418 with a non-terminal, it is invoked recursively with the DFA representing | |
| 419 the parsing rule for that as its initial state; when that DFA accepts, | |
| 420 the parser that invoked it continues. The parse tree constructed by the | |
| 421 recursively called parser is inserted as a child in the current parse tree. | |
| 422 | |
| 423 The DFA's can be constructed automatically from a more conventional | |
| 424 language description. An extended LL(1) grammar (ELL(1)) is suitable. | |
| 425 Certain restrictions make the parser's life easier: rules that can produce | |
| 426 the empty string should be outlawed (there are other ways to put loops | |
| 427 or optional parts in the language). To avoid the need to construct | |
| 428 FIRST sets, we can require that all but the last alternative of a rule | |
| 429 (really: arc going out of a DFA's state) must begin with a terminal | |
| 430 symbol. | |
| 431 | |
| 432 As an example, consider this grammar: | |
| 433 | |
|
61466
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
434 expr: term (OP term)* |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
435 term: CONSTANT | '(' expr ')' |
| 6 | 436 |
| 437 The DFA corresponding to the rule for expr is: | |
| 438 | |
| 439 ------->.---term-->.-------> | |
|
61466
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
440 ^ | |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
441 | | |
|
cab14be0ada1
Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
52641
diff
changeset
|
442 \----OP----/ |
| 6 | 443 |
| 444 The parse tree generated for the input a+b is: | |
| 445 | |
| 446 (expr: (term: (NAME: a)), (OP: +), (term: (NAME: b))) | |
| 447 | |
| 448 */ |
