Mercurial > cpython-lru_cache-weakref
annotate Parser/parser.c @ 49955:878d40468c4e legacy-trunk
Issue #4384: Added logging integration with warnings module using captureWarnings(). This change includes a NullHandler which does nothing; it will be of use to library developers who want to avoid the "No handlers could be found for logger XXX" message which can appear if the library user doesn't configure logging.
| author | Vinay Sajip <vinay_sajip@yahoo.co.uk> |
|---|---|
| date | Wed, 03 Dec 2008 23:22:58 +0000 |
| parents | e80d9e6e26bb |
| children | fd25030223b0 |
| rev | line source |
|---|---|
| 145 | 1 |
| 6 | 2 /* Parser implementation */ |
| 3 | |
| 4 /* For a description, see the comments at end of this file */ | |
| 5 | |
| 6 /* XXX To do: error recovery */ | |
| 7 | |
|
21181
944d6e02f359
The parser doesn't need its own implementation of assert, and having its
Tim Peters <tim.peters@gmail.com>
parents:
18739
diff
changeset
|
8 #include "Python.h" |
| 86 | 9 #include "pgenheaders.h" |
| 6 | 10 #include "token.h" |
| 11 #include "grammar.h" | |
| 12 #include "node.h" | |
| 13 #include "parser.h" | |
| 14 #include "errcode.h" | |
| 15 | |
| 16 | |
|
4839
4567e158d20c
Rename DEBUG macro to Py_DEBUG
Guido van Rossum <guido@python.org>
parents:
4503
diff
changeset
|
17 #ifdef Py_DEBUG |
|
5379
1c9a84e0981d
Another directory quickly renamed.
Guido van Rossum <guido@python.org>
parents:
4839
diff
changeset
|
18 extern int Py_DebugFlag; |
|
1c9a84e0981d
Another directory quickly renamed.
Guido van Rossum <guido@python.org>
parents:
4839
diff
changeset
|
19 #define D(x) if (!Py_DebugFlag); else x |
| 6 | 20 #else |
| 21 #define D(x) | |
| 22 #endif | |
| 23 | |
| 24 | |
| 25 /* STACK DATA TYPE */ | |
| 26 | |
|
13447
a89f7f268de4
Nuke all remaining occurrences of Py_PROTO and Py_FPROTO.
Tim Peters <tim.peters@gmail.com>
parents:
13256
diff
changeset
|
27 static void s_reset(stack *); |
| 6 | 28 |
| 29 static void | |
| 13659 | 30 s_reset(stack *s) |
| 6 | 31 { |
| 32 s->s_top = &s->s_base[MAXSTACK]; | |
| 33 } | |
| 34 | |
| 35 #define s_empty(s) ((s)->s_top == &(s)->s_base[MAXSTACK]) | |
| 36 | |
| 37 static int | |
| 13659 | 38 s_push(register stack *s, dfa *d, node *parent) |
| 6 | 39 { |
| 40 register stackentry *top; | |
| 41 if (s->s_top == s->s_base) { | |
| 42 fprintf(stderr, "s_push: parser stack overflow\n"); | |
|
14968
5336946b6495
Fix a bug in stack overflow error handling. This fixes half of Bug
Guido van Rossum <guido@python.org>
parents:
14375
diff
changeset
|
43 return E_NOMEM; |
| 6 | 44 } |
| 45 top = --s->s_top; | |
| 46 top->s_dfa = d; | |
| 47 top->s_parent = parent; | |
| 48 top->s_state = 0; | |
| 49 return 0; | |
| 50 } | |
| 51 | |
|
4839
4567e158d20c
Rename DEBUG macro to Py_DEBUG
Guido van Rossum <guido@python.org>
parents:
4503
diff
changeset
|
52 #ifdef Py_DEBUG |
| 6 | 53 |
| 54 static void | |
| 13659 | 55 s_pop(register stack *s) |
| 6 | 56 { |
|
2044
281e40652e23
Parser/tokenizer.c (tok_nextc): zap tok->buf after freeing;
Guido van Rossum <guido@python.org>
parents:
1848
diff
changeset
|
57 if (s_empty(s)) |
|
5379
1c9a84e0981d
Another directory quickly renamed.
Guido van Rossum <guido@python.org>
parents:
4839
diff
changeset
|
58 Py_FatalError("s_pop: parser stack underflow -- FATAL"); |
| 6 | 59 s->s_top++; |
| 60 } | |
| 61 | |
|
4839
4567e158d20c
Rename DEBUG macro to Py_DEBUG
Guido van Rossum <guido@python.org>
parents:
4503
diff
changeset
|
62 #else /* !Py_DEBUG */ |
| 6 | 63 |
| 64 #define s_pop(s) (s)->s_top++ | |
| 65 | |
| 66 #endif | |
| 67 | |
| 68 | |
| 69 /* PARSER CREATION */ | |
| 70 | |
| 71 parser_state * | |
| 13659 | 72 PyParser_New(grammar *g, int start) |
| 6 | 73 { |
| 74 parser_state *ps; | |
| 75 | |
| 76 if (!g->g_accel) | |
|
5379
1c9a84e0981d
Another directory quickly renamed.
Guido van Rossum <guido@python.org>
parents:
4839
diff
changeset
|
77 PyGrammar_AddAccelerators(g); |
|
37585
292854c43b99
Fix the code in Parser/ to also compile with C++. This was mostly casts for
Anthony Baxter <anthonybaxter@gmail.com>
parents:
37557
diff
changeset
|
78 ps = (parser_state *)PyMem_MALLOC(sizeof(parser_state)); |
| 6 | 79 if (ps == NULL) |
| 80 return NULL; | |
| 81 ps->p_grammar = g; | |
|
36726
bbf16fd024df
from __future__ import with_statement addon for 'with', mostly written by
Thomas Wouters <thomas@python.org>
parents:
22466
diff
changeset
|
82 #ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD |
|
bbf16fd024df
from __future__ import with_statement addon for 'with', mostly written by
Thomas Wouters <thomas@python.org>
parents:
22466
diff
changeset
|
83 ps->p_flags = 0; |
|
22466
1db1510abe21
Disable the parser hacks that allowed the "yield" keyword to be enabled
Neil Schemenauer <nascheme@enme.ucalgary.ca>
parents:
21181
diff
changeset
|
84 #endif |
|
5379
1c9a84e0981d
Another directory quickly renamed.
Guido van Rossum <guido@python.org>
parents:
4839
diff
changeset
|
85 ps->p_tree = PyNode_New(start); |
| 6 | 86 if (ps->p_tree == NULL) { |
|
37557
6ae4adefc573
SF patch #1467512, fix double free with triple quoted string in standard build.
Neal Norwitz <nnorwitz@gmail.com>
parents:
36786
diff
changeset
|
87 PyMem_FREE(ps); |
| 6 | 88 return NULL; |
| 89 } | |
| 90 s_reset(&ps->p_stack); | |
|
5379
1c9a84e0981d
Another directory quickly renamed.
Guido van Rossum <guido@python.org>
parents:
4839
diff
changeset
|
91 (void) s_push(&ps->p_stack, PyGrammar_FindDFA(g, start), ps->p_tree); |
| 6 | 92 return ps; |
| 93 } | |
| 94 | |
| 95 void | |
| 13659 | 96 PyParser_Delete(parser_state *ps) |
| 6 | 97 { |
|
79
6c43e5476de6
Free parse tree when deleting parser.
Guido van Rossum <guido@python.org>
parents:
6
diff
changeset
|
98 /* NB If you want to save the parse tree, |
|
6c43e5476de6
Free parse tree when deleting parser.
Guido van Rossum <guido@python.org>
parents:
6
diff
changeset
|
99 you must set p_tree to NULL before calling delparser! */ |
|
5379
1c9a84e0981d
Another directory quickly renamed.
Guido van Rossum <guido@python.org>
parents:
4839
diff
changeset
|
100 PyNode_Free(ps->p_tree); |
|
37557
6ae4adefc573
SF patch #1467512, fix double free with triple quoted string in standard build.
Neal Norwitz <nnorwitz@gmail.com>
parents:
36786
diff
changeset
|
101 PyMem_FREE(ps); |
| 6 | 102 } |
| 103 | |
| 104 | |
| 105 /* PARSER STACK OPERATIONS */ | |
| 106 | |
| 107 static int | |
|
36786
57322cb2ca11
Patch #1440601: Add col_offset attribute to AST nodes.
Martin v. Löwis <martin@v.loewis.de>
parents:
36742
diff
changeset
|
108 shift(register stack *s, int type, char *str, int newstate, int lineno, int col_offset) |
| 6 | 109 { |
|
12970
63436719d12e
Add new parser error code, E_OVERFLOW. This error is returned when
Jeremy Hylton <jeremy@alum.mit.edu>
parents:
5379
diff
changeset
|
110 int err; |
| 6 | 111 assert(!s_empty(s)); |
|
36786
57322cb2ca11
Patch #1440601: Add col_offset attribute to AST nodes.
Martin v. Löwis <martin@v.loewis.de>
parents:
36742
diff
changeset
|
112 err = PyNode_AddChild(s->s_top->s_parent, type, str, lineno, col_offset); |
|
12970
63436719d12e
Add new parser error code, E_OVERFLOW. This error is returned when
Jeremy Hylton <jeremy@alum.mit.edu>
parents:
5379
diff
changeset
|
113 if (err) |
|
63436719d12e
Add new parser error code, E_OVERFLOW. This error is returned when
Jeremy Hylton <jeremy@alum.mit.edu>
parents:
5379
diff
changeset
|
114 return err; |
| 6 | 115 s->s_top->s_state = newstate; |
| 116 return 0; | |
| 117 } | |
| 118 | |
| 119 static int | |
|
36786
57322cb2ca11
Patch #1440601: Add col_offset attribute to AST nodes.
Martin v. Löwis <martin@v.loewis.de>
parents:
36742
diff
changeset
|
120 push(register stack *s, int type, dfa *d, int newstate, int lineno, int col_offset) |
| 6 | 121 { |
|
12970
63436719d12e
Add new parser error code, E_OVERFLOW. This error is returned when
Jeremy Hylton <jeremy@alum.mit.edu>
parents:
5379
diff
changeset
|
122 int err; |
| 6 | 123 register node *n; |
| 124 n = s->s_top->s_parent; | |
| 125 assert(!s_empty(s)); | |
|
36786
57322cb2ca11
Patch #1440601: Add col_offset attribute to AST nodes.
Martin v. Löwis <martin@v.loewis.de>
parents:
36742
diff
changeset
|
126 err = PyNode_AddChild(n, type, (char *)NULL, lineno, col_offset); |
|
12970
63436719d12e
Add new parser error code, E_OVERFLOW. This error is returned when
Jeremy Hylton <jeremy@alum.mit.edu>
parents:
5379
diff
changeset
|
127 if (err) |
|
63436719d12e
Add new parser error code, E_OVERFLOW. This error is returned when
Jeremy Hylton <jeremy@alum.mit.edu>
parents:
5379
diff
changeset
|
128 return err; |
| 6 | 129 s->s_top->s_state = newstate; |
| 130 return s_push(s, d, CHILD(n, NCH(n)-1)); | |
| 131 } | |
| 132 | |
| 133 | |
| 134 /* PARSER PROPER */ | |
| 135 | |
| 136 static int | |
|
18609
59446d0f8341
Preliminary support for "from __future__ import generators" to enable
Guido van Rossum <guido@python.org>
parents:
14968
diff
changeset
|
137 classify(parser_state *ps, int type, char *str) |
| 6 | 138 { |
|
18609
59446d0f8341
Preliminary support for "from __future__ import generators" to enable
Guido van Rossum <guido@python.org>
parents:
14968
diff
changeset
|
139 grammar *g = ps->p_grammar; |
| 6 | 140 register int n = g->g_ll.ll_nlabels; |
| 141 | |
| 142 if (type == NAME) { | |
| 143 register char *s = str; | |
| 144 register label *l = g->g_ll.ll_label; | |
| 145 register int i; | |
| 146 for (i = n; i > 0; i--, l++) { | |
|
36742
658f8ff2a442
Make 'as' an actual keyword when with's future statement is used. Not
Thomas Wouters <thomas@python.org>
parents:
36726
diff
changeset
|
147 if (l->lb_type != NAME || l->lb_str == NULL || |
|
658f8ff2a442
Make 'as' an actual keyword when with's future statement is used. Not
Thomas Wouters <thomas@python.org>
parents:
36726
diff
changeset
|
148 l->lb_str[0] != s[0] || |
|
658f8ff2a442
Make 'as' an actual keyword when with's future statement is used. Not
Thomas Wouters <thomas@python.org>
parents:
36726
diff
changeset
|
149 strcmp(l->lb_str, s) != 0) |
|
658f8ff2a442
Make 'as' an actual keyword when with's future statement is used. Not
Thomas Wouters <thomas@python.org>
parents:
36726
diff
changeset
|
150 continue; |
|
36726
bbf16fd024df
from __future__ import with_statement addon for 'with', mostly written by
Thomas Wouters <thomas@python.org>
parents:
22466
diff
changeset
|
151 #ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD |
|
45891
727f743cf630
Backport of the print function, using a __future__ import.
Eric Smith <eric@trueblade.com>
parents:
39881
diff
changeset
|
152 if (ps->p_flags & CO_FUTURE_PRINT_FUNCTION && |
|
727f743cf630
Backport of the print function, using a __future__ import.
Eric Smith <eric@trueblade.com>
parents:
39881
diff
changeset
|
153 s[0] == 'p' && strcmp(s, "print") == 0) { |
|
727f743cf630
Backport of the print function, using a __future__ import.
Eric Smith <eric@trueblade.com>
parents:
39881
diff
changeset
|
154 break; /* no longer a keyword */ |
|
727f743cf630
Backport of the print function, using a __future__ import.
Eric Smith <eric@trueblade.com>
parents:
39881
diff
changeset
|
155 } |
|
22466
1db1510abe21
Disable the parser hacks that allowed the "yield" keyword to be enabled
Neil Schemenauer <nascheme@enme.ucalgary.ca>
parents:
21181
diff
changeset
|
156 #endif |
|
36742
658f8ff2a442
Make 'as' an actual keyword when with's future statement is used. Not
Thomas Wouters <thomas@python.org>
parents:
36726
diff
changeset
|
157 D(printf("It's a keyword\n")); |
|
658f8ff2a442
Make 'as' an actual keyword when with's future statement is used. Not
Thomas Wouters <thomas@python.org>
parents:
36726
diff
changeset
|
158 return n - i; |
| 6 | 159 } |
| 160 } | |
| 161 | |
| 162 { | |
| 163 register label *l = g->g_ll.ll_label; | |
| 164 register int i; | |
| 165 for (i = n; i > 0; i--, l++) { | |
| 166 if (l->lb_type == type && l->lb_str == NULL) { | |
| 167 D(printf("It's a token we know\n")); | |
| 168 return n - i; | |
| 169 } | |
| 170 } | |
| 171 } | |
| 172 | |
| 173 D(printf("Illegal token\n")); | |
| 174 return -1; | |
| 175 } | |
| 176 | |
|
36726
bbf16fd024df
from __future__ import with_statement addon for 'with', mostly written by
Thomas Wouters <thomas@python.org>
parents:
22466
diff
changeset
|
177 #ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD |
|
18609
59446d0f8341
Preliminary support for "from __future__ import generators" to enable
Guido van Rossum <guido@python.org>
parents:
14968
diff
changeset
|
178 static void |
|
59446d0f8341
Preliminary support for "from __future__ import generators" to enable
Guido van Rossum <guido@python.org>
parents:
14968
diff
changeset
|
179 future_hack(parser_state *ps) |
|
59446d0f8341
Preliminary support for "from __future__ import generators" to enable
Guido van Rossum <guido@python.org>
parents:
14968
diff
changeset
|
180 { |
|
59446d0f8341
Preliminary support for "from __future__ import generators" to enable
Guido van Rossum <guido@python.org>
parents:
14968
diff
changeset
|
181 node *n = ps->p_stack.s_top->s_parent; |
|
39881
25bcdf3cd194
Fix a bug in the parser's future statement handling that led to "with"
Georg Brandl <georg@python.org>
parents:
37585
diff
changeset
|
182 node *ch, *cch; |
|
18739
704486642079
Fis SF bug #442647: not all forms of legal future statements were
Guido van Rossum <guido@python.org>
parents:
18609
diff
changeset
|
183 int i; |
|
18609
59446d0f8341
Preliminary support for "from __future__ import generators" to enable
Guido van Rossum <guido@python.org>
parents:
14968
diff
changeset
|
184 |
|
36726
bbf16fd024df
from __future__ import with_statement addon for 'with', mostly written by
Thomas Wouters <thomas@python.org>
parents:
22466
diff
changeset
|
185 /* from __future__ import ..., must have at least 4 children */ |
|
bbf16fd024df
from __future__ import with_statement addon for 'with', mostly written by
Thomas Wouters <thomas@python.org>
parents:
22466
diff
changeset
|
186 n = CHILD(n, 0); |
|
bbf16fd024df
from __future__ import with_statement addon for 'with', mostly written by
Thomas Wouters <thomas@python.org>
parents:
22466
diff
changeset
|
187 if (NCH(n) < 4) |
|
bbf16fd024df
from __future__ import with_statement addon for 'with', mostly written by
Thomas Wouters <thomas@python.org>
parents:
22466
diff
changeset
|
188 return; |
|
bbf16fd024df
from __future__ import with_statement addon for 'with', mostly written by
Thomas Wouters <thomas@python.org>
parents:
22466
diff
changeset
|
189 ch = CHILD(n, 0); |
|
bbf16fd024df
from __future__ import with_statement addon for 'with', mostly written by
Thomas Wouters <thomas@python.org>
parents:
22466
diff
changeset
|
190 if (STR(ch) == NULL || strcmp(STR(ch), "from") != 0) |
|
18609
59446d0f8341
Preliminary support for "from __future__ import generators" to enable
Guido van Rossum <guido@python.org>
parents:
14968
diff
changeset
|
191 return; |
|
59446d0f8341
Preliminary support for "from __future__ import generators" to enable
Guido van Rossum <guido@python.org>
parents:
14968
diff
changeset
|
192 ch = CHILD(n, 1); |
|
36726
bbf16fd024df
from __future__ import with_statement addon for 'with', mostly written by
Thomas Wouters <thomas@python.org>
parents:
22466
diff
changeset
|
193 if (NCH(ch) == 1 && STR(CHILD(ch, 0)) && |
|
bbf16fd024df
from __future__ import with_statement addon for 'with', mostly written by
Thomas Wouters <thomas@python.org>
parents:
22466
diff
changeset
|
194 strcmp(STR(CHILD(ch, 0)), "__future__") != 0) |
|
18609
59446d0f8341
Preliminary support for "from __future__ import generators" to enable
Guido van Rossum <guido@python.org>
parents:
14968
diff
changeset
|
195 return; |
|
39881
25bcdf3cd194
Fix a bug in the parser's future statement handling that led to "with"
Georg Brandl <georg@python.org>
parents:
37585
diff
changeset
|
196 ch = CHILD(n, 3); |
|
25bcdf3cd194
Fix a bug in the parser's future statement handling that led to "with"
Georg Brandl <georg@python.org>
parents:
37585
diff
changeset
|
197 /* ch can be a star, a parenthesis or import_as_names */ |
|
25bcdf3cd194
Fix a bug in the parser's future statement handling that led to "with"
Georg Brandl <georg@python.org>
parents:
37585
diff
changeset
|
198 if (TYPE(ch) == STAR) |
|
25bcdf3cd194
Fix a bug in the parser's future statement handling that led to "with"
Georg Brandl <georg@python.org>
parents:
37585
diff
changeset
|
199 return; |
|
25bcdf3cd194
Fix a bug in the parser's future statement handling that led to "with"
Georg Brandl <georg@python.org>
parents:
37585
diff
changeset
|
200 if (TYPE(ch) == LPAR) |
|
25bcdf3cd194
Fix a bug in the parser's future statement handling that led to "with"
Georg Brandl <georg@python.org>
parents:
37585
diff
changeset
|
201 ch = CHILD(n, 4); |
|
25bcdf3cd194
Fix a bug in the parser's future statement handling that led to "with"
Georg Brandl <georg@python.org>
parents:
37585
diff
changeset
|
202 |
|
25bcdf3cd194
Fix a bug in the parser's future statement handling that led to "with"
Georg Brandl <georg@python.org>
parents:
37585
diff
changeset
|
203 for (i = 0; i < NCH(ch); i += 2) { |
|
25bcdf3cd194
Fix a bug in the parser's future statement handling that led to "with"
Georg Brandl <georg@python.org>
parents:
37585
diff
changeset
|
204 cch = CHILD(ch, i); |
|
46118
c78d3586c474
Patch #2477: Added from __future__ import unicode_literals
Christian Heimes <christian@cheimes.de>
parents:
45891
diff
changeset
|
205 if (NCH(cch) >= 1 && TYPE(CHILD(cch, 0)) == NAME) { |
|
c78d3586c474
Patch #2477: Added from __future__ import unicode_literals
Christian Heimes <christian@cheimes.de>
parents:
45891
diff
changeset
|
206 char *str_ch = STR(CHILD(cch, 0)); |
|
c78d3586c474
Patch #2477: Added from __future__ import unicode_literals
Christian Heimes <christian@cheimes.de>
parents:
45891
diff
changeset
|
207 if (strcmp(str_ch, FUTURE_WITH_STATEMENT) == 0) { |
|
c78d3586c474
Patch #2477: Added from __future__ import unicode_literals
Christian Heimes <christian@cheimes.de>
parents:
45891
diff
changeset
|
208 ps->p_flags |= CO_FUTURE_WITH_STATEMENT; |
|
c78d3586c474
Patch #2477: Added from __future__ import unicode_literals
Christian Heimes <christian@cheimes.de>
parents:
45891
diff
changeset
|
209 } else if (strcmp(str_ch, FUTURE_PRINT_FUNCTION) == 0) { |
|
c78d3586c474
Patch #2477: Added from __future__ import unicode_literals
Christian Heimes <christian@cheimes.de>
parents:
45891
diff
changeset
|
210 ps->p_flags |= CO_FUTURE_PRINT_FUNCTION; |
|
c78d3586c474
Patch #2477: Added from __future__ import unicode_literals
Christian Heimes <christian@cheimes.de>
parents:
45891
diff
changeset
|
211 } else if (strcmp(str_ch, FUTURE_UNICODE_LITERALS) == 0) { |
|
c78d3586c474
Patch #2477: Added from __future__ import unicode_literals
Christian Heimes <christian@cheimes.de>
parents:
45891
diff
changeset
|
212 ps->p_flags |= CO_FUTURE_UNICODE_LITERALS; |
|
c78d3586c474
Patch #2477: Added from __future__ import unicode_literals
Christian Heimes <christian@cheimes.de>
parents:
45891
diff
changeset
|
213 } |
|
18739
704486642079
Fis SF bug #442647: not all forms of legal future statements were
Guido van Rossum <guido@python.org>
parents:
18609
diff
changeset
|
214 } |
|
704486642079
Fis SF bug #442647: not all forms of legal future statements were
Guido van Rossum <guido@python.org>
parents:
18609
diff
changeset
|
215 } |
|
18609
59446d0f8341
Preliminary support for "from __future__ import generators" to enable
Guido van Rossum <guido@python.org>
parents:
14968
diff
changeset
|
216 } |
|
22466
1db1510abe21
Disable the parser hacks that allowed the "yield" keyword to be enabled
Neil Schemenauer <nascheme@enme.ucalgary.ca>
parents:
21181
diff
changeset
|
217 #endif /* future keyword */ |
|
18609
59446d0f8341
Preliminary support for "from __future__ import generators" to enable
Guido van Rossum <guido@python.org>
parents:
14968
diff
changeset
|
218 |
| 6 | 219 int |
| 13659 | 220 PyParser_AddToken(register parser_state *ps, register int type, char *str, |
|
36786
57322cb2ca11
Patch #1440601: Add col_offset attribute to AST nodes.
Martin v. Löwis <martin@v.loewis.de>
parents:
36742
diff
changeset
|
221 int lineno, int col_offset, int *expected_ret) |
| 6 | 222 { |
| 223 register int ilabel; | |
|
12970
63436719d12e
Add new parser error code, E_OVERFLOW. This error is returned when
Jeremy Hylton <jeremy@alum.mit.edu>
parents:
5379
diff
changeset
|
224 int err; |
| 6 | 225 |
|
5379
1c9a84e0981d
Another directory quickly renamed.
Guido van Rossum <guido@python.org>
parents:
4839
diff
changeset
|
226 D(printf("Token %s/'%s' ... ", _PyParser_TokenNames[type], str)); |
| 6 | 227 |
| 228 /* Find out which label this token is */ | |
|
18609
59446d0f8341
Preliminary support for "from __future__ import generators" to enable
Guido van Rossum <guido@python.org>
parents:
14968
diff
changeset
|
229 ilabel = classify(ps, type, str); |
| 6 | 230 if (ilabel < 0) |
| 231 return E_SYNTAX; | |
| 232 | |
| 233 /* Loop until the token is shifted or an error occurred */ | |
| 234 for (;;) { | |
| 235 /* Fetch the current dfa and state */ | |
| 236 register dfa *d = ps->p_stack.s_top->s_dfa; | |
| 237 register state *s = &d->d_state[ps->p_stack.s_top->s_state]; | |
| 238 | |
| 239 D(printf(" DFA '%s', state %d:", | |
| 240 d->d_name, ps->p_stack.s_top->s_state)); | |
| 241 | |
| 242 /* Check accelerator */ | |
| 243 if (s->s_lower <= ilabel && ilabel < s->s_upper) { | |
| 244 register int x = s->s_accel[ilabel - s->s_lower]; | |
| 245 if (x != -1) { | |
| 246 if (x & (1<<7)) { | |
| 247 /* Push non-terminal */ | |
| 248 int nt = (x >> 8) + NT_OFFSET; | |
| 249 int arrow = x & ((1<<7)-1); | |
|
5379
1c9a84e0981d
Another directory quickly renamed.
Guido van Rossum <guido@python.org>
parents:
4839
diff
changeset
|
250 dfa *d1 = PyGrammar_FindDFA( |
|
1c9a84e0981d
Another directory quickly renamed.
Guido van Rossum <guido@python.org>
parents:
4839
diff
changeset
|
251 ps->p_grammar, nt); |
|
12970
63436719d12e
Add new parser error code, E_OVERFLOW. This error is returned when
Jeremy Hylton <jeremy@alum.mit.edu>
parents:
5379
diff
changeset
|
252 if ((err = push(&ps->p_stack, nt, d1, |
|
36786
57322cb2ca11
Patch #1440601: Add col_offset attribute to AST nodes.
Martin v. Löwis <martin@v.loewis.de>
parents:
36742
diff
changeset
|
253 arrow, lineno, col_offset)) > 0) { |
|
5379
1c9a84e0981d
Another directory quickly renamed.
Guido van Rossum <guido@python.org>
parents:
4839
diff
changeset
|
254 D(printf(" MemError: push\n")); |
|
12970
63436719d12e
Add new parser error code, E_OVERFLOW. This error is returned when
Jeremy Hylton <jeremy@alum.mit.edu>
parents:
5379
diff
changeset
|
255 return err; |
| 6 | 256 } |
| 257 D(printf(" Push ...\n")); | |
| 258 continue; | |
| 259 } | |
| 260 | |
| 261 /* Shift the token */ | |
|
12970
63436719d12e
Add new parser error code, E_OVERFLOW. This error is returned when
Jeremy Hylton <jeremy@alum.mit.edu>
parents:
5379
diff
changeset
|
262 if ((err = shift(&ps->p_stack, type, str, |
|
36786
57322cb2ca11
Patch #1440601: Add col_offset attribute to AST nodes.
Martin v. Löwis <martin@v.loewis.de>
parents:
36742
diff
changeset
|
263 x, lineno, col_offset)) > 0) { |
| 6 | 264 D(printf(" MemError: shift.\n")); |
|
12970
63436719d12e
Add new parser error code, E_OVERFLOW. This error is returned when
Jeremy Hylton <jeremy@alum.mit.edu>
parents:
5379
diff
changeset
|
265 return err; |
| 6 | 266 } |
| 267 D(printf(" Shift.\n")); | |
| 268 /* Pop while we are in an accept-only state */ | |
| 269 while (s = &d->d_state | |
| 270 [ps->p_stack.s_top->s_state], | |
| 271 s->s_accept && s->s_narcs == 1) { | |
|
18609
59446d0f8341
Preliminary support for "from __future__ import generators" to enable
Guido van Rossum <guido@python.org>
parents:
14968
diff
changeset
|
272 D(printf(" DFA '%s', state %d: " |
|
59446d0f8341
Preliminary support for "from __future__ import generators" to enable
Guido van Rossum <guido@python.org>
parents:
14968
diff
changeset
|
273 "Direct pop.\n", |
|
59446d0f8341
Preliminary support for "from __future__ import generators" to enable
Guido van Rossum <guido@python.org>
parents:
14968
diff
changeset
|
274 d->d_name, |
|
59446d0f8341
Preliminary support for "from __future__ import generators" to enable
Guido van Rossum <guido@python.org>
parents:
14968
diff
changeset
|
275 ps->p_stack.s_top->s_state)); |
|
36726
bbf16fd024df
from __future__ import with_statement addon for 'with', mostly written by
Thomas Wouters <thomas@python.org>
parents:
22466
diff
changeset
|
276 #ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD |
|
18609
59446d0f8341
Preliminary support for "from __future__ import generators" to enable
Guido van Rossum <guido@python.org>
parents:
14968
diff
changeset
|
277 if (d->d_name[0] == 'i' && |
|
59446d0f8341
Preliminary support for "from __future__ import generators" to enable
Guido van Rossum <guido@python.org>
parents:
14968
diff
changeset
|
278 strcmp(d->d_name, |
|
59446d0f8341
Preliminary support for "from __future__ import generators" to enable
Guido van Rossum <guido@python.org>
parents:
14968
diff
changeset
|
279 "import_stmt") == 0) |
|
59446d0f8341
Preliminary support for "from __future__ import generators" to enable
Guido van Rossum <guido@python.org>
parents:
14968
diff
changeset
|
280 future_hack(ps); |
|
22466
1db1510abe21
Disable the parser hacks that allowed the "yield" keyword to be enabled
Neil Schemenauer <nascheme@enme.ucalgary.ca>
parents:
21181
diff
changeset
|
281 #endif |
| 6 | 282 s_pop(&ps->p_stack); |
| 283 if (s_empty(&ps->p_stack)) { | |
| 284 D(printf(" ACCEPT.\n")); | |
| 285 return E_DONE; | |
| 286 } | |
| 287 d = ps->p_stack.s_top->s_dfa; | |
| 288 } | |
| 289 return E_OK; | |
| 290 } | |
| 291 } | |
| 292 | |
| 293 if (s->s_accept) { | |
|
36726
bbf16fd024df
from __future__ import with_statement addon for 'with', mostly written by
Thomas Wouters <thomas@python.org>
parents:
22466
diff
changeset
|
294 #ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD |
|
18609
59446d0f8341
Preliminary support for "from __future__ import generators" to enable
Guido van Rossum <guido@python.org>
parents:
14968
diff
changeset
|
295 if (d->d_name[0] == 'i' && |
|
59446d0f8341
Preliminary support for "from __future__ import generators" to enable
Guido van Rossum <guido@python.org>
parents:
14968
diff
changeset
|
296 strcmp(d->d_name, "import_stmt") == 0) |
|
59446d0f8341
Preliminary support for "from __future__ import generators" to enable
Guido van Rossum <guido@python.org>
parents:
14968
diff
changeset
|
297 future_hack(ps); |
|
22466
1db1510abe21
Disable the parser hacks that allowed the "yield" keyword to be enabled
Neil Schemenauer <nascheme@enme.ucalgary.ca>
parents:
21181
diff
changeset
|
298 #endif |
| 6 | 299 /* Pop this dfa and try again */ |
| 300 s_pop(&ps->p_stack); | |
| 301 D(printf(" Pop ...\n")); | |
| 302 if (s_empty(&ps->p_stack)) { | |
| 303 D(printf(" Error: bottom of stack.\n")); | |
| 304 return E_SYNTAX; | |
| 305 } | |
| 306 continue; | |
| 307 } | |
| 308 | |
| 309 /* Stuck, report syntax error */ | |
| 310 D(printf(" Error.\n")); | |
|
13539
feab677269e0
Create two new exceptions: IndentationError and TabError. These are
Fred Drake <fdrake@acm.org>
parents:
13447
diff
changeset
|
311 if (expected_ret) { |
|
feab677269e0
Create two new exceptions: IndentationError and TabError. These are
Fred Drake <fdrake@acm.org>
parents:
13447
diff
changeset
|
312 if (s->s_lower == s->s_upper - 1) { |
|
feab677269e0
Create two new exceptions: IndentationError and TabError. These are
Fred Drake <fdrake@acm.org>
parents:
13447
diff
changeset
|
313 /* Only one possible expected token */ |
|
feab677269e0
Create two new exceptions: IndentationError and TabError. These are
Fred Drake <fdrake@acm.org>
parents:
13447
diff
changeset
|
314 *expected_ret = ps->p_grammar-> |
|
feab677269e0
Create two new exceptions: IndentationError and TabError. These are
Fred Drake <fdrake@acm.org>
parents:
13447
diff
changeset
|
315 g_ll.ll_label[s->s_lower].lb_type; |
|
feab677269e0
Create two new exceptions: IndentationError and TabError. These are
Fred Drake <fdrake@acm.org>
parents:
13447
diff
changeset
|
316 } |
|
feab677269e0
Create two new exceptions: IndentationError and TabError. These are
Fred Drake <fdrake@acm.org>
parents:
13447
diff
changeset
|
317 else |
|
feab677269e0
Create two new exceptions: IndentationError and TabError. These are
Fred Drake <fdrake@acm.org>
parents:
13447
diff
changeset
|
318 *expected_ret = -1; |
|
feab677269e0
Create two new exceptions: IndentationError and TabError. These are
Fred Drake <fdrake@acm.org>
parents:
13447
diff
changeset
|
319 } |
| 6 | 320 return E_SYNTAX; |
| 321 } | |
| 322 } | |
| 323 | |
| 324 | |
|
4839
4567e158d20c
Rename DEBUG macro to Py_DEBUG
Guido van Rossum <guido@python.org>
parents:
4503
diff
changeset
|
325 #ifdef Py_DEBUG |
| 6 | 326 |
| 327 /* DEBUG OUTPUT */ | |
| 328 | |
| 329 void | |
| 13659 | 330 dumptree(grammar *g, node *n) |
| 6 | 331 { |
| 332 int i; | |
| 333 | |
| 334 if (n == NULL) | |
| 335 printf("NIL"); | |
| 336 else { | |
| 337 label l; | |
| 338 l.lb_type = TYPE(n); | |
|
1027
513940eb4d7b
Correct debug printing code
Guido van Rossum <guido@python.org>
parents:
795
diff
changeset
|
339 l.lb_str = STR(n); |
|
5379
1c9a84e0981d
Another directory quickly renamed.
Guido van Rossum <guido@python.org>
parents:
4839
diff
changeset
|
340 printf("%s", PyGrammar_LabelRepr(&l)); |
| 6 | 341 if (ISNONTERMINAL(TYPE(n))) { |
| 342 printf("("); | |
| 343 for (i = 0; i < NCH(n); i++) { | |
| 344 if (i > 0) | |
| 345 printf(","); | |
| 346 dumptree(g, CHILD(n, i)); | |
| 347 } | |
| 348 printf(")"); | |
| 349 } | |
| 350 } | |
| 351 } | |
| 352 | |
| 353 void | |
| 13659 | 354 showtree(grammar *g, node *n) |
| 6 | 355 { |
| 356 int i; | |
| 357 | |
| 358 if (n == NULL) | |
| 359 return; | |
| 360 if (ISNONTERMINAL(TYPE(n))) { | |
| 361 for (i = 0; i < NCH(n); i++) | |
| 362 showtree(g, CHILD(n, i)); | |
| 363 } | |
| 364 else if (ISTERMINAL(TYPE(n))) { | |
|
5379
1c9a84e0981d
Another directory quickly renamed.
Guido van Rossum <guido@python.org>
parents:
4839
diff
changeset
|
365 printf("%s", _PyParser_TokenNames[TYPE(n)]); |
| 6 | 366 if (TYPE(n) == NUMBER || TYPE(n) == NAME) |
| 367 printf("(%s)", STR(n)); | |
| 368 printf(" "); | |
| 369 } | |
| 370 else | |
| 371 printf("? "); | |
| 372 } | |
| 373 | |
| 374 void | |
| 13659 | 375 printtree(parser_state *ps) |
| 6 | 376 { |
|
5379
1c9a84e0981d
Another directory quickly renamed.
Guido van Rossum <guido@python.org>
parents:
4839
diff
changeset
|
377 if (Py_DebugFlag) { |
| 6 | 378 printf("Parse tree:\n"); |
| 379 dumptree(ps->p_grammar, ps->p_tree); | |
| 380 printf("\n"); | |
| 381 printf("Tokens:\n"); | |
| 382 showtree(ps->p_grammar, ps->p_tree); | |
| 383 printf("\n"); | |
| 384 } | |
| 385 printf("Listing:\n"); | |
|
5379
1c9a84e0981d
Another directory quickly renamed.
Guido van Rossum <guido@python.org>
parents:
4839
diff
changeset
|
386 PyNode_ListTree(ps->p_tree); |
| 6 | 387 printf("\n"); |
| 388 } | |
| 389 | |
|
4839
4567e158d20c
Rename DEBUG macro to Py_DEBUG
Guido van Rossum <guido@python.org>
parents:
4503
diff
changeset
|
390 #endif /* Py_DEBUG */ |
| 6 | 391 |
| 392 /* | |
| 393 | |
| 394 Description | |
| 395 ----------- | |
| 396 | |
| 397 The parser's interface is different than usual: the function addtoken() | |
| 398 must be called for each token in the input. This makes it possible to | |
| 399 turn it into an incremental parsing system later. The parsing system | |
| 400 constructs a parse tree as it goes. | |
| 401 | |
| 402 A parsing rule is represented as a Deterministic Finite-state Automaton | |
| 403 (DFA). A node in a DFA represents a state of the parser; an arc represents | |
| 404 a transition. Transitions are either labeled with terminal symbols or | |
| 405 with non-terminals. When the parser decides to follow an arc labeled | |
| 406 with a non-terminal, it is invoked recursively with the DFA representing | |
| 407 the parsing rule for that as its initial state; when that DFA accepts, | |
| 408 the parser that invoked it continues. The parse tree constructed by the | |
| 409 recursively called parser is inserted as a child in the current parse tree. | |
| 410 | |
| 411 The DFA's can be constructed automatically from a more conventional | |
| 412 language description. An extended LL(1) grammar (ELL(1)) is suitable. | |
| 413 Certain restrictions make the parser's life easier: rules that can produce | |
| 414 the empty string should be outlawed (there are other ways to put loops | |
| 415 or optional parts in the language). To avoid the need to construct | |
| 416 FIRST sets, we can require that all but the last alternative of a rule | |
| 417 (really: arc going out of a DFA's state) must begin with a terminal | |
| 418 symbol. | |
| 419 | |
| 420 As an example, consider this grammar: | |
| 421 | |
| 422 expr: term (OP term)* | |
| 423 term: CONSTANT | '(' expr ')' | |
| 424 | |
| 425 The DFA corresponding to the rule for expr is: | |
| 426 | |
| 427 ------->.---term-->.-------> | |
| 428 ^ | | |
| 429 | | | |
| 430 \----OP----/ | |
| 431 | |
| 432 The parse tree generated for the input a+b is: | |
| 433 | |
| 434 (expr: (term: (NAME: a)), (OP: +), (term: (NAME: b))) | |
| 435 | |
| 436 */ |