Mercurial > cpython-lru_cache-weakref
annotate Parser/parser.c @ 58689:514b56efd89a 3.1
Merged revisions 77962 via svnmerge from
svn+ssh://pythondev@svn.python.org/python/branches/py3k
................
r77962 | r.david.murray | 2010-02-04 11:41:57 -0500 (Thu, 04 Feb 2010) | 17 lines
Merged revisions 77943,77959-77960 via svnmerge from
svn+ssh://pythondev@svn.python.org/python/trunk
........
r77943 | r.david.murray | 2010-02-03 08:36:23 -0500 (Wed, 03 Feb 2010) | 2 lines
Explicitly mention the default value for formatdate's usegmt parameter.
........
r77959 | nick.coghlan | 2010-02-04 07:43:58 -0500 (Thu, 04 Feb 2010) | 1 line
Issue 6760: Clarify args handling for subprocess.Popen. Patch by Chris Rebert
........
r77960 | r.david.murray | 2010-02-04 11:33:31 -0500 (Thu, 04 Feb 2010) | 2 lines
Add Chris Rebert to ACKS for issue 6760 Popen doc improvements.
........
................
| author | R. David Murray <rdmurray@bitdance.com> |
|---|---|
| date | Thu, 04 Feb 2010 16:44:31 +0000 |
| parents | 0530aadff696 |
| children | cab14be0ada1 |
| rev | line source |
|---|---|
| 145 | 1 |
| 6 | 2 /* Parser implementation */ |
| 3 | |
| 4 /* For a description, see the comments at end of this file */ | |
| 5 | |
| 6 /* XXX To do: error recovery */ | |
| 7 | |
|
21181
944d6e02f359
The parser doesn't need its own implementation of assert, and having its
Tim Peters <tim.peters@gmail.com>
parents:
18739
diff
changeset
|
8 #include "Python.h" |
| 86 | 9 #include "pgenheaders.h" |
| 6 | 10 #include "token.h" |
| 11 #include "grammar.h" | |
| 12 #include "node.h" | |
| 13 #include "parser.h" | |
| 14 #include "errcode.h" | |
| 15 | |
| 16 | |
|
4839
4567e158d20c
Rename DEBUG macro to Py_DEBUG
Guido van Rossum <guido@python.org>
parents:
4503
diff
changeset
|
17 #ifdef Py_DEBUG |
|
5379
1c9a84e0981d
Another directory quickly renamed.
Guido van Rossum <guido@python.org>
parents:
4839
diff
changeset
|
18 extern int Py_DebugFlag; |
|
1c9a84e0981d
Another directory quickly renamed.
Guido van Rossum <guido@python.org>
parents:
4839
diff
changeset
|
19 #define D(x) if (!Py_DebugFlag); else x |
| 6 | 20 #else |
| 21 #define D(x) | |
| 22 #endif | |
| 23 | |
| 24 | |
| 25 /* STACK DATA TYPE */ | |
| 26 | |
|
13447
a89f7f268de4
Nuke all remaining occurrences of Py_PROTO and Py_FPROTO.
Tim Peters <tim.peters@gmail.com>
parents:
13256
diff
changeset
|
27 static void s_reset(stack *); |
| 6 | 28 |
| 29 static void | |
| 13659 | 30 s_reset(stack *s) |
| 6 | 31 { |
| 32 s->s_top = &s->s_base[MAXSTACK]; | |
| 33 } | |
| 34 | |
| 35 #define s_empty(s) ((s)->s_top == &(s)->s_base[MAXSTACK]) | |
| 36 | |
| 37 static int | |
| 13659 | 38 s_push(register stack *s, dfa *d, node *parent) |
| 6 | 39 { |
| 40 register stackentry *top; | |
| 41 if (s->s_top == s->s_base) { | |
| 42 fprintf(stderr, "s_push: parser stack overflow\n"); | |
|
14968
5336946b6495
Fix a bug in stack overflow error handling. This fixes half of Bug
Guido van Rossum <guido@python.org>
parents:
14375
diff
changeset
|
43 return E_NOMEM; |
| 6 | 44 } |
| 45 top = --s->s_top; | |
| 46 top->s_dfa = d; | |
| 47 top->s_parent = parent; | |
| 48 top->s_state = 0; | |
| 49 return 0; | |
| 50 } | |
| 51 | |
|
4839
4567e158d20c
Rename DEBUG macro to Py_DEBUG
Guido van Rossum <guido@python.org>
parents:
4503
diff
changeset
|
52 #ifdef Py_DEBUG |
| 6 | 53 |
| 54 static void | |
| 13659 | 55 s_pop(register stack *s) |
| 6 | 56 { |
|
2044
281e40652e23
Parser/tokenizer.c (tok_nextc): zap tok->buf after freeing;
Guido van Rossum <guido@python.org>
parents:
1848
diff
changeset
|
57 if (s_empty(s)) |
|
5379
1c9a84e0981d
Another directory quickly renamed.
Guido van Rossum <guido@python.org>
parents:
4839
diff
changeset
|
58 Py_FatalError("s_pop: parser stack underflow -- FATAL"); |
| 6 | 59 s->s_top++; |
| 60 } | |
| 61 | |
|
4839
4567e158d20c
Rename DEBUG macro to Py_DEBUG
Guido van Rossum <guido@python.org>
parents:
4503
diff
changeset
|
62 #else /* !Py_DEBUG */ |
| 6 | 63 |
| 64 #define s_pop(s) (s)->s_top++ | |
| 65 | |
| 66 #endif | |
| 67 | |
| 68 | |
| 69 /* PARSER CREATION */ | |
| 70 | |
| 71 parser_state * | |
| 13659 | 72 PyParser_New(grammar *g, int start) |
| 6 | 73 { |
| 74 parser_state *ps; | |
| 75 | |
| 76 if (!g->g_accel) | |
|
5379
1c9a84e0981d
Another directory quickly renamed.
Guido van Rossum <guido@python.org>
parents:
4839
diff
changeset
|
77 PyGrammar_AddAccelerators(g); |
|
37879
f731859e9b3b
Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents:
36988
diff
changeset
|
78 ps = (parser_state *)PyMem_MALLOC(sizeof(parser_state)); |
| 6 | 79 if (ps == NULL) |
| 80 return NULL; | |
| 81 ps->p_grammar = g; | |
|
36726
bbf16fd024df
from __future__ import with_statement addon for 'with', mostly written by
Thomas Wouters <thomas@python.org>
parents:
22466
diff
changeset
|
82 #ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD |
|
bbf16fd024df
from __future__ import with_statement addon for 'with', mostly written by
Thomas Wouters <thomas@python.org>
parents:
22466
diff
changeset
|
83 ps->p_flags = 0; |
|
22466
1db1510abe21
Disable the parser hacks that allowed the "yield" keyword to be enabled
Neil Schemenauer <nascheme@enme.ucalgary.ca>
parents:
21181
diff
changeset
|
84 #endif |
|
5379
1c9a84e0981d
Another directory quickly renamed.
Guido van Rossum <guido@python.org>
parents:
4839
diff
changeset
|
85 ps->p_tree = PyNode_New(start); |
| 6 | 86 if (ps->p_tree == NULL) { |
|
37879
f731859e9b3b
Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents:
36988
diff
changeset
|
87 PyMem_FREE(ps); |
| 6 | 88 return NULL; |
| 89 } | |
| 90 s_reset(&ps->p_stack); | |
|
5379
1c9a84e0981d
Another directory quickly renamed.
Guido van Rossum <guido@python.org>
parents:
4839
diff
changeset
|
91 (void) s_push(&ps->p_stack, PyGrammar_FindDFA(g, start), ps->p_tree); |
| 6 | 92 return ps; |
| 93 } | |
| 94 | |
| 95 void | |
| 13659 | 96 PyParser_Delete(parser_state *ps) |
| 6 | 97 { |
|
79
6c43e5476de6
Free parse tree when deleting parser.
Guido van Rossum <guido@python.org>
parents:
6
diff
changeset
|
98 /* NB If you want to save the parse tree, |
|
6c43e5476de6
Free parse tree when deleting parser.
Guido van Rossum <guido@python.org>
parents:
6
diff
changeset
|
99 you must set p_tree to NULL before calling delparser! */ |
|
5379
1c9a84e0981d
Another directory quickly renamed.
Guido van Rossum <guido@python.org>
parents:
4839
diff
changeset
|
100 PyNode_Free(ps->p_tree); |
|
37879
f731859e9b3b
Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents:
36988
diff
changeset
|
101 PyMem_FREE(ps); |
| 6 | 102 } |
| 103 | |
| 104 | |
| 105 /* PARSER STACK OPERATIONS */ | |
| 106 | |
| 107 static int | |
|
36786
57322cb2ca11
Patch #1440601: Add col_offset attribute to AST nodes.
Martin v. Löwis <martin@v.loewis.de>
parents:
36742
diff
changeset
|
108 shift(register stack *s, int type, char *str, int newstate, int lineno, int col_offset) |
| 6 | 109 { |
|
12970
63436719d12e
Add new parser error code, E_OVERFLOW. This error is returned when
Jeremy Hylton <jeremy@alum.mit.edu>
parents:
5379
diff
changeset
|
110 int err; |
| 6 | 111 assert(!s_empty(s)); |
|
36786
57322cb2ca11
Patch #1440601: Add col_offset attribute to AST nodes.
Martin v. Löwis <martin@v.loewis.de>
parents:
36742
diff
changeset
|
112 err = PyNode_AddChild(s->s_top->s_parent, type, str, lineno, col_offset); |
|
12970
63436719d12e
Add new parser error code, E_OVERFLOW. This error is returned when
Jeremy Hylton <jeremy@alum.mit.edu>
parents:
5379
diff
changeset
|
113 if (err) |
|
63436719d12e
Add new parser error code, E_OVERFLOW. This error is returned when
Jeremy Hylton <jeremy@alum.mit.edu>
parents:
5379
diff
changeset
|
114 return err; |
| 6 | 115 s->s_top->s_state = newstate; |
| 116 return 0; | |
| 117 } | |
| 118 | |
| 119 static int | |
|
36786
57322cb2ca11
Patch #1440601: Add col_offset attribute to AST nodes.
Martin v. Löwis <martin@v.loewis.de>
parents:
36742
diff
changeset
|
120 push(register stack *s, int type, dfa *d, int newstate, int lineno, int col_offset) |
| 6 | 121 { |
|
12970
63436719d12e
Add new parser error code, E_OVERFLOW. This error is returned when
Jeremy Hylton <jeremy@alum.mit.edu>
parents:
5379
diff
changeset
|
122 int err; |
| 6 | 123 register node *n; |
| 124 n = s->s_top->s_parent; | |
| 125 assert(!s_empty(s)); | |
|
36786
57322cb2ca11
Patch #1440601: Add col_offset attribute to AST nodes.
Martin v. Löwis <martin@v.loewis.de>
parents:
36742
diff
changeset
|
126 err = PyNode_AddChild(n, type, (char *)NULL, lineno, col_offset); |
|
12970
63436719d12e
Add new parser error code, E_OVERFLOW. This error is returned when
Jeremy Hylton <jeremy@alum.mit.edu>
parents:
5379
diff
changeset
|
127 if (err) |
|
63436719d12e
Add new parser error code, E_OVERFLOW. This error is returned when
Jeremy Hylton <jeremy@alum.mit.edu>
parents:
5379
diff
changeset
|
128 return err; |
| 6 | 129 s->s_top->s_state = newstate; |
| 130 return s_push(s, d, CHILD(n, NCH(n)-1)); | |
| 131 } | |
| 132 | |
| 133 | |
| 134 /* PARSER PROPER */ | |
| 135 | |
| 136 static int | |
|
18609
59446d0f8341
Preliminary support for "from __future__ import generators" to enable
Guido van Rossum <guido@python.org>
parents:
14968
diff
changeset
|
137 classify(parser_state *ps, int type, char *str) |
| 6 | 138 { |
|
18609
59446d0f8341
Preliminary support for "from __future__ import generators" to enable
Guido van Rossum <guido@python.org>
parents:
14968
diff
changeset
|
139 grammar *g = ps->p_grammar; |
| 6 | 140 register int n = g->g_ll.ll_nlabels; |
| 141 | |
| 142 if (type == NAME) { | |
| 143 register char *s = str; | |
| 144 register label *l = g->g_ll.ll_label; | |
| 145 register int i; | |
| 146 for (i = n; i > 0; i--, l++) { | |
|
36742
658f8ff2a442
Make 'as' an actual keyword when with's future statement is used. Not
Thomas Wouters <thomas@python.org>
parents:
36726
diff
changeset
|
147 if (l->lb_type != NAME || l->lb_str == NULL || |
|
658f8ff2a442
Make 'as' an actual keyword when with's future statement is used. Not
Thomas Wouters <thomas@python.org>
parents:
36726
diff
changeset
|
148 l->lb_str[0] != s[0] || |
|
658f8ff2a442
Make 'as' an actual keyword when with's future statement is used. Not
Thomas Wouters <thomas@python.org>
parents:
36726
diff
changeset
|
149 strcmp(l->lb_str, s) != 0) |
|
658f8ff2a442
Make 'as' an actual keyword when with's future statement is used. Not
Thomas Wouters <thomas@python.org>
parents:
36726
diff
changeset
|
150 continue; |
|
36726
bbf16fd024df
from __future__ import with_statement addon for 'with', mostly written by
Thomas Wouters <thomas@python.org>
parents:
22466
diff
changeset
|
151 #ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD |
|
52641
0530aadff696
The BDFL has retired! Long live the FLUFL (Friendly Language Uncle For Life)!
Brett Cannon <bcannon@gmail.com>
parents:
49594
diff
changeset
|
152 #if 0 |
|
36988
49c6eb688906
Checkpoint. 218 tests are okay; 53 are failing. Done so far:
Guido van Rossum <guido@python.org>
parents:
36786
diff
changeset
|
153 /* Leaving this in as an example */ |
|
36742
658f8ff2a442
Make 'as' an actual keyword when with's future statement is used. Not
Thomas Wouters <thomas@python.org>
parents:
36726
diff
changeset
|
154 if (!(ps->p_flags & CO_FUTURE_WITH_STATEMENT)) { |
|
658f8ff2a442
Make 'as' an actual keyword when with's future statement is used. Not
Thomas Wouters <thomas@python.org>
parents:
36726
diff
changeset
|
155 if (s[0] == 'w' && strcmp(s, "with") == 0) |
|
658f8ff2a442
Make 'as' an actual keyword when with's future statement is used. Not
Thomas Wouters <thomas@python.org>
parents:
36726
diff
changeset
|
156 break; /* not a keyword yet */ |
|
658f8ff2a442
Make 'as' an actual keyword when with's future statement is used. Not
Thomas Wouters <thomas@python.org>
parents:
36726
diff
changeset
|
157 else if (s[0] == 'a' && strcmp(s, "as") == 0) |
|
658f8ff2a442
Make 'as' an actual keyword when with's future statement is used. Not
Thomas Wouters <thomas@python.org>
parents:
36726
diff
changeset
|
158 break; /* not a keyword yet */ |
|
658f8ff2a442
Make 'as' an actual keyword when with's future statement is used. Not
Thomas Wouters <thomas@python.org>
parents:
36726
diff
changeset
|
159 } |
|
22466
1db1510abe21
Disable the parser hacks that allowed the "yield" keyword to be enabled
Neil Schemenauer <nascheme@enme.ucalgary.ca>
parents:
21181
diff
changeset
|
160 #endif |
|
52641
0530aadff696
The BDFL has retired! Long live the FLUFL (Friendly Language Uncle For Life)!
Brett Cannon <bcannon@gmail.com>
parents:
49594
diff
changeset
|
161 #endif |
|
36742
658f8ff2a442
Make 'as' an actual keyword when with's future statement is used. Not
Thomas Wouters <thomas@python.org>
parents:
36726
diff
changeset
|
162 D(printf("It's a keyword\n")); |
|
658f8ff2a442
Make 'as' an actual keyword when with's future statement is used. Not
Thomas Wouters <thomas@python.org>
parents:
36726
diff
changeset
|
163 return n - i; |
| 6 | 164 } |
| 165 } | |
| 166 | |
| 167 { | |
| 168 register label *l = g->g_ll.ll_label; | |
| 169 register int i; | |
| 170 for (i = n; i > 0; i--, l++) { | |
| 171 if (l->lb_type == type && l->lb_str == NULL) { | |
| 172 D(printf("It's a token we know\n")); | |
| 173 return n - i; | |
| 174 } | |
| 175 } | |
| 176 } | |
| 177 | |
| 178 D(printf("Illegal token\n")); | |
| 179 return -1; | |
| 180 } | |
| 181 | |
|
36726
bbf16fd024df
from __future__ import with_statement addon for 'with', mostly written by
Thomas Wouters <thomas@python.org>
parents:
22466
diff
changeset
|
182 #ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD |
|
52641
0530aadff696
The BDFL has retired! Long live the FLUFL (Friendly Language Uncle For Life)!
Brett Cannon <bcannon@gmail.com>
parents:
49594
diff
changeset
|
183 #if 0 |
|
36988
49c6eb688906
Checkpoint. 218 tests are okay; 53 are failing. Done so far:
Guido van Rossum <guido@python.org>
parents:
36786
diff
changeset
|
184 /* Leaving this in as an example */ |
|
18609
59446d0f8341
Preliminary support for "from __future__ import generators" to enable
Guido van Rossum <guido@python.org>
parents:
14968
diff
changeset
|
185 static void |
|
59446d0f8341
Preliminary support for "from __future__ import generators" to enable
Guido van Rossum <guido@python.org>
parents:
14968
diff
changeset
|
186 future_hack(parser_state *ps) |
|
59446d0f8341
Preliminary support for "from __future__ import generators" to enable
Guido van Rossum <guido@python.org>
parents:
14968
diff
changeset
|
187 { |
|
59446d0f8341
Preliminary support for "from __future__ import generators" to enable
Guido van Rossum <guido@python.org>
parents:
14968
diff
changeset
|
188 node *n = ps->p_stack.s_top->s_parent; |
|
40450
cc992d75d5b3
Four months of trunk changes (including a few releases...)
Thomas Wouters <thomas@python.org>
parents:
37879
diff
changeset
|
189 node *ch, *cch; |
|
18739
704486642079
Fis SF bug #442647: not all forms of legal future statements were
Guido van Rossum <guido@python.org>
parents:
18609
diff
changeset
|
190 int i; |
|
18609
59446d0f8341
Preliminary support for "from __future__ import generators" to enable
Guido van Rossum <guido@python.org>
parents:
14968
diff
changeset
|
191 |
|
36726
bbf16fd024df
from __future__ import with_statement addon for 'with', mostly written by
Thomas Wouters <thomas@python.org>
parents:
22466
diff
changeset
|
192 /* from __future__ import ..., must have at least 4 children */ |
|
bbf16fd024df
from __future__ import with_statement addon for 'with', mostly written by
Thomas Wouters <thomas@python.org>
parents:
22466
diff
changeset
|
193 n = CHILD(n, 0); |
|
bbf16fd024df
from __future__ import with_statement addon for 'with', mostly written by
Thomas Wouters <thomas@python.org>
parents:
22466
diff
changeset
|
194 if (NCH(n) < 4) |
|
bbf16fd024df
from __future__ import with_statement addon for 'with', mostly written by
Thomas Wouters <thomas@python.org>
parents:
22466
diff
changeset
|
195 return; |
|
bbf16fd024df
from __future__ import with_statement addon for 'with', mostly written by
Thomas Wouters <thomas@python.org>
parents:
22466
diff
changeset
|
196 ch = CHILD(n, 0); |
|
bbf16fd024df
from __future__ import with_statement addon for 'with', mostly written by
Thomas Wouters <thomas@python.org>
parents:
22466
diff
changeset
|
197 if (STR(ch) == NULL || strcmp(STR(ch), "from") != 0) |
|
18609
59446d0f8341
Preliminary support for "from __future__ import generators" to enable
Guido van Rossum <guido@python.org>
parents:
14968
diff
changeset
|
198 return; |
|
59446d0f8341
Preliminary support for "from __future__ import generators" to enable
Guido van Rossum <guido@python.org>
parents:
14968
diff
changeset
|
199 ch = CHILD(n, 1); |
|
36726
bbf16fd024df
from __future__ import with_statement addon for 'with', mostly written by
Thomas Wouters <thomas@python.org>
parents:
22466
diff
changeset
|
200 if (NCH(ch) == 1 && STR(CHILD(ch, 0)) && |
|
bbf16fd024df
from __future__ import with_statement addon for 'with', mostly written by
Thomas Wouters <thomas@python.org>
parents:
22466
diff
changeset
|
201 strcmp(STR(CHILD(ch, 0)), "__future__") != 0) |
|
18609
59446d0f8341
Preliminary support for "from __future__ import generators" to enable
Guido van Rossum <guido@python.org>
parents:
14968
diff
changeset
|
202 return; |
|
40450
cc992d75d5b3
Four months of trunk changes (including a few releases...)
Thomas Wouters <thomas@python.org>
parents:
37879
diff
changeset
|
203 ch = CHILD(n, 3); |
|
cc992d75d5b3
Four months of trunk changes (including a few releases...)
Thomas Wouters <thomas@python.org>
parents:
37879
diff
changeset
|
204 /* ch can be a star, a parenthesis or import_as_names */ |
|
cc992d75d5b3
Four months of trunk changes (including a few releases...)
Thomas Wouters <thomas@python.org>
parents:
37879
diff
changeset
|
205 if (TYPE(ch) == STAR) |
|
cc992d75d5b3
Four months of trunk changes (including a few releases...)
Thomas Wouters <thomas@python.org>
parents:
37879
diff
changeset
|
206 return; |
|
cc992d75d5b3
Four months of trunk changes (including a few releases...)
Thomas Wouters <thomas@python.org>
parents:
37879
diff
changeset
|
207 if (TYPE(ch) == LPAR) |
|
cc992d75d5b3
Four months of trunk changes (including a few releases...)
Thomas Wouters <thomas@python.org>
parents:
37879
diff
changeset
|
208 ch = CHILD(n, 4); |
|
cc992d75d5b3
Four months of trunk changes (including a few releases...)
Thomas Wouters <thomas@python.org>
parents:
37879
diff
changeset
|
209 |
|
cc992d75d5b3
Four months of trunk changes (including a few releases...)
Thomas Wouters <thomas@python.org>
parents:
37879
diff
changeset
|
210 for (i = 0; i < NCH(ch); i += 2) { |
|
cc992d75d5b3
Four months of trunk changes (including a few releases...)
Thomas Wouters <thomas@python.org>
parents:
37879
diff
changeset
|
211 cch = CHILD(ch, i); |
|
46120
78504d604b44
Merged revisions 61952-61953 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
40450
diff
changeset
|
212 if (NCH(cch) >= 1 && TYPE(CHILD(cch, 0)) == NAME) { |
|
78504d604b44
Merged revisions 61952-61953 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
40450
diff
changeset
|
213 char *str_ch = STR(CHILD(cch, 0)); |
|
78504d604b44
Merged revisions 61952-61953 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
40450
diff
changeset
|
214 if (strcmp(str_ch, FUTURE_WITH_STATEMENT) == 0) { |
|
78504d604b44
Merged revisions 61952-61953 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
40450
diff
changeset
|
215 ps->p_flags |= CO_FUTURE_WITH_STATEMENT; |
|
78504d604b44
Merged revisions 61952-61953 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
40450
diff
changeset
|
216 } else if (strcmp(str_ch, FUTURE_PRINT_FUNCTION) == 0) { |
|
78504d604b44
Merged revisions 61952-61953 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
40450
diff
changeset
|
217 ps->p_flags |= CO_FUTURE_PRINT_FUNCTION; |
|
78504d604b44
Merged revisions 61952-61953 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
40450
diff
changeset
|
218 } else if (strcmp(str_ch, FUTURE_UNICODE_LITERALS) == 0) { |
|
78504d604b44
Merged revisions 61952-61953 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
40450
diff
changeset
|
219 ps->p_flags |= CO_FUTURE_UNICODE_LITERALS; |
|
78504d604b44
Merged revisions 61952-61953 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
40450
diff
changeset
|
220 } |
|
18739
704486642079
Fis SF bug #442647: not all forms of legal future statements were
Guido van Rossum <guido@python.org>
parents:
18609
diff
changeset
|
221 } |
|
704486642079
Fis SF bug #442647: not all forms of legal future statements were
Guido van Rossum <guido@python.org>
parents:
18609
diff
changeset
|
222 } |
|
18609
59446d0f8341
Preliminary support for "from __future__ import generators" to enable
Guido van Rossum <guido@python.org>
parents:
14968
diff
changeset
|
223 } |
|
52641
0530aadff696
The BDFL has retired! Long live the FLUFL (Friendly Language Uncle For Life)!
Brett Cannon <bcannon@gmail.com>
parents:
49594
diff
changeset
|
224 #endif |
|
22466
1db1510abe21
Disable the parser hacks that allowed the "yield" keyword to be enabled
Neil Schemenauer <nascheme@enme.ucalgary.ca>
parents:
21181
diff
changeset
|
225 #endif /* future keyword */ |
|
18609
59446d0f8341
Preliminary support for "from __future__ import generators" to enable
Guido van Rossum <guido@python.org>
parents:
14968
diff
changeset
|
226 |
| 6 | 227 int |
| 13659 | 228 PyParser_AddToken(register parser_state *ps, register int type, char *str, |
|
36786
57322cb2ca11
Patch #1440601: Add col_offset attribute to AST nodes.
Martin v. Löwis <martin@v.loewis.de>
parents:
36742
diff
changeset
|
229 int lineno, int col_offset, int *expected_ret) |
| 6 | 230 { |
| 231 register int ilabel; | |
|
12970
63436719d12e
Add new parser error code, E_OVERFLOW. This error is returned when
Jeremy Hylton <jeremy@alum.mit.edu>
parents:
5379
diff
changeset
|
232 int err; |
| 6 | 233 |
|
5379
1c9a84e0981d
Another directory quickly renamed.
Guido van Rossum <guido@python.org>
parents:
4839
diff
changeset
|
234 D(printf("Token %s/'%s' ... ", _PyParser_TokenNames[type], str)); |
| 6 | 235 |
| 236 /* Find out which label this token is */ | |
|
18609
59446d0f8341
Preliminary support for "from __future__ import generators" to enable
Guido van Rossum <guido@python.org>
parents:
14968
diff
changeset
|
237 ilabel = classify(ps, type, str); |
| 6 | 238 if (ilabel < 0) |
| 239 return E_SYNTAX; | |
| 240 | |
| 241 /* Loop until the token is shifted or an error occurred */ | |
| 242 for (;;) { | |
| 243 /* Fetch the current dfa and state */ | |
| 244 register dfa *d = ps->p_stack.s_top->s_dfa; | |
| 245 register state *s = &d->d_state[ps->p_stack.s_top->s_state]; | |
| 246 | |
| 247 D(printf(" DFA '%s', state %d:", | |
| 248 d->d_name, ps->p_stack.s_top->s_state)); | |
| 249 | |
| 250 /* Check accelerator */ | |
| 251 if (s->s_lower <= ilabel && ilabel < s->s_upper) { | |
| 252 register int x = s->s_accel[ilabel - s->s_lower]; | |
| 253 if (x != -1) { | |
| 254 if (x & (1<<7)) { | |
| 255 /* Push non-terminal */ | |
| 256 int nt = (x >> 8) + NT_OFFSET; | |
| 257 int arrow = x & ((1<<7)-1); | |
|
5379
1c9a84e0981d
Another directory quickly renamed.
Guido van Rossum <guido@python.org>
parents:
4839
diff
changeset
|
258 dfa *d1 = PyGrammar_FindDFA( |
|
1c9a84e0981d
Another directory quickly renamed.
Guido van Rossum <guido@python.org>
parents:
4839
diff
changeset
|
259 ps->p_grammar, nt); |
|
12970
63436719d12e
Add new parser error code, E_OVERFLOW. This error is returned when
Jeremy Hylton <jeremy@alum.mit.edu>
parents:
5379
diff
changeset
|
260 if ((err = push(&ps->p_stack, nt, d1, |
|
36786
57322cb2ca11
Patch #1440601: Add col_offset attribute to AST nodes.
Martin v. Löwis <martin@v.loewis.de>
parents:
36742
diff
changeset
|
261 arrow, lineno, col_offset)) > 0) { |
|
5379
1c9a84e0981d
Another directory quickly renamed.
Guido van Rossum <guido@python.org>
parents:
4839
diff
changeset
|
262 D(printf(" MemError: push\n")); |
|
12970
63436719d12e
Add new parser error code, E_OVERFLOW. This error is returned when
Jeremy Hylton <jeremy@alum.mit.edu>
parents:
5379
diff
changeset
|
263 return err; |
| 6 | 264 } |
| 265 D(printf(" Push ...\n")); | |
| 266 continue; | |
| 267 } | |
| 268 | |
| 269 /* Shift the token */ | |
|
12970
63436719d12e
Add new parser error code, E_OVERFLOW. This error is returned when
Jeremy Hylton <jeremy@alum.mit.edu>
parents:
5379
diff
changeset
|
270 if ((err = shift(&ps->p_stack, type, str, |
|
36786
57322cb2ca11
Patch #1440601: Add col_offset attribute to AST nodes.
Martin v. Löwis <martin@v.loewis.de>
parents:
36742
diff
changeset
|
271 x, lineno, col_offset)) > 0) { |
| 6 | 272 D(printf(" MemError: shift.\n")); |
|
12970
63436719d12e
Add new parser error code, E_OVERFLOW. This error is returned when
Jeremy Hylton <jeremy@alum.mit.edu>
parents:
5379
diff
changeset
|
273 return err; |
| 6 | 274 } |
| 275 D(printf(" Shift.\n")); | |
| 276 /* Pop while we are in an accept-only state */ | |
| 277 while (s = &d->d_state | |
| 278 [ps->p_stack.s_top->s_state], | |
| 279 s->s_accept && s->s_narcs == 1) { | |
|
18609
59446d0f8341
Preliminary support for "from __future__ import generators" to enable
Guido van Rossum <guido@python.org>
parents:
14968
diff
changeset
|
280 D(printf(" DFA '%s', state %d: " |
|
59446d0f8341
Preliminary support for "from __future__ import generators" to enable
Guido van Rossum <guido@python.org>
parents:
14968
diff
changeset
|
281 "Direct pop.\n", |
|
59446d0f8341
Preliminary support for "from __future__ import generators" to enable
Guido van Rossum <guido@python.org>
parents:
14968
diff
changeset
|
282 d->d_name, |
|
59446d0f8341
Preliminary support for "from __future__ import generators" to enable
Guido van Rossum <guido@python.org>
parents:
14968
diff
changeset
|
283 ps->p_stack.s_top->s_state)); |
|
36726
bbf16fd024df
from __future__ import with_statement addon for 'with', mostly written by
Thomas Wouters <thomas@python.org>
parents:
22466
diff
changeset
|
284 #ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD |
|
52641
0530aadff696
The BDFL has retired! Long live the FLUFL (Friendly Language Uncle For Life)!
Brett Cannon <bcannon@gmail.com>
parents:
49594
diff
changeset
|
285 #if 0 |
|
18609
59446d0f8341
Preliminary support for "from __future__ import generators" to enable
Guido van Rossum <guido@python.org>
parents:
14968
diff
changeset
|
286 if (d->d_name[0] == 'i' && |
|
59446d0f8341
Preliminary support for "from __future__ import generators" to enable
Guido van Rossum <guido@python.org>
parents:
14968
diff
changeset
|
287 strcmp(d->d_name, |
|
59446d0f8341
Preliminary support for "from __future__ import generators" to enable
Guido van Rossum <guido@python.org>
parents:
14968
diff
changeset
|
288 "import_stmt") == 0) |
|
59446d0f8341
Preliminary support for "from __future__ import generators" to enable
Guido van Rossum <guido@python.org>
parents:
14968
diff
changeset
|
289 future_hack(ps); |
|
22466
1db1510abe21
Disable the parser hacks that allowed the "yield" keyword to be enabled
Neil Schemenauer <nascheme@enme.ucalgary.ca>
parents:
21181
diff
changeset
|
290 #endif |
|
52641
0530aadff696
The BDFL has retired! Long live the FLUFL (Friendly Language Uncle For Life)!
Brett Cannon <bcannon@gmail.com>
parents:
49594
diff
changeset
|
291 #endif |
| 6 | 292 s_pop(&ps->p_stack); |
| 293 if (s_empty(&ps->p_stack)) { | |
| 294 D(printf(" ACCEPT.\n")); | |
| 295 return E_DONE; | |
| 296 } | |
| 297 d = ps->p_stack.s_top->s_dfa; | |
| 298 } | |
| 299 return E_OK; | |
| 300 } | |
| 301 } | |
| 302 | |
| 303 if (s->s_accept) { | |
|
36726
bbf16fd024df
from __future__ import with_statement addon for 'with', mostly written by
Thomas Wouters <thomas@python.org>
parents:
22466
diff
changeset
|
304 #ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD |
|
52641
0530aadff696
The BDFL has retired! Long live the FLUFL (Friendly Language Uncle For Life)!
Brett Cannon <bcannon@gmail.com>
parents:
49594
diff
changeset
|
305 #if 0 |
|
18609
59446d0f8341
Preliminary support for "from __future__ import generators" to enable
Guido van Rossum <guido@python.org>
parents:
14968
diff
changeset
|
306 if (d->d_name[0] == 'i' && |
|
59446d0f8341
Preliminary support for "from __future__ import generators" to enable
Guido van Rossum <guido@python.org>
parents:
14968
diff
changeset
|
307 strcmp(d->d_name, "import_stmt") == 0) |
|
59446d0f8341
Preliminary support for "from __future__ import generators" to enable
Guido van Rossum <guido@python.org>
parents:
14968
diff
changeset
|
308 future_hack(ps); |
|
22466
1db1510abe21
Disable the parser hacks that allowed the "yield" keyword to be enabled
Neil Schemenauer <nascheme@enme.ucalgary.ca>
parents:
21181
diff
changeset
|
309 #endif |
|
52641
0530aadff696
The BDFL has retired! Long live the FLUFL (Friendly Language Uncle For Life)!
Brett Cannon <bcannon@gmail.com>
parents:
49594
diff
changeset
|
310 #endif |
| 6 | 311 /* Pop this dfa and try again */ |
| 312 s_pop(&ps->p_stack); | |
| 313 D(printf(" Pop ...\n")); | |
| 314 if (s_empty(&ps->p_stack)) { | |
| 315 D(printf(" Error: bottom of stack.\n")); | |
| 316 return E_SYNTAX; | |
| 317 } | |
| 318 continue; | |
| 319 } | |
| 320 | |
| 321 /* Stuck, report syntax error */ | |
| 322 D(printf(" Error.\n")); | |
|
13539
feab677269e0
Create two new exceptions: IndentationError and TabError. These are
Fred Drake <fdrake@acm.org>
parents:
13447
diff
changeset
|
323 if (expected_ret) { |
|
feab677269e0
Create two new exceptions: IndentationError and TabError. These are
Fred Drake <fdrake@acm.org>
parents:
13447
diff
changeset
|
324 if (s->s_lower == s->s_upper - 1) { |
|
feab677269e0
Create two new exceptions: IndentationError and TabError. These are
Fred Drake <fdrake@acm.org>
parents:
13447
diff
changeset
|
325 /* Only one possible expected token */ |
|
feab677269e0
Create two new exceptions: IndentationError and TabError. These are
Fred Drake <fdrake@acm.org>
parents:
13447
diff
changeset
|
326 *expected_ret = ps->p_grammar-> |
|
feab677269e0
Create two new exceptions: IndentationError and TabError. These are
Fred Drake <fdrake@acm.org>
parents:
13447
diff
changeset
|
327 g_ll.ll_label[s->s_lower].lb_type; |
|
feab677269e0
Create two new exceptions: IndentationError and TabError. These are
Fred Drake <fdrake@acm.org>
parents:
13447
diff
changeset
|
328 } |
|
feab677269e0
Create two new exceptions: IndentationError and TabError. These are
Fred Drake <fdrake@acm.org>
parents:
13447
diff
changeset
|
329 else |
|
feab677269e0
Create two new exceptions: IndentationError and TabError. These are
Fred Drake <fdrake@acm.org>
parents:
13447
diff
changeset
|
330 *expected_ret = -1; |
|
feab677269e0
Create two new exceptions: IndentationError and TabError. These are
Fred Drake <fdrake@acm.org>
parents:
13447
diff
changeset
|
331 } |
| 6 | 332 return E_SYNTAX; |
| 333 } | |
| 334 } | |
| 335 | |
| 336 | |
|
4839
4567e158d20c
Rename DEBUG macro to Py_DEBUG
Guido van Rossum <guido@python.org>
parents:
4503
diff
changeset
|
337 #ifdef Py_DEBUG |
| 6 | 338 |
| 339 /* DEBUG OUTPUT */ | |
| 340 | |
| 341 void | |
| 13659 | 342 dumptree(grammar *g, node *n) |
| 6 | 343 { |
| 344 int i; | |
| 345 | |
| 346 if (n == NULL) | |
| 347 printf("NIL"); | |
| 348 else { | |
| 349 label l; | |
| 350 l.lb_type = TYPE(n); | |
|
1027
513940eb4d7b
Correct debug printing code
Guido van Rossum <guido@python.org>
parents:
795
diff
changeset
|
351 l.lb_str = STR(n); |
|
5379
1c9a84e0981d
Another directory quickly renamed.
Guido van Rossum <guido@python.org>
parents:
4839
diff
changeset
|
352 printf("%s", PyGrammar_LabelRepr(&l)); |
| 6 | 353 if (ISNONTERMINAL(TYPE(n))) { |
| 354 printf("("); | |
| 355 for (i = 0; i < NCH(n); i++) { | |
| 356 if (i > 0) | |
| 357 printf(","); | |
| 358 dumptree(g, CHILD(n, i)); | |
| 359 } | |
| 360 printf(")"); | |
| 361 } | |
| 362 } | |
| 363 } | |
| 364 | |
| 365 void | |
| 13659 | 366 showtree(grammar *g, node *n) |
| 6 | 367 { |
| 368 int i; | |
| 369 | |
| 370 if (n == NULL) | |
| 371 return; | |
| 372 if (ISNONTERMINAL(TYPE(n))) { | |
| 373 for (i = 0; i < NCH(n); i++) | |
| 374 showtree(g, CHILD(n, i)); | |
| 375 } | |
| 376 else if (ISTERMINAL(TYPE(n))) { | |
|
5379
1c9a84e0981d
Another directory quickly renamed.
Guido van Rossum <guido@python.org>
parents:
4839
diff
changeset
|
377 printf("%s", _PyParser_TokenNames[TYPE(n)]); |
| 6 | 378 if (TYPE(n) == NUMBER || TYPE(n) == NAME) |
| 379 printf("(%s)", STR(n)); | |
| 380 printf(" "); | |
| 381 } | |
| 382 else | |
| 383 printf("? "); | |
| 384 } | |
| 385 | |
| 386 void | |
| 13659 | 387 printtree(parser_state *ps) |
| 6 | 388 { |
|
5379
1c9a84e0981d
Another directory quickly renamed.
Guido van Rossum <guido@python.org>
parents:
4839
diff
changeset
|
389 if (Py_DebugFlag) { |
| 6 | 390 printf("Parse tree:\n"); |
| 391 dumptree(ps->p_grammar, ps->p_tree); | |
| 392 printf("\n"); | |
| 393 printf("Tokens:\n"); | |
| 394 showtree(ps->p_grammar, ps->p_tree); | |
| 395 printf("\n"); | |
| 396 } | |
| 397 printf("Listing:\n"); | |
|
5379
1c9a84e0981d
Another directory quickly renamed.
Guido van Rossum <guido@python.org>
parents:
4839
diff
changeset
|
398 PyNode_ListTree(ps->p_tree); |
| 6 | 399 printf("\n"); |
| 400 } | |
| 401 | |
|
4839
4567e158d20c
Rename DEBUG macro to Py_DEBUG
Guido van Rossum <guido@python.org>
parents:
4503
diff
changeset
|
402 #endif /* Py_DEBUG */ |
| 6 | 403 |
| 404 /* | |
| 405 | |
| 406 Description | |
| 407 ----------- | |
| 408 | |
| 409 The parser's interface is different than usual: the function addtoken() | |
| 410 must be called for each token in the input. This makes it possible to | |
| 411 turn it into an incremental parsing system later. The parsing system | |
| 412 constructs a parse tree as it goes. | |
| 413 | |
| 414 A parsing rule is represented as a Deterministic Finite-state Automaton | |
| 415 (DFA). A node in a DFA represents a state of the parser; an arc represents | |
| 416 a transition. Transitions are either labeled with terminal symbols or | |
| 417 with non-terminals. When the parser decides to follow an arc labeled | |
| 418 with a non-terminal, it is invoked recursively with the DFA representing | |
| 419 the parsing rule for that as its initial state; when that DFA accepts, | |
| 420 the parser that invoked it continues. The parse tree constructed by the | |
| 421 recursively called parser is inserted as a child in the current parse tree. | |
| 422 | |
| 423 The DFA's can be constructed automatically from a more conventional | |
| 424 language description. An extended LL(1) grammar (ELL(1)) is suitable. | |
| 425 Certain restrictions make the parser's life easier: rules that can produce | |
| 426 the empty string should be outlawed (there are other ways to put loops | |
| 427 or optional parts in the language). To avoid the need to construct | |
| 428 FIRST sets, we can require that all but the last alternative of a rule | |
| 429 (really: arc going out of a DFA's state) must begin with a terminal | |
| 430 symbol. | |
| 431 | |
| 432 As an example, consider this grammar: | |
| 433 | |
| 434 expr: term (OP term)* | |
| 435 term: CONSTANT | '(' expr ')' | |
| 436 | |
| 437 The DFA corresponding to the rule for expr is: | |
| 438 | |
| 439 ------->.---term-->.-------> | |
| 440 ^ | | |
| 441 | | | |
| 442 \----OP----/ | |
| 443 | |
| 444 The parse tree generated for the input a+b is: | |
| 445 | |
| 446 (expr: (term: (NAME: a)), (OP: +), (term: (NAME: b))) | |
| 447 | |
| 448 */ |