Mercurial > cpython-lru_cache-weakref
annotate Parser/parser.c @ 58664:993f703a61ba
Merged revisions 77484,77487,77561,77570,77593,77603,77608,77667,77702-77703,77739,77858,77887,77889 via svnmerge from
svn+ssh://pythondev@svn.python.org/python/trunk
........
r77484 | skip.montanaro | 2010-01-13 19:12:34 -0600 (Wed, 13 Jan 2010) | 4 lines
Update PyEval_EvalFrame to PyEval_EvalFrameEx. This looks to have been done
partially before. Also add a comment describing how this might have to work
with different versions of the interpreter.
........
r77487 | ezio.melotti | 2010-01-14 05:34:10 -0600 (Thu, 14 Jan 2010) | 1 line
Fixed typo
........
r77561 | georg.brandl | 2010-01-17 02:42:30 -0600 (Sun, 17 Jan 2010) | 1 line
#7699: improve datetime docs: straightforward linking to strftime/strptime section, mark classmethods as such.
........
r77570 | georg.brandl | 2010-01-17 06:14:42 -0600 (Sun, 17 Jan 2010) | 1 line
Add note about usage of STRINGLIB_EMPTY.
........
r77593 | georg.brandl | 2010-01-17 17:33:53 -0600 (Sun, 17 Jan 2010) | 1 line
Fix internal reference.
........
r77603 | benjamin.peterson | 2010-01-18 17:07:56 -0600 (Mon, 18 Jan 2010) | 8 lines
data descriptors do not override the class dictionary if __get__ is not defined
Adjust documentation and add a test to verify this behavior.
See http://mail.python.org/pipermail/python-dev/2010-January/095637.html for
discussion.
........
r77608 | gregory.p.smith | 2010-01-19 02:19:03 -0600 (Tue, 19 Jan 2010) | 6 lines
Do not compile stubs for the sha2 series hashes in the openssl hashlib
module when the openssl version is too old to support them. That
leads both compiled code bloat and to unittests attempting to test
implementations that don't exist for comparison purposes on such
platforms.
........
r77667 | mark.dickinson | 2010-01-21 12:32:27 -0600 (Thu, 21 Jan 2010) | 1 line
Add two more test_strtod test values.
........
r77702 | georg.brandl | 2010-01-23 02:43:31 -0600 (Sat, 23 Jan 2010) | 1 line
#7762: fix refcount annotation of PyUnicode_Tailmatch().
........
r77703 | georg.brandl | 2010-01-23 02:47:54 -0600 (Sat, 23 Jan 2010) | 1 line
#7725: fix referencing issue.
........
r77739 | benjamin.peterson | 2010-01-24 21:52:52 -0600 (Sun, 24 Jan 2010) | 1 line
mention from_float() in error message
........
r77858 | georg.brandl | 2010-01-30 11:57:48 -0600 (Sat, 30 Jan 2010) | 1 line
#7802: fix invalid example (heh).
........
r77887 | georg.brandl | 2010-01-31 12:51:49 -0600 (Sun, 31 Jan 2010) | 5 lines
Fix-up ftplib documentation:
move exception descriptions to toplevel, not inside a class
remove attribution in "versionadded"
spell and grammar check docstring of FTP_TLS
........
r77889 | michael.foord | 2010-01-31 13:59:26 -0600 (Sun, 31 Jan 2010) | 1 line
Minor modification to unittest documentation.
........
| author | Benjamin Peterson <benjamin@python.org> |
|---|---|
| date | Wed, 03 Feb 2010 02:35:45 +0000 |
| parents | 0530aadff696 |
| children | cab14be0ada1 |
| rev | line source |
|---|---|
| 145 | 1 |
| 6 | 2 /* Parser implementation */ |
| 3 | |
| 4 /* For a description, see the comments at end of this file */ | |
| 5 | |
| 6 /* XXX To do: error recovery */ | |
| 7 | |
|
21181
944d6e02f359
The parser doesn't need its own implementation of assert, and having its
Tim Peters <tim.peters@gmail.com>
parents:
18739
diff
changeset
|
8 #include "Python.h" |
| 86 | 9 #include "pgenheaders.h" |
| 6 | 10 #include "token.h" |
| 11 #include "grammar.h" | |
| 12 #include "node.h" | |
| 13 #include "parser.h" | |
| 14 #include "errcode.h" | |
| 15 | |
| 16 | |
|
4839
4567e158d20c
Rename DEBUG macro to Py_DEBUG
Guido van Rossum <guido@python.org>
parents:
4503
diff
changeset
|
17 #ifdef Py_DEBUG |
|
5379
1c9a84e0981d
Another directory quickly renamed.
Guido van Rossum <guido@python.org>
parents:
4839
diff
changeset
|
18 extern int Py_DebugFlag; |
|
1c9a84e0981d
Another directory quickly renamed.
Guido van Rossum <guido@python.org>
parents:
4839
diff
changeset
|
19 #define D(x) if (!Py_DebugFlag); else x |
| 6 | 20 #else |
| 21 #define D(x) | |
| 22 #endif | |
| 23 | |
| 24 | |
| 25 /* STACK DATA TYPE */ | |
| 26 | |
|
13447
a89f7f268de4
Nuke all remaining occurrences of Py_PROTO and Py_FPROTO.
Tim Peters <tim.peters@gmail.com>
parents:
13256
diff
changeset
|
27 static void s_reset(stack *); |
| 6 | 28 |
| 29 static void | |
| 13659 | 30 s_reset(stack *s) |
| 6 | 31 { |
| 32 s->s_top = &s->s_base[MAXSTACK]; | |
| 33 } | |
| 34 | |
| 35 #define s_empty(s) ((s)->s_top == &(s)->s_base[MAXSTACK]) | |
| 36 | |
| 37 static int | |
| 13659 | 38 s_push(register stack *s, dfa *d, node *parent) |
| 6 | 39 { |
| 40 register stackentry *top; | |
| 41 if (s->s_top == s->s_base) { | |
| 42 fprintf(stderr, "s_push: parser stack overflow\n"); | |
|
14968
5336946b6495
Fix a bug in stack overflow error handling. This fixes half of Bug
Guido van Rossum <guido@python.org>
parents:
14375
diff
changeset
|
43 return E_NOMEM; |
| 6 | 44 } |
| 45 top = --s->s_top; | |
| 46 top->s_dfa = d; | |
| 47 top->s_parent = parent; | |
| 48 top->s_state = 0; | |
| 49 return 0; | |
| 50 } | |
| 51 | |
|
4839
4567e158d20c
Rename DEBUG macro to Py_DEBUG
Guido van Rossum <guido@python.org>
parents:
4503
diff
changeset
|
52 #ifdef Py_DEBUG |
| 6 | 53 |
| 54 static void | |
| 13659 | 55 s_pop(register stack *s) |
| 6 | 56 { |
|
2044
281e40652e23
Parser/tokenizer.c (tok_nextc): zap tok->buf after freeing;
Guido van Rossum <guido@python.org>
parents:
1848
diff
changeset
|
57 if (s_empty(s)) |
|
5379
1c9a84e0981d
Another directory quickly renamed.
Guido van Rossum <guido@python.org>
parents:
4839
diff
changeset
|
58 Py_FatalError("s_pop: parser stack underflow -- FATAL"); |
| 6 | 59 s->s_top++; |
| 60 } | |
| 61 | |
|
4839
4567e158d20c
Rename DEBUG macro to Py_DEBUG
Guido van Rossum <guido@python.org>
parents:
4503
diff
changeset
|
62 #else /* !Py_DEBUG */ |
| 6 | 63 |
| 64 #define s_pop(s) (s)->s_top++ | |
| 65 | |
| 66 #endif | |
| 67 | |
| 68 | |
| 69 /* PARSER CREATION */ | |
| 70 | |
| 71 parser_state * | |
| 13659 | 72 PyParser_New(grammar *g, int start) |
| 6 | 73 { |
| 74 parser_state *ps; | |
| 75 | |
| 76 if (!g->g_accel) | |
|
5379
1c9a84e0981d
Another directory quickly renamed.
Guido van Rossum <guido@python.org>
parents:
4839
diff
changeset
|
77 PyGrammar_AddAccelerators(g); |
|
37879
f731859e9b3b
Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents:
36988
diff
changeset
|
78 ps = (parser_state *)PyMem_MALLOC(sizeof(parser_state)); |
| 6 | 79 if (ps == NULL) |
| 80 return NULL; | |
| 81 ps->p_grammar = g; | |
|
36726
bbf16fd024df
from __future__ import with_statement addon for 'with', mostly written by
Thomas Wouters <thomas@python.org>
parents:
22466
diff
changeset
|
82 #ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD |
|
bbf16fd024df
from __future__ import with_statement addon for 'with', mostly written by
Thomas Wouters <thomas@python.org>
parents:
22466
diff
changeset
|
83 ps->p_flags = 0; |
|
22466
1db1510abe21
Disable the parser hacks that allowed the "yield" keyword to be enabled
Neil Schemenauer <nascheme@enme.ucalgary.ca>
parents:
21181
diff
changeset
|
84 #endif |
|
5379
1c9a84e0981d
Another directory quickly renamed.
Guido van Rossum <guido@python.org>
parents:
4839
diff
changeset
|
85 ps->p_tree = PyNode_New(start); |
| 6 | 86 if (ps->p_tree == NULL) { |
|
37879
f731859e9b3b
Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents:
36988
diff
changeset
|
87 PyMem_FREE(ps); |
| 6 | 88 return NULL; |
| 89 } | |
| 90 s_reset(&ps->p_stack); | |
|
5379
1c9a84e0981d
Another directory quickly renamed.
Guido van Rossum <guido@python.org>
parents:
4839
diff
changeset
|
91 (void) s_push(&ps->p_stack, PyGrammar_FindDFA(g, start), ps->p_tree); |
| 6 | 92 return ps; |
| 93 } | |
| 94 | |
| 95 void | |
| 13659 | 96 PyParser_Delete(parser_state *ps) |
| 6 | 97 { |
|
79
6c43e5476de6
Free parse tree when deleting parser.
Guido van Rossum <guido@python.org>
parents:
6
diff
changeset
|
98 /* NB If you want to save the parse tree, |
|
6c43e5476de6
Free parse tree when deleting parser.
Guido van Rossum <guido@python.org>
parents:
6
diff
changeset
|
99 you must set p_tree to NULL before calling delparser! */ |
|
5379
1c9a84e0981d
Another directory quickly renamed.
Guido van Rossum <guido@python.org>
parents:
4839
diff
changeset
|
100 PyNode_Free(ps->p_tree); |
|
37879
f731859e9b3b
Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents:
36988
diff
changeset
|
101 PyMem_FREE(ps); |
| 6 | 102 } |
| 103 | |
| 104 | |
| 105 /* PARSER STACK OPERATIONS */ | |
| 106 | |
| 107 static int | |
|
36786
57322cb2ca11
Patch #1440601: Add col_offset attribute to AST nodes.
Martin v. Löwis <martin@v.loewis.de>
parents:
36742
diff
changeset
|
108 shift(register stack *s, int type, char *str, int newstate, int lineno, int col_offset) |
| 6 | 109 { |
|
12970
63436719d12e
Add new parser error code, E_OVERFLOW. This error is returned when
Jeremy Hylton <jeremy@alum.mit.edu>
parents:
5379
diff
changeset
|
110 int err; |
| 6 | 111 assert(!s_empty(s)); |
|
36786
57322cb2ca11
Patch #1440601: Add col_offset attribute to AST nodes.
Martin v. Löwis <martin@v.loewis.de>
parents:
36742
diff
changeset
|
112 err = PyNode_AddChild(s->s_top->s_parent, type, str, lineno, col_offset); |
|
12970
63436719d12e
Add new parser error code, E_OVERFLOW. This error is returned when
Jeremy Hylton <jeremy@alum.mit.edu>
parents:
5379
diff
changeset
|
113 if (err) |
|
63436719d12e
Add new parser error code, E_OVERFLOW. This error is returned when
Jeremy Hylton <jeremy@alum.mit.edu>
parents:
5379
diff
changeset
|
114 return err; |
| 6 | 115 s->s_top->s_state = newstate; |
| 116 return 0; | |
| 117 } | |
| 118 | |
| 119 static int | |
|
36786
57322cb2ca11
Patch #1440601: Add col_offset attribute to AST nodes.
Martin v. Löwis <martin@v.loewis.de>
parents:
36742
diff
changeset
|
120 push(register stack *s, int type, dfa *d, int newstate, int lineno, int col_offset) |
| 6 | 121 { |
|
12970
63436719d12e
Add new parser error code, E_OVERFLOW. This error is returned when
Jeremy Hylton <jeremy@alum.mit.edu>
parents:
5379
diff
changeset
|
122 int err; |
| 6 | 123 register node *n; |
| 124 n = s->s_top->s_parent; | |
| 125 assert(!s_empty(s)); | |
|
36786
57322cb2ca11
Patch #1440601: Add col_offset attribute to AST nodes.
Martin v. Löwis <martin@v.loewis.de>
parents:
36742
diff
changeset
|
126 err = PyNode_AddChild(n, type, (char *)NULL, lineno, col_offset); |
|
12970
63436719d12e
Add new parser error code, E_OVERFLOW. This error is returned when
Jeremy Hylton <jeremy@alum.mit.edu>
parents:
5379
diff
changeset
|
127 if (err) |
|
63436719d12e
Add new parser error code, E_OVERFLOW. This error is returned when
Jeremy Hylton <jeremy@alum.mit.edu>
parents:
5379
diff
changeset
|
128 return err; |
| 6 | 129 s->s_top->s_state = newstate; |
| 130 return s_push(s, d, CHILD(n, NCH(n)-1)); | |
| 131 } | |
| 132 | |
| 133 | |
| 134 /* PARSER PROPER */ | |
| 135 | |
| 136 static int | |
|
18609
59446d0f8341
Preliminary support for "from __future__ import generators" to enable
Guido van Rossum <guido@python.org>
parents:
14968
diff
changeset
|
137 classify(parser_state *ps, int type, char *str) |
| 6 | 138 { |
|
18609
59446d0f8341
Preliminary support for "from __future__ import generators" to enable
Guido van Rossum <guido@python.org>
parents:
14968
diff
changeset
|
139 grammar *g = ps->p_grammar; |
| 6 | 140 register int n = g->g_ll.ll_nlabels; |
| 141 | |
| 142 if (type == NAME) { | |
| 143 register char *s = str; | |
| 144 register label *l = g->g_ll.ll_label; | |
| 145 register int i; | |
| 146 for (i = n; i > 0; i--, l++) { | |
|
36742
658f8ff2a442
Make 'as' an actual keyword when with's future statement is used. Not
Thomas Wouters <thomas@python.org>
parents:
36726
diff
changeset
|
147 if (l->lb_type != NAME || l->lb_str == NULL || |
|
658f8ff2a442
Make 'as' an actual keyword when with's future statement is used. Not
Thomas Wouters <thomas@python.org>
parents:
36726
diff
changeset
|
148 l->lb_str[0] != s[0] || |
|
658f8ff2a442
Make 'as' an actual keyword when with's future statement is used. Not
Thomas Wouters <thomas@python.org>
parents:
36726
diff
changeset
|
149 strcmp(l->lb_str, s) != 0) |
|
658f8ff2a442
Make 'as' an actual keyword when with's future statement is used. Not
Thomas Wouters <thomas@python.org>
parents:
36726
diff
changeset
|
150 continue; |
|
36726
bbf16fd024df
from __future__ import with_statement addon for 'with', mostly written by
Thomas Wouters <thomas@python.org>
parents:
22466
diff
changeset
|
151 #ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD |
|
52641
0530aadff696
The BDFL has retired! Long live the FLUFL (Friendly Language Uncle For Life)!
Brett Cannon <bcannon@gmail.com>
parents:
49594
diff
changeset
|
152 #if 0 |
|
36988
49c6eb688906
Checkpoint. 218 tests are okay; 53 are failing. Done so far:
Guido van Rossum <guido@python.org>
parents:
36786
diff
changeset
|
153 /* Leaving this in as an example */ |
|
36742
658f8ff2a442
Make 'as' an actual keyword when with's future statement is used. Not
Thomas Wouters <thomas@python.org>
parents:
36726
diff
changeset
|
154 if (!(ps->p_flags & CO_FUTURE_WITH_STATEMENT)) { |
|
658f8ff2a442
Make 'as' an actual keyword when with's future statement is used. Not
Thomas Wouters <thomas@python.org>
parents:
36726
diff
changeset
|
155 if (s[0] == 'w' && strcmp(s, "with") == 0) |
|
658f8ff2a442
Make 'as' an actual keyword when with's future statement is used. Not
Thomas Wouters <thomas@python.org>
parents:
36726
diff
changeset
|
156 break; /* not a keyword yet */ |
|
658f8ff2a442
Make 'as' an actual keyword when with's future statement is used. Not
Thomas Wouters <thomas@python.org>
parents:
36726
diff
changeset
|
157 else if (s[0] == 'a' && strcmp(s, "as") == 0) |
|
658f8ff2a442
Make 'as' an actual keyword when with's future statement is used. Not
Thomas Wouters <thomas@python.org>
parents:
36726
diff
changeset
|
158 break; /* not a keyword yet */ |
|
658f8ff2a442
Make 'as' an actual keyword when with's future statement is used. Not
Thomas Wouters <thomas@python.org>
parents:
36726
diff
changeset
|
159 } |
|
22466
1db1510abe21
Disable the parser hacks that allowed the "yield" keyword to be enabled
Neil Schemenauer <nascheme@enme.ucalgary.ca>
parents:
21181
diff
changeset
|
160 #endif |
|
52641
0530aadff696
The BDFL has retired! Long live the FLUFL (Friendly Language Uncle For Life)!
Brett Cannon <bcannon@gmail.com>
parents:
49594
diff
changeset
|
161 #endif |
|
36742
658f8ff2a442
Make 'as' an actual keyword when with's future statement is used. Not
Thomas Wouters <thomas@python.org>
parents:
36726
diff
changeset
|
162 D(printf("It's a keyword\n")); |
|
658f8ff2a442
Make 'as' an actual keyword when with's future statement is used. Not
Thomas Wouters <thomas@python.org>
parents:
36726
diff
changeset
|
163 return n - i; |
| 6 | 164 } |
| 165 } | |
| 166 | |
| 167 { | |
| 168 register label *l = g->g_ll.ll_label; | |
| 169 register int i; | |
| 170 for (i = n; i > 0; i--, l++) { | |
| 171 if (l->lb_type == type && l->lb_str == NULL) { | |
| 172 D(printf("It's a token we know\n")); | |
| 173 return n - i; | |
| 174 } | |
| 175 } | |
| 176 } | |
| 177 | |
| 178 D(printf("Illegal token\n")); | |
| 179 return -1; | |
| 180 } | |
| 181 | |
|
36726
bbf16fd024df
from __future__ import with_statement addon for 'with', mostly written by
Thomas Wouters <thomas@python.org>
parents:
22466
diff
changeset
|
182 #ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD |
|
52641
0530aadff696
The BDFL has retired! Long live the FLUFL (Friendly Language Uncle For Life)!
Brett Cannon <bcannon@gmail.com>
parents:
49594
diff
changeset
|
183 #if 0 |
|
36988
49c6eb688906
Checkpoint. 218 tests are okay; 53 are failing. Done so far:
Guido van Rossum <guido@python.org>
parents:
36786
diff
changeset
|
184 /* Leaving this in as an example */ |
|
18609
59446d0f8341
Preliminary support for "from __future__ import generators" to enable
Guido van Rossum <guido@python.org>
parents:
14968
diff
changeset
|
185 static void |
|
59446d0f8341
Preliminary support for "from __future__ import generators" to enable
Guido van Rossum <guido@python.org>
parents:
14968
diff
changeset
|
186 future_hack(parser_state *ps) |
|
59446d0f8341
Preliminary support for "from __future__ import generators" to enable
Guido van Rossum <guido@python.org>
parents:
14968
diff
changeset
|
187 { |
|
59446d0f8341
Preliminary support for "from __future__ import generators" to enable
Guido van Rossum <guido@python.org>
parents:
14968
diff
changeset
|
188 node *n = ps->p_stack.s_top->s_parent; |
|
40450
cc992d75d5b3
Four months of trunk changes (including a few releases...)
Thomas Wouters <thomas@python.org>
parents:
37879
diff
changeset
|
189 node *ch, *cch; |
|
18739
704486642079
Fis SF bug #442647: not all forms of legal future statements were
Guido van Rossum <guido@python.org>
parents:
18609
diff
changeset
|
190 int i; |
|
18609
59446d0f8341
Preliminary support for "from __future__ import generators" to enable
Guido van Rossum <guido@python.org>
parents:
14968
diff
changeset
|
191 |
|
36726
bbf16fd024df
from __future__ import with_statement addon for 'with', mostly written by
Thomas Wouters <thomas@python.org>
parents:
22466
diff
changeset
|
192 /* from __future__ import ..., must have at least 4 children */ |
|
bbf16fd024df
from __future__ import with_statement addon for 'with', mostly written by
Thomas Wouters <thomas@python.org>
parents:
22466
diff
changeset
|
193 n = CHILD(n, 0); |
|
bbf16fd024df
from __future__ import with_statement addon for 'with', mostly written by
Thomas Wouters <thomas@python.org>
parents:
22466
diff
changeset
|
194 if (NCH(n) < 4) |
|
bbf16fd024df
from __future__ import with_statement addon for 'with', mostly written by
Thomas Wouters <thomas@python.org>
parents:
22466
diff
changeset
|
195 return; |
|
bbf16fd024df
from __future__ import with_statement addon for 'with', mostly written by
Thomas Wouters <thomas@python.org>
parents:
22466
diff
changeset
|
196 ch = CHILD(n, 0); |
|
bbf16fd024df
from __future__ import with_statement addon for 'with', mostly written by
Thomas Wouters <thomas@python.org>
parents:
22466
diff
changeset
|
197 if (STR(ch) == NULL || strcmp(STR(ch), "from") != 0) |
|
18609
59446d0f8341
Preliminary support for "from __future__ import generators" to enable
Guido van Rossum <guido@python.org>
parents:
14968
diff
changeset
|
198 return; |
|
59446d0f8341
Preliminary support for "from __future__ import generators" to enable
Guido van Rossum <guido@python.org>
parents:
14968
diff
changeset
|
199 ch = CHILD(n, 1); |
|
36726
bbf16fd024df
from __future__ import with_statement addon for 'with', mostly written by
Thomas Wouters <thomas@python.org>
parents:
22466
diff
changeset
|
200 if (NCH(ch) == 1 && STR(CHILD(ch, 0)) && |
|
bbf16fd024df
from __future__ import with_statement addon for 'with', mostly written by
Thomas Wouters <thomas@python.org>
parents:
22466
diff
changeset
|
201 strcmp(STR(CHILD(ch, 0)), "__future__") != 0) |
|
18609
59446d0f8341
Preliminary support for "from __future__ import generators" to enable
Guido van Rossum <guido@python.org>
parents:
14968
diff
changeset
|
202 return; |
|
40450
cc992d75d5b3
Four months of trunk changes (including a few releases...)
Thomas Wouters <thomas@python.org>
parents:
37879
diff
changeset
|
203 ch = CHILD(n, 3); |
|
cc992d75d5b3
Four months of trunk changes (including a few releases...)
Thomas Wouters <thomas@python.org>
parents:
37879
diff
changeset
|
204 /* ch can be a star, a parenthesis or import_as_names */ |
|
cc992d75d5b3
Four months of trunk changes (including a few releases...)
Thomas Wouters <thomas@python.org>
parents:
37879
diff
changeset
|
205 if (TYPE(ch) == STAR) |
|
cc992d75d5b3
Four months of trunk changes (including a few releases...)
Thomas Wouters <thomas@python.org>
parents:
37879
diff
changeset
|
206 return; |
|
cc992d75d5b3
Four months of trunk changes (including a few releases...)
Thomas Wouters <thomas@python.org>
parents:
37879
diff
changeset
|
207 if (TYPE(ch) == LPAR) |
|
cc992d75d5b3
Four months of trunk changes (including a few releases...)
Thomas Wouters <thomas@python.org>
parents:
37879
diff
changeset
|
208 ch = CHILD(n, 4); |
|
cc992d75d5b3
Four months of trunk changes (including a few releases...)
Thomas Wouters <thomas@python.org>
parents:
37879
diff
changeset
|
209 |
|
cc992d75d5b3
Four months of trunk changes (including a few releases...)
Thomas Wouters <thomas@python.org>
parents:
37879
diff
changeset
|
210 for (i = 0; i < NCH(ch); i += 2) { |
|
cc992d75d5b3
Four months of trunk changes (including a few releases...)
Thomas Wouters <thomas@python.org>
parents:
37879
diff
changeset
|
211 cch = CHILD(ch, i); |
|
46120
78504d604b44
Merged revisions 61952-61953 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
40450
diff
changeset
|
212 if (NCH(cch) >= 1 && TYPE(CHILD(cch, 0)) == NAME) { |
|
78504d604b44
Merged revisions 61952-61953 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
40450
diff
changeset
|
213 char *str_ch = STR(CHILD(cch, 0)); |
|
78504d604b44
Merged revisions 61952-61953 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
40450
diff
changeset
|
214 if (strcmp(str_ch, FUTURE_WITH_STATEMENT) == 0) { |
|
78504d604b44
Merged revisions 61952-61953 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
40450
diff
changeset
|
215 ps->p_flags |= CO_FUTURE_WITH_STATEMENT; |
|
78504d604b44
Merged revisions 61952-61953 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
40450
diff
changeset
|
216 } else if (strcmp(str_ch, FUTURE_PRINT_FUNCTION) == 0) { |
|
78504d604b44
Merged revisions 61952-61953 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
40450
diff
changeset
|
217 ps->p_flags |= CO_FUTURE_PRINT_FUNCTION; |
|
78504d604b44
Merged revisions 61952-61953 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
40450
diff
changeset
|
218 } else if (strcmp(str_ch, FUTURE_UNICODE_LITERALS) == 0) { |
|
78504d604b44
Merged revisions 61952-61953 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
40450
diff
changeset
|
219 ps->p_flags |= CO_FUTURE_UNICODE_LITERALS; |
|
78504d604b44
Merged revisions 61952-61953 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
40450
diff
changeset
|
220 } |
|
18739
704486642079
Fis SF bug #442647: not all forms of legal future statements were
Guido van Rossum <guido@python.org>
parents:
18609
diff
changeset
|
221 } |
|
704486642079
Fis SF bug #442647: not all forms of legal future statements were
Guido van Rossum <guido@python.org>
parents:
18609
diff
changeset
|
222 } |
|
18609
59446d0f8341
Preliminary support for "from __future__ import generators" to enable
Guido van Rossum <guido@python.org>
parents:
14968
diff
changeset
|
223 } |
|
52641
0530aadff696
The BDFL has retired! Long live the FLUFL (Friendly Language Uncle For Life)!
Brett Cannon <bcannon@gmail.com>
parents:
49594
diff
changeset
|
224 #endif |
|
22466
1db1510abe21
Disable the parser hacks that allowed the "yield" keyword to be enabled
Neil Schemenauer <nascheme@enme.ucalgary.ca>
parents:
21181
diff
changeset
|
225 #endif /* future keyword */ |
|
18609
59446d0f8341
Preliminary support for "from __future__ import generators" to enable
Guido van Rossum <guido@python.org>
parents:
14968
diff
changeset
|
226 |
| 6 | 227 int |
| 13659 | 228 PyParser_AddToken(register parser_state *ps, register int type, char *str, |
|
36786
57322cb2ca11
Patch #1440601: Add col_offset attribute to AST nodes.
Martin v. Löwis <martin@v.loewis.de>
parents:
36742
diff
changeset
|
229 int lineno, int col_offset, int *expected_ret) |
| 6 | 230 { |
| 231 register int ilabel; | |
|
12970
63436719d12e
Add new parser error code, E_OVERFLOW. This error is returned when
Jeremy Hylton <jeremy@alum.mit.edu>
parents:
5379
diff
changeset
|
232 int err; |
| 6 | 233 |
|
5379
1c9a84e0981d
Another directory quickly renamed.
Guido van Rossum <guido@python.org>
parents:
4839
diff
changeset
|
234 D(printf("Token %s/'%s' ... ", _PyParser_TokenNames[type], str)); |
| 6 | 235 |
| 236 /* Find out which label this token is */ | |
|
18609
59446d0f8341
Preliminary support for "from __future__ import generators" to enable
Guido van Rossum <guido@python.org>
parents:
14968
diff
changeset
|
237 ilabel = classify(ps, type, str); |
| 6 | 238 if (ilabel < 0) |
| 239 return E_SYNTAX; | |
| 240 | |
| 241 /* Loop until the token is shifted or an error occurred */ | |
| 242 for (;;) { | |
| 243 /* Fetch the current dfa and state */ | |
| 244 register dfa *d = ps->p_stack.s_top->s_dfa; | |
| 245 register state *s = &d->d_state[ps->p_stack.s_top->s_state]; | |
| 246 | |
| 247 D(printf(" DFA '%s', state %d:", | |
| 248 d->d_name, ps->p_stack.s_top->s_state)); | |
| 249 | |
| 250 /* Check accelerator */ | |
| 251 if (s->s_lower <= ilabel && ilabel < s->s_upper) { | |
| 252 register int x = s->s_accel[ilabel - s->s_lower]; | |
| 253 if (x != -1) { | |
| 254 if (x & (1<<7)) { | |
| 255 /* Push non-terminal */ | |
| 256 int nt = (x >> 8) + NT_OFFSET; | |
| 257 int arrow = x & ((1<<7)-1); | |
|
5379
1c9a84e0981d
Another directory quickly renamed.
Guido van Rossum <guido@python.org>
parents:
4839
diff
changeset
|
258 dfa *d1 = PyGrammar_FindDFA( |
|
1c9a84e0981d
Another directory quickly renamed.
Guido van Rossum <guido@python.org>
parents:
4839
diff
changeset
|
259 ps->p_grammar, nt); |
|
12970
63436719d12e
Add new parser error code, E_OVERFLOW. This error is returned when
Jeremy Hylton <jeremy@alum.mit.edu>
parents:
5379
diff
changeset
|
260 if ((err = push(&ps->p_stack, nt, d1, |
|
36786
57322cb2ca11
Patch #1440601: Add col_offset attribute to AST nodes.
Martin v. Löwis <martin@v.loewis.de>
parents:
36742
diff
changeset
|
261 arrow, lineno, col_offset)) > 0) { |
|
5379
1c9a84e0981d
Another directory quickly renamed.
Guido van Rossum <guido@python.org>
parents:
4839
diff
changeset
|
262 D(printf(" MemError: push\n")); |
|
12970
63436719d12e
Add new parser error code, E_OVERFLOW. This error is returned when
Jeremy Hylton <jeremy@alum.mit.edu>
parents:
5379
diff
changeset
|
263 return err; |
| 6 | 264 } |
| 265 D(printf(" Push ...\n")); | |
| 266 continue; | |
| 267 } | |
| 268 | |
| 269 /* Shift the token */ | |
|
12970
63436719d12e
Add new parser error code, E_OVERFLOW. This error is returned when
Jeremy Hylton <jeremy@alum.mit.edu>
parents:
5379
diff
changeset
|
270 if ((err = shift(&ps->p_stack, type, str, |
|
36786
57322cb2ca11
Patch #1440601: Add col_offset attribute to AST nodes.
Martin v. Löwis <martin@v.loewis.de>
parents:
36742
diff
changeset
|
271 x, lineno, col_offset)) > 0) { |
| 6 | 272 D(printf(" MemError: shift.\n")); |
|
12970
63436719d12e
Add new parser error code, E_OVERFLOW. This error is returned when
Jeremy Hylton <jeremy@alum.mit.edu>
parents:
5379
diff
changeset
|
273 return err; |
| 6 | 274 } |
| 275 D(printf(" Shift.\n")); | |
| 276 /* Pop while we are in an accept-only state */ | |
| 277 while (s = &d->d_state | |
| 278 [ps->p_stack.s_top->s_state], | |
| 279 s->s_accept && s->s_narcs == 1) { | |
|
18609
59446d0f8341
Preliminary support for "from __future__ import generators" to enable
Guido van Rossum <guido@python.org>
parents:
14968
diff
changeset
|
280 D(printf(" DFA '%s', state %d: " |
|
59446d0f8341
Preliminary support for "from __future__ import generators" to enable
Guido van Rossum <guido@python.org>
parents:
14968
diff
changeset
|
281 "Direct pop.\n", |
|
59446d0f8341
Preliminary support for "from __future__ import generators" to enable
Guido van Rossum <guido@python.org>
parents:
14968
diff
changeset
|
282 d->d_name, |
|
59446d0f8341
Preliminary support for "from __future__ import generators" to enable
Guido van Rossum <guido@python.org>
parents:
14968
diff
changeset
|
283 ps->p_stack.s_top->s_state)); |
|
36726
bbf16fd024df
from __future__ import with_statement addon for 'with', mostly written by
Thomas Wouters <thomas@python.org>
parents:
22466
diff
changeset
|
284 #ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD |
|
52641
0530aadff696
The BDFL has retired! Long live the FLUFL (Friendly Language Uncle For Life)!
Brett Cannon <bcannon@gmail.com>
parents:
49594
diff
changeset
|
285 #if 0 |
|
18609
59446d0f8341
Preliminary support for "from __future__ import generators" to enable
Guido van Rossum <guido@python.org>
parents:
14968
diff
changeset
|
286 if (d->d_name[0] == 'i' && |
|
59446d0f8341
Preliminary support for "from __future__ import generators" to enable
Guido van Rossum <guido@python.org>
parents:
14968
diff
changeset
|
287 strcmp(d->d_name, |
|
59446d0f8341
Preliminary support for "from __future__ import generators" to enable
Guido van Rossum <guido@python.org>
parents:
14968
diff
changeset
|
288 "import_stmt") == 0) |
|
59446d0f8341
Preliminary support for "from __future__ import generators" to enable
Guido van Rossum <guido@python.org>
parents:
14968
diff
changeset
|
289 future_hack(ps); |
|
22466
1db1510abe21
Disable the parser hacks that allowed the "yield" keyword to be enabled
Neil Schemenauer <nascheme@enme.ucalgary.ca>
parents:
21181
diff
changeset
|
290 #endif |
|
52641
0530aadff696
The BDFL has retired! Long live the FLUFL (Friendly Language Uncle For Life)!
Brett Cannon <bcannon@gmail.com>
parents:
49594
diff
changeset
|
291 #endif |
| 6 | 292 s_pop(&ps->p_stack); |
| 293 if (s_empty(&ps->p_stack)) { | |
| 294 D(printf(" ACCEPT.\n")); | |
| 295 return E_DONE; | |
| 296 } | |
| 297 d = ps->p_stack.s_top->s_dfa; | |
| 298 } | |
| 299 return E_OK; | |
| 300 } | |
| 301 } | |
| 302 | |
| 303 if (s->s_accept) { | |
|
36726
bbf16fd024df
from __future__ import with_statement addon for 'with', mostly written by
Thomas Wouters <thomas@python.org>
parents:
22466
diff
changeset
|
304 #ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD |
|
52641
0530aadff696
The BDFL has retired! Long live the FLUFL (Friendly Language Uncle For Life)!
Brett Cannon <bcannon@gmail.com>
parents:
49594
diff
changeset
|
305 #if 0 |
|
18609
59446d0f8341
Preliminary support for "from __future__ import generators" to enable
Guido van Rossum <guido@python.org>
parents:
14968
diff
changeset
|
306 if (d->d_name[0] == 'i' && |
|
59446d0f8341
Preliminary support for "from __future__ import generators" to enable
Guido van Rossum <guido@python.org>
parents:
14968
diff
changeset
|
307 strcmp(d->d_name, "import_stmt") == 0) |
|
59446d0f8341
Preliminary support for "from __future__ import generators" to enable
Guido van Rossum <guido@python.org>
parents:
14968
diff
changeset
|
308 future_hack(ps); |
|
22466
1db1510abe21
Disable the parser hacks that allowed the "yield" keyword to be enabled
Neil Schemenauer <nascheme@enme.ucalgary.ca>
parents:
21181
diff
changeset
|
309 #endif |
|
52641
0530aadff696
The BDFL has retired! Long live the FLUFL (Friendly Language Uncle For Life)!
Brett Cannon <bcannon@gmail.com>
parents:
49594
diff
changeset
|
310 #endif |
| 6 | 311 /* Pop this dfa and try again */ |
| 312 s_pop(&ps->p_stack); | |
| 313 D(printf(" Pop ...\n")); | |
| 314 if (s_empty(&ps->p_stack)) { | |
| 315 D(printf(" Error: bottom of stack.\n")); | |
| 316 return E_SYNTAX; | |
| 317 } | |
| 318 continue; | |
| 319 } | |
| 320 | |
| 321 /* Stuck, report syntax error */ | |
| 322 D(printf(" Error.\n")); | |
|
13539
feab677269e0
Create two new exceptions: IndentationError and TabError. These are
Fred Drake <fdrake@acm.org>
parents:
13447
diff
changeset
|
323 if (expected_ret) { |
|
feab677269e0
Create two new exceptions: IndentationError and TabError. These are
Fred Drake <fdrake@acm.org>
parents:
13447
diff
changeset
|
324 if (s->s_lower == s->s_upper - 1) { |
|
feab677269e0
Create two new exceptions: IndentationError and TabError. These are
Fred Drake <fdrake@acm.org>
parents:
13447
diff
changeset
|
325 /* Only one possible expected token */ |
|
feab677269e0
Create two new exceptions: IndentationError and TabError. These are
Fred Drake <fdrake@acm.org>
parents:
13447
diff
changeset
|
326 *expected_ret = ps->p_grammar-> |
|
feab677269e0
Create two new exceptions: IndentationError and TabError. These are
Fred Drake <fdrake@acm.org>
parents:
13447
diff
changeset
|
327 g_ll.ll_label[s->s_lower].lb_type; |
|
feab677269e0
Create two new exceptions: IndentationError and TabError. These are
Fred Drake <fdrake@acm.org>
parents:
13447
diff
changeset
|
328 } |
|
feab677269e0
Create two new exceptions: IndentationError and TabError. These are
Fred Drake <fdrake@acm.org>
parents:
13447
diff
changeset
|
329 else |
|
feab677269e0
Create two new exceptions: IndentationError and TabError. These are
Fred Drake <fdrake@acm.org>
parents:
13447
diff
changeset
|
330 *expected_ret = -1; |
|
feab677269e0
Create two new exceptions: IndentationError and TabError. These are
Fred Drake <fdrake@acm.org>
parents:
13447
diff
changeset
|
331 } |
| 6 | 332 return E_SYNTAX; |
| 333 } | |
| 334 } | |
| 335 | |
| 336 | |
|
4839
4567e158d20c
Rename DEBUG macro to Py_DEBUG
Guido van Rossum <guido@python.org>
parents:
4503
diff
changeset
|
337 #ifdef Py_DEBUG |
| 6 | 338 |
| 339 /* DEBUG OUTPUT */ | |
| 340 | |
| 341 void | |
| 13659 | 342 dumptree(grammar *g, node *n) |
| 6 | 343 { |
| 344 int i; | |
| 345 | |
| 346 if (n == NULL) | |
| 347 printf("NIL"); | |
| 348 else { | |
| 349 label l; | |
| 350 l.lb_type = TYPE(n); | |
|
1027
513940eb4d7b
Correct debug printing code
Guido van Rossum <guido@python.org>
parents:
795
diff
changeset
|
351 l.lb_str = STR(n); |
|
5379
1c9a84e0981d
Another directory quickly renamed.
Guido van Rossum <guido@python.org>
parents:
4839
diff
changeset
|
352 printf("%s", PyGrammar_LabelRepr(&l)); |
| 6 | 353 if (ISNONTERMINAL(TYPE(n))) { |
| 354 printf("("); | |
| 355 for (i = 0; i < NCH(n); i++) { | |
| 356 if (i > 0) | |
| 357 printf(","); | |
| 358 dumptree(g, CHILD(n, i)); | |
| 359 } | |
| 360 printf(")"); | |
| 361 } | |
| 362 } | |
| 363 } | |
| 364 | |
| 365 void | |
| 13659 | 366 showtree(grammar *g, node *n) |
| 6 | 367 { |
| 368 int i; | |
| 369 | |
| 370 if (n == NULL) | |
| 371 return; | |
| 372 if (ISNONTERMINAL(TYPE(n))) { | |
| 373 for (i = 0; i < NCH(n); i++) | |
| 374 showtree(g, CHILD(n, i)); | |
| 375 } | |
| 376 else if (ISTERMINAL(TYPE(n))) { | |
|
5379
1c9a84e0981d
Another directory quickly renamed.
Guido van Rossum <guido@python.org>
parents:
4839
diff
changeset
|
377 printf("%s", _PyParser_TokenNames[TYPE(n)]); |
| 6 | 378 if (TYPE(n) == NUMBER || TYPE(n) == NAME) |
| 379 printf("(%s)", STR(n)); | |
| 380 printf(" "); | |
| 381 } | |
| 382 else | |
| 383 printf("? "); | |
| 384 } | |
| 385 | |
| 386 void | |
| 13659 | 387 printtree(parser_state *ps) |
| 6 | 388 { |
|
5379
1c9a84e0981d
Another directory quickly renamed.
Guido van Rossum <guido@python.org>
parents:
4839
diff
changeset
|
389 if (Py_DebugFlag) { |
| 6 | 390 printf("Parse tree:\n"); |
| 391 dumptree(ps->p_grammar, ps->p_tree); | |
| 392 printf("\n"); | |
| 393 printf("Tokens:\n"); | |
| 394 showtree(ps->p_grammar, ps->p_tree); | |
| 395 printf("\n"); | |
| 396 } | |
| 397 printf("Listing:\n"); | |
|
5379
1c9a84e0981d
Another directory quickly renamed.
Guido van Rossum <guido@python.org>
parents:
4839
diff
changeset
|
398 PyNode_ListTree(ps->p_tree); |
| 6 | 399 printf("\n"); |
| 400 } | |
| 401 | |
|
4839
4567e158d20c
Rename DEBUG macro to Py_DEBUG
Guido van Rossum <guido@python.org>
parents:
4503
diff
changeset
|
402 #endif /* Py_DEBUG */ |
| 6 | 403 |
| 404 /* | |
| 405 | |
| 406 Description | |
| 407 ----------- | |
| 408 | |
| 409 The parser's interface is different than usual: the function addtoken() | |
| 410 must be called for each token in the input. This makes it possible to | |
| 411 turn it into an incremental parsing system later. The parsing system | |
| 412 constructs a parse tree as it goes. | |
| 413 | |
| 414 A parsing rule is represented as a Deterministic Finite-state Automaton | |
| 415 (DFA). A node in a DFA represents a state of the parser; an arc represents | |
| 416 a transition. Transitions are either labeled with terminal symbols or | |
| 417 with non-terminals. When the parser decides to follow an arc labeled | |
| 418 with a non-terminal, it is invoked recursively with the DFA representing | |
| 419 the parsing rule for that as its initial state; when that DFA accepts, | |
| 420 the parser that invoked it continues. The parse tree constructed by the | |
| 421 recursively called parser is inserted as a child in the current parse tree. | |
| 422 | |
| 423 The DFA's can be constructed automatically from a more conventional | |
| 424 language description. An extended LL(1) grammar (ELL(1)) is suitable. | |
| 425 Certain restrictions make the parser's life easier: rules that can produce | |
| 426 the empty string should be outlawed (there are other ways to put loops | |
| 427 or optional parts in the language). To avoid the need to construct | |
| 428 FIRST sets, we can require that all but the last alternative of a rule | |
| 429 (really: arc going out of a DFA's state) must begin with a terminal | |
| 430 symbol. | |
| 431 | |
| 432 As an example, consider this grammar: | |
| 433 | |
| 434 expr: term (OP term)* | |
| 435 term: CONSTANT | '(' expr ')' | |
| 436 | |
| 437 The DFA corresponding to the rule for expr is: | |
| 438 | |
| 439 ------->.---term-->.-------> | |
| 440 ^ | | |
| 441 | | | |
| 442 \----OP----/ | |
| 443 | |
| 444 The parse tree generated for the input a+b is: | |
| 445 | |
| 446 (expr: (term: (NAME: a)), (OP: +), (term: (NAME: b))) | |
| 447 | |
| 448 */ |