Mercurial > cpython-lru_cache-weakref

annotate Parser/parser.c @ 80661:94de77bd0b4b 2.7

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Issue #16194: document imp.load_dynamic problems when read several extension modules from the same binary object file.
author Andrew Svetlov <andrew.svetlov@gmail.com>
date Thu, 29 Nov 2012 19:51:28 +0200
parents fd25030223b0
children
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
145
853b736a6a9e Added copyright notice.
Guido van Rossum <guido@python.org>
parents: 86
diff changeset
1
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
2 /* Parser implementation */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
3
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
4 /* For a description, see the comments at end of this file */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
5
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
6 /* XXX To do: error recovery */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
7
21181
944d6e02f359 The parser doesn't need its own implementation of assert, and having its
Tim Peters <tim.peters@gmail.com>
parents: 18739
diff changeset
8 #include "Python.h"
86
a6934380c6e7 "Compiling" version
Guido van Rossum <guido@python.org>
parents: 79
diff changeset
9 #include "pgenheaders.h"
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
10 #include "token.h"
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
11 #include "grammar.h"
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
12 #include "node.h"
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
13 #include "parser.h"
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
14 #include "errcode.h"
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
15
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
16
4839
4567e158d20c Rename DEBUG macro to Py_DEBUG
Guido van Rossum <guido@python.org>
parents: 4503
diff changeset
17 #ifdef Py_DEBUG
5379
1c9a84e0981d Another directory quickly renamed.
Guido van Rossum <guido@python.org>
parents: 4839
diff changeset
18 extern int Py_DebugFlag;
1c9a84e0981d Another directory quickly renamed.
Guido van Rossum <guido@python.org>
parents: 4839
diff changeset
19 #define D(x) if (!Py_DebugFlag); else x
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
20 #else
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
21 #define D(x)
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
22 #endif
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
23
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
24
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
25 /* STACK DATA TYPE */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
26
13447
a89f7f268de4 Nuke all remaining occurrences of Py_PROTO and Py_FPROTO.
Tim Peters <tim.peters@gmail.com>
parents: 13256
diff changeset
27 static void s_reset(stack *);
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
28
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
29 static void
13659
e78153e179fb Mass ANSIfication.
Thomas Wouters <thomas@python.org>
parents: 13539
diff changeset
30 s_reset(stack *s)
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
31 {
61463
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
32 s->s_top = &s->s_base[MAXSTACK];
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
33 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
34
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
35 #define s_empty(s) ((s)->s_top == &(s)->s_base[MAXSTACK])
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
36
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
37 static int
13659
e78153e179fb Mass ANSIfication.
Thomas Wouters <thomas@python.org>
parents: 13539
diff changeset
38 s_push(register stack *s, dfa *d, node *parent)
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
39 {
61463
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
40 register stackentry *top;
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
41 if (s->s_top == s->s_base) {
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
42 fprintf(stderr, "s_push: parser stack overflow\n");
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
43 return E_NOMEM;
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
44 }
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
45 top = --s->s_top;
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
46 top->s_dfa = d;
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
47 top->s_parent = parent;
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
48 top->s_state = 0;
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
49 return 0;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
50 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
51
4839
4567e158d20c Rename DEBUG macro to Py_DEBUG
Guido van Rossum <guido@python.org>
parents: 4503
diff changeset
52 #ifdef Py_DEBUG
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
53
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
54 static void
13659
e78153e179fb Mass ANSIfication.
Thomas Wouters <thomas@python.org>
parents: 13539
diff changeset
55 s_pop(register stack *s)
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
56 {
61463
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
57 if (s_empty(s))
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
58 Py_FatalError("s_pop: parser stack underflow -- FATAL");
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
59 s->s_top++;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
60 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
61
4839
4567e158d20c Rename DEBUG macro to Py_DEBUG
Guido van Rossum <guido@python.org>
parents: 4503
diff changeset
62 #else /* !Py_DEBUG */
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
63
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
64 #define s_pop(s) (s)->s_top++
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
65
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
66 #endif
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
67
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
68
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
69 /* PARSER CREATION */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
70
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
71 parser_state *
13659
e78153e179fb Mass ANSIfication.
Thomas Wouters <thomas@python.org>
parents: 13539
diff changeset
72 PyParser_New(grammar *g, int start)
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
73 {
61463
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
74 parser_state *ps;
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
75
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
76 if (!g->g_accel)
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
77 PyGrammar_AddAccelerators(g);
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
78 ps = (parser_state *)PyMem_MALLOC(sizeof(parser_state));
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
79 if (ps == NULL)
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
80 return NULL;
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
81 ps->p_grammar = g;
36726
bbf16fd024df from __future__ import with_statement addon for 'with', mostly written by
Thomas Wouters <thomas@python.org>
parents: 22466
diff changeset
82 #ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD
61463
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
83 ps->p_flags = 0;
22466
1db1510abe21 Disable the parser hacks that allowed the "yield" keyword to be enabled
Neil Schemenauer <nascheme@enme.ucalgary.ca>
parents: 21181
diff changeset
84 #endif
61463
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
85 ps->p_tree = PyNode_New(start);
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
86 if (ps->p_tree == NULL) {
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
87 PyMem_FREE(ps);
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
88 return NULL;
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
89 }
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
90 s_reset(&ps->p_stack);
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
91 (void) s_push(&ps->p_stack, PyGrammar_FindDFA(g, start), ps->p_tree);
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
92 return ps;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
93 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
94
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
95 void
13659
e78153e179fb Mass ANSIfication.
Thomas Wouters <thomas@python.org>
parents: 13539
diff changeset
96 PyParser_Delete(parser_state *ps)
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
97 {
61463
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
98 /* NB If you want to save the parse tree,
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
99 you must set p_tree to NULL before calling delparser! */
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
100 PyNode_Free(ps->p_tree);
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
101 PyMem_FREE(ps);
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
102 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
103
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
104
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
105 /* PARSER STACK OPERATIONS */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
106
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
107 static int
36786
57322cb2ca11 Patch #1440601: Add col_offset attribute to AST nodes.
Martin v. Löwis <martin@v.loewis.de>
parents: 36742
diff changeset
108 shift(register stack *s, int type, char *str, int newstate, int lineno, int col_offset)
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
109 {
61463
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
110 int err;
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
111 assert(!s_empty(s));
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
112 err = PyNode_AddChild(s->s_top->s_parent, type, str, lineno, col_offset);
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
113 if (err)
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
114 return err;
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
115 s->s_top->s_state = newstate;
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
116 return 0;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
117 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
118
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
119 static int
36786
57322cb2ca11 Patch #1440601: Add col_offset attribute to AST nodes.
Martin v. Löwis <martin@v.loewis.de>
parents: 36742
diff changeset
120 push(register stack *s, int type, dfa *d, int newstate, int lineno, int col_offset)
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
121 {
61463
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
122 int err;
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
123 register node *n;
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
124 n = s->s_top->s_parent;
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
125 assert(!s_empty(s));
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
126 err = PyNode_AddChild(n, type, (char *)NULL, lineno, col_offset);
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
127 if (err)
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
128 return err;
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
129 s->s_top->s_state = newstate;
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
130 return s_push(s, d, CHILD(n, NCH(n)-1));
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
131 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
132
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
133
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
134 /* PARSER PROPER */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
135
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
136 static int
18609
59446d0f8341 Preliminary support for "from __future__ import generators" to enable
Guido van Rossum <guido@python.org>
parents: 14968
diff changeset
137 classify(parser_state *ps, int type, char *str)
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
138 {
61463
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
139 grammar *g = ps->p_grammar;
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
140 register int n = g->g_ll.ll_nlabels;
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
141
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
142 if (type == NAME) {
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
143 register char *s = str;
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
144 register label *l = g->g_ll.ll_label;
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
145 register int i;
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
146 for (i = n; i > 0; i--, l++) {
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
147 if (l->lb_type != NAME || l->lb_str == NULL ||
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
148 l->lb_str[0] != s[0] ||
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
149 strcmp(l->lb_str, s) != 0)
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
150 continue;
36726
bbf16fd024df from __future__ import with_statement addon for 'with', mostly written by
Thomas Wouters <thomas@python.org>
parents: 22466
diff changeset
151 #ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD
61463
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
152 if (ps->p_flags & CO_FUTURE_PRINT_FUNCTION &&
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
153 s[0] == 'p' && strcmp(s, "print") == 0) {
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
154 break; /* no longer a keyword */
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
155 }
22466
1db1510abe21 Disable the parser hacks that allowed the "yield" keyword to be enabled
Neil Schemenauer <nascheme@enme.ucalgary.ca>
parents: 21181
diff changeset
156 #endif
61463
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
157 D(printf("It's a keyword\n"));
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
158 return n - i;
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
159 }
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
160 }
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
161
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
162 {
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
163 register label *l = g->g_ll.ll_label;
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
164 register int i;
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
165 for (i = n; i > 0; i--, l++) {
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
166 if (l->lb_type == type && l->lb_str == NULL) {
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
167 D(printf("It's a token we know\n"));
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
168 return n - i;
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
169 }
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
170 }
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
171 }
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
172
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
173 D(printf("Illegal token\n"));
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
174 return -1;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
175 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
176
36726
bbf16fd024df from __future__ import with_statement addon for 'with', mostly written by
Thomas Wouters <thomas@python.org>
parents: 22466
diff changeset
177 #ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD
18609
59446d0f8341 Preliminary support for "from __future__ import generators" to enable
Guido van Rossum <guido@python.org>
parents: 14968
diff changeset
178 static void
59446d0f8341 Preliminary support for "from __future__ import generators" to enable
Guido van Rossum <guido@python.org>
parents: 14968
diff changeset
179 future_hack(parser_state *ps)
59446d0f8341 Preliminary support for "from __future__ import generators" to enable
Guido van Rossum <guido@python.org>
parents: 14968
diff changeset
180 {
61463
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
181 node *n = ps->p_stack.s_top->s_parent;
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
182 node *ch, *cch;
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
183 int i;
18609
59446d0f8341 Preliminary support for "from __future__ import generators" to enable
Guido van Rossum <guido@python.org>
parents: 14968
diff changeset
184
61463
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
185 /* from __future__ import ..., must have at least 4 children */
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
186 n = CHILD(n, 0);
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
187 if (NCH(n) < 4)
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
188 return;
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
189 ch = CHILD(n, 0);
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
190 if (STR(ch) == NULL || strcmp(STR(ch), "from") != 0)
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
191 return;
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
192 ch = CHILD(n, 1);
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
193 if (NCH(ch) == 1 && STR(CHILD(ch, 0)) &&
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
194 strcmp(STR(CHILD(ch, 0)), "__future__") != 0)
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
195 return;
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
196 ch = CHILD(n, 3);
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
197 /* ch can be a star, a parenthesis or import_as_names */
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
198 if (TYPE(ch) == STAR)
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
199 return;
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
200 if (TYPE(ch) == LPAR)
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
201 ch = CHILD(n, 4);
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
202
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
203 for (i = 0; i < NCH(ch); i += 2) {
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
204 cch = CHILD(ch, i);
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
205 if (NCH(cch) >= 1 && TYPE(CHILD(cch, 0)) == NAME) {
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
206 char *str_ch = STR(CHILD(cch, 0));
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
207 if (strcmp(str_ch, FUTURE_WITH_STATEMENT) == 0) {
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
208 ps->p_flags |= CO_FUTURE_WITH_STATEMENT;
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
209 } else if (strcmp(str_ch, FUTURE_PRINT_FUNCTION) == 0) {
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
210 ps->p_flags |= CO_FUTURE_PRINT_FUNCTION;
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
211 } else if (strcmp(str_ch, FUTURE_UNICODE_LITERALS) == 0) {
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
212 ps->p_flags |= CO_FUTURE_UNICODE_LITERALS;
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
213 }
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
214 }
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
215 }
18609
59446d0f8341 Preliminary support for "from __future__ import generators" to enable
Guido van Rossum <guido@python.org>
parents: 14968
diff changeset
216 }
22466
1db1510abe21 Disable the parser hacks that allowed the "yield" keyword to be enabled
Neil Schemenauer <nascheme@enme.ucalgary.ca>
parents: 21181
diff changeset
217 #endif /* future keyword */
18609
59446d0f8341 Preliminary support for "from __future__ import generators" to enable
Guido van Rossum <guido@python.org>
parents: 14968
diff changeset
218
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
219 int
13659
e78153e179fb Mass ANSIfication.
Thomas Wouters <thomas@python.org>
parents: 13539
diff changeset
220 PyParser_AddToken(register parser_state *ps, register int type, char *str,
61463
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
221 int lineno, int col_offset, int *expected_ret)
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
222 {
61463
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
223 register int ilabel;
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
224 int err;
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
225
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
226 D(printf("Token %s/'%s' ... ", _PyParser_TokenNames[type], str));
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
227
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
228 /* Find out which label this token is */
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
229 ilabel = classify(ps, type, str);
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
230 if (ilabel < 0)
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
231 return E_SYNTAX;
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
232
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
233 /* Loop until the token is shifted or an error occurred */
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
234 for (;;) {
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
235 /* Fetch the current dfa and state */
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
236 register dfa *d = ps->p_stack.s_top->s_dfa;
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
237 register state *s = &d->d_state[ps->p_stack.s_top->s_state];
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
238
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
239 D(printf(" DFA '%s', state %d:",
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
240 d->d_name, ps->p_stack.s_top->s_state));
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
241
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
242 /* Check accelerator */
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
243 if (s->s_lower <= ilabel && ilabel < s->s_upper) {
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
244 register int x = s->s_accel[ilabel - s->s_lower];
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
245 if (x != -1) {
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
246 if (x & (1<<7)) {
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
247 /* Push non-terminal */
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
248 int nt = (x >> 8) + NT_OFFSET;
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
249 int arrow = x & ((1<<7)-1);
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
250 dfa *d1 = PyGrammar_FindDFA(
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
251 ps->p_grammar, nt);
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
252 if ((err = push(&ps->p_stack, nt, d1,
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
253 arrow, lineno, col_offset)) > 0) {
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
254 D(printf(" MemError: push\n"));
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
255 return err;
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
256 }
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
257 D(printf(" Push ...\n"));
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
258 continue;
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
259 }
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
260
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
261 /* Shift the token */
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
262 if ((err = shift(&ps->p_stack, type, str,
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
263 x, lineno, col_offset)) > 0) {
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
264 D(printf(" MemError: shift.\n"));
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
265 return err;
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
266 }
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
267 D(printf(" Shift.\n"));
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
268 /* Pop while we are in an accept-only state */
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
269 while (s = &d->d_state
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
270 [ps->p_stack.s_top->s_state],
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
271 s->s_accept && s->s_narcs == 1) {
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
272 D(printf(" DFA '%s', state %d: "
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
273 "Direct pop.\n",
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
274 d->d_name,
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
275 ps->p_stack.s_top->s_state));
36726
bbf16fd024df from __future__ import with_statement addon for 'with', mostly written by
Thomas Wouters <thomas@python.org>
parents: 22466
diff changeset
276 #ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD
61463
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
277 if (d->d_name[0] == 'i' &&
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
278 strcmp(d->d_name,
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
279 "import_stmt") == 0)
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
280 future_hack(ps);
22466
1db1510abe21 Disable the parser hacks that allowed the "yield" keyword to be enabled
Neil Schemenauer <nascheme@enme.ucalgary.ca>
parents: 21181
diff changeset
281 #endif
61463
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
282 s_pop(&ps->p_stack);
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
283 if (s_empty(&ps->p_stack)) {
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
284 D(printf(" ACCEPT.\n"));
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
285 return E_DONE;
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
286 }
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
287 d = ps->p_stack.s_top->s_dfa;
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
288 }
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
289 return E_OK;
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
290 }
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
291 }
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
292
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
293 if (s->s_accept) {
36726
bbf16fd024df from __future__ import with_statement addon for 'with', mostly written by
Thomas Wouters <thomas@python.org>
parents: 22466
diff changeset
294 #ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD
61463
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
295 if (d->d_name[0] == 'i' &&
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
296 strcmp(d->d_name, "import_stmt") == 0)
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
297 future_hack(ps);
22466
1db1510abe21 Disable the parser hacks that allowed the "yield" keyword to be enabled
Neil Schemenauer <nascheme@enme.ucalgary.ca>
parents: 21181
diff changeset
298 #endif
61463
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
299 /* Pop this dfa and try again */
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
300 s_pop(&ps->p_stack);
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
301 D(printf(" Pop ...\n"));
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
302 if (s_empty(&ps->p_stack)) {
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
303 D(printf(" Error: bottom of stack.\n"));
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
304 return E_SYNTAX;
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
305 }
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
306 continue;
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
307 }
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
308
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
309 /* Stuck, report syntax error */
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
310 D(printf(" Error.\n"));
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
311 if (expected_ret) {
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
312 if (s->s_lower == s->s_upper - 1) {
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
313 /* Only one possible expected token */
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
314 *expected_ret = ps->p_grammar->
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
315 g_ll.ll_label[s->s_lower].lb_type;
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
316 }
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
317 else
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
318 *expected_ret = -1;
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
319 }
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
320 return E_SYNTAX;
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
321 }
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
322 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
323
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
324
4839
4567e158d20c Rename DEBUG macro to Py_DEBUG
Guido van Rossum <guido@python.org>
parents: 4503
diff changeset
325 #ifdef Py_DEBUG
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
326
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
327 /* DEBUG OUTPUT */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
328
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
329 void
13659
e78153e179fb Mass ANSIfication.
Thomas Wouters <thomas@python.org>
parents: 13539
diff changeset
330 dumptree(grammar *g, node *n)
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
331 {
61463
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
332 int i;
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
333
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
334 if (n == NULL)
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
335 printf("NIL");
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
336 else {
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
337 label l;
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
338 l.lb_type = TYPE(n);
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
339 l.lb_str = STR(n);
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
340 printf("%s", PyGrammar_LabelRepr(&l));
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
341 if (ISNONTERMINAL(TYPE(n))) {
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
342 printf("(");
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
343 for (i = 0; i < NCH(n); i++) {
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
344 if (i > 0)
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
345 printf(",");
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
346 dumptree(g, CHILD(n, i));
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
347 }
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
348 printf(")");
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
349 }
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
350 }
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
351 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
352
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
353 void
13659
e78153e179fb Mass ANSIfication.
Thomas Wouters <thomas@python.org>
parents: 13539
diff changeset
354 showtree(grammar *g, node *n)
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
355 {
61463
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
356 int i;
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
357
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
358 if (n == NULL)
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
359 return;
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
360 if (ISNONTERMINAL(TYPE(n))) {
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
361 for (i = 0; i < NCH(n); i++)
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
362 showtree(g, CHILD(n, i));
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
363 }
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
364 else if (ISTERMINAL(TYPE(n))) {
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
365 printf("%s", _PyParser_TokenNames[TYPE(n)]);
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
366 if (TYPE(n) == NUMBER || TYPE(n) == NAME)
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
367 printf("(%s)", STR(n));
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
368 printf(" ");
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
369 }
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
370 else
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
371 printf("? ");
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
372 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
373
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
374 void
13659
e78153e179fb Mass ANSIfication.
Thomas Wouters <thomas@python.org>
parents: 13539
diff changeset
375 printtree(parser_state *ps)
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
376 {
61463
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
377 if (Py_DebugFlag) {
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
378 printf("Parse tree:\n");
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
379 dumptree(ps->p_grammar, ps->p_tree);
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
380 printf("\n");
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
381 printf("Tokens:\n");
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
382 showtree(ps->p_grammar, ps->p_tree);
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
383 printf("\n");
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
384 }
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
385 printf("Listing:\n");
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
386 PyNode_ListTree(ps->p_tree);
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
387 printf("\n");
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
388 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
389
4839
4567e158d20c Rename DEBUG macro to Py_DEBUG
Guido van Rossum <guido@python.org>
parents: 4503
diff changeset
390 #endif /* Py_DEBUG */
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
391
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
392 /*
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
393
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
394 Description
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
395 -----------
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
396
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
397 The parser's interface is different than usual: the function addtoken()
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
398 must be called for each token in the input. This makes it possible to
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
399 turn it into an incremental parsing system later. The parsing system
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
400 constructs a parse tree as it goes.
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
401
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
402 A parsing rule is represented as a Deterministic Finite-state Automaton
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
403 (DFA). A node in a DFA represents a state of the parser; an arc represents
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
404 a transition. Transitions are either labeled with terminal symbols or
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
405 with non-terminals. When the parser decides to follow an arc labeled
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
406 with a non-terminal, it is invoked recursively with the DFA representing
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
407 the parsing rule for that as its initial state; when that DFA accepts,
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
408 the parser that invoked it continues. The parse tree constructed by the
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
409 recursively called parser is inserted as a child in the current parse tree.
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
410
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
411 The DFA's can be constructed automatically from a more conventional
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
412 language description. An extended LL(1) grammar (ELL(1)) is suitable.
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
413 Certain restrictions make the parser's life easier: rules that can produce
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
414 the empty string should be outlawed (there are other ways to put loops
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
415 or optional parts in the language). To avoid the need to construct
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
416 FIRST sets, we can require that all but the last alternative of a rule
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
417 (really: arc going out of a DFA's state) must begin with a terminal
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
418 symbol.
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
419
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
420 As an example, consider this grammar:
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
421
61463
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
422 expr: term (OP term)*
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
423 term: CONSTANT | '(' expr ')'
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
424
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
425 The DFA corresponding to the rule for expr is:
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
426
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
427 ------->.---term-->.------->
61463
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
428 ^ |
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
429 | |
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 49595
diff changeset
430 \----OP----/
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
431
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
432 The parse tree generated for the input a+b is:
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
433
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
434 (expr: (term: (NAME: a)), (OP: +), (term: (NAME: b)))
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
435
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
436 */