Mercurial > cpython-lru_cache-weakref

annotate Parser/tokenizer.c @ 27998:e27f58a50737 legacy-trunk

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
SF bug #690377: Dud footnote reference in distutils docs
author Raymond Hettinger <python@rcn.com>
date Fri, 21 Feb 2003 02:47:09 +0000
parents be371ffc4cd0
children 86fdf8fe7f17
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
145
853b736a6a9e Added copyright notice.
Guido van Rossum <guido@python.org>
parents: 96
diff changeset
1
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
2 /* Tokenizer implementation */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
3
23029
8291473fde3e Mass checkin of universal newline support.
Jack Jansen <jack.jansen@cwi.nl>
parents: 19621
diff changeset
4 #include "Python.h"
86
a6934380c6e7 "Compiling" version
Guido van Rossum <guido@python.org>
parents: 67
diff changeset
5 #include "pgenheaders.h"
a6934380c6e7 "Compiling" version
Guido van Rossum <guido@python.org>
parents: 67
diff changeset
6
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
7 #include <ctype.h>
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
8 #include <assert.h>
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
9
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
10 #include "tokenizer.h"
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
11 #include "errcode.h"
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
12
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
13 #ifndef PGEN
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
14 #include "unicodeobject.h"
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
15 #include "stringobject.h"
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
16 #include "fileobject.h"
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
17 #include "codecs.h"
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
18 #include "abstract.h"
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
19 #endif /* PGEN */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
20
25924
dc4a0336a2a3 Patch #512981: Update readline input stream on sys.stdin/out change.
Martin v. Löwis <martin@v.loewis.de>
parents: 25101
diff changeset
21 extern char *PyOS_Readline(FILE *, FILE *, char *);
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
22 /* Return malloc'ed string including trailing \n;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
23 empty malloc'ed string for EOF;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
24 NULL if interrupted */
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
25
678
205b947b1d3c Make tabs always 8 spaces wide -- it's more portable.
Guido van Rossum <guido@python.org>
parents: 477
diff changeset
26 /* Don't ever change this -- it would break the portability of Python code */
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
27 #define TABSIZE 8
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
28
7118
e9108e97a5a0 tok_nextc() should return unsigned characters, to avoid mistaking
Guido van Rossum <guido@python.org>
parents: 5379
diff changeset
29 /* Convert a possibly signed character to a nonnegative int */
e9108e97a5a0 tok_nextc() should return unsigned characters, to avoid mistaking
Guido van Rossum <guido@python.org>
parents: 5379
diff changeset
30 /* XXX This assumes characters are 8 bits wide */
e9108e97a5a0 tok_nextc() should return unsigned characters, to avoid mistaking
Guido van Rossum <guido@python.org>
parents: 5379
diff changeset
31 #ifdef __CHAR_UNSIGNED__
e9108e97a5a0 tok_nextc() should return unsigned characters, to avoid mistaking
Guido van Rossum <guido@python.org>
parents: 5379
diff changeset
32 #define Py_CHARMASK(c) (c)
e9108e97a5a0 tok_nextc() should return unsigned characters, to avoid mistaking
Guido van Rossum <guido@python.org>
parents: 5379
diff changeset
33 #else
e9108e97a5a0 tok_nextc() should return unsigned characters, to avoid mistaking
Guido van Rossum <guido@python.org>
parents: 5379
diff changeset
34 #define Py_CHARMASK(c) ((c) & 0xff)
e9108e97a5a0 tok_nextc() should return unsigned characters, to avoid mistaking
Guido van Rossum <guido@python.org>
parents: 5379
diff changeset
35 #endif
e9108e97a5a0 tok_nextc() should return unsigned characters, to avoid mistaking
Guido van Rossum <guido@python.org>
parents: 5379
diff changeset
36
86
a6934380c6e7 "Compiling" version
Guido van Rossum <guido@python.org>
parents: 67
diff changeset
37 /* Forward */
13447
a89f7f268de4 Nuke all remaining occurrences of Py_PROTO and Py_FPROTO.
Tim Peters <tim.peters@gmail.com>
parents: 13256
diff changeset
38 static struct tok_state *tok_new(void);
a89f7f268de4 Nuke all remaining occurrences of Py_PROTO and Py_FPROTO.
Tim Peters <tim.peters@gmail.com>
parents: 13256
diff changeset
39 static int tok_nextc(struct tok_state *tok);
a89f7f268de4 Nuke all remaining occurrences of Py_PROTO and Py_FPROTO.
Tim Peters <tim.peters@gmail.com>
parents: 13256
diff changeset
40 static void tok_backup(struct tok_state *tok, int c);
86
a6934380c6e7 "Compiling" version
Guido van Rossum <guido@python.org>
parents: 67
diff changeset
41
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
42 /* Token names */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
43
5379
1c9a84e0981d Another directory quickly renamed.
Guido van Rossum <guido@python.org>
parents: 5341
diff changeset
44 char *_PyParser_TokenNames[] = {
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
45 "ENDMARKER",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
46 "NAME",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
47 "NUMBER",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
48 "STRING",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
49 "NEWLINE",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
50 "INDENT",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
51 "DEDENT",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
52 "LPAR",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
53 "RPAR",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
54 "LSQB",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
55 "RSQB",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
56 "COLON",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
57 "COMMA",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
58 "SEMI",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
59 "PLUS",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
60 "MINUS",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
61 "STAR",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
62 "SLASH",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
63 "VBAR",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
64 "AMPER",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
65 "LESS",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
66 "GREATER",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
67 "EQUAL",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
68 "DOT",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
69 "PERCENT",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
70 "BACKQUOTE",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
71 "LBRACE",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
72 "RBRACE",
400
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
73 "EQEQUAL",
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
74 "NOTEQUAL",
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
75 "LESSEQUAL",
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
76 "GREATEREQUAL",
410
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
77 "TILDE",
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
78 "CIRCUMFLEX",
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
79 "LEFTSHIFT",
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
80 "RIGHTSHIFT",
3333
4b5bff95804e changes for pow(**) and complex
Guido van Rossum <guido@python.org>
parents: 3116
diff changeset
81 "DOUBLESTAR",
14163
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
82 "PLUSEQUAL",
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
83 "MINEQUAL",
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
84 "STAREQUAL",
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
85 "SLASHEQUAL",
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
86 "PERCENTEQUAL",
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
87 "AMPEREQUAL",
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
88 "VBAREQUAL",
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
89 "CIRCUMFLEXEQUAL",
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
90 "LEFTSHIFTEQUAL",
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
91 "RIGHTSHIFTEQUAL",
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
92 "DOUBLESTAREQUAL",
19051
f20727cc3d0f Implement PEP 238 in its (almost) full glory.
Guido van Rossum <guido@python.org>
parents: 17600
diff changeset
93 "DOUBLESLASH",
f20727cc3d0f Implement PEP 238 in its (almost) full glory.
Guido van Rossum <guido@python.org>
parents: 17600
diff changeset
94 "DOUBLESLASHEQUAL",
400
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
95 /* This table must match the #defines in token.h! */
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
96 "OP",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
97 "<ERRORTOKEN>",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
98 "<N_TOKENS>"
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
99 };
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
100
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
101
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
102 /* Create and initialize a new tok_state structure */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
103
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
104 static struct tok_state *
13659
e78153e179fb Mass ANSIfication.
Thomas Wouters <thomas@python.org>
parents: 13539
diff changeset
105 tok_new(void)
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
106 {
5379
1c9a84e0981d Another directory quickly renamed.
Guido van Rossum <guido@python.org>
parents: 5341
diff changeset
107 struct tok_state *tok = PyMem_NEW(struct tok_state, 1);
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
108 if (tok == NULL)
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
109 return NULL;
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
110 tok->buf = tok->cur = tok->end = tok->inp = tok->start = NULL;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
111 tok->done = E_OK;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
112 tok->fp = NULL;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
113 tok->tabsize = TABSIZE;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
114 tok->indent = 0;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
115 tok->indstack[0] = 0;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
116 tok->atbol = 1;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
117 tok->pendin = 0;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
118 tok->prompt = tok->nextprompt = NULL;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
119 tok->lineno = 0;
1284
dbce72afe261 * selectmodule.c: fix (another!) two memory leaks -- this time in list2set
Guido van Rossum <guido@python.org>
parents: 1282
diff changeset
120 tok->level = 0;
7945
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
121 tok->filename = NULL;
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
122 tok->altwarning = 0;
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
123 tok->alterror = 0;
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
124 tok->alttabsize = 1;
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
125 tok->altindstack[0] = 0;
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
126 tok->decoding_state = 0;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
127 tok->decoding_erred = 0;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
128 tok->read_coding_spec = 0;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
129 tok->issued_encoding_warning = 0;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
130 tok->encoding = NULL;
25098
c02a668488d0 Ignore encoding declarations inside strings. Fixes #603509.
Martin v. Löwis <martin@v.loewis.de>
parents: 24867
diff changeset
131 tok->cont_line = 0;
24544
0bd521eb714e Make pgen compile with pydebug. Duplicate normalized names, as it may
Martin v. Löwis <martin@v.loewis.de>
parents: 24543
diff changeset
132 #ifndef PGEN
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
133 tok->decoding_readline = NULL;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
134 tok->decoding_buffer = NULL;
24544
0bd521eb714e Make pgen compile with pydebug. Duplicate normalized names, as it may
Martin v. Löwis <martin@v.loewis.de>
parents: 24543
diff changeset
135 #endif
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
136 return tok;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
137 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
138
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
139 #ifdef PGEN
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
140
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
141 static char *
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
142 decoding_fgets(char *s, int size, struct tok_state *tok)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
143 {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
144 return fgets(s, size, tok->fp);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
145 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
146
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
147 static int
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
148 decoding_feof(struct tok_state *tok)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
149 {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
150 return feof(tok->fp);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
151 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
152
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
153 static const char *
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
154 decode_str(const char *str, struct tok_state *tok)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
155 {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
156 return str;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
157 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
158
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
159 #else /* PGEN */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
160
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
161 static char *
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
162 error_ret(struct tok_state *tok) /* XXX */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
163 {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
164 tok->decoding_erred = 1;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
165 if (tok->fp != NULL && tok->buf != NULL) /* see PyTokenizer_Free */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
166 PyMem_DEL(tok->buf);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
167 tok->buf = NULL;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
168 return NULL; /* as if it were EOF */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
169 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
170
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
171 static char *
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
172 new_string(const char *s, int len)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
173 {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
174 char* result = PyMem_NEW(char, len + 1);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
175 if (result != NULL) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
176 memcpy(result, s, len);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
177 result[len] = '\0';
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
178 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
179 return result;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
180 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
181
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
182 static char *
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
183 get_normal_name(char *s) /* for utf-8 and latin-1 */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
184 {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
185 char buf[13];
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
186 int i;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
187 for (i = 0; i < 12; i++) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
188 int c = s[i];
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
189 if (c == '\0') break;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
190 else if (c == '_') buf[i] = '-';
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
191 else buf[i] = tolower(c);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
192 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
193 buf[i] = '\0';
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
194 if (strcmp(buf, "utf-8") == 0 ||
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
195 strncmp(buf, "utf-8-", 6) == 0) return "utf-8";
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
196 else if (strcmp(buf, "latin-1") == 0 ||
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
197 strcmp(buf, "iso-8859-1") == 0 ||
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
198 strcmp(buf, "iso-latin-1") == 0 ||
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
199 strncmp(buf, "latin-1-", 8) == 0 ||
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
200 strncmp(buf, "iso-8859-1-", 11) == 0 ||
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
201 strncmp(buf, "iso-latin-1-", 12) == 0) return "iso-8859-1";
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
202 else return s;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
203 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
204
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
205 /* Return the coding spec in S, or NULL if none is found. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
206
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
207 static char *
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
208 get_coding_spec(const char *s, int size)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
209 {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
210 int i;
25098
c02a668488d0 Ignore encoding declarations inside strings. Fixes #603509.
Martin v. Löwis <martin@v.loewis.de>
parents: 24867
diff changeset
211 /* Coding spec must be in a comment, and that comment must be
c02a668488d0 Ignore encoding declarations inside strings. Fixes #603509.
Martin v. Löwis <martin@v.loewis.de>
parents: 24867
diff changeset
212 * the only statement on the source code line. */
c02a668488d0 Ignore encoding declarations inside strings. Fixes #603509.
Martin v. Löwis <martin@v.loewis.de>
parents: 24867
diff changeset
213 for (i = 0; i < size - 6; i++) {
c02a668488d0 Ignore encoding declarations inside strings. Fixes #603509.
Martin v. Löwis <martin@v.loewis.de>
parents: 24867
diff changeset
214 if (s[i] == '#')
c02a668488d0 Ignore encoding declarations inside strings. Fixes #603509.
Martin v. Löwis <martin@v.loewis.de>
parents: 24867
diff changeset
215 break;
c02a668488d0 Ignore encoding declarations inside strings. Fixes #603509.
Martin v. Löwis <martin@v.loewis.de>
parents: 24867
diff changeset
216 if (s[i] != ' ' && s[i] != '\t' && s[i] != '\014')
c02a668488d0 Ignore encoding declarations inside strings. Fixes #603509.
Martin v. Löwis <martin@v.loewis.de>
parents: 24867
diff changeset
217 return NULL;
c02a668488d0 Ignore encoding declarations inside strings. Fixes #603509.
Martin v. Löwis <martin@v.loewis.de>
parents: 24867
diff changeset
218 }
c02a668488d0 Ignore encoding declarations inside strings. Fixes #603509.
Martin v. Löwis <martin@v.loewis.de>
parents: 24867
diff changeset
219 for (; i < size - 6; i++) { /* XXX inefficient search */
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
220 const char* t = s + i;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
221 if (strncmp(t, "coding", 6) == 0) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
222 const char* begin = NULL;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
223 t += 6;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
224 if (t[0] != ':' && t[0] != '=')
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
225 continue;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
226 do {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
227 t++;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
228 } while (t[0] == '\x20' || t[0] == '\t');
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
229
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
230 begin = t;
25995
1aaa12c08b35 Fix compiler warning on HP-UX.
Neal Norwitz <nnorwitz@gmail.com>
parents: 25924
diff changeset
231 while (isalnum((int)t[0]) ||
1aaa12c08b35 Fix compiler warning on HP-UX.
Neal Norwitz <nnorwitz@gmail.com>
parents: 25924
diff changeset
232 t[0] == '-' || t[0] == '_' || t[0] == '.')
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
233 t++;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
234
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
235 if (begin < t) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
236 char* r = new_string(begin, t - begin);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
237 char* q = get_normal_name(r);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
238 if (r != q) {
24544
0bd521eb714e Make pgen compile with pydebug. Duplicate normalized names, as it may
Martin v. Löwis <martin@v.loewis.de>
parents: 24543
diff changeset
239 PyMem_DEL(r);
0bd521eb714e Make pgen compile with pydebug. Duplicate normalized names, as it may
Martin v. Löwis <martin@v.loewis.de>
parents: 24543
diff changeset
240 r = new_string(q, strlen(q));
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
241 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
242 return r;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
243 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
244 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
245 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
246 return NULL;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
247 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
248
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
249 /* Check whether the line contains a coding spec. If it does,
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
250 invoke the set_readline function for the new encoding.
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
251 This function receives the tok_state and the new encoding.
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
252 Return 1 on success, 0 on failure. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
253
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
254 static int
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
255 check_coding_spec(const char* line, int size, struct tok_state *tok,
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
256 int set_readline(struct tok_state *, const char *))
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
257 {
25101
32331d26d7cf Removed reliance on gcc/C99 extension.
Tim Peters <tim.peters@gmail.com>
parents: 25098
diff changeset
258 char * cs;
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
259 int r = 1;
25101
32331d26d7cf Removed reliance on gcc/C99 extension.
Tim Peters <tim.peters@gmail.com>
parents: 25098
diff changeset
260
25098
c02a668488d0 Ignore encoding declarations inside strings. Fixes #603509.
Martin v. Löwis <martin@v.loewis.de>
parents: 24867
diff changeset
261 if (tok->cont_line)
c02a668488d0 Ignore encoding declarations inside strings. Fixes #603509.
Martin v. Löwis <martin@v.loewis.de>
parents: 24867
diff changeset
262 /* It's a continuation line, so it can't be a coding spec. */
c02a668488d0 Ignore encoding declarations inside strings. Fixes #603509.
Martin v. Löwis <martin@v.loewis.de>
parents: 24867
diff changeset
263 return 1;
25101
32331d26d7cf Removed reliance on gcc/C99 extension.
Tim Peters <tim.peters@gmail.com>
parents: 25098
diff changeset
264 cs = get_coding_spec(line, size);
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
265 if (cs != NULL) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
266 tok->read_coding_spec = 1;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
267 if (tok->encoding == NULL) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
268 assert(tok->decoding_state == 1); /* raw */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
269 if (strcmp(cs, "utf-8") == 0 ||
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
270 strcmp(cs, "iso-8859-1") == 0) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
271 tok->encoding = cs;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
272 } else {
24624
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
273 #ifdef Py_USING_UNICODE
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
274 r = set_readline(tok, cs);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
275 if (r) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
276 tok->encoding = cs;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
277 tok->decoding_state = -1;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
278 }
24624
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
279 #else
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
280 /* Without Unicode support, we cannot
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
281 process the coding spec. Since there
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
282 won't be any Unicode literals, that
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
283 won't matter. */
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
284 #endif
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
285 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
286 } else { /* then, compare cs with BOM */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
287 r = (strcmp(tok->encoding, cs) == 0);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
288 PyMem_DEL(cs);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
289 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
290 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
291 return r;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
292 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
293
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
294 /* See whether the file starts with a BOM. If it does,
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
295 invoke the set_readline function with the new encoding.
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
296 Return 1 on success, 0 on failure. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
297
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
298 static int
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
299 check_bom(int get_char(struct tok_state *),
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
300 void unget_char(int, struct tok_state *),
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
301 int set_readline(struct tok_state *, const char *),
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
302 struct tok_state *tok)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
303 {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
304 int ch = get_char(tok);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
305 tok->decoding_state = 1;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
306 if (ch == EOF) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
307 return 1;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
308 } else if (ch == 0xEF) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
309 ch = get_char(tok); if (ch != 0xBB) goto NON_BOM;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
310 ch = get_char(tok); if (ch != 0xBF) goto NON_BOM;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
311 #if 0
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
312 /* Disable support for UTF-16 BOMs until a decision
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
313 is made whether this needs to be supported. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
314 } else if (ch == 0xFE) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
315 ch = get_char(tok); if (ch != 0xFF) goto NON_BOM;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
316 if (!set_readline(tok, "utf-16-be")) return 0;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
317 tok->decoding_state = -1;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
318 } else if (ch == 0xFF) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
319 ch = get_char(tok); if (ch != 0xFE) goto NON_BOM;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
320 if (!set_readline(tok, "utf-16-le")) return 0;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
321 tok->decoding_state = -1;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
322 #endif
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
323 } else {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
324 unget_char(ch, tok);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
325 return 1;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
326 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
327 tok->encoding = new_string("utf-8", 5); /* resulting is in utf-8 */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
328 return 1;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
329 NON_BOM:
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
330 /* any token beginning with '\xEF', '\xFE', '\xFF' is a bad token */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
331 unget_char(0xFF, tok); /* XXX this will cause a syntax error */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
332 return 1;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
333 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
334
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
335 /* Read a line of text from TOK into S, using the stream in TOK.
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
336 Return NULL on failure, else S. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
337
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
338 static char *
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
339 fp_readl(char *s, int size, struct tok_state *tok)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
340 {
24624
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
341 #ifndef Py_USING_UNICODE
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
342 /* In a non-Unicode built, this should never be called. */
24630
43caa212b959 Use Py_FatalError instead of abort.
Martin v. Löwis <martin@v.loewis.de>
parents: 24624
diff changeset
343 Py_FatalError("fp_readl should not be called in this build.");
24867
255d1d3e66a3 Squash a few calls to the hideously expensive PyObject_CallObject(o,a)
Guido van Rossum <guido@python.org>
parents: 24839
diff changeset
344 return NULL; /* Keep compiler happy (not reachable) */
24624
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
345 #else
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
346 PyObject* utf8;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
347 PyObject* buf = tok->decoding_buffer;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
348 if (buf == NULL) {
27247
5ddb90f8169a Fix [ 665014 ] files with long lines and an encoding crash.
Mark Hammond <mhammond@skippinet.com.au>
parents: 26601
diff changeset
349 /* Ask for one less byte so we can terminate it */
5ddb90f8169a Fix [ 665014 ] files with long lines and an encoding crash.
Mark Hammond <mhammond@skippinet.com.au>
parents: 26601
diff changeset
350 PyObject *args = Py_BuildValue("(i)", size-1);
24867
255d1d3e66a3 Squash a few calls to the hideously expensive PyObject_CallObject(o,a)
Guido van Rossum <guido@python.org>
parents: 24839
diff changeset
351 if (args == NULL)
255d1d3e66a3 Squash a few calls to the hideously expensive PyObject_CallObject(o,a)
Guido van Rossum <guido@python.org>
parents: 24839
diff changeset
352 return error_ret(tok);
255d1d3e66a3 Squash a few calls to the hideously expensive PyObject_CallObject(o,a)
Guido van Rossum <guido@python.org>
parents: 24839
diff changeset
353 buf = PyObject_Call(tok->decoding_readline, args, NULL);
255d1d3e66a3 Squash a few calls to the hideously expensive PyObject_CallObject(o,a)
Guido van Rossum <guido@python.org>
parents: 24839
diff changeset
354 Py_DECREF(args);
24543
c00df23e856e Group statements properly.
Martin v. Löwis <martin@v.loewis.de>
parents: 24542
diff changeset
355 if (buf == NULL)
c00df23e856e Group statements properly.
Martin v. Löwis <martin@v.loewis.de>
parents: 24542
diff changeset
356 return error_ret(tok);
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
357 } else {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
358 tok->decoding_buffer = NULL;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
359 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
360 utf8 = PyUnicode_AsUTF8String(buf);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
361 Py_DECREF(buf);
24543
c00df23e856e Group statements properly.
Martin v. Löwis <martin@v.loewis.de>
parents: 24542
diff changeset
362 if (utf8 == NULL)
c00df23e856e Group statements properly.
Martin v. Löwis <martin@v.loewis.de>
parents: 24542
diff changeset
363 return error_ret(tok);
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
364 else {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
365 const char* str = PyString_AsString(utf8);
24541
ff9efa692594 Squash compiler wng about signed-vs-unsigned mismatch.
Tim Peters <tim.peters@gmail.com>
parents: 24539
diff changeset
366 assert(strlen(str) < (size_t)size); /* XXX */
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
367 strcpy(s, str);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
368 Py_DECREF(utf8);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
369 if (s[0] == '\0') return NULL; /* EOF */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
370 return s;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
371 }
24624
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
372 #endif
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
373 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
374
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
375 /* Set the readline function for TOK to a StreamReader's
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
376 readline function. The StreamReader is named ENC.
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
377
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
378 This function is called from check_bom and check_coding_spec.
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
379
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
380 ENC is usually identical to the future value of tok->encoding,
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
381 except for the (currently unsupported) case of UTF-16.
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
382
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
383 Return 1 on success, 0 on failure. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
384
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
385 static int
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
386 fp_setreadl(struct tok_state *tok, const char* enc)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
387 {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
388 PyObject *reader, *stream, *readline;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
389
26601
cc05c2c8660e Constify filenames and scripts. Fixes #651362.
Martin v. Löwis <martin@v.loewis.de>
parents: 25995
diff changeset
390 /* XXX: constify filename argument. */
cc05c2c8660e Constify filenames and scripts. Fixes #651362.
Martin v. Löwis <martin@v.loewis.de>
parents: 25995
diff changeset
391 stream = PyFile_FromFile(tok->fp, (char*)tok->filename, "rb", NULL);
24543
c00df23e856e Group statements properly.
Martin v. Löwis <martin@v.loewis.de>
parents: 24542
diff changeset
392 if (stream == NULL)
c00df23e856e Group statements properly.
Martin v. Löwis <martin@v.loewis.de>
parents: 24542
diff changeset
393 return 0;
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
394
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
395 reader = PyCodec_StreamReader(enc, stream, NULL);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
396 Py_DECREF(stream);
24543
c00df23e856e Group statements properly.
Martin v. Löwis <martin@v.loewis.de>
parents: 24542
diff changeset
397 if (reader == NULL)
c00df23e856e Group statements properly.
Martin v. Löwis <martin@v.loewis.de>
parents: 24542
diff changeset
398 return 0;
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
399
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
400 readline = PyObject_GetAttrString(reader, "readline");
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
401 Py_DECREF(reader);
24543
c00df23e856e Group statements properly.
Martin v. Löwis <martin@v.loewis.de>
parents: 24542
diff changeset
402 if (readline == NULL)
c00df23e856e Group statements properly.
Martin v. Löwis <martin@v.loewis.de>
parents: 24542
diff changeset
403 return 0;
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
404
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
405 tok->decoding_readline = readline;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
406 return 1;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
407 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
408
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
409 /* Fetch the next byte from TOK. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
410
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
411 static int fp_getc(struct tok_state *tok) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
412 return getc(tok->fp);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
413 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
414
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
415 /* Unfetch the last byte back into TOK. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
416
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
417 static void fp_ungetc(int c, struct tok_state *tok) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
418 ungetc(c, tok->fp);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
419 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
420
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
421 /* Read a line of input from TOK. Determine encoding
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
422 if necessary. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
423
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
424 static char *
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
425 decoding_fgets(char *s, int size, struct tok_state *tok)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
426 {
24630
43caa212b959 Use Py_FatalError instead of abort.
Martin v. Löwis <martin@v.loewis.de>
parents: 24624
diff changeset
427 char *line = NULL;
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
428 int warn = 0, badchar = 0;
24543
c00df23e856e Group statements properly.
Martin v. Löwis <martin@v.loewis.de>
parents: 24542
diff changeset
429 for (;;) {
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
430 if (tok->decoding_state < 0) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
431 /* We already have a codec associated with
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
432 this input. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
433 line = fp_readl(s, size, tok);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
434 break;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
435 } else if (tok->decoding_state > 0) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
436 /* We want a 'raw' read. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
437 line = Py_UniversalNewlineFgets(s, size,
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
438 tok->fp, NULL);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
439 warn = 1;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
440 break;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
441 } else {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
442 /* We have not yet determined the encoding.
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
443 If an encoding is found, use the file-pointer
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
444 reader functions from now on. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
445 if (!check_bom(fp_getc, fp_ungetc, fp_setreadl, tok))
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
446 return error_ret(tok);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
447 assert(tok->decoding_state != 0);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
448 }
24543
c00df23e856e Group statements properly.
Martin v. Löwis <martin@v.loewis.de>
parents: 24542
diff changeset
449 }
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
450 if (line != NULL && tok->lineno < 2 && !tok->read_coding_spec) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
451 if (!check_coding_spec(line, strlen(line), tok, fp_setreadl)) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
452 return error_ret(tok);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
453 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
454 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
455 #ifndef PGEN
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
456 if (warn && line && !tok->issued_encoding_warning && !tok->encoding) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
457 unsigned char *c;
24565
c8d694b641f1 Added a cast to shut up a compiler warning.
Jack Jansen <jack.jansen@cwi.nl>
parents: 24558
diff changeset
458 for (c = (unsigned char *)line; *c; c++)
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
459 if (*c > 127) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
460 badchar = *c;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
461 break;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
462 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
463 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
464 if (badchar) {
27918
be371ffc4cd0 Add URL for PEP to the source code encoding warning.
Marc-André Lemburg <mal@egenix.com>
parents: 27772
diff changeset
465 char buf[500];
24558
d26b62f0c6db Add 1 to lineno in deprecation warning. Fixes #590888.
Martin v. Löwis <martin@v.loewis.de>
parents: 24544
diff changeset
466 /* Need to add 1 to the line number, since this line
d26b62f0c6db Add 1 to lineno in deprecation warning. Fixes #590888.
Martin v. Löwis <martin@v.loewis.de>
parents: 24544
diff changeset
467 has not been counted, yet. */
27918
be371ffc4cd0 Add URL for PEP to the source code encoding warning.
Marc-André Lemburg <mal@egenix.com>
parents: 27772
diff changeset
468 sprintf(buf,
be371ffc4cd0 Add URL for PEP to the source code encoding warning.
Marc-André Lemburg <mal@egenix.com>
parents: 27772
diff changeset
469 "Non-ASCII character '\\x%.2x' "
be371ffc4cd0 Add URL for PEP to the source code encoding warning.
Marc-André Lemburg <mal@egenix.com>
parents: 27772
diff changeset
470 "in file %.200s on line %i, "
be371ffc4cd0 Add URL for PEP to the source code encoding warning.
Marc-André Lemburg <mal@egenix.com>
parents: 27772
diff changeset
471 "but no encoding declared; "
be371ffc4cd0 Add URL for PEP to the source code encoding warning.
Marc-André Lemburg <mal@egenix.com>
parents: 27772
diff changeset
472 "see http://www.python.org/peps/pep-0263.html for details",
be371ffc4cd0 Add URL for PEP to the source code encoding warning.
Marc-André Lemburg <mal@egenix.com>
parents: 27772
diff changeset
473 badchar, tok->filename, tok->lineno + 1);
be371ffc4cd0 Add URL for PEP to the source code encoding warning.
Marc-André Lemburg <mal@egenix.com>
parents: 27772
diff changeset
474 /* We don't use PyErr_WarnExplicit() here because
be371ffc4cd0 Add URL for PEP to the source code encoding warning.
Marc-André Lemburg <mal@egenix.com>
parents: 27772
diff changeset
475 printing the line in question to e.g. a log file
be371ffc4cd0 Add URL for PEP to the source code encoding warning.
Marc-André Lemburg <mal@egenix.com>
parents: 27772
diff changeset
476 could result in sensitive information being
be371ffc4cd0 Add URL for PEP to the source code encoding warning.
Marc-André Lemburg <mal@egenix.com>
parents: 27772
diff changeset
477 exposed. */
be371ffc4cd0 Add URL for PEP to the source code encoding warning.
Marc-André Lemburg <mal@egenix.com>
parents: 27772
diff changeset
478 PyErr_Warn(PyExc_DeprecationWarning, buf);
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
479 tok->issued_encoding_warning = 1;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
480 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
481 #endif
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
482 return line;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
483 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
484
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
485 static int
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
486 decoding_feof(struct tok_state *tok)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
487 {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
488 if (tok->decoding_state >= 0) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
489 return feof(tok->fp);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
490 } else {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
491 PyObject* buf = tok->decoding_buffer;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
492 if (buf == NULL) {
24867
255d1d3e66a3 Squash a few calls to the hideously expensive PyObject_CallObject(o,a)
Guido van Rossum <guido@python.org>
parents: 24839
diff changeset
493 PyObject *args = PyTuple_New(0);
255d1d3e66a3 Squash a few calls to the hideously expensive PyObject_CallObject(o,a)
Guido van Rossum <guido@python.org>
parents: 24839
diff changeset
494 if (args == NULL) {
255d1d3e66a3 Squash a few calls to the hideously expensive PyObject_CallObject(o,a)
Guido van Rossum <guido@python.org>
parents: 24839
diff changeset
495 error_ret(tok);
255d1d3e66a3 Squash a few calls to the hideously expensive PyObject_CallObject(o,a)
Guido van Rossum <guido@python.org>
parents: 24839
diff changeset
496 return 1;
255d1d3e66a3 Squash a few calls to the hideously expensive PyObject_CallObject(o,a)
Guido van Rossum <guido@python.org>
parents: 24839
diff changeset
497 }
255d1d3e66a3 Squash a few calls to the hideously expensive PyObject_CallObject(o,a)
Guido van Rossum <guido@python.org>
parents: 24839
diff changeset
498 buf = PyObject_Call(tok->decoding_readline,
255d1d3e66a3 Squash a few calls to the hideously expensive PyObject_CallObject(o,a)
Guido van Rossum <guido@python.org>
parents: 24839
diff changeset
499 args, NULL);
255d1d3e66a3 Squash a few calls to the hideously expensive PyObject_CallObject(o,a)
Guido van Rossum <guido@python.org>
parents: 24839
diff changeset
500 Py_DECREF(args);
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
501 if (buf == NULL) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
502 error_ret(tok);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
503 return 1;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
504 } else {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
505 tok->decoding_buffer = buf;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
506 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
507 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
508 return PyObject_Length(buf) == 0;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
509 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
510 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
511
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
512 /* Fetch a byte from TOK, using the string buffer. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
513
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
514 static int buf_getc(struct tok_state *tok) {
27772
3566b84e66b9 patch 680474 that fixes bug 679880: compile/eval/exec refused utf-8 bom
Just van Rossum <just@letterror.com>
parents: 27247
diff changeset
515 return Py_CHARMASK(*tok->str++);
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
516 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
517
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
518 /* Unfetch a byte from TOK, using the string buffer. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
519
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
520 static void buf_ungetc(int c, struct tok_state *tok) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
521 tok->str--;
27772
3566b84e66b9 patch 680474 that fixes bug 679880: compile/eval/exec refused utf-8 bom
Just van Rossum <just@letterror.com>
parents: 27247
diff changeset
522 assert(Py_CHARMASK(*tok->str) == c); /* tok->cur may point to read-only segment */
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
523 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
524
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
525 /* Set the readline function for TOK to ENC. For the string-based
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
526 tokenizer, this means to just record the encoding. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
527
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
528 static int buf_setreadl(struct tok_state *tok, const char* enc) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
529 tok->enc = enc;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
530 return 1;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
531 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
532
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
533 /* Return a UTF-8 encoding Python string object from the
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
534 C byte string STR, which is encoded with ENC. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
535
24624
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
536 #ifdef Py_USING_UNICODE
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
537 static PyObject *
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
538 translate_into_utf8(const char* str, const char* enc) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
539 PyObject *utf8;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
540 PyObject* buf = PyUnicode_Decode(str, strlen(str), enc, NULL);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
541 if (buf == NULL)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
542 return NULL;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
543 utf8 = PyUnicode_AsUTF8String(buf);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
544 Py_DECREF(buf);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
545 return utf8;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
546 }
24624
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
547 #endif
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
548
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
549 /* Decode a byte string STR for use as the buffer of TOK.
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
550 Look for encoding declarations inside STR, and record them
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
551 inside TOK. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
552
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
553 static const char *
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
554 decode_str(const char *str, struct tok_state *tok)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
555 {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
556 PyObject* utf8 = NULL;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
557 const char *s;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
558 int lineno = 0;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
559 tok->enc = NULL;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
560 tok->str = str;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
561 if (!check_bom(buf_getc, buf_ungetc, buf_setreadl, tok))
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
562 return NULL;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
563 str = tok->str; /* string after BOM if any */
24542
d0d3f760d067 Repaired a fatal compiler error in the debug build: it's not clear what
Tim Peters <tim.peters@gmail.com>
parents: 24541
diff changeset
564 assert(str);
24624
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
565 #ifdef Py_USING_UNICODE
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
566 if (tok->enc != NULL) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
567 utf8 = translate_into_utf8(str, tok->enc);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
568 if (utf8 == NULL)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
569 return NULL;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
570 str = PyString_AsString(utf8);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
571 }
24624
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
572 #endif
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
573 for (s = str;; s++) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
574 if (*s == '\0') break;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
575 else if (*s == '\n') {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
576 lineno++;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
577 if (lineno == 2) break;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
578 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
579 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
580 tok->enc = NULL;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
581 if (!check_coding_spec(str, s - str, tok, buf_setreadl))
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
582 return NULL;
24624
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
583 #ifdef Py_USING_UNICODE
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
584 if (tok->enc != NULL) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
585 assert(utf8 == NULL);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
586 utf8 = translate_into_utf8(str, tok->enc);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
587 if (utf8 == NULL)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
588 return NULL;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
589 str = PyString_AsString(utf8);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
590 }
24624
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
591 #endif
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
592 assert(tok->decoding_buffer == NULL);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
593 tok->decoding_buffer = utf8; /* CAUTION */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
594 return str;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
595 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
596
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
597 #endif /* PGEN */
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
598
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
599 /* Set up tokenizer for string */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
600
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
601 struct tok_state *
26601
cc05c2c8660e Constify filenames and scripts. Fixes #651362.
Martin v. Löwis <martin@v.loewis.de>
parents: 25995
diff changeset
602 PyTokenizer_FromString(const char *str)
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
603 {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
604 struct tok_state *tok = tok_new();
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
605 if (tok == NULL)
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
606 return NULL;
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
607 str = (char *)decode_str(str, tok);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
608 if (str == NULL)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
609 return NULL;
26601
cc05c2c8660e Constify filenames and scripts. Fixes #651362.
Martin v. Löwis <martin@v.loewis.de>
parents: 25995
diff changeset
610 /* XXX: constify members. */
cc05c2c8660e Constify filenames and scripts. Fixes #651362.
Martin v. Löwis <martin@v.loewis.de>
parents: 25995
diff changeset
611 tok->buf = tok->cur = tok->end = tok->inp = (char*)str;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
612 return tok;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
613 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
614
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
615
326
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
616 /* Set up tokenizer for file */
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
617
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
618 struct tok_state *
13659
e78153e179fb Mass ANSIfication.
Thomas Wouters <thomas@python.org>
parents: 13539
diff changeset
619 PyTokenizer_FromFile(FILE *fp, char *ps1, char *ps2)
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
620 {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
621 struct tok_state *tok = tok_new();
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
622 if (tok == NULL)
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
623 return NULL;
5379
1c9a84e0981d Another directory quickly renamed.
Guido van Rossum <guido@python.org>
parents: 5341
diff changeset
624 if ((tok->buf = PyMem_NEW(char, BUFSIZ)) == NULL) {
1c9a84e0981d Another directory quickly renamed.
Guido van Rossum <guido@python.org>
parents: 5341
diff changeset
625 PyMem_DEL(tok);
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
626 return NULL;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
627 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
628 tok->cur = tok->inp = tok->buf;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
629 tok->end = tok->buf + BUFSIZ;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
630 tok->fp = fp;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
631 tok->prompt = ps1;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
632 tok->nextprompt = ps2;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
633 return tok;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
634 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
635
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
636
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
637 /* Free a tok_state structure */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
638
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
639 void
13659
e78153e179fb Mass ANSIfication.
Thomas Wouters <thomas@python.org>
parents: 13539
diff changeset
640 PyTokenizer_Free(struct tok_state *tok)
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
641 {
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
642 if (tok->encoding != NULL)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
643 PyMem_DEL(tok->encoding);
24544
0bd521eb714e Make pgen compile with pydebug. Duplicate normalized names, as it may
Martin v. Löwis <martin@v.loewis.de>
parents: 24543
diff changeset
644 #ifndef PGEN
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
645 Py_XDECREF(tok->decoding_readline);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
646 Py_XDECREF(tok->decoding_buffer);
24544
0bd521eb714e Make pgen compile with pydebug. Duplicate normalized names, as it may
Martin v. Löwis <martin@v.loewis.de>
parents: 24543
diff changeset
647 #endif
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
648 if (tok->fp != NULL && tok->buf != NULL)
5379
1c9a84e0981d Another directory quickly renamed.
Guido van Rossum <guido@python.org>
parents: 5341
diff changeset
649 PyMem_DEL(tok->buf);
1c9a84e0981d Another directory quickly renamed.
Guido van Rossum <guido@python.org>
parents: 5341
diff changeset
650 PyMem_DEL(tok);
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
651 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
652
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
653
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
654 /* Get next char, updating state; error code goes into tok->done */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
655
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
656 static int
13659
e78153e179fb Mass ANSIfication.
Thomas Wouters <thomas@python.org>
parents: 13539
diff changeset
657 tok_nextc(register struct tok_state *tok)
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
658 {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
659 for (;;) {
1897
3fbda8768855 * Parser/tokenizer.c (tok_nextc): count line numbers when parsing
Guido van Rossum <guido@python.org>
parents: 1835
diff changeset
660 if (tok->cur != tok->inp) {
7118
e9108e97a5a0 tok_nextc() should return unsigned characters, to avoid mistaking
Guido van Rossum <guido@python.org>
parents: 5379
diff changeset
661 return Py_CHARMASK(*tok->cur++); /* Fast path */
1897
3fbda8768855 * Parser/tokenizer.c (tok_nextc): count line numbers when parsing
Guido van Rossum <guido@python.org>
parents: 1835
diff changeset
662 }
1282
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
663 if (tok->done != E_OK)
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
664 return EOF;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
665 if (tok->fp == NULL) {
1897
3fbda8768855 * Parser/tokenizer.c (tok_nextc): count line numbers when parsing
Guido van Rossum <guido@python.org>
parents: 1835
diff changeset
666 char *end = strchr(tok->inp, '\n');
3fbda8768855 * Parser/tokenizer.c (tok_nextc): count line numbers when parsing
Guido van Rossum <guido@python.org>
parents: 1835
diff changeset
667 if (end != NULL)
3fbda8768855 * Parser/tokenizer.c (tok_nextc): count line numbers when parsing
Guido van Rossum <guido@python.org>
parents: 1835
diff changeset
668 end++;
3fbda8768855 * Parser/tokenizer.c (tok_nextc): count line numbers when parsing
Guido van Rossum <guido@python.org>
parents: 1835
diff changeset
669 else {
3fbda8768855 * Parser/tokenizer.c (tok_nextc): count line numbers when parsing
Guido van Rossum <guido@python.org>
parents: 1835
diff changeset
670 end = strchr(tok->inp, '\0');
3fbda8768855 * Parser/tokenizer.c (tok_nextc): count line numbers when parsing
Guido van Rossum <guido@python.org>
parents: 1835
diff changeset
671 if (end == tok->inp) {
3fbda8768855 * Parser/tokenizer.c (tok_nextc): count line numbers when parsing
Guido van Rossum <guido@python.org>
parents: 1835
diff changeset
672 tok->done = E_EOF;
3fbda8768855 * Parser/tokenizer.c (tok_nextc): count line numbers when parsing
Guido van Rossum <guido@python.org>
parents: 1835
diff changeset
673 return EOF;
3fbda8768855 * Parser/tokenizer.c (tok_nextc): count line numbers when parsing
Guido van Rossum <guido@python.org>
parents: 1835
diff changeset
674 }
3fbda8768855 * Parser/tokenizer.c (tok_nextc): count line numbers when parsing
Guido van Rossum <guido@python.org>
parents: 1835
diff changeset
675 }
3fbda8768855 * Parser/tokenizer.c (tok_nextc): count line numbers when parsing
Guido van Rossum <guido@python.org>
parents: 1835
diff changeset
676 if (tok->start == NULL)
3fbda8768855 * Parser/tokenizer.c (tok_nextc): count line numbers when parsing
Guido van Rossum <guido@python.org>
parents: 1835
diff changeset
677 tok->buf = tok->cur;
3fbda8768855 * Parser/tokenizer.c (tok_nextc): count line numbers when parsing
Guido van Rossum <guido@python.org>
parents: 1835
diff changeset
678 tok->lineno++;
3fbda8768855 * Parser/tokenizer.c (tok_nextc): count line numbers when parsing
Guido van Rossum <guido@python.org>
parents: 1835
diff changeset
679 tok->inp = end;
7118
e9108e97a5a0 tok_nextc() should return unsigned characters, to avoid mistaking
Guido van Rossum <guido@python.org>
parents: 5379
diff changeset
680 return Py_CHARMASK(*tok->cur++);
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
681 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
682 if (tok->prompt != NULL) {
25924
dc4a0336a2a3 Patch #512981: Update readline input stream on sys.stdin/out change.
Martin v. Löwis <martin@v.loewis.de>
parents: 25101
diff changeset
683 char *new = PyOS_Readline(stdin, stdout, tok->prompt);
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
684 if (tok->nextprompt != NULL)
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
685 tok->prompt = tok->nextprompt;
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
686 if (new == NULL)
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
687 tok->done = E_INTR;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
688 else if (*new == '\0') {
12576
1c7c2dd1beb1 Vladimir Marangozov's long-awaited malloc restructuring.
Guido van Rossum <guido@python.org>
parents: 12224
diff changeset
689 PyMem_FREE(new);
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
690 tok->done = E_EOF;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
691 }
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
692 else if (tok->start != NULL) {
13093
cbe02b8aee07 Trent Mick: familiar simple Win64 patches
Guido van Rossum <guido@python.org>
parents: 12576
diff changeset
693 size_t start = tok->start - tok->buf;
cbe02b8aee07 Trent Mick: familiar simple Win64 patches
Guido van Rossum <guido@python.org>
parents: 12576
diff changeset
694 size_t oldlen = tok->cur - tok->buf;
cbe02b8aee07 Trent Mick: familiar simple Win64 patches
Guido van Rossum <guido@python.org>
parents: 12576
diff changeset
695 size_t newlen = oldlen + strlen(new);
12576
1c7c2dd1beb1 Vladimir Marangozov's long-awaited malloc restructuring.
Guido van Rossum <guido@python.org>
parents: 12224
diff changeset
696 char *buf = tok->buf;
1c7c2dd1beb1 Vladimir Marangozov's long-awaited malloc restructuring.
Guido van Rossum <guido@python.org>
parents: 12224
diff changeset
697 PyMem_RESIZE(buf, char, newlen+1);
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
698 tok->lineno++;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
699 if (buf == NULL) {
12576
1c7c2dd1beb1 Vladimir Marangozov's long-awaited malloc restructuring.
Guido van Rossum <guido@python.org>
parents: 12224
diff changeset
700 PyMem_DEL(tok->buf);
2044
281e40652e23 Parser/tokenizer.c (tok_nextc): zap tok->buf after freeing;
Guido van Rossum <guido@python.org>
parents: 1897
diff changeset
701 tok->buf = NULL;
12576
1c7c2dd1beb1 Vladimir Marangozov's long-awaited malloc restructuring.
Guido van Rossum <guido@python.org>
parents: 12224
diff changeset
702 PyMem_FREE(new);
1282
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
703 tok->done = E_NOMEM;
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
704 return EOF;
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
705 }
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
706 tok->buf = buf;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
707 tok->cur = tok->buf + oldlen;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
708 strcpy(tok->buf + oldlen, new);
12576
1c7c2dd1beb1 Vladimir Marangozov's long-awaited malloc restructuring.
Guido van Rossum <guido@python.org>
parents: 12224
diff changeset
709 PyMem_FREE(new);
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
710 tok->inp = tok->buf + newlen;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
711 tok->end = tok->inp + 1;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
712 tok->start = tok->buf + start;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
713 }
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
714 else {
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
715 tok->lineno++;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
716 if (tok->buf != NULL)
12576
1c7c2dd1beb1 Vladimir Marangozov's long-awaited malloc restructuring.
Guido van Rossum <guido@python.org>
parents: 12224
diff changeset
717 PyMem_DEL(tok->buf);
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
718 tok->buf = new;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
719 tok->cur = tok->buf;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
720 tok->inp = strchr(tok->buf, '\0');
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
721 tok->end = tok->inp + 1;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
722 }
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
723 }
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
724 else {
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
725 int done = 0;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
726 int cur = 0;
2732
daa150b3a6e4 replace "\r\n" with "\n" at line end (Jim Ahlstrom)
Guido van Rossum <guido@python.org>
parents: 2149
diff changeset
727 char *pt;
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
728 if (tok->start == NULL) {
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
729 if (tok->buf == NULL) {
5379
1c9a84e0981d Another directory quickly renamed.
Guido van Rossum <guido@python.org>
parents: 5341
diff changeset
730 tok->buf = PyMem_NEW(char, BUFSIZ);
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
731 if (tok->buf == NULL) {
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
732 tok->done = E_NOMEM;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
733 return EOF;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
734 }
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
735 tok->end = tok->buf + BUFSIZ;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
736 }
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
737 if (decoding_fgets(tok->buf, (int)(tok->end - tok->buf),
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
738 tok) == NULL) {
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
739 tok->done = E_EOF;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
740 done = 1;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
741 }
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
742 else {
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
743 tok->done = E_OK;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
744 tok->inp = strchr(tok->buf, '\0');
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
745 done = tok->inp[-1] == '\n';
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
746 }
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
747 }
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
748 else {
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
749 cur = tok->cur - tok->buf;
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
750 if (decoding_feof(tok)) {
2149
6060d7fb07e3 fix loop on unterminated triple quotes
Guido van Rossum <guido@python.org>
parents: 2060
diff changeset
751 tok->done = E_EOF;
6060d7fb07e3 fix loop on unterminated triple quotes
Guido van Rossum <guido@python.org>
parents: 2060
diff changeset
752 done = 1;
6060d7fb07e3 fix loop on unterminated triple quotes
Guido van Rossum <guido@python.org>
parents: 2060
diff changeset
753 }
6060d7fb07e3 fix loop on unterminated triple quotes
Guido van Rossum <guido@python.org>
parents: 2060
diff changeset
754 else
6060d7fb07e3 fix loop on unterminated triple quotes
Guido van Rossum <guido@python.org>
parents: 2060
diff changeset
755 tok->done = E_OK;
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
756 }
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
757 tok->lineno++;
1282
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
758 /* Read until '\n' or EOF */
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
759 while (!done) {
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
760 int curstart = tok->start == NULL ? -1 :
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
761 tok->start - tok->buf;
1282
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
762 int curvalid = tok->inp - tok->buf;
3116
db139a4112f3 fix bogus resize length in nextc
Guido van Rossum <guido@python.org>
parents: 2801
diff changeset
763 int newsize = curvalid + BUFSIZ;
1282
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
764 char *newbuf = tok->buf;
5379
1c9a84e0981d Another directory quickly renamed.
Guido van Rossum <guido@python.org>
parents: 5341
diff changeset
765 PyMem_RESIZE(newbuf, char, newsize);
1282
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
766 if (newbuf == NULL) {
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
767 tok->done = E_NOMEM;
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
768 tok->cur = tok->inp;
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
769 return EOF;
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
770 }
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
771 tok->buf = newbuf;
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
772 tok->inp = tok->buf + curvalid;
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
773 tok->end = tok->buf + newsize;
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
774 tok->start = curstart < 0 ? NULL :
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
775 tok->buf + curstart;
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
776 if (decoding_fgets(tok->inp,
1282
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
777 (int)(tok->end - tok->inp),
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
778 tok) == NULL) {
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
779 /* Last line does not end in \n,
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
780 fake one */
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
781 strcpy(tok->inp, "\n");
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
782 }
1282
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
783 tok->inp = strchr(tok->inp, '\0');
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
784 done = tok->inp[-1] == '\n';
1282
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
785 }
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
786 tok->cur = tok->buf + cur;
5311
782597cf6151 (Jack:) On the Mac, give syntax error on \r.
Guido van Rossum <guido@python.org>
parents: 5265
diff changeset
787 #ifndef macintosh
2732
daa150b3a6e4 replace "\r\n" with "\n" at line end (Jim Ahlstrom)
Guido van Rossum <guido@python.org>
parents: 2149
diff changeset
788 /* replace "\r\n" with "\n" */
5311
782597cf6151 (Jack:) On the Mac, give syntax error on \r.
Guido van Rossum <guido@python.org>
parents: 5265
diff changeset
789 /* For Mac we leave the \r, giving a syntax error */
2732
daa150b3a6e4 replace "\r\n" with "\n" at line end (Jim Ahlstrom)
Guido van Rossum <guido@python.org>
parents: 2149
diff changeset
790 pt = tok->inp - 2;
daa150b3a6e4 replace "\r\n" with "\n" at line end (Jim Ahlstrom)
Guido van Rossum <guido@python.org>
parents: 2149
diff changeset
791 if (pt >= tok->buf && *pt == '\r') {
daa150b3a6e4 replace "\r\n" with "\n" at line end (Jim Ahlstrom)
Guido van Rossum <guido@python.org>
parents: 2149
diff changeset
792 *pt++ = '\n';
daa150b3a6e4 replace "\r\n" with "\n" at line end (Jim Ahlstrom)
Guido van Rossum <guido@python.org>
parents: 2149
diff changeset
793 *pt = '\0';
daa150b3a6e4 replace "\r\n" with "\n" at line end (Jim Ahlstrom)
Guido van Rossum <guido@python.org>
parents: 2149
diff changeset
794 tok->inp = pt;
daa150b3a6e4 replace "\r\n" with "\n" at line end (Jim Ahlstrom)
Guido van Rossum <guido@python.org>
parents: 2149
diff changeset
795 }
5311
782597cf6151 (Jack:) On the Mac, give syntax error on \r.
Guido van Rossum <guido@python.org>
parents: 5265
diff changeset
796 #endif
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
797 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
798 if (tok->done != E_OK) {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
799 if (tok->prompt != NULL)
9047
6d2eb58f0418 Replace all calls to fprintf(stderr, ...) with PySys_WriteStderr(...).
Guido van Rossum <guido@python.org>
parents: 7945
diff changeset
800 PySys_WriteStderr("\n");
1282
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
801 tok->cur = tok->inp;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
802 return EOF;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
803 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
804 }
1282
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
805 /*NOTREACHED*/
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
806 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
807
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
808
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
809 /* Back-up one character */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
810
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
811 static void
13659
e78153e179fb Mass ANSIfication.
Thomas Wouters <thomas@python.org>
parents: 13539
diff changeset
812 tok_backup(register struct tok_state *tok, register int c)
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
813 {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
814 if (c != EOF) {
2044
281e40652e23 Parser/tokenizer.c (tok_nextc): zap tok->buf after freeing;
Guido van Rossum <guido@python.org>
parents: 1897
diff changeset
815 if (--tok->cur < tok->buf)
5379
1c9a84e0981d Another directory quickly renamed.
Guido van Rossum <guido@python.org>
parents: 5341
diff changeset
816 Py_FatalError("tok_backup: begin of buffer");
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
817 if (*tok->cur != c)
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
818 *tok->cur = c;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
819 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
820 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
821
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
822
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
823 /* Return the token corresponding to a single character */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
824
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
825 int
13659
e78153e179fb Mass ANSIfication.
Thomas Wouters <thomas@python.org>
parents: 13539
diff changeset
826 PyToken_OneChar(int c)
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
827 {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
828 switch (c) {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
829 case '(': return LPAR;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
830 case ')': return RPAR;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
831 case '[': return LSQB;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
832 case ']': return RSQB;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
833 case ':': return COLON;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
834 case ',': return COMMA;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
835 case ';': return SEMI;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
836 case '+': return PLUS;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
837 case '-': return MINUS;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
838 case '*': return STAR;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
839 case '/': return SLASH;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
840 case '|': return VBAR;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
841 case '&': return AMPER;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
842 case '<': return LESS;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
843 case '>': return GREATER;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
844 case '=': return EQUAL;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
845 case '.': return DOT;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
846 case '%': return PERCENT;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
847 case '`': return BACKQUOTE;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
848 case '{': return LBRACE;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
849 case '}': return RBRACE;
410
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
850 case '^': return CIRCUMFLEX;
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
851 case '~': return TILDE;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
852 default: return OP;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
853 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
854 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
855
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
856
400
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
857 int
13659
e78153e179fb Mass ANSIfication.
Thomas Wouters <thomas@python.org>
parents: 13539
diff changeset
858 PyToken_TwoChars(int c1, int c2)
400
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
859 {
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
860 switch (c1) {
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
861 case '=':
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
862 switch (c2) {
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
863 case '=': return EQEQUAL;
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
864 }
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
865 break;
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
866 case '!':
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
867 switch (c2) {
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
868 case '=': return NOTEQUAL;
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
869 }
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
870 break;
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
871 case '<':
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
872 switch (c2) {
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
873 case '>': return NOTEQUAL;
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
874 case '=': return LESSEQUAL;
410
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
875 case '<': return LEFTSHIFT;
400
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
876 }
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
877 break;
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
878 case '>':
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
879 switch (c2) {
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
880 case '=': return GREATEREQUAL;
410
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
881 case '>': return RIGHTSHIFT;
400
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
882 }
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
883 break;
14163
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
884 case '+':
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
885 switch (c2) {
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
886 case '=': return PLUSEQUAL;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
887 }
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
888 break;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
889 case '-':
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
890 switch (c2) {
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
891 case '=': return MINEQUAL;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
892 }
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
893 break;
3333
4b5bff95804e changes for pow(**) and complex
Guido van Rossum <guido@python.org>
parents: 3116
diff changeset
894 case '*':
4b5bff95804e changes for pow(**) and complex
Guido van Rossum <guido@python.org>
parents: 3116
diff changeset
895 switch (c2) {
4b5bff95804e changes for pow(**) and complex
Guido van Rossum <guido@python.org>
parents: 3116
diff changeset
896 case '*': return DOUBLESTAR;
14163
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
897 case '=': return STAREQUAL;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
898 }
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
899 break;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
900 case '/':
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
901 switch (c2) {
19051
f20727cc3d0f Implement PEP 238 in its (almost) full glory.
Guido van Rossum <guido@python.org>
parents: 17600
diff changeset
902 case '/': return DOUBLESLASH;
14163
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
903 case '=': return SLASHEQUAL;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
904 }
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
905 break;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
906 case '|':
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
907 switch (c2) {
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
908 case '=': return VBAREQUAL;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
909 }
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
910 break;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
911 case '%':
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
912 switch (c2) {
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
913 case '=': return PERCENTEQUAL;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
914 }
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
915 break;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
916 case '&':
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
917 switch (c2) {
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
918 case '=': return AMPEREQUAL;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
919 }
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
920 break;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
921 case '^':
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
922 switch (c2) {
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
923 case '=': return CIRCUMFLEXEQUAL;
3333
4b5bff95804e changes for pow(**) and complex
Guido van Rossum <guido@python.org>
parents: 3116
diff changeset
924 }
4b5bff95804e changes for pow(**) and complex
Guido van Rossum <guido@python.org>
parents: 3116
diff changeset
925 break;
400
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
926 }
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
927 return OP;
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
928 }
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
929
14163
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
930 int
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
931 PyToken_ThreeChars(int c1, int c2, int c3)
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
932 {
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
933 switch (c1) {
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
934 case '<':
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
935 switch (c2) {
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
936 case '<':
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
937 switch (c3) {
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
938 case '=':
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
939 return LEFTSHIFTEQUAL;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
940 }
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
941 break;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
942 }
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
943 break;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
944 case '>':
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
945 switch (c2) {
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
946 case '>':
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
947 switch (c3) {
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
948 case '=':
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
949 return RIGHTSHIFTEQUAL;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
950 }
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
951 break;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
952 }
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
953 break;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
954 case '*':
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
955 switch (c2) {
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
956 case '*':
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
957 switch (c3) {
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
958 case '=':
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
959 return DOUBLESTAREQUAL;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
960 }
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
961 break;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
962 }
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
963 break;
19051
f20727cc3d0f Implement PEP 238 in its (almost) full glory.
Guido van Rossum <guido@python.org>
parents: 17600
diff changeset
964 case '/':
f20727cc3d0f Implement PEP 238 in its (almost) full glory.
Guido van Rossum <guido@python.org>
parents: 17600
diff changeset
965 switch (c2) {
f20727cc3d0f Implement PEP 238 in its (almost) full glory.
Guido van Rossum <guido@python.org>
parents: 17600
diff changeset
966 case '/':
f20727cc3d0f Implement PEP 238 in its (almost) full glory.
Guido van Rossum <guido@python.org>
parents: 17600
diff changeset
967 switch (c3) {
f20727cc3d0f Implement PEP 238 in its (almost) full glory.
Guido van Rossum <guido@python.org>
parents: 17600
diff changeset
968 case '=':
f20727cc3d0f Implement PEP 238 in its (almost) full glory.
Guido van Rossum <guido@python.org>
parents: 17600
diff changeset
969 return DOUBLESLASHEQUAL;
f20727cc3d0f Implement PEP 238 in its (almost) full glory.
Guido van Rossum <guido@python.org>
parents: 17600
diff changeset
970 }
f20727cc3d0f Implement PEP 238 in its (almost) full glory.
Guido van Rossum <guido@python.org>
parents: 17600
diff changeset
971 break;
f20727cc3d0f Implement PEP 238 in its (almost) full glory.
Guido van Rossum <guido@python.org>
parents: 17600
diff changeset
972 }
f20727cc3d0f Implement PEP 238 in its (almost) full glory.
Guido van Rossum <guido@python.org>
parents: 17600
diff changeset
973 break;
14163
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
974 }
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
975 return OP;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
976 }
400
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
977
7945
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
978 static int
13659
e78153e179fb Mass ANSIfication.
Thomas Wouters <thomas@python.org>
parents: 13539
diff changeset
979 indenterror(struct tok_state *tok)
7945
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
980 {
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
981 if (tok->alterror) {
13539
feab677269e0 Create two new exceptions: IndentationError and TabError. These are
Fred Drake <fdrake@acm.org>
parents: 13447
diff changeset
982 tok->done = E_TABSPACE;
7945
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
983 tok->cur = tok->inp;
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
984 return 1;
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
985 }
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
986 if (tok->altwarning) {
13539
feab677269e0 Create two new exceptions: IndentationError and TabError. These are
Fred Drake <fdrake@acm.org>
parents: 13447
diff changeset
987 PySys_WriteStderr("%s: inconsistent use of tabs and spaces "
feab677269e0 Create two new exceptions: IndentationError and TabError. These are
Fred Drake <fdrake@acm.org>
parents: 13447
diff changeset
988 "in indentation\n", tok->filename);
7945
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
989 tok->altwarning = 0;
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
990 }
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
991 return 0;
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
992 }
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
993
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
994
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
995 /* Get next token, after space stripping etc. */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
996
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
997 static int
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
998 tok_get(register struct tok_state *tok, char **p_start, char **p_end)
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
999 {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1000 register int c;
326
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1001 int blankline;
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1002
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1003 *p_start = *p_end = NULL;
326
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1004 nextline:
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1005 tok->start = NULL;
326
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1006 blankline = 0;
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1007
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1008 /* Get indentation level */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1009 if (tok->atbol) {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1010 register int col = 0;
7945
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1011 register int altcol = 0;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1012 tok->atbol = 0;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1013 for (;;) {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1014 c = tok_nextc(tok);
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1015 if (c == ' ')
7945
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1016 col++, altcol++;
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1017 else if (c == '\t') {
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1018 col = (col/tok->tabsize + 1) * tok->tabsize;
7945
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1019 altcol = (altcol/tok->alttabsize + 1)
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1020 * tok->alttabsize;
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1021 }
2801
e911f2fb179c ignore control-l in whitespace
Guido van Rossum <guido@python.org>
parents: 2732
diff changeset
1022 else if (c == '\014') /* Control-L (formfeed) */
7945
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1023 col = altcol = 0; /* For Emacs users */
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1024 else
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1025 break;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1026 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1027 tok_backup(tok, c);
326
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1028 if (c == '#' || c == '\n') {
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1029 /* Lines with only whitespace and/or comments
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1030 shouldn't affect the indentation and are
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1031 not passed to the parser as NEWLINE tokens,
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1032 except *totally* empty lines in interactive
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1033 mode, which signal the end of a command group. */
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1034 if (col == 0 && c == '\n' && tok->prompt != NULL)
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1035 blankline = 0; /* Let it through */
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1036 else
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1037 blankline = 1; /* Ignore completely */
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1038 /* We can't jump back right here since we still
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1039 may need to skip to the end of a comment */
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1040 }
1284
dbce72afe261 * selectmodule.c: fix (another!) two memory leaks -- this time in list2set
Guido van Rossum <guido@python.org>
parents: 1282
diff changeset
1041 if (!blankline && tok->level == 0) {
326
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1042 if (col == tok->indstack[tok->indent]) {
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1043 /* No change */
7945
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1044 if (altcol != tok->altindstack[tok->indent]) {
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1045 if (indenterror(tok))
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1046 return ERRORTOKEN;
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1047 }
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1048 }
326
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1049 else if (col > tok->indstack[tok->indent]) {
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1050 /* Indent -- always one */
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1051 if (tok->indent+1 >= MAXINDENT) {
13539
feab677269e0 Create two new exceptions: IndentationError and TabError. These are
Fred Drake <fdrake@acm.org>
parents: 13447
diff changeset
1052 tok->done = E_TOODEEP;
1282
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
1053 tok->cur = tok->inp;
326
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1054 return ERRORTOKEN;
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1055 }
7945
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1056 if (altcol <= tok->altindstack[tok->indent]) {
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1057 if (indenterror(tok))
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1058 return ERRORTOKEN;
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1059 }
326
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1060 tok->pendin++;
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1061 tok->indstack[++tok->indent] = col;
7945
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1062 tok->altindstack[tok->indent] = altcol;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1063 }
326
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1064 else /* col < tok->indstack[tok->indent] */ {
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1065 /* Dedent -- any number, must be consistent */
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1066 while (tok->indent > 0 &&
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1067 col < tok->indstack[tok->indent]) {
7324
55125b1144fa Swap two statements in the dedent check loop. This makes absolutely
Guido van Rossum <guido@python.org>
parents: 7315
diff changeset
1068 tok->pendin--;
326
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1069 tok->indent--;
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1070 }
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1071 if (col != tok->indstack[tok->indent]) {
13539
feab677269e0 Create two new exceptions: IndentationError and TabError. These are
Fred Drake <fdrake@acm.org>
parents: 13447
diff changeset
1072 tok->done = E_DEDENT;
1282
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
1073 tok->cur = tok->inp;
326
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1074 return ERRORTOKEN;
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1075 }
7945
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1076 if (altcol != tok->altindstack[tok->indent]) {
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1077 if (indenterror(tok))
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1078 return ERRORTOKEN;
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1079 }
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1080 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1081 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1082 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1083
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1084 tok->start = tok->cur;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1085
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1086 /* Return pending indents/dedents */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1087 if (tok->pendin != 0) {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1088 if (tok->pendin < 0) {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1089 tok->pendin++;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1090 return DEDENT;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1091 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1092 else {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1093 tok->pendin--;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1094 return INDENT;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1095 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1096 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1097
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1098 again:
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1099 tok->start = NULL;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1100 /* Skip spaces */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1101 do {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1102 c = tok_nextc(tok);
2801
e911f2fb179c ignore control-l in whitespace
Guido van Rossum <guido@python.org>
parents: 2732
diff changeset
1103 } while (c == ' ' || c == '\t' || c == '\014');
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1104
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1105 /* Set start of current token */
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1106 tok->start = tok->cur - 1;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1107
12142
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1108 /* Skip comment, while looking for tab-setting magic */
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1109 if (c == '#') {
12142
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1110 static char *tabforms[] = {
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1111 "tab-width:", /* Emacs */
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1112 ":tabstop=", /* vim, full form */
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1113 ":ts=", /* vim, abbreviated form */
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1114 "set tabsize=", /* will vi never die? */
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1115 /* more templates can be added here to support other editors */
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1116 };
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1117 char cbuf[80];
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1118 char *tp, **cp;
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1119 tp = cbuf;
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1120 do {
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1121 *tp++ = c = tok_nextc(tok);
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1122 } while (c != EOF && c != '\n' &&
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1123 tp - cbuf + 1 < sizeof(cbuf));
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1124 *tp = '\0';
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1125 for (cp = tabforms;
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1126 cp < tabforms + sizeof(tabforms)/sizeof(tabforms[0]);
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1127 cp++) {
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1128 if ((tp = strstr(cbuf, *cp))) {
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1129 int newsize = atoi(tp + strlen(*cp));
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1130
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1131 if (newsize >= 1 && newsize <= 40) {
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1132 tok->tabsize = newsize;
12224
2dc1bdb4088e Only write message about changed Tab size with -v.
Guido van Rossum <guido@python.org>
parents: 12142
diff changeset
1133 if (Py_VerboseFlag)
2dc1bdb4088e Only write message about changed Tab size with -v.
Guido van Rossum <guido@python.org>
parents: 12142
diff changeset
1134 PySys_WriteStderr(
12142
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1135 "Tab size set to %d\n",
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1136 newsize);
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1137 }
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1138 }
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1139 }
12142
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1140 while (c != EOF && c != '\n')
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1141 c = tok_nextc(tok);
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1142 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1143
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1144 /* Check for EOF and errors now */
410
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
1145 if (c == EOF) {
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1146 return tok->done == E_EOF ? ENDMARKER : ERRORTOKEN;
410
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
1147 }
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1148
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1149 /* Identifier (most frequent token!) */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1150 if (isalpha(c) || c == '_') {
11971
574283db2b3d Marc-Andre Lemburg: add new string token types u"..." and ur"..."
Guido van Rossum <guido@python.org>
parents: 9590
diff changeset
1151 /* Process r"", u"" and ur"" */
5341
67e5237ce141 Now that the string-sig has settled on r"obin" strings, restrict the
Guido van Rossum <guido@python.org>
parents: 5311
diff changeset
1152 switch (c) {
67e5237ce141 Now that the string-sig has settled on r"obin" strings, restrict the
Guido van Rossum <guido@python.org>
parents: 5311
diff changeset
1153 case 'r':
67e5237ce141 Now that the string-sig has settled on r"obin" strings, restrict the
Guido van Rossum <guido@python.org>
parents: 5311
diff changeset
1154 case 'R':
67e5237ce141 Now that the string-sig has settled on r"obin" strings, restrict the
Guido van Rossum <guido@python.org>
parents: 5311
diff changeset
1155 c = tok_nextc(tok);
67e5237ce141 Now that the string-sig has settled on r"obin" strings, restrict the
Guido van Rossum <guido@python.org>
parents: 5311
diff changeset
1156 if (c == '"' || c == '\'')
67e5237ce141 Now that the string-sig has settled on r"obin" strings, restrict the
Guido van Rossum <guido@python.org>
parents: 5311
diff changeset
1157 goto letter_quote;
11971
574283db2b3d Marc-Andre Lemburg: add new string token types u"..." and ur"..."
Guido van Rossum <guido@python.org>
parents: 9590
diff changeset
1158 break;
574283db2b3d Marc-Andre Lemburg: add new string token types u"..." and ur"..."
Guido van Rossum <guido@python.org>
parents: 9590
diff changeset
1159 case 'u':
574283db2b3d Marc-Andre Lemburg: add new string token types u"..." and ur"..."
Guido van Rossum <guido@python.org>
parents: 9590
diff changeset
1160 case 'U':
574283db2b3d Marc-Andre Lemburg: add new string token types u"..." and ur"..."
Guido van Rossum <guido@python.org>
parents: 9590
diff changeset
1161 c = tok_nextc(tok);
574283db2b3d Marc-Andre Lemburg: add new string token types u"..." and ur"..."
Guido van Rossum <guido@python.org>
parents: 9590
diff changeset
1162 if (c == 'r' || c == 'R')
574283db2b3d Marc-Andre Lemburg: add new string token types u"..." and ur"..."
Guido van Rossum <guido@python.org>
parents: 9590
diff changeset
1163 c = tok_nextc(tok);
574283db2b3d Marc-Andre Lemburg: add new string token types u"..." and ur"..."
Guido van Rossum <guido@python.org>
parents: 9590
diff changeset
1164 if (c == '"' || c == '\'')
574283db2b3d Marc-Andre Lemburg: add new string token types u"..." and ur"..."
Guido van Rossum <guido@python.org>
parents: 9590
diff changeset
1165 goto letter_quote;
574283db2b3d Marc-Andre Lemburg: add new string token types u"..." and ur"..."
Guido van Rossum <guido@python.org>
parents: 9590
diff changeset
1166 break;
5341
67e5237ce141 Now that the string-sig has settled on r"obin" strings, restrict the
Guido van Rossum <guido@python.org>
parents: 5311
diff changeset
1167 }
5265
3c1788ab9761 Support for alternative string quotes (a"xx", b"xx", c"xx", ...).
Guido van Rossum <guido@python.org>
parents: 4839
diff changeset
1168 while (isalnum(c) || c == '_') {
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1169 c = tok_nextc(tok);
5265
3c1788ab9761 Support for alternative string quotes (a"xx", b"xx", c"xx", ...).
Guido van Rossum <guido@python.org>
parents: 4839
diff changeset
1170 }
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1171 tok_backup(tok, c);
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1172 *p_start = tok->start;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1173 *p_end = tok->cur;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1174 return NAME;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1175 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1176
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1177 /* Newline */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1178 if (c == '\n') {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1179 tok->atbol = 1;
1284
dbce72afe261 * selectmodule.c: fix (another!) two memory leaks -- this time in list2set
Guido van Rossum <guido@python.org>
parents: 1282
diff changeset
1180 if (blankline || tok->level > 0)
326
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1181 goto nextline;
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1182 *p_start = tok->start;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1183 *p_end = tok->cur - 1; /* Leave '\n' out of the string */
25098
c02a668488d0 Ignore encoding declarations inside strings. Fixes #603509.
Martin v. Löwis <martin@v.loewis.de>
parents: 24867
diff changeset
1184 tok->cont_line = 0;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1185 return NEWLINE;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1186 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1187
5311
782597cf6151 (Jack:) On the Mac, give syntax error on \r.
Guido van Rossum <guido@python.org>
parents: 5265
diff changeset
1188 #ifdef macintosh
782597cf6151 (Jack:) On the Mac, give syntax error on \r.
Guido van Rossum <guido@python.org>
parents: 5265
diff changeset
1189 if (c == '\r') {
9047
6d2eb58f0418 Replace all calls to fprintf(stderr, ...) with PySys_WriteStderr(...).
Guido van Rossum <guido@python.org>
parents: 7945
diff changeset
1190 PySys_WriteStderr(
5379
1c9a84e0981d Another directory quickly renamed.
Guido van Rossum <guido@python.org>
parents: 5341
diff changeset
1191 "File contains \\r characters (incorrect line endings?)\n");
5311
782597cf6151 (Jack:) On the Mac, give syntax error on \r.
Guido van Rossum <guido@python.org>
parents: 5265
diff changeset
1192 tok->done = E_TOKEN;
782597cf6151 (Jack:) On the Mac, give syntax error on \r.
Guido van Rossum <guido@python.org>
parents: 5265
diff changeset
1193 tok->cur = tok->inp;
782597cf6151 (Jack:) On the Mac, give syntax error on \r.
Guido van Rossum <guido@python.org>
parents: 5265
diff changeset
1194 return ERRORTOKEN;
782597cf6151 (Jack:) On the Mac, give syntax error on \r.
Guido van Rossum <guido@python.org>
parents: 5265
diff changeset
1195 }
782597cf6151 (Jack:) On the Mac, give syntax error on \r.
Guido van Rossum <guido@python.org>
parents: 5265
diff changeset
1196 #endif
410
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
1197 /* Period or number starting with period? */
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
1198 if (c == '.') {
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
1199 c = tok_nextc(tok);
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
1200 if (isdigit(c)) {
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
1201 goto fraction;
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
1202 }
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
1203 else {
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
1204 tok_backup(tok, c);
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1205 *p_start = tok->start;
410
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
1206 *p_end = tok->cur;
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
1207 return DOT;
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
1208 }
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
1209 }
3333
4b5bff95804e changes for pow(**) and complex
Guido van Rossum <guido@python.org>
parents: 3116
diff changeset
1210
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1211 /* Number */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1212 if (isdigit(c)) {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1213 if (c == '0') {
19621
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1214 /* Hex or octal -- maybe. */
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1215 c = tok_nextc(tok);
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1216 if (c == '.')
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1217 goto fraction;
3333
4b5bff95804e changes for pow(**) and complex
Guido van Rossum <guido@python.org>
parents: 3116
diff changeset
1218 #ifndef WITHOUT_COMPLEX
3364
bc17e8313ea0 use only j for imaginary constants
Guido van Rossum <guido@python.org>
parents: 3333
diff changeset
1219 if (c == 'j' || c == 'J')
3333
4b5bff95804e changes for pow(**) and complex
Guido van Rossum <guido@python.org>
parents: 3116
diff changeset
1220 goto imaginary;
4b5bff95804e changes for pow(**) and complex
Guido van Rossum <guido@python.org>
parents: 3116
diff changeset
1221 #endif
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1222 if (c == 'x' || c == 'X') {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1223 /* Hex */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1224 do {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1225 c = tok_nextc(tok);
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1226 } while (isxdigit(c));
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1227 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1228 else {
19621
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1229 int found_decimal = 0;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1230 /* Octal; c is first char of it */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1231 /* There's no 'isoctdigit' macro, sigh */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1232 while ('0' <= c && c < '8') {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1233 c = tok_nextc(tok);
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1234 }
19621
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1235 if (isdigit(c)) {
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1236 found_decimal = 1;
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1237 do {
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1238 c = tok_nextc(tok);
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1239 } while (isdigit(c));
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1240 }
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1241 if (c == '.')
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1242 goto fraction;
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1243 else if (c == 'e' || c == 'E')
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1244 goto exponent;
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1245 #ifndef WITHOUT_COMPLEX
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1246 else if (c == 'j' || c == 'J')
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1247 goto imaginary;
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1248 #endif
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1249 else if (found_decimal) {
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1250 tok->done = E_TOKEN;
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1251 tok_backup(tok, c);
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1252 return ERRORTOKEN;
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1253 }
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1254 }
236
0a92b6a1b9c0 Added recognition of 'l' or 'L' as long integer suffix
Guido van Rossum <guido@python.org>
parents: 145
diff changeset
1255 if (c == 'l' || c == 'L')
0a92b6a1b9c0 Added recognition of 'l' or 'L' as long integer suffix
Guido van Rossum <guido@python.org>
parents: 145
diff changeset
1256 c = tok_nextc(tok);
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1257 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1258 else {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1259 /* Decimal */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1260 do {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1261 c = tok_nextc(tok);
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1262 } while (isdigit(c));
236
0a92b6a1b9c0 Added recognition of 'l' or 'L' as long integer suffix
Guido van Rossum <guido@python.org>
parents: 145
diff changeset
1263 if (c == 'l' || c == 'L')
0a92b6a1b9c0 Added recognition of 'l' or 'L' as long integer suffix
Guido van Rossum <guido@python.org>
parents: 145
diff changeset
1264 c = tok_nextc(tok);
0a92b6a1b9c0 Added recognition of 'l' or 'L' as long integer suffix
Guido van Rossum <guido@python.org>
parents: 145
diff changeset
1265 else {
19501
ce4f1e98ba7e SF bug [#455775] float parsing discrepancy.
Tim Peters <tim.peters@gmail.com>
parents: 19051
diff changeset
1266 /* Accept floating point numbers. */
236
0a92b6a1b9c0 Added recognition of 'l' or 'L' as long integer suffix
Guido van Rossum <guido@python.org>
parents: 145
diff changeset
1267 if (c == '.') {
0a92b6a1b9c0 Added recognition of 'l' or 'L' as long integer suffix
Guido van Rossum <guido@python.org>
parents: 145
diff changeset
1268 fraction:
0a92b6a1b9c0 Added recognition of 'l' or 'L' as long integer suffix
Guido van Rossum <guido@python.org>
parents: 145
diff changeset
1269 /* Fraction */
0a92b6a1b9c0 Added recognition of 'l' or 'L' as long integer suffix
Guido van Rossum <guido@python.org>
parents: 145
diff changeset
1270 do {
0a92b6a1b9c0 Added recognition of 'l' or 'L' as long integer suffix
Guido van Rossum <guido@python.org>
parents: 145
diff changeset
1271 c = tok_nextc(tok);
0a92b6a1b9c0 Added recognition of 'l' or 'L' as long integer suffix
Guido van Rossum <guido@python.org>
parents: 145
diff changeset
1272 } while (isdigit(c));
0a92b6a1b9c0 Added recognition of 'l' or 'L' as long integer suffix
Guido van Rossum <guido@python.org>
parents: 145
diff changeset
1273 }
0a92b6a1b9c0 Added recognition of 'l' or 'L' as long integer suffix
Guido van Rossum <guido@python.org>
parents: 145
diff changeset
1274 if (c == 'e' || c == 'E') {
19621
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1275 exponent:
236
0a92b6a1b9c0 Added recognition of 'l' or 'L' as long integer suffix
Guido van Rossum <guido@python.org>
parents: 145
diff changeset
1276 /* Exponent part */
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1277 c = tok_nextc(tok);
236
0a92b6a1b9c0 Added recognition of 'l' or 'L' as long integer suffix
Guido van Rossum <guido@python.org>
parents: 145
diff changeset
1278 if (c == '+' || c == '-')
0a92b6a1b9c0 Added recognition of 'l' or 'L' as long integer suffix
Guido van Rossum <guido@python.org>
parents: 145
diff changeset
1279 c = tok_nextc(tok);
19501
ce4f1e98ba7e SF bug [#455775] float parsing discrepancy.
Tim Peters <tim.peters@gmail.com>
parents: 19051
diff changeset
1280 if (!isdigit(c)) {
ce4f1e98ba7e SF bug [#455775] float parsing discrepancy.
Tim Peters <tim.peters@gmail.com>
parents: 19051
diff changeset
1281 tok->done = E_TOKEN;
ce4f1e98ba7e SF bug [#455775] float parsing discrepancy.
Tim Peters <tim.peters@gmail.com>
parents: 19051
diff changeset
1282 tok_backup(tok, c);
ce4f1e98ba7e SF bug [#455775] float parsing discrepancy.
Tim Peters <tim.peters@gmail.com>
parents: 19051
diff changeset
1283 return ERRORTOKEN;
ce4f1e98ba7e SF bug [#455775] float parsing discrepancy.
Tim Peters <tim.peters@gmail.com>
parents: 19051
diff changeset
1284 }
ce4f1e98ba7e SF bug [#455775] float parsing discrepancy.
Tim Peters <tim.peters@gmail.com>
parents: 19051
diff changeset
1285 do {
236
0a92b6a1b9c0 Added recognition of 'l' or 'L' as long integer suffix
Guido van Rossum <guido@python.org>
parents: 145
diff changeset
1286 c = tok_nextc(tok);
19501
ce4f1e98ba7e SF bug [#455775] float parsing discrepancy.
Tim Peters <tim.peters@gmail.com>
parents: 19051
diff changeset
1287 } while (isdigit(c));
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1288 }
3333
4b5bff95804e changes for pow(**) and complex
Guido van Rossum <guido@python.org>
parents: 3116
diff changeset
1289 #ifndef WITHOUT_COMPLEX
3364
bc17e8313ea0 use only j for imaginary constants
Guido van Rossum <guido@python.org>
parents: 3333
diff changeset
1290 if (c == 'j' || c == 'J')
3333
4b5bff95804e changes for pow(**) and complex
Guido van Rossum <guido@python.org>
parents: 3116
diff changeset
1291 /* Imaginary part */
4b5bff95804e changes for pow(**) and complex
Guido van Rossum <guido@python.org>
parents: 3116
diff changeset
1292 imaginary:
4b5bff95804e changes for pow(**) and complex
Guido van Rossum <guido@python.org>
parents: 3116
diff changeset
1293 c = tok_nextc(tok);
4b5bff95804e changes for pow(**) and complex
Guido van Rossum <guido@python.org>
parents: 3116
diff changeset
1294 #endif
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1295 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1296 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1297 tok_backup(tok, c);
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1298 *p_start = tok->start;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1299 *p_end = tok->cur;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1300 return NUMBER;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1301 }
5265
3c1788ab9761 Support for alternative string quotes (a"xx", b"xx", c"xx", ...).
Guido van Rossum <guido@python.org>
parents: 4839
diff changeset
1302
3c1788ab9761 Support for alternative string quotes (a"xx", b"xx", c"xx", ...).
Guido van Rossum <guido@python.org>
parents: 4839
diff changeset
1303 letter_quote:
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1304 /* String */
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1305 if (c == '\'' || c == '"') {
7315
3a55fbb99f20 Fixed the bug in searching for triple quotes -- change the 'quote2'
Guido van Rossum <guido@python.org>
parents: 7118
diff changeset
1306 int quote2 = tok->cur - tok->start + 1;
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1307 int quote = c;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1308 int triple = 0;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1309 int tripcount = 0;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1310 for (;;) {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1311 c = tok_nextc(tok);
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1312 if (c == '\n') {
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1313 if (!triple) {
24839
17e95cc06f69 provide less mysterious error messages when seeing end-of-line in
Skip Montanaro <skip@pobox.com>
parents: 24630
diff changeset
1314 tok->done = E_EOLS;
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1315 tok_backup(tok, c);
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1316 return ERRORTOKEN;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1317 }
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1318 tripcount = 0;
25098
c02a668488d0 Ignore encoding declarations inside strings. Fixes #603509.
Martin v. Löwis <martin@v.loewis.de>
parents: 24867
diff changeset
1319 tok->cont_line = 1; /* multiline string. */
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1320 }
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1321 else if (c == EOF) {
24839
17e95cc06f69 provide less mysterious error messages when seeing end-of-line in
Skip Montanaro <skip@pobox.com>
parents: 24630
diff changeset
1322 if (triple)
17e95cc06f69 provide less mysterious error messages when seeing end-of-line in
Skip Montanaro <skip@pobox.com>
parents: 24630
diff changeset
1323 tok->done = E_EOFS;
17e95cc06f69 provide less mysterious error messages when seeing end-of-line in
Skip Montanaro <skip@pobox.com>
parents: 24630
diff changeset
1324 else
17e95cc06f69 provide less mysterious error messages when seeing end-of-line in
Skip Montanaro <skip@pobox.com>
parents: 24630
diff changeset
1325 tok->done = E_EOLS;
1282
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
1326 tok->cur = tok->inp;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1327 return ERRORTOKEN;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1328 }
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1329 else if (c == quote) {
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1330 tripcount++;
7315
3a55fbb99f20 Fixed the bug in searching for triple quotes -- change the 'quote2'
Guido van Rossum <guido@python.org>
parents: 7118
diff changeset
1331 if (tok->cur - tok->start == quote2) {
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1332 c = tok_nextc(tok);
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1333 if (c == quote) {
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1334 triple = 1;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1335 tripcount = 0;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1336 continue;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1337 }
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1338 tok_backup(tok, c);
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1339 }
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1340 if (!triple || tripcount == 3)
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1341 break;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1342 }
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1343 else if (c == '\\') {
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1344 tripcount = 0;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1345 c = tok_nextc(tok);
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1346 if (c == EOF) {
24839
17e95cc06f69 provide less mysterious error messages when seeing end-of-line in
Skip Montanaro <skip@pobox.com>
parents: 24630
diff changeset
1347 tok->done = E_EOLS;
1282
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
1348 tok->cur = tok->inp;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1349 return ERRORTOKEN;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1350 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1351 }
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1352 else
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1353 tripcount = 0;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1354 }
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1355 *p_start = tok->start;
1364
132602d854d8 Changes to accept double-quoted strings on input.
Guido van Rossum <guido@python.org>
parents: 1284
diff changeset
1356 *p_end = tok->cur;
132602d854d8 Changes to accept double-quoted strings on input.
Guido van Rossum <guido@python.org>
parents: 1284
diff changeset
1357 return STRING;
132602d854d8 Changes to accept double-quoted strings on input.
Guido van Rossum <guido@python.org>
parents: 1284
diff changeset
1358 }
132602d854d8 Changes to accept double-quoted strings on input.
Guido van Rossum <guido@python.org>
parents: 1284
diff changeset
1359
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1360 /* Line continuation */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1361 if (c == '\\') {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1362 c = tok_nextc(tok);
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1363 if (c != '\n') {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1364 tok->done = E_TOKEN;
1282
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
1365 tok->cur = tok->inp;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1366 return ERRORTOKEN;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1367 }
25098
c02a668488d0 Ignore encoding declarations inside strings. Fixes #603509.
Martin v. Löwis <martin@v.loewis.de>
parents: 24867
diff changeset
1368 tok->cont_line = 1;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1369 goto again; /* Read next line */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1370 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1371
400
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1372 /* Check for two-character token */
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1373 {
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1374 int c2 = tok_nextc(tok);
5379
1c9a84e0981d Another directory quickly renamed.
Guido van Rossum <guido@python.org>
parents: 5341
diff changeset
1375 int token = PyToken_TwoChars(c, c2);
400
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1376 if (token != OP) {
14163
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1377 int c3 = tok_nextc(tok);
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1378 int token3 = PyToken_ThreeChars(c, c2, c3);
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1379 if (token3 != OP) {
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1380 token = token3;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1381 } else {
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1382 tok_backup(tok, c3);
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1383 }
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1384 *p_start = tok->start;
400
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1385 *p_end = tok->cur;
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1386 return token;
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1387 }
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1388 tok_backup(tok, c2);
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1389 }
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1390
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1391 /* Keep track of parentheses nesting level */
1284
dbce72afe261 * selectmodule.c: fix (another!) two memory leaks -- this time in list2set
Guido van Rossum <guido@python.org>
parents: 1282
diff changeset
1392 switch (c) {
dbce72afe261 * selectmodule.c: fix (another!) two memory leaks -- this time in list2set
Guido van Rossum <guido@python.org>
parents: 1282
diff changeset
1393 case '(':
dbce72afe261 * selectmodule.c: fix (another!) two memory leaks -- this time in list2set
Guido van Rossum <guido@python.org>
parents: 1282
diff changeset
1394 case '[':
dbce72afe261 * selectmodule.c: fix (another!) two memory leaks -- this time in list2set
Guido van Rossum <guido@python.org>
parents: 1282
diff changeset
1395 case '{':
dbce72afe261 * selectmodule.c: fix (another!) two memory leaks -- this time in list2set
Guido van Rossum <guido@python.org>
parents: 1282
diff changeset
1396 tok->level++;
dbce72afe261 * selectmodule.c: fix (another!) two memory leaks -- this time in list2set
Guido van Rossum <guido@python.org>
parents: 1282
diff changeset
1397 break;
dbce72afe261 * selectmodule.c: fix (another!) two memory leaks -- this time in list2set
Guido van Rossum <guido@python.org>
parents: 1282
diff changeset
1398 case ')':
dbce72afe261 * selectmodule.c: fix (another!) two memory leaks -- this time in list2set
Guido van Rossum <guido@python.org>
parents: 1282
diff changeset
1399 case ']':
dbce72afe261 * selectmodule.c: fix (another!) two memory leaks -- this time in list2set
Guido van Rossum <guido@python.org>
parents: 1282
diff changeset
1400 case '}':
dbce72afe261 * selectmodule.c: fix (another!) two memory leaks -- this time in list2set
Guido van Rossum <guido@python.org>
parents: 1282
diff changeset
1401 tok->level--;
dbce72afe261 * selectmodule.c: fix (another!) two memory leaks -- this time in list2set
Guido van Rossum <guido@python.org>
parents: 1282
diff changeset
1402 break;
dbce72afe261 * selectmodule.c: fix (another!) two memory leaks -- this time in list2set
Guido van Rossum <guido@python.org>
parents: 1282
diff changeset
1403 }
dbce72afe261 * selectmodule.c: fix (another!) two memory leaks -- this time in list2set
Guido van Rossum <guido@python.org>
parents: 1282
diff changeset
1404
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1405 /* Punctuation character */
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1406 *p_start = tok->start;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1407 *p_end = tok->cur;
5379
1c9a84e0981d Another directory quickly renamed.
Guido van Rossum <guido@python.org>
parents: 5341
diff changeset
1408 return PyToken_OneChar(c);
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1409 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1410
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
1411 int
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
1412 PyTokenizer_Get(struct tok_state *tok, char **p_start, char **p_end)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
1413 {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
1414 int result = tok_get(tok, p_start, p_end);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
1415 if (tok->decoding_erred) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
1416 result = ERRORTOKEN;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
1417 tok->done = E_DECODE;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
1418 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
1419 return result;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
1420 }
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1421
4839
4567e158d20c Rename DEBUG macro to Py_DEBUG
Guido van Rossum <guido@python.org>
parents: 4577
diff changeset
1422 #ifdef Py_DEBUG
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1423
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1424 void
13659
e78153e179fb Mass ANSIfication.
Thomas Wouters <thomas@python.org>
parents: 13539
diff changeset
1425 tok_dump(int type, char *start, char *end)
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1426 {
5379
1c9a84e0981d Another directory quickly renamed.
Guido van Rossum <guido@python.org>
parents: 5341
diff changeset
1427 printf("%s", _PyParser_TokenNames[type]);
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1428 if (type == NAME || type == NUMBER || type == STRING || type == OP)
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1429 printf("(%.*s)", (int)(end - start), start);
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1430 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1431
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1432 #endif