Mercurial > cpython-lru_cache-weakref

annotate Parser/tokenizer.c @ 39042:c02fa187e0f4 legacy-trunk

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Part of SF patch #1484695. This removes dead code. The chksum was already verified in .frombuf() on the lines above. If there was a problem an exception is raised, so there was no way this condition could have been true.
author Neal Norwitz <nnorwitz@gmail.com>
date Mon, 10 Jul 2006 00:23:17 +0000
parents 92d8cdc4b90c
children 425127c1d113 5d83994cd8cd
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
145
853b736a6a9e Added copyright notice.
Guido van Rossum <guido@python.org>
parents: 96
diff changeset
1
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
2 /* Tokenizer implementation */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
3
23029
8291473fde3e Mass checkin of universal newline support.
Jack Jansen <jack.jansen@cwi.nl>
parents: 19621
diff changeset
4 #include "Python.h"
86
a6934380c6e7 "Compiling" version
Guido van Rossum <guido@python.org>
parents: 67
diff changeset
5 #include "pgenheaders.h"
a6934380c6e7 "Compiling" version
Guido van Rossum <guido@python.org>
parents: 67
diff changeset
6
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
7 #include <ctype.h>
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
8 #include <assert.h>
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
9
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
10 #include "tokenizer.h"
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
11 #include "errcode.h"
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
12
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
13 #ifndef PGEN
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
14 #include "unicodeobject.h"
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
15 #include "stringobject.h"
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
16 #include "fileobject.h"
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
17 #include "codecs.h"
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
18 #include "abstract.h"
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
19 #endif /* PGEN */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
20
25924
dc4a0336a2a3 Patch #512981: Update readline input stream on sys.stdin/out change.
Martin v. Löwis <martin@v.loewis.de>
parents: 25101
diff changeset
21 extern char *PyOS_Readline(FILE *, FILE *, char *);
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
22 /* Return malloc'ed string including trailing \n;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
23 empty malloc'ed string for EOF;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
24 NULL if interrupted */
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
25
678
205b947b1d3c Make tabs always 8 spaces wide -- it's more portable.
Guido van Rossum <guido@python.org>
parents: 477
diff changeset
26 /* Don't ever change this -- it would break the portability of Python code */
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
27 #define TABSIZE 8
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
28
7118
e9108e97a5a0 tok_nextc() should return unsigned characters, to avoid mistaking
Guido van Rossum <guido@python.org>
parents: 5379
diff changeset
29 /* Convert a possibly signed character to a nonnegative int */
e9108e97a5a0 tok_nextc() should return unsigned characters, to avoid mistaking
Guido van Rossum <guido@python.org>
parents: 5379
diff changeset
30 /* XXX This assumes characters are 8 bits wide */
e9108e97a5a0 tok_nextc() should return unsigned characters, to avoid mistaking
Guido van Rossum <guido@python.org>
parents: 5379
diff changeset
31 #ifdef __CHAR_UNSIGNED__
e9108e97a5a0 tok_nextc() should return unsigned characters, to avoid mistaking
Guido van Rossum <guido@python.org>
parents: 5379
diff changeset
32 #define Py_CHARMASK(c) (c)
e9108e97a5a0 tok_nextc() should return unsigned characters, to avoid mistaking
Guido van Rossum <guido@python.org>
parents: 5379
diff changeset
33 #else
e9108e97a5a0 tok_nextc() should return unsigned characters, to avoid mistaking
Guido van Rossum <guido@python.org>
parents: 5379
diff changeset
34 #define Py_CHARMASK(c) ((c) & 0xff)
e9108e97a5a0 tok_nextc() should return unsigned characters, to avoid mistaking
Guido van Rossum <guido@python.org>
parents: 5379
diff changeset
35 #endif
e9108e97a5a0 tok_nextc() should return unsigned characters, to avoid mistaking
Guido van Rossum <guido@python.org>
parents: 5379
diff changeset
36
86
a6934380c6e7 "Compiling" version
Guido van Rossum <guido@python.org>
parents: 67
diff changeset
37 /* Forward */
13447
a89f7f268de4 Nuke all remaining occurrences of Py_PROTO and Py_FPROTO.
Tim Peters <tim.peters@gmail.com>
parents: 13256
diff changeset
38 static struct tok_state *tok_new(void);
a89f7f268de4 Nuke all remaining occurrences of Py_PROTO and Py_FPROTO.
Tim Peters <tim.peters@gmail.com>
parents: 13256
diff changeset
39 static int tok_nextc(struct tok_state *tok);
a89f7f268de4 Nuke all remaining occurrences of Py_PROTO and Py_FPROTO.
Tim Peters <tim.peters@gmail.com>
parents: 13256
diff changeset
40 static void tok_backup(struct tok_state *tok, int c);
86
a6934380c6e7 "Compiling" version
Guido van Rossum <guido@python.org>
parents: 67
diff changeset
41
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
42 /* Token names */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
43
5379
1c9a84e0981d Another directory quickly renamed.
Guido van Rossum <guido@python.org>
parents: 5341
diff changeset
44 char *_PyParser_TokenNames[] = {
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
45 "ENDMARKER",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
46 "NAME",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
47 "NUMBER",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
48 "STRING",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
49 "NEWLINE",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
50 "INDENT",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
51 "DEDENT",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
52 "LPAR",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
53 "RPAR",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
54 "LSQB",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
55 "RSQB",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
56 "COLON",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
57 "COMMA",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
58 "SEMI",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
59 "PLUS",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
60 "MINUS",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
61 "STAR",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
62 "SLASH",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
63 "VBAR",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
64 "AMPER",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
65 "LESS",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
66 "GREATER",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
67 "EQUAL",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
68 "DOT",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
69 "PERCENT",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
70 "BACKQUOTE",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
71 "LBRACE",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
72 "RBRACE",
400
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
73 "EQEQUAL",
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
74 "NOTEQUAL",
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
75 "LESSEQUAL",
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
76 "GREATEREQUAL",
410
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
77 "TILDE",
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
78 "CIRCUMFLEX",
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
79 "LEFTSHIFT",
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
80 "RIGHTSHIFT",
3333
4b5bff95804e changes for pow(**) and complex
Guido van Rossum <guido@python.org>
parents: 3116
diff changeset
81 "DOUBLESTAR",
14163
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
82 "PLUSEQUAL",
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
83 "MINEQUAL",
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
84 "STAREQUAL",
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
85 "SLASHEQUAL",
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
86 "PERCENTEQUAL",
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
87 "AMPEREQUAL",
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
88 "VBAREQUAL",
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
89 "CIRCUMFLEXEQUAL",
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
90 "LEFTSHIFTEQUAL",
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
91 "RIGHTSHIFTEQUAL",
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
92 "DOUBLESTAREQUAL",
19051
f20727cc3d0f Implement PEP 238 in its (almost) full glory.
Guido van Rossum <guido@python.org>
parents: 17600
diff changeset
93 "DOUBLESLASH",
f20727cc3d0f Implement PEP 238 in its (almost) full glory.
Guido van Rossum <guido@python.org>
parents: 17600
diff changeset
94 "DOUBLESLASHEQUAL",
32888
c7e03cf8333e PEP-0318, @decorator-style. In Guido's words:
Anthony Baxter <anthonybaxter@gmail.com>
parents: 30979
diff changeset
95 "AT",
400
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
96 /* This table must match the #defines in token.h! */
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
97 "OP",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
98 "<ERRORTOKEN>",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
99 "<N_TOKENS>"
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
100 };
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
101
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
102
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
103 /* Create and initialize a new tok_state structure */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
104
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
105 static struct tok_state *
13659
e78153e179fb Mass ANSIfication.
Thomas Wouters <thomas@python.org>
parents: 13539
diff changeset
106 tok_new(void)
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
107 {
37585
292854c43b99 Fix the code in Parser/ to also compile with C++. This was mostly casts for
Anthony Baxter <anthonybaxter@gmail.com>
parents: 37557
diff changeset
108 struct tok_state *tok = (struct tok_state *)PyMem_MALLOC(
292854c43b99 Fix the code in Parser/ to also compile with C++. This was mostly casts for
Anthony Baxter <anthonybaxter@gmail.com>
parents: 37557
diff changeset
109 sizeof(struct tok_state));
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
110 if (tok == NULL)
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
111 return NULL;
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
112 tok->buf = tok->cur = tok->end = tok->inp = tok->start = NULL;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
113 tok->done = E_OK;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
114 tok->fp = NULL;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
115 tok->tabsize = TABSIZE;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
116 tok->indent = 0;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
117 tok->indstack[0] = 0;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
118 tok->atbol = 1;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
119 tok->pendin = 0;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
120 tok->prompt = tok->nextprompt = NULL;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
121 tok->lineno = 0;
1284
dbce72afe261 * selectmodule.c: fix (another!) two memory leaks -- this time in list2set
Guido van Rossum <guido@python.org>
parents: 1282
diff changeset
122 tok->level = 0;
7945
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
123 tok->filename = NULL;
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
124 tok->altwarning = 0;
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
125 tok->alterror = 0;
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
126 tok->alttabsize = 1;
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
127 tok->altindstack[0] = 0;
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
128 tok->decoding_state = 0;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
129 tok->decoding_erred = 0;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
130 tok->read_coding_spec = 0;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
131 tok->encoding = NULL;
25098
c02a668488d0 Ignore encoding declarations inside strings. Fixes #603509.
Martin v. Löwis <martin@v.loewis.de>
parents: 24867
diff changeset
132 tok->cont_line = 0;
24544
0bd521eb714e Make pgen compile with pydebug. Duplicate normalized names, as it may
Martin v. Löwis <martin@v.loewis.de>
parents: 24543
diff changeset
133 #ifndef PGEN
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
134 tok->decoding_readline = NULL;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
135 tok->decoding_buffer = NULL;
24544
0bd521eb714e Make pgen compile with pydebug. Duplicate normalized names, as it may
Martin v. Löwis <martin@v.loewis.de>
parents: 24543
diff changeset
136 #endif
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
137 return tok;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
138 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
139
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
140 #ifdef PGEN
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
141
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
142 static char *
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
143 decoding_fgets(char *s, int size, struct tok_state *tok)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
144 {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
145 return fgets(s, size, tok->fp);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
146 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
147
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
148 static int
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
149 decoding_feof(struct tok_state *tok)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
150 {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
151 return feof(tok->fp);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
152 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
153
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
154 static const char *
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
155 decode_str(const char *str, struct tok_state *tok)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
156 {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
157 return str;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
158 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
159
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
160 #else /* PGEN */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
161
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
162 static char *
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
163 error_ret(struct tok_state *tok) /* XXX */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
164 {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
165 tok->decoding_erred = 1;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
166 if (tok->fp != NULL && tok->buf != NULL) /* see PyTokenizer_Free */
37600
c4c5eeb07739 As discussed on python-dev, really fix the PyMem_*/PyObject_* memory API
Neal Norwitz <nnorwitz@gmail.com>
parents: 37585
diff changeset
167 PyMem_FREE(tok->buf);
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
168 tok->buf = NULL;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
169 return NULL; /* as if it were EOF */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
170 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
171
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
172 static char *
36501
7fdc639bc5b4 Merge ssize_t branch.
Martin v. Löwis <martin@v.loewis.de>
parents: 36090
diff changeset
173 new_string(const char *s, Py_ssize_t len)
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
174 {
37600
c4c5eeb07739 As discussed on python-dev, really fix the PyMem_*/PyObject_* memory API
Neal Norwitz <nnorwitz@gmail.com>
parents: 37585
diff changeset
175 char* result = (char *)PyMem_MALLOC(len + 1);
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
176 if (result != NULL) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
177 memcpy(result, s, len);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
178 result[len] = '\0';
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
179 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
180 return result;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
181 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
182
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
183 static char *
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
184 get_normal_name(char *s) /* for utf-8 and latin-1 */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
185 {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
186 char buf[13];
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
187 int i;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
188 for (i = 0; i < 12; i++) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
189 int c = s[i];
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
190 if (c == '\0') break;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
191 else if (c == '_') buf[i] = '-';
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
192 else buf[i] = tolower(c);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
193 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
194 buf[i] = '\0';
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
195 if (strcmp(buf, "utf-8") == 0 ||
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
196 strncmp(buf, "utf-8-", 6) == 0) return "utf-8";
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
197 else if (strcmp(buf, "latin-1") == 0 ||
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
198 strcmp(buf, "iso-8859-1") == 0 ||
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
199 strcmp(buf, "iso-latin-1") == 0 ||
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
200 strncmp(buf, "latin-1-", 8) == 0 ||
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
201 strncmp(buf, "iso-8859-1-", 11) == 0 ||
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
202 strncmp(buf, "iso-latin-1-", 12) == 0) return "iso-8859-1";
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
203 else return s;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
204 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
205
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
206 /* Return the coding spec in S, or NULL if none is found. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
207
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
208 static char *
36501
7fdc639bc5b4 Merge ssize_t branch.
Martin v. Löwis <martin@v.loewis.de>
parents: 36090
diff changeset
209 get_coding_spec(const char *s, Py_ssize_t size)
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
210 {
36501
7fdc639bc5b4 Merge ssize_t branch.
Martin v. Löwis <martin@v.loewis.de>
parents: 36090
diff changeset
211 Py_ssize_t i;
25098
c02a668488d0 Ignore encoding declarations inside strings. Fixes #603509.
Martin v. Löwis <martin@v.loewis.de>
parents: 24867
diff changeset
212 /* Coding spec must be in a comment, and that comment must be
c02a668488d0 Ignore encoding declarations inside strings. Fixes #603509.
Martin v. Löwis <martin@v.loewis.de>
parents: 24867
diff changeset
213 * the only statement on the source code line. */
c02a668488d0 Ignore encoding declarations inside strings. Fixes #603509.
Martin v. Löwis <martin@v.loewis.de>
parents: 24867
diff changeset
214 for (i = 0; i < size - 6; i++) {
c02a668488d0 Ignore encoding declarations inside strings. Fixes #603509.
Martin v. Löwis <martin@v.loewis.de>
parents: 24867
diff changeset
215 if (s[i] == '#')
c02a668488d0 Ignore encoding declarations inside strings. Fixes #603509.
Martin v. Löwis <martin@v.loewis.de>
parents: 24867
diff changeset
216 break;
c02a668488d0 Ignore encoding declarations inside strings. Fixes #603509.
Martin v. Löwis <martin@v.loewis.de>
parents: 24867
diff changeset
217 if (s[i] != ' ' && s[i] != '\t' && s[i] != '\014')
c02a668488d0 Ignore encoding declarations inside strings. Fixes #603509.
Martin v. Löwis <martin@v.loewis.de>
parents: 24867
diff changeset
218 return NULL;
c02a668488d0 Ignore encoding declarations inside strings. Fixes #603509.
Martin v. Löwis <martin@v.loewis.de>
parents: 24867
diff changeset
219 }
c02a668488d0 Ignore encoding declarations inside strings. Fixes #603509.
Martin v. Löwis <martin@v.loewis.de>
parents: 24867
diff changeset
220 for (; i < size - 6; i++) { /* XXX inefficient search */
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
221 const char* t = s + i;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
222 if (strncmp(t, "coding", 6) == 0) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
223 const char* begin = NULL;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
224 t += 6;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
225 if (t[0] != ':' && t[0] != '=')
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
226 continue;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
227 do {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
228 t++;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
229 } while (t[0] == '\x20' || t[0] == '\t');
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
230
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
231 begin = t;
36090
8cb686dd6e64 Fix SF bug #1072182, problems with signed characters.
Neal Norwitz <nnorwitz@gmail.com>
parents: 36078
diff changeset
232 while (isalnum(Py_CHARMASK(t[0])) ||
25995
1aaa12c08b35 Fix compiler warning on HP-UX.
Neal Norwitz <nnorwitz@gmail.com>
parents: 25924
diff changeset
233 t[0] == '-' || t[0] == '_' || t[0] == '.')
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
234 t++;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
235
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
236 if (begin < t) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
237 char* r = new_string(begin, t - begin);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
238 char* q = get_normal_name(r);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
239 if (r != q) {
37600
c4c5eeb07739 As discussed on python-dev, really fix the PyMem_*/PyObject_* memory API
Neal Norwitz <nnorwitz@gmail.com>
parents: 37585
diff changeset
240 PyMem_FREE(r);
24544
0bd521eb714e Make pgen compile with pydebug. Duplicate normalized names, as it may
Martin v. Löwis <martin@v.loewis.de>
parents: 24543
diff changeset
241 r = new_string(q, strlen(q));
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
242 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
243 return r;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
244 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
245 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
246 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
247 return NULL;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
248 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
249
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
250 /* Check whether the line contains a coding spec. If it does,
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
251 invoke the set_readline function for the new encoding.
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
252 This function receives the tok_state and the new encoding.
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
253 Return 1 on success, 0 on failure. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
254
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
255 static int
36501
7fdc639bc5b4 Merge ssize_t branch.
Martin v. Löwis <martin@v.loewis.de>
parents: 36090
diff changeset
256 check_coding_spec(const char* line, Py_ssize_t size, struct tok_state *tok,
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
257 int set_readline(struct tok_state *, const char *))
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
258 {
25101
32331d26d7cf Removed reliance on gcc/C99 extension.
Tim Peters <tim.peters@gmail.com>
parents: 25098
diff changeset
259 char * cs;
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
260 int r = 1;
25101
32331d26d7cf Removed reliance on gcc/C99 extension.
Tim Peters <tim.peters@gmail.com>
parents: 25098
diff changeset
261
25098
c02a668488d0 Ignore encoding declarations inside strings. Fixes #603509.
Martin v. Löwis <martin@v.loewis.de>
parents: 24867
diff changeset
262 if (tok->cont_line)
c02a668488d0 Ignore encoding declarations inside strings. Fixes #603509.
Martin v. Löwis <martin@v.loewis.de>
parents: 24867
diff changeset
263 /* It's a continuation line, so it can't be a coding spec. */
c02a668488d0 Ignore encoding declarations inside strings. Fixes #603509.
Martin v. Löwis <martin@v.loewis.de>
parents: 24867
diff changeset
264 return 1;
25101
32331d26d7cf Removed reliance on gcc/C99 extension.
Tim Peters <tim.peters@gmail.com>
parents: 25098
diff changeset
265 cs = get_coding_spec(line, size);
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
266 if (cs != NULL) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
267 tok->read_coding_spec = 1;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
268 if (tok->encoding == NULL) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
269 assert(tok->decoding_state == 1); /* raw */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
270 if (strcmp(cs, "utf-8") == 0 ||
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
271 strcmp(cs, "iso-8859-1") == 0) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
272 tok->encoding = cs;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
273 } else {
24624
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
274 #ifdef Py_USING_UNICODE
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
275 r = set_readline(tok, cs);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
276 if (r) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
277 tok->encoding = cs;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
278 tok->decoding_state = -1;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
279 }
35751
e56144704808 Free coding spec (cs) if there was an error to prevent mem leak. Maybe backport candidate
Neal Norwitz <nnorwitz@gmail.com>
parents: 35666
diff changeset
280 else
37600
c4c5eeb07739 As discussed on python-dev, really fix the PyMem_*/PyObject_* memory API
Neal Norwitz <nnorwitz@gmail.com>
parents: 37585
diff changeset
281 PyMem_FREE(cs);
24624
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
282 #else
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
283 /* Without Unicode support, we cannot
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
284 process the coding spec. Since there
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
285 won't be any Unicode literals, that
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
286 won't matter. */
37600
c4c5eeb07739 As discussed on python-dev, really fix the PyMem_*/PyObject_* memory API
Neal Norwitz <nnorwitz@gmail.com>
parents: 37585
diff changeset
287 PyMem_FREE(cs);
24624
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
288 #endif
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
289 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
290 } else { /* then, compare cs with BOM */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
291 r = (strcmp(tok->encoding, cs) == 0);
37600
c4c5eeb07739 As discussed on python-dev, really fix the PyMem_*/PyObject_* memory API
Neal Norwitz <nnorwitz@gmail.com>
parents: 37585
diff changeset
292 PyMem_FREE(cs);
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
293 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
294 }
36078
1ff4790b7605 Fix Bug #1378022, UTF-8 files with a leading BOM crashed the interpreter.
Neal Norwitz <nnorwitz@gmail.com>
parents: 35878
diff changeset
295 if (!r) {
1ff4790b7605 Fix Bug #1378022, UTF-8 files with a leading BOM crashed the interpreter.
Neal Norwitz <nnorwitz@gmail.com>
parents: 35878
diff changeset
296 cs = tok->encoding;
1ff4790b7605 Fix Bug #1378022, UTF-8 files with a leading BOM crashed the interpreter.
Neal Norwitz <nnorwitz@gmail.com>
parents: 35878
diff changeset
297 if (!cs)
1ff4790b7605 Fix Bug #1378022, UTF-8 files with a leading BOM crashed the interpreter.
Neal Norwitz <nnorwitz@gmail.com>
parents: 35878
diff changeset
298 cs = "with BOM";
1ff4790b7605 Fix Bug #1378022, UTF-8 files with a leading BOM crashed the interpreter.
Neal Norwitz <nnorwitz@gmail.com>
parents: 35878
diff changeset
299 PyErr_Format(PyExc_SyntaxError, "encoding problem: %s", cs);
1ff4790b7605 Fix Bug #1378022, UTF-8 files with a leading BOM crashed the interpreter.
Neal Norwitz <nnorwitz@gmail.com>
parents: 35878
diff changeset
300 }
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
301 return r;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
302 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
303
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
304 /* See whether the file starts with a BOM. If it does,
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
305 invoke the set_readline function with the new encoding.
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
306 Return 1 on success, 0 on failure. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
307
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
308 static int
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
309 check_bom(int get_char(struct tok_state *),
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
310 void unget_char(int, struct tok_state *),
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
311 int set_readline(struct tok_state *, const char *),
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
312 struct tok_state *tok)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
313 {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
314 int ch = get_char(tok);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
315 tok->decoding_state = 1;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
316 if (ch == EOF) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
317 return 1;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
318 } else if (ch == 0xEF) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
319 ch = get_char(tok); if (ch != 0xBB) goto NON_BOM;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
320 ch = get_char(tok); if (ch != 0xBF) goto NON_BOM;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
321 #if 0
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
322 /* Disable support for UTF-16 BOMs until a decision
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
323 is made whether this needs to be supported. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
324 } else if (ch == 0xFE) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
325 ch = get_char(tok); if (ch != 0xFF) goto NON_BOM;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
326 if (!set_readline(tok, "utf-16-be")) return 0;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
327 tok->decoding_state = -1;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
328 } else if (ch == 0xFF) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
329 ch = get_char(tok); if (ch != 0xFE) goto NON_BOM;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
330 if (!set_readline(tok, "utf-16-le")) return 0;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
331 tok->decoding_state = -1;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
332 #endif
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
333 } else {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
334 unget_char(ch, tok);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
335 return 1;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
336 }
35878
94a409da4bd4 Fix some more memory leaks.
Neal Norwitz <nnorwitz@gmail.com>
parents: 35751
diff changeset
337 if (tok->encoding != NULL)
37600
c4c5eeb07739 As discussed on python-dev, really fix the PyMem_*/PyObject_* memory API
Neal Norwitz <nnorwitz@gmail.com>
parents: 37585
diff changeset
338 PyMem_FREE(tok->encoding);
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
339 tok->encoding = new_string("utf-8", 5); /* resulting is in utf-8 */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
340 return 1;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
341 NON_BOM:
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
342 /* any token beginning with '\xEF', '\xFE', '\xFF' is a bad token */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
343 unget_char(0xFF, tok); /* XXX this will cause a syntax error */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
344 return 1;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
345 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
346
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
347 /* Read a line of text from TOK into S, using the stream in TOK.
35231
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
348 Return NULL on failure, else S.
37229
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
349
35231
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
350 On entry, tok->decoding_buffer will be one of:
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
351 1) NULL: need to call tok->decoding_readline to get a new line
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
352 2) PyUnicodeObject *: decoding_feof has called tok->decoding_readline and
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
353 stored the result in tok->decoding_buffer
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
354 3) PyStringObject *: previous call to fp_readl did not have enough room
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
355 (in the s buffer) to copy entire contents of the line read
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
356 by tok->decoding_readline. tok->decoding_buffer has the overflow.
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
357 In this case, fp_readl is called in a loop (with an expanded buffer)
37229
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
358 until the buffer ends with a '\n' (or until the end of the file is
35231
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
359 reached): see tok_nextc and its calls to decoding_fgets.
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
360 */
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
361
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
362 static char *
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
363 fp_readl(char *s, int size, struct tok_state *tok)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
364 {
24624
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
365 #ifndef Py_USING_UNICODE
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
366 /* In a non-Unicode built, this should never be called. */
24630
43caa212b959 Use Py_FatalError instead of abort.
Martin v. Löwis <martin@v.loewis.de>
parents: 24624
diff changeset
367 Py_FatalError("fp_readl should not be called in this build.");
24867
255d1d3e66a3 Squash a few calls to the hideously expensive PyObject_CallObject(o,a)
Guido van Rossum <guido@python.org>
parents: 24839
diff changeset
368 return NULL; /* Keep compiler happy (not reachable) */
24624
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
369 #else
35231
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
370 PyObject* utf8 = NULL;
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
371 PyObject* buf = tok->decoding_buffer;
35231
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
372 char *str;
36520
26470cdeea3d Use Py_ssize_t to count the length.
Martin v. Löwis <martin@v.loewis.de>
parents: 36501
diff changeset
373 Py_ssize_t utf8len;
35231
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
374
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
375 /* Ask for one less byte so we can terminate it */
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
376 assert(size > 0);
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
377 size--;
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
378
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
379 if (buf == NULL) {
35231
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
380 buf = PyObject_CallObject(tok->decoding_readline, NULL);
24543
c00df23e856e Group statements properly.
Martin v. Löwis <martin@v.loewis.de>
parents: 24542
diff changeset
381 if (buf == NULL)
c00df23e856e Group statements properly.
Martin v. Löwis <martin@v.loewis.de>
parents: 24542
diff changeset
382 return error_ret(tok);
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
383 } else {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
384 tok->decoding_buffer = NULL;
35231
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
385 if (PyString_CheckExact(buf))
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
386 utf8 = buf;
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
387 }
35231
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
388 if (utf8 == NULL) {
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
389 utf8 = PyUnicode_AsUTF8String(buf);
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
390 Py_DECREF(buf);
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
391 if (utf8 == NULL)
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
392 return error_ret(tok);
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
393 }
35231
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
394 str = PyString_AsString(utf8);
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
395 utf8len = PyString_GET_SIZE(utf8);
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
396 if (utf8len > size) {
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
397 tok->decoding_buffer = PyString_FromStringAndSize(str+size, utf8len-size);
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
398 if (tok->decoding_buffer == NULL) {
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
399 Py_DECREF(utf8);
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
400 return error_ret(tok);
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
401 }
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
402 utf8len = size;
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
403 }
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
404 memcpy(s, str, utf8len);
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
405 s[utf8len] = '\0';
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
406 Py_DECREF(utf8);
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
407 if (utf8len == 0) return NULL; /* EOF */
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
408 return s;
24624
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
409 #endif
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
410 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
411
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
412 /* Set the readline function for TOK to a StreamReader's
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
413 readline function. The StreamReader is named ENC.
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
414
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
415 This function is called from check_bom and check_coding_spec.
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
416
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
417 ENC is usually identical to the future value of tok->encoding,
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
418 except for the (currently unsupported) case of UTF-16.
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
419
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
420 Return 1 on success, 0 on failure. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
421
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
422 static int
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
423 fp_setreadl(struct tok_state *tok, const char* enc)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
424 {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
425 PyObject *reader, *stream, *readline;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
426
26601
cc05c2c8660e Constify filenames and scripts. Fixes #651362.
Martin v. Löwis <martin@v.loewis.de>
parents: 25995
diff changeset
427 /* XXX: constify filename argument. */
cc05c2c8660e Constify filenames and scripts. Fixes #651362.
Martin v. Löwis <martin@v.loewis.de>
parents: 25995
diff changeset
428 stream = PyFile_FromFile(tok->fp, (char*)tok->filename, "rb", NULL);
24543
c00df23e856e Group statements properly.
Martin v. Löwis <martin@v.loewis.de>
parents: 24542
diff changeset
429 if (stream == NULL)
c00df23e856e Group statements properly.
Martin v. Löwis <martin@v.loewis.de>
parents: 24542
diff changeset
430 return 0;
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
431
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
432 reader = PyCodec_StreamReader(enc, stream, NULL);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
433 Py_DECREF(stream);
24543
c00df23e856e Group statements properly.
Martin v. Löwis <martin@v.loewis.de>
parents: 24542
diff changeset
434 if (reader == NULL)
c00df23e856e Group statements properly.
Martin v. Löwis <martin@v.loewis.de>
parents: 24542
diff changeset
435 return 0;
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
436
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
437 readline = PyObject_GetAttrString(reader, "readline");
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
438 Py_DECREF(reader);
24543
c00df23e856e Group statements properly.
Martin v. Löwis <martin@v.loewis.de>
parents: 24542
diff changeset
439 if (readline == NULL)
c00df23e856e Group statements properly.
Martin v. Löwis <martin@v.loewis.de>
parents: 24542
diff changeset
440 return 0;
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
441
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
442 tok->decoding_readline = readline;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
443 return 1;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
444 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
445
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
446 /* Fetch the next byte from TOK. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
447
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
448 static int fp_getc(struct tok_state *tok) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
449 return getc(tok->fp);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
450 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
451
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
452 /* Unfetch the last byte back into TOK. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
453
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
454 static void fp_ungetc(int c, struct tok_state *tok) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
455 ungetc(c, tok->fp);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
456 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
457
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
458 /* Read a line of input from TOK. Determine encoding
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
459 if necessary. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
460
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
461 static char *
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
462 decoding_fgets(char *s, int size, struct tok_state *tok)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
463 {
24630
43caa212b959 Use Py_FatalError instead of abort.
Martin v. Löwis <martin@v.loewis.de>
parents: 24624
diff changeset
464 char *line = NULL;
36741
c0982b8155c6 Change non-ASCII warning into a SyntaxError.
Martin v. Löwis <martin@v.loewis.de>
parents: 36520
diff changeset
465 int badchar = 0;
24543
c00df23e856e Group statements properly.
Martin v. Löwis <martin@v.loewis.de>
parents: 24542
diff changeset
466 for (;;) {
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
467 if (tok->decoding_state < 0) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
468 /* We already have a codec associated with
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
469 this input. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
470 line = fp_readl(s, size, tok);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
471 break;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
472 } else if (tok->decoding_state > 0) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
473 /* We want a 'raw' read. */
37229
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
474 line = Py_UniversalNewlineFgets(s, size,
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
475 tok->fp, NULL);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
476 break;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
477 } else {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
478 /* We have not yet determined the encoding.
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
479 If an encoding is found, use the file-pointer
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
480 reader functions from now on. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
481 if (!check_bom(fp_getc, fp_ungetc, fp_setreadl, tok))
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
482 return error_ret(tok);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
483 assert(tok->decoding_state != 0);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
484 }
24543
c00df23e856e Group statements properly.
Martin v. Löwis <martin@v.loewis.de>
parents: 24542
diff changeset
485 }
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
486 if (line != NULL && tok->lineno < 2 && !tok->read_coding_spec) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
487 if (!check_coding_spec(line, strlen(line), tok, fp_setreadl)) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
488 return error_ret(tok);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
489 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
490 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
491 #ifndef PGEN
36741
c0982b8155c6 Change non-ASCII warning into a SyntaxError.
Martin v. Löwis <martin@v.loewis.de>
parents: 36520
diff changeset
492 /* The default encoding is ASCII, so make sure we don't have any
c0982b8155c6 Change non-ASCII warning into a SyntaxError.
Martin v. Löwis <martin@v.loewis.de>
parents: 36520
diff changeset
493 non-ASCII bytes in it. */
c0982b8155c6 Change non-ASCII warning into a SyntaxError.
Martin v. Löwis <martin@v.loewis.de>
parents: 36520
diff changeset
494 if (line && !tok->encoding) {
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
495 unsigned char *c;
24565
c8d694b641f1 Added a cast to shut up a compiler warning.
Jack Jansen <jack.jansen@cwi.nl>
parents: 24558
diff changeset
496 for (c = (unsigned char *)line; *c; c++)
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
497 if (*c > 127) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
498 badchar = *c;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
499 break;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
500 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
501 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
502 if (badchar) {
27918
be371ffc4cd0 Add URL for PEP to the source code encoding warning.
Marc-André Lemburg <mal@egenix.com>
parents: 27772
diff changeset
503 char buf[500];
24558
d26b62f0c6db Add 1 to lineno in deprecation warning. Fixes #590888.
Martin v. Löwis <martin@v.loewis.de>
parents: 24544
diff changeset
504 /* Need to add 1 to the line number, since this line
d26b62f0c6db Add 1 to lineno in deprecation warning. Fixes #590888.
Martin v. Löwis <martin@v.loewis.de>
parents: 24544
diff changeset
505 has not been counted, yet. */
37229
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
506 sprintf(buf,
27918
be371ffc4cd0 Add URL for PEP to the source code encoding warning.
Marc-André Lemburg <mal@egenix.com>
parents: 27772
diff changeset
507 "Non-ASCII character '\\x%.2x' "
be371ffc4cd0 Add URL for PEP to the source code encoding warning.
Marc-André Lemburg <mal@egenix.com>
parents: 27772
diff changeset
508 "in file %.200s on line %i, "
be371ffc4cd0 Add URL for PEP to the source code encoding warning.
Marc-André Lemburg <mal@egenix.com>
parents: 27772
diff changeset
509 "but no encoding declared; "
37229
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
510 "see http://www.python.org/peps/pep-0263.html for details",
27918
be371ffc4cd0 Add URL for PEP to the source code encoding warning.
Marc-André Lemburg <mal@egenix.com>
parents: 27772
diff changeset
511 badchar, tok->filename, tok->lineno + 1);
36741
c0982b8155c6 Change non-ASCII warning into a SyntaxError.
Martin v. Löwis <martin@v.loewis.de>
parents: 36520
diff changeset
512 PyErr_SetString(PyExc_SyntaxError, buf);
c0982b8155c6 Change non-ASCII warning into a SyntaxError.
Martin v. Löwis <martin@v.loewis.de>
parents: 36520
diff changeset
513 return error_ret(tok);
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
514 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
515 #endif
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
516 return line;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
517 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
518
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
519 static int
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
520 decoding_feof(struct tok_state *tok)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
521 {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
522 if (tok->decoding_state >= 0) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
523 return feof(tok->fp);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
524 } else {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
525 PyObject* buf = tok->decoding_buffer;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
526 if (buf == NULL) {
35231
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
527 buf = PyObject_CallObject(tok->decoding_readline, NULL);
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
528 if (buf == NULL) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
529 error_ret(tok);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
530 return 1;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
531 } else {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
532 tok->decoding_buffer = buf;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
533 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
534 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
535 return PyObject_Length(buf) == 0;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
536 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
537 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
538
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
539 /* Fetch a byte from TOK, using the string buffer. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
540
37229
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
541 static int
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
542 buf_getc(struct tok_state *tok) {
27772
3566b84e66b9 patch 680474 that fixes bug 679880: compile/eval/exec refused utf-8 bom
Just van Rossum <just@letterror.com>
parents: 27247
diff changeset
543 return Py_CHARMASK(*tok->str++);
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
544 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
545
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
546 /* Unfetch a byte from TOK, using the string buffer. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
547
37229
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
548 static void
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
549 buf_ungetc(int c, struct tok_state *tok) {
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
550 tok->str--;
27772
3566b84e66b9 patch 680474 that fixes bug 679880: compile/eval/exec refused utf-8 bom
Just van Rossum <just@letterror.com>
parents: 27247
diff changeset
551 assert(Py_CHARMASK(*tok->str) == c); /* tok->cur may point to read-only segment */
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
552 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
553
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
554 /* Set the readline function for TOK to ENC. For the string-based
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
555 tokenizer, this means to just record the encoding. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
556
37229
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
557 static int
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
558 buf_setreadl(struct tok_state *tok, const char* enc) {
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
559 tok->enc = enc;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
560 return 1;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
561 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
562
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
563 /* Return a UTF-8 encoding Python string object from the
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
564 C byte string STR, which is encoded with ENC. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
565
24624
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
566 #ifdef Py_USING_UNICODE
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
567 static PyObject *
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
568 translate_into_utf8(const char* str, const char* enc) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
569 PyObject *utf8;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
570 PyObject* buf = PyUnicode_Decode(str, strlen(str), enc, NULL);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
571 if (buf == NULL)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
572 return NULL;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
573 utf8 = PyUnicode_AsUTF8String(buf);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
574 Py_DECREF(buf);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
575 return utf8;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
576 }
24624
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
577 #endif
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
578
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
579 /* Decode a byte string STR for use as the buffer of TOK.
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
580 Look for encoding declarations inside STR, and record them
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
581 inside TOK. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
582
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
583 static const char *
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
584 decode_str(const char *str, struct tok_state *tok)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
585 {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
586 PyObject* utf8 = NULL;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
587 const char *s;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
588 int lineno = 0;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
589 tok->enc = NULL;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
590 tok->str = str;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
591 if (!check_bom(buf_getc, buf_ungetc, buf_setreadl, tok))
35878
94a409da4bd4 Fix some more memory leaks.
Neal Norwitz <nnorwitz@gmail.com>
parents: 35751
diff changeset
592 return error_ret(tok);
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
593 str = tok->str; /* string after BOM if any */
24542
d0d3f760d067 Repaired a fatal compiler error in the debug build: it's not clear what
Tim Peters <tim.peters@gmail.com>
parents: 24541
diff changeset
594 assert(str);
24624
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
595 #ifdef Py_USING_UNICODE
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
596 if (tok->enc != NULL) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
597 utf8 = translate_into_utf8(str, tok->enc);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
598 if (utf8 == NULL)
35878
94a409da4bd4 Fix some more memory leaks.
Neal Norwitz <nnorwitz@gmail.com>
parents: 35751
diff changeset
599 return error_ret(tok);
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
600 str = PyString_AsString(utf8);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
601 }
24624
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
602 #endif
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
603 for (s = str;; s++) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
604 if (*s == '\0') break;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
605 else if (*s == '\n') {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
606 lineno++;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
607 if (lineno == 2) break;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
608 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
609 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
610 tok->enc = NULL;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
611 if (!check_coding_spec(str, s - str, tok, buf_setreadl))
35878
94a409da4bd4 Fix some more memory leaks.
Neal Norwitz <nnorwitz@gmail.com>
parents: 35751
diff changeset
612 return error_ret(tok);
24624
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
613 #ifdef Py_USING_UNICODE
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
614 if (tok->enc != NULL) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
615 assert(utf8 == NULL);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
616 utf8 = translate_into_utf8(str, tok->enc);
35666
b8e571e6de2e - Fix segfault with invalid coding.
Neal Norwitz <nnorwitz@gmail.com>
parents: 35231
diff changeset
617 if (utf8 == NULL) {
b8e571e6de2e - Fix segfault with invalid coding.
Neal Norwitz <nnorwitz@gmail.com>
parents: 35231
diff changeset
618 PyErr_Format(PyExc_SyntaxError,
b8e571e6de2e - Fix segfault with invalid coding.
Neal Norwitz <nnorwitz@gmail.com>
parents: 35231
diff changeset
619 "unknown encoding: %s", tok->enc);
35878
94a409da4bd4 Fix some more memory leaks.
Neal Norwitz <nnorwitz@gmail.com>
parents: 35751
diff changeset
620 return error_ret(tok);
35666
b8e571e6de2e - Fix segfault with invalid coding.
Neal Norwitz <nnorwitz@gmail.com>
parents: 35231
diff changeset
621 }
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
622 str = PyString_AsString(utf8);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
623 }
24624
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
624 #endif
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
625 assert(tok->decoding_buffer == NULL);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
626 tok->decoding_buffer = utf8; /* CAUTION */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
627 return str;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
628 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
629
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
630 #endif /* PGEN */
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
631
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
632 /* Set up tokenizer for string */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
633
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
634 struct tok_state *
26601
cc05c2c8660e Constify filenames and scripts. Fixes #651362.
Martin v. Löwis <martin@v.loewis.de>
parents: 25995
diff changeset
635 PyTokenizer_FromString(const char *str)
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
636 {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
637 struct tok_state *tok = tok_new();
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
638 if (tok == NULL)
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
639 return NULL;
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
640 str = (char *)decode_str(str, tok);
35878
94a409da4bd4 Fix some more memory leaks.
Neal Norwitz <nnorwitz@gmail.com>
parents: 35751
diff changeset
641 if (str == NULL) {
94a409da4bd4 Fix some more memory leaks.
Neal Norwitz <nnorwitz@gmail.com>
parents: 35751
diff changeset
642 PyTokenizer_Free(tok);
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
643 return NULL;
35878
94a409da4bd4 Fix some more memory leaks.
Neal Norwitz <nnorwitz@gmail.com>
parents: 35751
diff changeset
644 }
94a409da4bd4 Fix some more memory leaks.
Neal Norwitz <nnorwitz@gmail.com>
parents: 35751
diff changeset
645
26601
cc05c2c8660e Constify filenames and scripts. Fixes #651362.
Martin v. Löwis <martin@v.loewis.de>
parents: 25995
diff changeset
646 /* XXX: constify members. */
cc05c2c8660e Constify filenames and scripts. Fixes #651362.
Martin v. Löwis <martin@v.loewis.de>
parents: 25995
diff changeset
647 tok->buf = tok->cur = tok->end = tok->inp = (char*)str;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
648 return tok;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
649 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
650
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
651
326
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
652 /* Set up tokenizer for file */
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
653
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
654 struct tok_state *
13659
e78153e179fb Mass ANSIfication.
Thomas Wouters <thomas@python.org>
parents: 13539
diff changeset
655 PyTokenizer_FromFile(FILE *fp, char *ps1, char *ps2)
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
656 {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
657 struct tok_state *tok = tok_new();
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
658 if (tok == NULL)
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
659 return NULL;
37600
c4c5eeb07739 As discussed on python-dev, really fix the PyMem_*/PyObject_* memory API
Neal Norwitz <nnorwitz@gmail.com>
parents: 37585
diff changeset
660 if ((tok->buf = (char *)PyMem_MALLOC(BUFSIZ)) == NULL) {
35878
94a409da4bd4 Fix some more memory leaks.
Neal Norwitz <nnorwitz@gmail.com>
parents: 35751
diff changeset
661 PyTokenizer_Free(tok);
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
662 return NULL;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
663 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
664 tok->cur = tok->inp = tok->buf;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
665 tok->end = tok->buf + BUFSIZ;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
666 tok->fp = fp;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
667 tok->prompt = ps1;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
668 tok->nextprompt = ps2;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
669 return tok;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
670 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
671
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
672
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
673 /* Free a tok_state structure */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
674
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
675 void
13659
e78153e179fb Mass ANSIfication.
Thomas Wouters <thomas@python.org>
parents: 13539
diff changeset
676 PyTokenizer_Free(struct tok_state *tok)
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
677 {
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
678 if (tok->encoding != NULL)
37600
c4c5eeb07739 As discussed on python-dev, really fix the PyMem_*/PyObject_* memory API
Neal Norwitz <nnorwitz@gmail.com>
parents: 37585
diff changeset
679 PyMem_FREE(tok->encoding);
24544
0bd521eb714e Make pgen compile with pydebug. Duplicate normalized names, as it may
Martin v. Löwis <martin@v.loewis.de>
parents: 24543
diff changeset
680 #ifndef PGEN
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
681 Py_XDECREF(tok->decoding_readline);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
682 Py_XDECREF(tok->decoding_buffer);
24544
0bd521eb714e Make pgen compile with pydebug. Duplicate normalized names, as it may
Martin v. Löwis <martin@v.loewis.de>
parents: 24543
diff changeset
683 #endif
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
684 if (tok->fp != NULL && tok->buf != NULL)
37600
c4c5eeb07739 As discussed on python-dev, really fix the PyMem_*/PyObject_* memory API
Neal Norwitz <nnorwitz@gmail.com>
parents: 37585
diff changeset
685 PyMem_FREE(tok->buf);
37229
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
686 PyMem_FREE(tok);
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
687 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
688
32961
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
689 #if !defined(PGEN) && defined(Py_USING_UNICODE)
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
690 static int
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
691 tok_stdin_decode(struct tok_state *tok, char **inp)
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
692 {
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
693 PyObject *enc, *sysstdin, *decoded, *utf8;
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
694 const char *encoding;
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
695 char *converted;
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
696
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
697 if (PySys_GetFile((char *)"stdin", NULL) != stdin)
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
698 return 0;
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
699 sysstdin = PySys_GetObject("stdin");
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
700 if (sysstdin == NULL || !PyFile_Check(sysstdin))
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
701 return 0;
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
702
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
703 enc = ((PyFileObject *)sysstdin)->f_encoding;
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
704 if (enc == NULL || !PyString_Check(enc))
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
705 return 0;
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
706 Py_INCREF(enc);
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
707
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
708 encoding = PyString_AsString(enc);
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
709 decoded = PyUnicode_Decode(*inp, strlen(*inp), encoding, NULL);
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
710 if (decoded == NULL)
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
711 goto error_clear;
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
712
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
713 utf8 = PyUnicode_AsEncodedString(decoded, "utf-8", NULL);
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
714 Py_DECREF(decoded);
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
715 if (utf8 == NULL)
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
716 goto error_clear;
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
717
37086
3b7b295cca90 Use macro versions instead of function versions when we already know the type.
Neal Norwitz <nnorwitz@gmail.com>
parents: 36809
diff changeset
718 assert(PyString_Check(utf8));
3b7b295cca90 Use macro versions instead of function versions when we already know the type.
Neal Norwitz <nnorwitz@gmail.com>
parents: 36809
diff changeset
719 converted = new_string(PyString_AS_STRING(utf8),
3b7b295cca90 Use macro versions instead of function versions when we already know the type.
Neal Norwitz <nnorwitz@gmail.com>
parents: 36809
diff changeset
720 PyString_GET_SIZE(utf8));
32961
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
721 Py_DECREF(utf8);
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
722 if (converted == NULL)
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
723 goto error_nomem;
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
724
37600
c4c5eeb07739 As discussed on python-dev, really fix the PyMem_*/PyObject_* memory API
Neal Norwitz <nnorwitz@gmail.com>
parents: 37585
diff changeset
725 PyMem_FREE(*inp);
32961
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
726 *inp = converted;
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
727 if (tok->encoding != NULL)
37600
c4c5eeb07739 As discussed on python-dev, really fix the PyMem_*/PyObject_* memory API
Neal Norwitz <nnorwitz@gmail.com>
parents: 37585
diff changeset
728 PyMem_FREE(tok->encoding);
32961
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
729 tok->encoding = new_string(encoding, strlen(encoding));
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
730 if (tok->encoding == NULL)
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
731 goto error_nomem;
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
732
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
733 Py_DECREF(enc);
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
734 return 0;
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
735
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
736 error_nomem:
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
737 Py_DECREF(enc);
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
738 tok->done = E_NOMEM;
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
739 return -1;
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
740
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
741 error_clear:
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
742 /* Fallback to iso-8859-1: for backward compatibility */
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
743 Py_DECREF(enc);
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
744 PyErr_Clear();
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
745 return 0;
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
746 }
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
747 #endif
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
748
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
749 /* Get next char, updating state; error code goes into tok->done */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
750
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
751 static int
13659
e78153e179fb Mass ANSIfication.
Thomas Wouters <thomas@python.org>
parents: 13539
diff changeset
752 tok_nextc(register struct tok_state *tok)
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
753 {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
754 for (;;) {
1897
3fbda8768855 * Parser/tokenizer.c (tok_nextc): count line numbers when parsing
Guido van Rossum <guido@python.org>
parents: 1835
diff changeset
755 if (tok->cur != tok->inp) {
7118
e9108e97a5a0 tok_nextc() should return unsigned characters, to avoid mistaking
Guido van Rossum <guido@python.org>
parents: 5379
diff changeset
756 return Py_CHARMASK(*tok->cur++); /* Fast path */
1897
3fbda8768855 * Parser/tokenizer.c (tok_nextc): count line numbers when parsing
Guido van Rossum <guido@python.org>
parents: 1835
diff changeset
757 }
1282
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
758 if (tok->done != E_OK)
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
759 return EOF;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
760 if (tok->fp == NULL) {
1897
3fbda8768855 * Parser/tokenizer.c (tok_nextc): count line numbers when parsing
Guido van Rossum <guido@python.org>
parents: 1835
diff changeset
761 char *end = strchr(tok->inp, '\n');
3fbda8768855 * Parser/tokenizer.c (tok_nextc): count line numbers when parsing
Guido van Rossum <guido@python.org>
parents: 1835
diff changeset
762 if (end != NULL)
3fbda8768855 * Parser/tokenizer.c (tok_nextc): count line numbers when parsing
Guido van Rossum <guido@python.org>
parents: 1835
diff changeset
763 end++;
3fbda8768855 * Parser/tokenizer.c (tok_nextc): count line numbers when parsing
Guido van Rossum <guido@python.org>
parents: 1835
diff changeset
764 else {
3fbda8768855 * Parser/tokenizer.c (tok_nextc): count line numbers when parsing
Guido van Rossum <guido@python.org>
parents: 1835
diff changeset
765 end = strchr(tok->inp, '\0');
3fbda8768855 * Parser/tokenizer.c (tok_nextc): count line numbers when parsing
Guido van Rossum <guido@python.org>
parents: 1835
diff changeset
766 if (end == tok->inp) {
3fbda8768855 * Parser/tokenizer.c (tok_nextc): count line numbers when parsing
Guido van Rossum <guido@python.org>
parents: 1835
diff changeset
767 tok->done = E_EOF;
3fbda8768855 * Parser/tokenizer.c (tok_nextc): count line numbers when parsing
Guido van Rossum <guido@python.org>
parents: 1835
diff changeset
768 return EOF;
3fbda8768855 * Parser/tokenizer.c (tok_nextc): count line numbers when parsing
Guido van Rossum <guido@python.org>
parents: 1835
diff changeset
769 }
3fbda8768855 * Parser/tokenizer.c (tok_nextc): count line numbers when parsing
Guido van Rossum <guido@python.org>
parents: 1835
diff changeset
770 }
3fbda8768855 * Parser/tokenizer.c (tok_nextc): count line numbers when parsing
Guido van Rossum <guido@python.org>
parents: 1835
diff changeset
771 if (tok->start == NULL)
3fbda8768855 * Parser/tokenizer.c (tok_nextc): count line numbers when parsing
Guido van Rossum <guido@python.org>
parents: 1835
diff changeset
772 tok->buf = tok->cur;
36786
57322cb2ca11 Patch #1440601: Add col_offset attribute to AST nodes.
Martin v. Löwis <martin@v.loewis.de>
parents: 36741
diff changeset
773 tok->line_start = tok->cur;
1897
3fbda8768855 * Parser/tokenizer.c (tok_nextc): count line numbers when parsing
Guido van Rossum <guido@python.org>
parents: 1835
diff changeset
774 tok->lineno++;
3fbda8768855 * Parser/tokenizer.c (tok_nextc): count line numbers when parsing
Guido van Rossum <guido@python.org>
parents: 1835
diff changeset
775 tok->inp = end;
7118
e9108e97a5a0 tok_nextc() should return unsigned characters, to avoid mistaking
Guido van Rossum <guido@python.org>
parents: 5379
diff changeset
776 return Py_CHARMASK(*tok->cur++);
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
777 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
778 if (tok->prompt != NULL) {
37585
292854c43b99 Fix the code in Parser/ to also compile with C++. This was mostly casts for
Anthony Baxter <anthonybaxter@gmail.com>
parents: 37557
diff changeset
779 char *newtok = PyOS_Readline(stdin, stdout, tok->prompt);
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
780 if (tok->nextprompt != NULL)
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
781 tok->prompt = tok->nextprompt;
37585
292854c43b99 Fix the code in Parser/ to also compile with C++. This was mostly casts for
Anthony Baxter <anthonybaxter@gmail.com>
parents: 37557
diff changeset
782 if (newtok == NULL)
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
783 tok->done = E_INTR;
37585
292854c43b99 Fix the code in Parser/ to also compile with C++. This was mostly casts for
Anthony Baxter <anthonybaxter@gmail.com>
parents: 37557
diff changeset
784 else if (*newtok == '\0') {
37600
c4c5eeb07739 As discussed on python-dev, really fix the PyMem_*/PyObject_* memory API
Neal Norwitz <nnorwitz@gmail.com>
parents: 37585
diff changeset
785 PyMem_FREE(newtok);
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
786 tok->done = E_EOF;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
787 }
32961
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
788 #if !defined(PGEN) && defined(Py_USING_UNICODE)
37585
292854c43b99 Fix the code in Parser/ to also compile with C++. This was mostly casts for
Anthony Baxter <anthonybaxter@gmail.com>
parents: 37557
diff changeset
789 else if (tok_stdin_decode(tok, &newtok) != 0)
37600
c4c5eeb07739 As discussed on python-dev, really fix the PyMem_*/PyObject_* memory API
Neal Norwitz <nnorwitz@gmail.com>
parents: 37585
diff changeset
790 PyMem_FREE(newtok);
32961
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
791 #endif
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
792 else if (tok->start != NULL) {
13093
cbe02b8aee07 Trent Mick: familiar simple Win64 patches
Guido van Rossum <guido@python.org>
parents: 12576
diff changeset
793 size_t start = tok->start - tok->buf;
cbe02b8aee07 Trent Mick: familiar simple Win64 patches
Guido van Rossum <guido@python.org>
parents: 12576
diff changeset
794 size_t oldlen = tok->cur - tok->buf;
37585
292854c43b99 Fix the code in Parser/ to also compile with C++. This was mostly casts for
Anthony Baxter <anthonybaxter@gmail.com>
parents: 37557
diff changeset
795 size_t newlen = oldlen + strlen(newtok);
12576
1c7c2dd1beb1 Vladimir Marangozov's long-awaited malloc restructuring.
Guido van Rossum <guido@python.org>
parents: 12224
diff changeset
796 char *buf = tok->buf;
37600
c4c5eeb07739 As discussed on python-dev, really fix the PyMem_*/PyObject_* memory API
Neal Norwitz <nnorwitz@gmail.com>
parents: 37585
diff changeset
797 buf = (char *)PyMem_REALLOC(buf, newlen+1);
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
798 tok->lineno++;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
799 if (buf == NULL) {
37600
c4c5eeb07739 As discussed on python-dev, really fix the PyMem_*/PyObject_* memory API
Neal Norwitz <nnorwitz@gmail.com>
parents: 37585
diff changeset
800 PyMem_FREE(tok->buf);
2044
281e40652e23 Parser/tokenizer.c (tok_nextc): zap tok->buf after freeing;
Guido van Rossum <guido@python.org>
parents: 1897
diff changeset
801 tok->buf = NULL;
37600
c4c5eeb07739 As discussed on python-dev, really fix the PyMem_*/PyObject_* memory API
Neal Norwitz <nnorwitz@gmail.com>
parents: 37585
diff changeset
802 PyMem_FREE(newtok);
1282
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
803 tok->done = E_NOMEM;
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
804 return EOF;
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
805 }
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
806 tok->buf = buf;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
807 tok->cur = tok->buf + oldlen;
36786
57322cb2ca11 Patch #1440601: Add col_offset attribute to AST nodes.
Martin v. Löwis <martin@v.loewis.de>
parents: 36741
diff changeset
808 tok->line_start = tok->cur;
37585
292854c43b99 Fix the code in Parser/ to also compile with C++. This was mostly casts for
Anthony Baxter <anthonybaxter@gmail.com>
parents: 37557
diff changeset
809 strcpy(tok->buf + oldlen, newtok);
37600
c4c5eeb07739 As discussed on python-dev, really fix the PyMem_*/PyObject_* memory API
Neal Norwitz <nnorwitz@gmail.com>
parents: 37585
diff changeset
810 PyMem_FREE(newtok);
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
811 tok->inp = tok->buf + newlen;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
812 tok->end = tok->inp + 1;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
813 tok->start = tok->buf + start;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
814 }
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
815 else {
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
816 tok->lineno++;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
817 if (tok->buf != NULL)
37600
c4c5eeb07739 As discussed on python-dev, really fix the PyMem_*/PyObject_* memory API
Neal Norwitz <nnorwitz@gmail.com>
parents: 37585
diff changeset
818 PyMem_FREE(tok->buf);
37585
292854c43b99 Fix the code in Parser/ to also compile with C++. This was mostly casts for
Anthony Baxter <anthonybaxter@gmail.com>
parents: 37557
diff changeset
819 tok->buf = newtok;
36786
57322cb2ca11 Patch #1440601: Add col_offset attribute to AST nodes.
Martin v. Löwis <martin@v.loewis.de>
parents: 36741
diff changeset
820 tok->line_start = tok->buf;
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
821 tok->cur = tok->buf;
36786
57322cb2ca11 Patch #1440601: Add col_offset attribute to AST nodes.
Martin v. Löwis <martin@v.loewis.de>
parents: 36741
diff changeset
822 tok->line_start = tok->buf;
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
823 tok->inp = strchr(tok->buf, '\0');
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
824 tok->end = tok->inp + 1;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
825 }
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
826 }
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
827 else {
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
828 int done = 0;
36501
7fdc639bc5b4 Merge ssize_t branch.
Martin v. Löwis <martin@v.loewis.de>
parents: 36090
diff changeset
829 Py_ssize_t cur = 0;
2732
daa150b3a6e4 replace "\r\n" with "\n" at line end (Jim Ahlstrom)
Guido van Rossum <guido@python.org>
parents: 2149
diff changeset
830 char *pt;
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
831 if (tok->start == NULL) {
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
832 if (tok->buf == NULL) {
37229
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
833 tok->buf = (char *)
37600
c4c5eeb07739 As discussed on python-dev, really fix the PyMem_*/PyObject_* memory API
Neal Norwitz <nnorwitz@gmail.com>
parents: 37585
diff changeset
834 PyMem_MALLOC(BUFSIZ);
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
835 if (tok->buf == NULL) {
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
836 tok->done = E_NOMEM;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
837 return EOF;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
838 }
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
839 tok->end = tok->buf + BUFSIZ;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
840 }
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
841 if (decoding_fgets(tok->buf, (int)(tok->end - tok->buf),
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
842 tok) == NULL) {
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
843 tok->done = E_EOF;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
844 done = 1;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
845 }
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
846 else {
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
847 tok->done = E_OK;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
848 tok->inp = strchr(tok->buf, '\0');
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
849 done = tok->inp[-1] == '\n';
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
850 }
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
851 }
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
852 else {
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
853 cur = tok->cur - tok->buf;
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
854 if (decoding_feof(tok)) {
2149
6060d7fb07e3 fix loop on unterminated triple quotes
Guido van Rossum <guido@python.org>
parents: 2060
diff changeset
855 tok->done = E_EOF;
6060d7fb07e3 fix loop on unterminated triple quotes
Guido van Rossum <guido@python.org>
parents: 2060
diff changeset
856 done = 1;
6060d7fb07e3 fix loop on unterminated triple quotes
Guido van Rossum <guido@python.org>
parents: 2060
diff changeset
857 }
6060d7fb07e3 fix loop on unterminated triple quotes
Guido van Rossum <guido@python.org>
parents: 2060
diff changeset
858 else
6060d7fb07e3 fix loop on unterminated triple quotes
Guido van Rossum <guido@python.org>
parents: 2060
diff changeset
859 tok->done = E_OK;
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
860 }
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
861 tok->lineno++;
1282
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
862 /* Read until '\n' or EOF */
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
863 while (!done) {
36501
7fdc639bc5b4 Merge ssize_t branch.
Martin v. Löwis <martin@v.loewis.de>
parents: 36090
diff changeset
864 Py_ssize_t curstart = tok->start == NULL ? -1 :
7fdc639bc5b4 Merge ssize_t branch.
Martin v. Löwis <martin@v.loewis.de>
parents: 36090
diff changeset
865 tok->start - tok->buf;
7fdc639bc5b4 Merge ssize_t branch.
Martin v. Löwis <martin@v.loewis.de>
parents: 36090
diff changeset
866 Py_ssize_t curvalid = tok->inp - tok->buf;
7fdc639bc5b4 Merge ssize_t branch.
Martin v. Löwis <martin@v.loewis.de>
parents: 36090
diff changeset
867 Py_ssize_t newsize = curvalid + BUFSIZ;
1282
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
868 char *newbuf = tok->buf;
37600
c4c5eeb07739 As discussed on python-dev, really fix the PyMem_*/PyObject_* memory API
Neal Norwitz <nnorwitz@gmail.com>
parents: 37585
diff changeset
869 newbuf = (char *)PyMem_REALLOC(newbuf,
c4c5eeb07739 As discussed on python-dev, really fix the PyMem_*/PyObject_* memory API
Neal Norwitz <nnorwitz@gmail.com>
parents: 37585
diff changeset
870 newsize);
1282
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
871 if (newbuf == NULL) {
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
872 tok->done = E_NOMEM;
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
873 tok->cur = tok->inp;
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
874 return EOF;
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
875 }
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
876 tok->buf = newbuf;
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
877 tok->inp = tok->buf + curvalid;
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
878 tok->end = tok->buf + newsize;
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
879 tok->start = curstart < 0 ? NULL :
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
880 tok->buf + curstart;
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
881 if (decoding_fgets(tok->inp,
1282
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
882 (int)(tok->end - tok->inp),
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
883 tok) == NULL) {
36809
2c2dffe070df Fix crashing bug in tokenizer, when tokenizing files with non-ASCII bytes
Thomas Wouters <thomas@python.org>
parents: 36786
diff changeset
884 /* Break out early on decoding
2c2dffe070df Fix crashing bug in tokenizer, when tokenizing files with non-ASCII bytes
Thomas Wouters <thomas@python.org>
parents: 36786
diff changeset
885 errors, as tok->buf will be NULL
2c2dffe070df Fix crashing bug in tokenizer, when tokenizing files with non-ASCII bytes
Thomas Wouters <thomas@python.org>
parents: 36786
diff changeset
886 */
2c2dffe070df Fix crashing bug in tokenizer, when tokenizing files with non-ASCII bytes
Thomas Wouters <thomas@python.org>
parents: 36786
diff changeset
887 if (tok->decoding_erred)
2c2dffe070df Fix crashing bug in tokenizer, when tokenizing files with non-ASCII bytes
Thomas Wouters <thomas@python.org>
parents: 36786
diff changeset
888 return EOF;
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
889 /* Last line does not end in \n,
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
890 fake one */
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
891 strcpy(tok->inp, "\n");
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
892 }
1282
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
893 tok->inp = strchr(tok->inp, '\0');
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
894 done = tok->inp[-1] == '\n';
1282
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
895 }
38544
697dd848e1f1 Patch #1357836:
Neal Norwitz <nnorwitz@gmail.com>
parents: 37817
diff changeset
896 if (tok->buf != NULL) {
697dd848e1f1 Patch #1357836:
Neal Norwitz <nnorwitz@gmail.com>
parents: 37817
diff changeset
897 tok->cur = tok->buf + cur;
697dd848e1f1 Patch #1357836:
Neal Norwitz <nnorwitz@gmail.com>
parents: 37817
diff changeset
898 tok->line_start = tok->cur;
697dd848e1f1 Patch #1357836:
Neal Norwitz <nnorwitz@gmail.com>
parents: 37817
diff changeset
899 /* replace "\r\n" with "\n" */
697dd848e1f1 Patch #1357836:
Neal Norwitz <nnorwitz@gmail.com>
parents: 37817
diff changeset
900 /* For Mac leave the \r, giving syntax error */
697dd848e1f1 Patch #1357836:
Neal Norwitz <nnorwitz@gmail.com>
parents: 37817
diff changeset
901 pt = tok->inp - 2;
697dd848e1f1 Patch #1357836:
Neal Norwitz <nnorwitz@gmail.com>
parents: 37817
diff changeset
902 if (pt >= tok->buf && *pt == '\r') {
697dd848e1f1 Patch #1357836:
Neal Norwitz <nnorwitz@gmail.com>
parents: 37817
diff changeset
903 *pt++ = '\n';
697dd848e1f1 Patch #1357836:
Neal Norwitz <nnorwitz@gmail.com>
parents: 37817
diff changeset
904 *pt = '\0';
697dd848e1f1 Patch #1357836:
Neal Norwitz <nnorwitz@gmail.com>
parents: 37817
diff changeset
905 tok->inp = pt;
697dd848e1f1 Patch #1357836:
Neal Norwitz <nnorwitz@gmail.com>
parents: 37817
diff changeset
906 }
2732
daa150b3a6e4 replace "\r\n" with "\n" at line end (Jim Ahlstrom)
Guido van Rossum <guido@python.org>
parents: 2149
diff changeset
907 }
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
908 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
909 if (tok->done != E_OK) {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
910 if (tok->prompt != NULL)
9047
6d2eb58f0418 Replace all calls to fprintf(stderr, ...) with PySys_WriteStderr(...).
Guido van Rossum <guido@python.org>
parents: 7945
diff changeset
911 PySys_WriteStderr("\n");
1282
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
912 tok->cur = tok->inp;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
913 return EOF;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
914 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
915 }
1282
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
916 /*NOTREACHED*/
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
917 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
918
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
919
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
920 /* Back-up one character */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
921
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
922 static void
13659
e78153e179fb Mass ANSIfication.
Thomas Wouters <thomas@python.org>
parents: 13539
diff changeset
923 tok_backup(register struct tok_state *tok, register int c)
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
924 {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
925 if (c != EOF) {
2044
281e40652e23 Parser/tokenizer.c (tok_nextc): zap tok->buf after freeing;
Guido van Rossum <guido@python.org>
parents: 1897
diff changeset
926 if (--tok->cur < tok->buf)
5379
1c9a84e0981d Another directory quickly renamed.
Guido van Rossum <guido@python.org>
parents: 5341
diff changeset
927 Py_FatalError("tok_backup: begin of buffer");
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
928 if (*tok->cur != c)
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
929 *tok->cur = c;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
930 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
931 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
932
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
933
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
934 /* Return the token corresponding to a single character */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
935
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
936 int
13659
e78153e179fb Mass ANSIfication.
Thomas Wouters <thomas@python.org>
parents: 13539
diff changeset
937 PyToken_OneChar(int c)
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
938 {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
939 switch (c) {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
940 case '(': return LPAR;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
941 case ')': return RPAR;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
942 case '[': return LSQB;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
943 case ']': return RSQB;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
944 case ':': return COLON;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
945 case ',': return COMMA;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
946 case ';': return SEMI;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
947 case '+': return PLUS;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
948 case '-': return MINUS;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
949 case '*': return STAR;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
950 case '/': return SLASH;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
951 case '|': return VBAR;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
952 case '&': return AMPER;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
953 case '<': return LESS;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
954 case '>': return GREATER;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
955 case '=': return EQUAL;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
956 case '.': return DOT;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
957 case '%': return PERCENT;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
958 case '`': return BACKQUOTE;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
959 case '{': return LBRACE;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
960 case '}': return RBRACE;
410
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
961 case '^': return CIRCUMFLEX;
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
962 case '~': return TILDE;
32888
c7e03cf8333e PEP-0318, @decorator-style. In Guido's words:
Anthony Baxter <anthonybaxter@gmail.com>
parents: 30979
diff changeset
963 case '@': return AT;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
964 default: return OP;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
965 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
966 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
967
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
968
400
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
969 int
13659
e78153e179fb Mass ANSIfication.
Thomas Wouters <thomas@python.org>
parents: 13539
diff changeset
970 PyToken_TwoChars(int c1, int c2)
400
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
971 {
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
972 switch (c1) {
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
973 case '=':
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
974 switch (c2) {
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
975 case '=': return EQEQUAL;
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
976 }
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
977 break;
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
978 case '!':
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
979 switch (c2) {
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
980 case '=': return NOTEQUAL;
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
981 }
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
982 break;
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
983 case '<':
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
984 switch (c2) {
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
985 case '>': return NOTEQUAL;
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
986 case '=': return LESSEQUAL;
410
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
987 case '<': return LEFTSHIFT;
400
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
988 }
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
989 break;
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
990 case '>':
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
991 switch (c2) {
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
992 case '=': return GREATEREQUAL;
410
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
993 case '>': return RIGHTSHIFT;
400
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
994 }
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
995 break;
14163
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
996 case '+':
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
997 switch (c2) {
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
998 case '=': return PLUSEQUAL;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
999 }
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1000 break;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1001 case '-':
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1002 switch (c2) {
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1003 case '=': return MINEQUAL;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1004 }
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1005 break;
3333
4b5bff95804e changes for pow(**) and complex
Guido van Rossum <guido@python.org>
parents: 3116
diff changeset
1006 case '*':
4b5bff95804e changes for pow(**) and complex
Guido van Rossum <guido@python.org>
parents: 3116
diff changeset
1007 switch (c2) {
4b5bff95804e changes for pow(**) and complex
Guido van Rossum <guido@python.org>
parents: 3116
diff changeset
1008 case '*': return DOUBLESTAR;
14163
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1009 case '=': return STAREQUAL;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1010 }
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1011 break;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1012 case '/':
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1013 switch (c2) {
19051
f20727cc3d0f Implement PEP 238 in its (almost) full glory.
Guido van Rossum <guido@python.org>
parents: 17600
diff changeset
1014 case '/': return DOUBLESLASH;
14163
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1015 case '=': return SLASHEQUAL;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1016 }
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1017 break;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1018 case '|':
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1019 switch (c2) {
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1020 case '=': return VBAREQUAL;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1021 }
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1022 break;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1023 case '%':
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1024 switch (c2) {
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1025 case '=': return PERCENTEQUAL;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1026 }
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1027 break;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1028 case '&':
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1029 switch (c2) {
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1030 case '=': return AMPEREQUAL;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1031 }
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1032 break;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1033 case '^':
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1034 switch (c2) {
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1035 case '=': return CIRCUMFLEXEQUAL;
3333
4b5bff95804e changes for pow(**) and complex
Guido van Rossum <guido@python.org>
parents: 3116
diff changeset
1036 }
4b5bff95804e changes for pow(**) and complex
Guido van Rossum <guido@python.org>
parents: 3116
diff changeset
1037 break;
400
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1038 }
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1039 return OP;
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1040 }
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1041
14163
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1042 int
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1043 PyToken_ThreeChars(int c1, int c2, int c3)
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1044 {
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1045 switch (c1) {
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1046 case '<':
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1047 switch (c2) {
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1048 case '<':
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1049 switch (c3) {
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1050 case '=':
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1051 return LEFTSHIFTEQUAL;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1052 }
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1053 break;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1054 }
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1055 break;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1056 case '>':
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1057 switch (c2) {
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1058 case '>':
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1059 switch (c3) {
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1060 case '=':
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1061 return RIGHTSHIFTEQUAL;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1062 }
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1063 break;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1064 }
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1065 break;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1066 case '*':
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1067 switch (c2) {
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1068 case '*':
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1069 switch (c3) {
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1070 case '=':
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1071 return DOUBLESTAREQUAL;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1072 }
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1073 break;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1074 }
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1075 break;
19051
f20727cc3d0f Implement PEP 238 in its (almost) full glory.
Guido van Rossum <guido@python.org>
parents: 17600
diff changeset
1076 case '/':
f20727cc3d0f Implement PEP 238 in its (almost) full glory.
Guido van Rossum <guido@python.org>
parents: 17600
diff changeset
1077 switch (c2) {
f20727cc3d0f Implement PEP 238 in its (almost) full glory.
Guido van Rossum <guido@python.org>
parents: 17600
diff changeset
1078 case '/':
f20727cc3d0f Implement PEP 238 in its (almost) full glory.
Guido van Rossum <guido@python.org>
parents: 17600
diff changeset
1079 switch (c3) {
f20727cc3d0f Implement PEP 238 in its (almost) full glory.
Guido van Rossum <guido@python.org>
parents: 17600
diff changeset
1080 case '=':
f20727cc3d0f Implement PEP 238 in its (almost) full glory.
Guido van Rossum <guido@python.org>
parents: 17600
diff changeset
1081 return DOUBLESLASHEQUAL;
f20727cc3d0f Implement PEP 238 in its (almost) full glory.
Guido van Rossum <guido@python.org>
parents: 17600
diff changeset
1082 }
f20727cc3d0f Implement PEP 238 in its (almost) full glory.
Guido van Rossum <guido@python.org>
parents: 17600
diff changeset
1083 break;
f20727cc3d0f Implement PEP 238 in its (almost) full glory.
Guido van Rossum <guido@python.org>
parents: 17600
diff changeset
1084 }
f20727cc3d0f Implement PEP 238 in its (almost) full glory.
Guido van Rossum <guido@python.org>
parents: 17600
diff changeset
1085 break;
14163
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1086 }
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1087 return OP;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1088 }
400
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1089
7945
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1090 static int
13659
e78153e179fb Mass ANSIfication.
Thomas Wouters <thomas@python.org>
parents: 13539
diff changeset
1091 indenterror(struct tok_state *tok)
7945
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1092 {
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1093 if (tok->alterror) {
13539
feab677269e0 Create two new exceptions: IndentationError and TabError. These are
Fred Drake <fdrake@acm.org>
parents: 13447
diff changeset
1094 tok->done = E_TABSPACE;
7945
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1095 tok->cur = tok->inp;
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1096 return 1;
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1097 }
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1098 if (tok->altwarning) {
13539
feab677269e0 Create two new exceptions: IndentationError and TabError. These are
Fred Drake <fdrake@acm.org>
parents: 13447
diff changeset
1099 PySys_WriteStderr("%s: inconsistent use of tabs and spaces "
feab677269e0 Create two new exceptions: IndentationError and TabError. These are
Fred Drake <fdrake@acm.org>
parents: 13447
diff changeset
1100 "in indentation\n", tok->filename);
7945
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1101 tok->altwarning = 0;
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1102 }
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1103 return 0;
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1104 }
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1105
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1106
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1107 /* Get next token, after space stripping etc. */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1108
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
1109 static int
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
1110 tok_get(register struct tok_state *tok, char **p_start, char **p_end)
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1111 {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1112 register int c;
326
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1113 int blankline;
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1114
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1115 *p_start = *p_end = NULL;
326
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1116 nextline:
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1117 tok->start = NULL;
326
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1118 blankline = 0;
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1119
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1120 /* Get indentation level */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1121 if (tok->atbol) {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1122 register int col = 0;
7945
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1123 register int altcol = 0;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1124 tok->atbol = 0;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1125 for (;;) {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1126 c = tok_nextc(tok);
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1127 if (c == ' ')
7945
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1128 col++, altcol++;
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1129 else if (c == '\t') {
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1130 col = (col/tok->tabsize + 1) * tok->tabsize;
7945
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1131 altcol = (altcol/tok->alttabsize + 1)
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1132 * tok->alttabsize;
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1133 }
2801
e911f2fb179c ignore control-l in whitespace
Guido van Rossum <guido@python.org>
parents: 2732
diff changeset
1134 else if (c == '\014') /* Control-L (formfeed) */
7945
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1135 col = altcol = 0; /* For Emacs users */
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1136 else
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1137 break;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1138 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1139 tok_backup(tok, c);
326
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1140 if (c == '#' || c == '\n') {
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1141 /* Lines with only whitespace and/or comments
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1142 shouldn't affect the indentation and are
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1143 not passed to the parser as NEWLINE tokens,
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1144 except *totally* empty lines in interactive
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1145 mode, which signal the end of a command group. */
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1146 if (col == 0 && c == '\n' && tok->prompt != NULL)
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1147 blankline = 0; /* Let it through */
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1148 else
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1149 blankline = 1; /* Ignore completely */
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1150 /* We can't jump back right here since we still
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1151 may need to skip to the end of a comment */
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1152 }
1284
dbce72afe261 * selectmodule.c: fix (another!) two memory leaks -- this time in list2set
Guido van Rossum <guido@python.org>
parents: 1282
diff changeset
1153 if (!blankline && tok->level == 0) {
326
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1154 if (col == tok->indstack[tok->indent]) {
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1155 /* No change */
7945
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1156 if (altcol != tok->altindstack[tok->indent]) {
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1157 if (indenterror(tok))
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1158 return ERRORTOKEN;
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1159 }
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1160 }
326
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1161 else if (col > tok->indstack[tok->indent]) {
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1162 /* Indent -- always one */
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1163 if (tok->indent+1 >= MAXINDENT) {
13539
feab677269e0 Create two new exceptions: IndentationError and TabError. These are
Fred Drake <fdrake@acm.org>
parents: 13447
diff changeset
1164 tok->done = E_TOODEEP;
1282
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
1165 tok->cur = tok->inp;
326
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1166 return ERRORTOKEN;
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1167 }
7945
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1168 if (altcol <= tok->altindstack[tok->indent]) {
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1169 if (indenterror(tok))
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1170 return ERRORTOKEN;
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1171 }
326
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1172 tok->pendin++;
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1173 tok->indstack[++tok->indent] = col;
7945
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1174 tok->altindstack[tok->indent] = altcol;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1175 }
326
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1176 else /* col < tok->indstack[tok->indent] */ {
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1177 /* Dedent -- any number, must be consistent */
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1178 while (tok->indent > 0 &&
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1179 col < tok->indstack[tok->indent]) {
7324
55125b1144fa Swap two statements in the dedent check loop. This makes absolutely
Guido van Rossum <guido@python.org>
parents: 7315
diff changeset
1180 tok->pendin--;
326
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1181 tok->indent--;
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1182 }
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1183 if (col != tok->indstack[tok->indent]) {
13539
feab677269e0 Create two new exceptions: IndentationError and TabError. These are
Fred Drake <fdrake@acm.org>
parents: 13447
diff changeset
1184 tok->done = E_DEDENT;
1282
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
1185 tok->cur = tok->inp;
326
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1186 return ERRORTOKEN;
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1187 }
7945
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1188 if (altcol != tok->altindstack[tok->indent]) {
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1189 if (indenterror(tok))
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1190 return ERRORTOKEN;
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1191 }
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1192 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1193 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1194 }
37229
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
1195
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1196 tok->start = tok->cur;
37229
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
1197
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1198 /* Return pending indents/dedents */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1199 if (tok->pendin != 0) {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1200 if (tok->pendin < 0) {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1201 tok->pendin++;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1202 return DEDENT;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1203 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1204 else {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1205 tok->pendin--;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1206 return INDENT;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1207 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1208 }
37229
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
1209
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1210 again:
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1211 tok->start = NULL;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1212 /* Skip spaces */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1213 do {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1214 c = tok_nextc(tok);
2801
e911f2fb179c ignore control-l in whitespace
Guido van Rossum <guido@python.org>
parents: 2732
diff changeset
1215 } while (c == ' ' || c == '\t' || c == '\014');
37229
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
1216
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1217 /* Set start of current token */
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1218 tok->start = tok->cur - 1;
37229
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
1219
12142
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1220 /* Skip comment, while looking for tab-setting magic */
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1221 if (c == '#') {
12142
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1222 static char *tabforms[] = {
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1223 "tab-width:", /* Emacs */
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1224 ":tabstop=", /* vim, full form */
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1225 ":ts=", /* vim, abbreviated form */
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1226 "set tabsize=", /* will vi never die? */
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1227 /* more templates can be added here to support other editors */
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1228 };
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1229 char cbuf[80];
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1230 char *tp, **cp;
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1231 tp = cbuf;
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1232 do {
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1233 *tp++ = c = tok_nextc(tok);
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1234 } while (c != EOF && c != '\n' &&
38762
92d8cdc4b90c Don't truncate if size_t is bigger than uint
Neal Norwitz <nnorwitz@gmail.com>
parents: 38544
diff changeset
1235 (size_t)(tp - cbuf + 1) < sizeof(cbuf));
12142
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1236 *tp = '\0';
37229
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
1237 for (cp = tabforms;
12142
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1238 cp < tabforms + sizeof(tabforms)/sizeof(tabforms[0]);
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1239 cp++) {
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1240 if ((tp = strstr(cbuf, *cp))) {
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1241 int newsize = atoi(tp + strlen(*cp));
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1242
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1243 if (newsize >= 1 && newsize <= 40) {
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1244 tok->tabsize = newsize;
12224
2dc1bdb4088e Only write message about changed Tab size with -v.
Guido van Rossum <guido@python.org>
parents: 12142
diff changeset
1245 if (Py_VerboseFlag)
2dc1bdb4088e Only write message about changed Tab size with -v.
Guido van Rossum <guido@python.org>
parents: 12142
diff changeset
1246 PySys_WriteStderr(
12142
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1247 "Tab size set to %d\n",
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1248 newsize);
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1249 }
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1250 }
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1251 }
12142
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1252 while (c != EOF && c != '\n')
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1253 c = tok_nextc(tok);
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1254 }
37229
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
1255
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1256 /* Check for EOF and errors now */
410
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
1257 if (c == EOF) {
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1258 return tok->done == E_EOF ? ENDMARKER : ERRORTOKEN;
410
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
1259 }
37229
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
1260
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1261 /* Identifier (most frequent token!) */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1262 if (isalpha(c) || c == '_') {
11971
574283db2b3d Marc-Andre Lemburg: add new string token types u"..." and ur"..."
Guido van Rossum <guido@python.org>
parents: 9590
diff changeset
1263 /* Process r"", u"" and ur"" */
5341
67e5237ce141 Now that the string-sig has settled on r"obin" strings, restrict the
Guido van Rossum <guido@python.org>
parents: 5311
diff changeset
1264 switch (c) {
67e5237ce141 Now that the string-sig has settled on r"obin" strings, restrict the
Guido van Rossum <guido@python.org>
parents: 5311
diff changeset
1265 case 'r':
67e5237ce141 Now that the string-sig has settled on r"obin" strings, restrict the
Guido van Rossum <guido@python.org>
parents: 5311
diff changeset
1266 case 'R':
67e5237ce141 Now that the string-sig has settled on r"obin" strings, restrict the
Guido van Rossum <guido@python.org>
parents: 5311
diff changeset
1267 c = tok_nextc(tok);
67e5237ce141 Now that the string-sig has settled on r"obin" strings, restrict the
Guido van Rossum <guido@python.org>
parents: 5311
diff changeset
1268 if (c == '"' || c == '\'')
67e5237ce141 Now that the string-sig has settled on r"obin" strings, restrict the
Guido van Rossum <guido@python.org>
parents: 5311
diff changeset
1269 goto letter_quote;
11971
574283db2b3d Marc-Andre Lemburg: add new string token types u"..." and ur"..."
Guido van Rossum <guido@python.org>
parents: 9590
diff changeset
1270 break;
574283db2b3d Marc-Andre Lemburg: add new string token types u"..." and ur"..."
Guido van Rossum <guido@python.org>
parents: 9590
diff changeset
1271 case 'u':
574283db2b3d Marc-Andre Lemburg: add new string token types u"..." and ur"..."
Guido van Rossum <guido@python.org>
parents: 9590
diff changeset
1272 case 'U':
574283db2b3d Marc-Andre Lemburg: add new string token types u"..." and ur"..."
Guido van Rossum <guido@python.org>
parents: 9590
diff changeset
1273 c = tok_nextc(tok);
574283db2b3d Marc-Andre Lemburg: add new string token types u"..." and ur"..."
Guido van Rossum <guido@python.org>
parents: 9590
diff changeset
1274 if (c == 'r' || c == 'R')
574283db2b3d Marc-Andre Lemburg: add new string token types u"..." and ur"..."
Guido van Rossum <guido@python.org>
parents: 9590
diff changeset
1275 c = tok_nextc(tok);
574283db2b3d Marc-Andre Lemburg: add new string token types u"..." and ur"..."
Guido van Rossum <guido@python.org>
parents: 9590
diff changeset
1276 if (c == '"' || c == '\'')
574283db2b3d Marc-Andre Lemburg: add new string token types u"..." and ur"..."
Guido van Rossum <guido@python.org>
parents: 9590
diff changeset
1277 goto letter_quote;
574283db2b3d Marc-Andre Lemburg: add new string token types u"..." and ur"..."
Guido van Rossum <guido@python.org>
parents: 9590
diff changeset
1278 break;
5341
67e5237ce141 Now that the string-sig has settled on r"obin" strings, restrict the
Guido van Rossum <guido@python.org>
parents: 5311
diff changeset
1279 }
5265
3c1788ab9761 Support for alternative string quotes (a"xx", b"xx", c"xx", ...).
Guido van Rossum <guido@python.org>
parents: 4839
diff changeset
1280 while (isalnum(c) || c == '_') {
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1281 c = tok_nextc(tok);
5265
3c1788ab9761 Support for alternative string quotes (a"xx", b"xx", c"xx", ...).
Guido van Rossum <guido@python.org>
parents: 4839
diff changeset
1282 }
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1283 tok_backup(tok, c);
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1284 *p_start = tok->start;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1285 *p_end = tok->cur;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1286 return NAME;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1287 }
37229
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
1288
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1289 /* Newline */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1290 if (c == '\n') {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1291 tok->atbol = 1;
1284
dbce72afe261 * selectmodule.c: fix (another!) two memory leaks -- this time in list2set
Guido van Rossum <guido@python.org>
parents: 1282
diff changeset
1292 if (blankline || tok->level > 0)
326
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1293 goto nextline;
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1294 *p_start = tok->start;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1295 *p_end = tok->cur - 1; /* Leave '\n' out of the string */
25098
c02a668488d0 Ignore encoding declarations inside strings. Fixes #603509.
Martin v. Löwis <martin@v.loewis.de>
parents: 24867
diff changeset
1296 tok->cont_line = 0;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1297 return NEWLINE;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1298 }
37229
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
1299
410
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
1300 /* Period or number starting with period? */
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
1301 if (c == '.') {
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
1302 c = tok_nextc(tok);
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
1303 if (isdigit(c)) {
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
1304 goto fraction;
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
1305 }
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
1306 else {
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
1307 tok_backup(tok, c);
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1308 *p_start = tok->start;
410
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
1309 *p_end = tok->cur;
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
1310 return DOT;
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
1311 }
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
1312 }
3333
4b5bff95804e changes for pow(**) and complex
Guido van Rossum <guido@python.org>
parents: 3116
diff changeset
1313
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1314 /* Number */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1315 if (isdigit(c)) {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1316 if (c == '0') {
19621
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1317 /* Hex or octal -- maybe. */
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1318 c = tok_nextc(tok);
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1319 if (c == '.')
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1320 goto fraction;
3333
4b5bff95804e changes for pow(**) and complex
Guido van Rossum <guido@python.org>
parents: 3116
diff changeset
1321 #ifndef WITHOUT_COMPLEX
3364
bc17e8313ea0 use only j for imaginary constants
Guido van Rossum <guido@python.org>
parents: 3333
diff changeset
1322 if (c == 'j' || c == 'J')
3333
4b5bff95804e changes for pow(**) and complex
Guido van Rossum <guido@python.org>
parents: 3116
diff changeset
1323 goto imaginary;
4b5bff95804e changes for pow(**) and complex
Guido van Rossum <guido@python.org>
parents: 3116
diff changeset
1324 #endif
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1325 if (c == 'x' || c == 'X') {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1326 /* Hex */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1327 do {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1328 c = tok_nextc(tok);
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1329 } while (isxdigit(c));
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1330 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1331 else {
19621
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1332 int found_decimal = 0;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1333 /* Octal; c is first char of it */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1334 /* There's no 'isoctdigit' macro, sigh */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1335 while ('0' <= c && c < '8') {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1336 c = tok_nextc(tok);
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1337 }
19621
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1338 if (isdigit(c)) {
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1339 found_decimal = 1;
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1340 do {
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1341 c = tok_nextc(tok);
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1342 } while (isdigit(c));
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1343 }
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1344 if (c == '.')
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1345 goto fraction;
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1346 else if (c == 'e' || c == 'E')
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1347 goto exponent;
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1348 #ifndef WITHOUT_COMPLEX
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1349 else if (c == 'j' || c == 'J')
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1350 goto imaginary;
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1351 #endif
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1352 else if (found_decimal) {
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1353 tok->done = E_TOKEN;
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1354 tok_backup(tok, c);
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1355 return ERRORTOKEN;
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1356 }
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1357 }
236
0a92b6a1b9c0 Added recognition of 'l' or 'L' as long integer suffix
Guido van Rossum <guido@python.org>
parents: 145
diff changeset
1358 if (c == 'l' || c == 'L')
0a92b6a1b9c0 Added recognition of 'l' or 'L' as long integer suffix
Guido van Rossum <guido@python.org>
parents: 145
diff changeset
1359 c = tok_nextc(tok);
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1360 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1361 else {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1362 /* Decimal */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1363 do {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1364 c = tok_nextc(tok);
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1365 } while (isdigit(c));
236
0a92b6a1b9c0 Added recognition of 'l' or 'L' as long integer suffix
Guido van Rossum <guido@python.org>
parents: 145
diff changeset
1366 if (c == 'l' || c == 'L')
0a92b6a1b9c0 Added recognition of 'l' or 'L' as long integer suffix
Guido van Rossum <guido@python.org>
parents: 145
diff changeset
1367 c = tok_nextc(tok);
0a92b6a1b9c0 Added recognition of 'l' or 'L' as long integer suffix
Guido van Rossum <guido@python.org>
parents: 145
diff changeset
1368 else {
19501
ce4f1e98ba7e SF bug [#455775] float parsing discrepancy.
Tim Peters <tim.peters@gmail.com>
parents: 19051
diff changeset
1369 /* Accept floating point numbers. */
236
0a92b6a1b9c0 Added recognition of 'l' or 'L' as long integer suffix
Guido van Rossum <guido@python.org>
parents: 145
diff changeset
1370 if (c == '.') {
0a92b6a1b9c0 Added recognition of 'l' or 'L' as long integer suffix
Guido van Rossum <guido@python.org>
parents: 145
diff changeset
1371 fraction:
0a92b6a1b9c0 Added recognition of 'l' or 'L' as long integer suffix
Guido van Rossum <guido@python.org>
parents: 145
diff changeset
1372 /* Fraction */
0a92b6a1b9c0 Added recognition of 'l' or 'L' as long integer suffix
Guido van Rossum <guido@python.org>
parents: 145
diff changeset
1373 do {
0a92b6a1b9c0 Added recognition of 'l' or 'L' as long integer suffix
Guido van Rossum <guido@python.org>
parents: 145
diff changeset
1374 c = tok_nextc(tok);
0a92b6a1b9c0 Added recognition of 'l' or 'L' as long integer suffix
Guido van Rossum <guido@python.org>
parents: 145
diff changeset
1375 } while (isdigit(c));
0a92b6a1b9c0 Added recognition of 'l' or 'L' as long integer suffix
Guido van Rossum <guido@python.org>
parents: 145
diff changeset
1376 }
0a92b6a1b9c0 Added recognition of 'l' or 'L' as long integer suffix
Guido van Rossum <guido@python.org>
parents: 145
diff changeset
1377 if (c == 'e' || c == 'E') {
19621
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1378 exponent:
236
0a92b6a1b9c0 Added recognition of 'l' or 'L' as long integer suffix
Guido van Rossum <guido@python.org>
parents: 145
diff changeset
1379 /* Exponent part */
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1380 c = tok_nextc(tok);
236
0a92b6a1b9c0 Added recognition of 'l' or 'L' as long integer suffix
Guido van Rossum <guido@python.org>
parents: 145
diff changeset
1381 if (c == '+' || c == '-')
0a92b6a1b9c0 Added recognition of 'l' or 'L' as long integer suffix
Guido van Rossum <guido@python.org>
parents: 145
diff changeset
1382 c = tok_nextc(tok);
19501
ce4f1e98ba7e SF bug [#455775] float parsing discrepancy.
Tim Peters <tim.peters@gmail.com>
parents: 19051
diff changeset
1383 if (!isdigit(c)) {
ce4f1e98ba7e SF bug [#455775] float parsing discrepancy.
Tim Peters <tim.peters@gmail.com>
parents: 19051
diff changeset
1384 tok->done = E_TOKEN;
ce4f1e98ba7e SF bug [#455775] float parsing discrepancy.
Tim Peters <tim.peters@gmail.com>
parents: 19051
diff changeset
1385 tok_backup(tok, c);
ce4f1e98ba7e SF bug [#455775] float parsing discrepancy.
Tim Peters <tim.peters@gmail.com>
parents: 19051
diff changeset
1386 return ERRORTOKEN;
ce4f1e98ba7e SF bug [#455775] float parsing discrepancy.
Tim Peters <tim.peters@gmail.com>
parents: 19051
diff changeset
1387 }
ce4f1e98ba7e SF bug [#455775] float parsing discrepancy.
Tim Peters <tim.peters@gmail.com>
parents: 19051
diff changeset
1388 do {
236
0a92b6a1b9c0 Added recognition of 'l' or 'L' as long integer suffix
Guido van Rossum <guido@python.org>
parents: 145
diff changeset
1389 c = tok_nextc(tok);
19501
ce4f1e98ba7e SF bug [#455775] float parsing discrepancy.
Tim Peters <tim.peters@gmail.com>
parents: 19051
diff changeset
1390 } while (isdigit(c));
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1391 }
3333
4b5bff95804e changes for pow(**) and complex
Guido van Rossum <guido@python.org>
parents: 3116
diff changeset
1392 #ifndef WITHOUT_COMPLEX
3364
bc17e8313ea0 use only j for imaginary constants
Guido van Rossum <guido@python.org>
parents: 3333
diff changeset
1393 if (c == 'j' || c == 'J')
3333
4b5bff95804e changes for pow(**) and complex
Guido van Rossum <guido@python.org>
parents: 3116
diff changeset
1394 /* Imaginary part */
4b5bff95804e changes for pow(**) and complex
Guido van Rossum <guido@python.org>
parents: 3116
diff changeset
1395 imaginary:
4b5bff95804e changes for pow(**) and complex
Guido van Rossum <guido@python.org>
parents: 3116
diff changeset
1396 c = tok_nextc(tok);
4b5bff95804e changes for pow(**) and complex
Guido van Rossum <guido@python.org>
parents: 3116
diff changeset
1397 #endif
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1398 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1399 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1400 tok_backup(tok, c);
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1401 *p_start = tok->start;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1402 *p_end = tok->cur;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1403 return NUMBER;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1404 }
5265
3c1788ab9761 Support for alternative string quotes (a"xx", b"xx", c"xx", ...).
Guido van Rossum <guido@python.org>
parents: 4839
diff changeset
1405
3c1788ab9761 Support for alternative string quotes (a"xx", b"xx", c"xx", ...).
Guido van Rossum <guido@python.org>
parents: 4839
diff changeset
1406 letter_quote:
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1407 /* String */
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1408 if (c == '\'' || c == '"') {
36501
7fdc639bc5b4 Merge ssize_t branch.
Martin v. Löwis <martin@v.loewis.de>
parents: 36090
diff changeset
1409 Py_ssize_t quote2 = tok->cur - tok->start + 1;
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1410 int quote = c;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1411 int triple = 0;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1412 int tripcount = 0;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1413 for (;;) {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1414 c = tok_nextc(tok);
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1415 if (c == '\n') {
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1416 if (!triple) {
24839
17e95cc06f69 provide less mysterious error messages when seeing end-of-line in
Skip Montanaro <skip@pobox.com>
parents: 24630
diff changeset
1417 tok->done = E_EOLS;
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1418 tok_backup(tok, c);
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1419 return ERRORTOKEN;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1420 }
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1421 tripcount = 0;
25098
c02a668488d0 Ignore encoding declarations inside strings. Fixes #603509.
Martin v. Löwis <martin@v.loewis.de>
parents: 24867
diff changeset
1422 tok->cont_line = 1; /* multiline string. */
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1423 }
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1424 else if (c == EOF) {
24839
17e95cc06f69 provide less mysterious error messages when seeing end-of-line in
Skip Montanaro <skip@pobox.com>
parents: 24630
diff changeset
1425 if (triple)
17e95cc06f69 provide less mysterious error messages when seeing end-of-line in
Skip Montanaro <skip@pobox.com>
parents: 24630
diff changeset
1426 tok->done = E_EOFS;
17e95cc06f69 provide less mysterious error messages when seeing end-of-line in
Skip Montanaro <skip@pobox.com>
parents: 24630
diff changeset
1427 else
17e95cc06f69 provide less mysterious error messages when seeing end-of-line in
Skip Montanaro <skip@pobox.com>
parents: 24630
diff changeset
1428 tok->done = E_EOLS;
1282
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
1429 tok->cur = tok->inp;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1430 return ERRORTOKEN;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1431 }
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1432 else if (c == quote) {
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1433 tripcount++;
7315
3a55fbb99f20 Fixed the bug in searching for triple quotes -- change the 'quote2'
Guido van Rossum <guido@python.org>
parents: 7118
diff changeset
1434 if (tok->cur - tok->start == quote2) {
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1435 c = tok_nextc(tok);
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1436 if (c == quote) {
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1437 triple = 1;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1438 tripcount = 0;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1439 continue;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1440 }
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1441 tok_backup(tok, c);
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1442 }
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1443 if (!triple || tripcount == 3)
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1444 break;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1445 }
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1446 else if (c == '\\') {
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1447 tripcount = 0;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1448 c = tok_nextc(tok);
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1449 if (c == EOF) {
24839
17e95cc06f69 provide less mysterious error messages when seeing end-of-line in
Skip Montanaro <skip@pobox.com>
parents: 24630
diff changeset
1450 tok->done = E_EOLS;
1282
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
1451 tok->cur = tok->inp;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1452 return ERRORTOKEN;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1453 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1454 }
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1455 else
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1456 tripcount = 0;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1457 }
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1458 *p_start = tok->start;
1364
132602d854d8 Changes to accept double-quoted strings on input.
Guido van Rossum <guido@python.org>
parents: 1284
diff changeset
1459 *p_end = tok->cur;
132602d854d8 Changes to accept double-quoted strings on input.
Guido van Rossum <guido@python.org>
parents: 1284
diff changeset
1460 return STRING;
132602d854d8 Changes to accept double-quoted strings on input.
Guido van Rossum <guido@python.org>
parents: 1284
diff changeset
1461 }
37229
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
1462
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1463 /* Line continuation */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1464 if (c == '\\') {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1465 c = tok_nextc(tok);
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1466 if (c != '\n') {
34664
3785610ff763 Patch #802188: better parser error message for non-EOL following line cont.
Martin v. Löwis <martin@v.loewis.de>
parents: 32961
diff changeset
1467 tok->done = E_LINECONT;
1282
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
1468 tok->cur = tok->inp;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1469 return ERRORTOKEN;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1470 }
25098
c02a668488d0 Ignore encoding declarations inside strings. Fixes #603509.
Martin v. Löwis <martin@v.loewis.de>
parents: 24867
diff changeset
1471 tok->cont_line = 1;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1472 goto again; /* Read next line */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1473 }
37229
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
1474
400
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1475 /* Check for two-character token */
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1476 {
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1477 int c2 = tok_nextc(tok);
5379
1c9a84e0981d Another directory quickly renamed.
Guido van Rossum <guido@python.org>
parents: 5341
diff changeset
1478 int token = PyToken_TwoChars(c, c2);
400
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1479 if (token != OP) {
14163
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1480 int c3 = tok_nextc(tok);
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1481 int token3 = PyToken_ThreeChars(c, c2, c3);
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1482 if (token3 != OP) {
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1483 token = token3;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1484 } else {
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1485 tok_backup(tok, c3);
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1486 }
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1487 *p_start = tok->start;
400
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1488 *p_end = tok->cur;
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1489 return token;
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1490 }
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1491 tok_backup(tok, c2);
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1492 }
37229
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
1493
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1494 /* Keep track of parentheses nesting level */
1284
dbce72afe261 * selectmodule.c: fix (another!) two memory leaks -- this time in list2set
Guido van Rossum <guido@python.org>
parents: 1282
diff changeset
1495 switch (c) {
dbce72afe261 * selectmodule.c: fix (another!) two memory leaks -- this time in list2set
Guido van Rossum <guido@python.org>
parents: 1282
diff changeset
1496 case '(':
dbce72afe261 * selectmodule.c: fix (another!) two memory leaks -- this time in list2set
Guido van Rossum <guido@python.org>
parents: 1282
diff changeset
1497 case '[':
dbce72afe261 * selectmodule.c: fix (another!) two memory leaks -- this time in list2set
Guido van Rossum <guido@python.org>
parents: 1282
diff changeset
1498 case '{':
dbce72afe261 * selectmodule.c: fix (another!) two memory leaks -- this time in list2set
Guido van Rossum <guido@python.org>
parents: 1282
diff changeset
1499 tok->level++;
dbce72afe261 * selectmodule.c: fix (another!) two memory leaks -- this time in list2set
Guido van Rossum <guido@python.org>
parents: 1282
diff changeset
1500 break;
dbce72afe261 * selectmodule.c: fix (another!) two memory leaks -- this time in list2set
Guido van Rossum <guido@python.org>
parents: 1282
diff changeset
1501 case ')':
dbce72afe261 * selectmodule.c: fix (another!) two memory leaks -- this time in list2set
Guido van Rossum <guido@python.org>
parents: 1282
diff changeset
1502 case ']':
dbce72afe261 * selectmodule.c: fix (another!) two memory leaks -- this time in list2set
Guido van Rossum <guido@python.org>
parents: 1282
diff changeset
1503 case '}':
dbce72afe261 * selectmodule.c: fix (another!) two memory leaks -- this time in list2set
Guido van Rossum <guido@python.org>
parents: 1282
diff changeset
1504 tok->level--;
dbce72afe261 * selectmodule.c: fix (another!) two memory leaks -- this time in list2set
Guido van Rossum <guido@python.org>
parents: 1282
diff changeset
1505 break;
dbce72afe261 * selectmodule.c: fix (another!) two memory leaks -- this time in list2set
Guido van Rossum <guido@python.org>
parents: 1282
diff changeset
1506 }
37229
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
1507
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1508 /* Punctuation character */
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1509 *p_start = tok->start;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1510 *p_end = tok->cur;
5379
1c9a84e0981d Another directory quickly renamed.
Guido van Rossum <guido@python.org>
parents: 5341
diff changeset
1511 return PyToken_OneChar(c);
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1512 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1513
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
1514 int
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
1515 PyTokenizer_Get(struct tok_state *tok, char **p_start, char **p_end)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
1516 {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
1517 int result = tok_get(tok, p_start, p_end);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
1518 if (tok->decoding_erred) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
1519 result = ERRORTOKEN;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
1520 tok->done = E_DECODE;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
1521 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
1522 return result;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
1523 }
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1524
4839
4567e158d20c Rename DEBUG macro to Py_DEBUG
Guido van Rossum <guido@python.org>
parents: 4577
diff changeset
1525 #ifdef Py_DEBUG
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1526
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1527 void
13659
e78153e179fb Mass ANSIfication.
Thomas Wouters <thomas@python.org>
parents: 13539
diff changeset
1528 tok_dump(int type, char *start, char *end)
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1529 {
5379
1c9a84e0981d Another directory quickly renamed.
Guido van Rossum <guido@python.org>
parents: 5341
diff changeset
1530 printf("%s", _PyParser_TokenNames[type]);
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1531 if (type == NAME || type == NUMBER || type == STRING || type == OP)
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1532 printf("(%.*s)", (int)(end - start), start);
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1533 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1534
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1535 #endif