Mercurial > cpython-lru_cache-weakref

annotate Parser/tokenizer.c @ 58765:2ba7677bfa95 legacy-trunk

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
testCopysign was defined twice in test_math; combine the definitions
author Mark Dickinson <dickinsm@gmail.com>
date Sat, 06 Feb 2010 23:11:25 +0000
parents eea803d235d3
children b044405d62e5
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
145
853b736a6a9e Added copyright notice.
Guido van Rossum <guido@python.org>
parents: 96
diff changeset
1
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
2 /* Tokenizer implementation */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
3
23029
8291473fde3e Mass checkin of universal newline support.
Jack Jansen <jack.jansen@cwi.nl>
parents: 19621
diff changeset
4 #include "Python.h"
86
a6934380c6e7 "Compiling" version
Guido van Rossum <guido@python.org>
parents: 67
diff changeset
5 #include "pgenheaders.h"
a6934380c6e7 "Compiling" version
Guido van Rossum <guido@python.org>
parents: 67
diff changeset
6
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
7 #include <ctype.h>
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
8 #include <assert.h>
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
9
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
10 #include "tokenizer.h"
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
11 #include "errcode.h"
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
12
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
13 #ifndef PGEN
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
14 #include "unicodeobject.h"
47527
52294af4b663 This reverts r63675 based on the discussion in this thread:
Gregory P. Smith <greg@mad-scientist.com>
parents: 47294
diff changeset
15 #include "stringobject.h"
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
16 #include "fileobject.h"
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
17 #include "codecs.h"
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
18 #include "abstract.h"
43982
5f6a8e2b9e9c Applied patch #1754273 and #1754271 from Thomas Glee
Christian Heimes <christian@cheimes.de>
parents: 43572
diff changeset
19 #include "pydebug.h"
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
20 #endif /* PGEN */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
21
25924
dc4a0336a2a3 Patch #512981: Update readline input stream on sys.stdin/out change.
Martin v. Löwis <martin@v.loewis.de>
parents: 25101
diff changeset
22 extern char *PyOS_Readline(FILE *, FILE *, char *);
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
23 /* Return malloc'ed string including trailing \n;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
24 empty malloc'ed string for EOF;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
25 NULL if interrupted */
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
26
678
205b947b1d3c Make tabs always 8 spaces wide -- it's more portable.
Guido van Rossum <guido@python.org>
parents: 477
diff changeset
27 /* Don't ever change this -- it would break the portability of Python code */
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
28 #define TABSIZE 8
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
29
86
a6934380c6e7 "Compiling" version
Guido van Rossum <guido@python.org>
parents: 67
diff changeset
30 /* Forward */
13447
a89f7f268de4 Nuke all remaining occurrences of Py_PROTO and Py_FPROTO.
Tim Peters <tim.peters@gmail.com>
parents: 13256
diff changeset
31 static struct tok_state *tok_new(void);
a89f7f268de4 Nuke all remaining occurrences of Py_PROTO and Py_FPROTO.
Tim Peters <tim.peters@gmail.com>
parents: 13256
diff changeset
32 static int tok_nextc(struct tok_state *tok);
a89f7f268de4 Nuke all remaining occurrences of Py_PROTO and Py_FPROTO.
Tim Peters <tim.peters@gmail.com>
parents: 13256
diff changeset
33 static void tok_backup(struct tok_state *tok, int c);
86
a6934380c6e7 "Compiling" version
Guido van Rossum <guido@python.org>
parents: 67
diff changeset
34
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
35 /* Token names */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
36
5379
1c9a84e0981d Another directory quickly renamed.
Guido van Rossum <guido@python.org>
parents: 5341
diff changeset
37 char *_PyParser_TokenNames[] = {
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
38 "ENDMARKER",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
39 "NAME",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
40 "NUMBER",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
41 "STRING",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
42 "NEWLINE",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
43 "INDENT",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
44 "DEDENT",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
45 "LPAR",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
46 "RPAR",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
47 "LSQB",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
48 "RSQB",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
49 "COLON",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
50 "COMMA",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
51 "SEMI",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
52 "PLUS",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
53 "MINUS",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
54 "STAR",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
55 "SLASH",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
56 "VBAR",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
57 "AMPER",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
58 "LESS",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
59 "GREATER",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
60 "EQUAL",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
61 "DOT",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
62 "PERCENT",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
63 "BACKQUOTE",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
64 "LBRACE",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
65 "RBRACE",
400
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
66 "EQEQUAL",
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
67 "NOTEQUAL",
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
68 "LESSEQUAL",
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
69 "GREATEREQUAL",
410
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
70 "TILDE",
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
71 "CIRCUMFLEX",
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
72 "LEFTSHIFT",
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
73 "RIGHTSHIFT",
3333
4b5bff95804e changes for pow(**) and complex
Guido van Rossum <guido@python.org>
parents: 3116
diff changeset
74 "DOUBLESTAR",
14163
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
75 "PLUSEQUAL",
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
76 "MINEQUAL",
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
77 "STAREQUAL",
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
78 "SLASHEQUAL",
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
79 "PERCENTEQUAL",
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
80 "AMPEREQUAL",
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
81 "VBAREQUAL",
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
82 "CIRCUMFLEXEQUAL",
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
83 "LEFTSHIFTEQUAL",
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
84 "RIGHTSHIFTEQUAL",
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
85 "DOUBLESTAREQUAL",
19051
f20727cc3d0f Implement PEP 238 in its (almost) full glory.
Guido van Rossum <guido@python.org>
parents: 17600
diff changeset
86 "DOUBLESLASH",
f20727cc3d0f Implement PEP 238 in its (almost) full glory.
Guido van Rossum <guido@python.org>
parents: 17600
diff changeset
87 "DOUBLESLASHEQUAL",
32888
c7e03cf8333e PEP-0318, @decorator-style. In Guido's words:
Anthony Baxter <anthonybaxter@gmail.com>
parents: 30979
diff changeset
88 "AT",
400
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
89 /* This table must match the #defines in token.h! */
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
90 "OP",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
91 "<ERRORTOKEN>",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
92 "<N_TOKENS>"
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
93 };
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
94
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
95
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
96 /* Create and initialize a new tok_state structure */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
97
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
98 static struct tok_state *
13659
e78153e179fb Mass ANSIfication.
Thomas Wouters <thomas@python.org>
parents: 13539
diff changeset
99 tok_new(void)
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
100 {
37585
292854c43b99 Fix the code in Parser/ to also compile with C++. This was mostly casts for
Anthony Baxter <anthonybaxter@gmail.com>
parents: 37557
diff changeset
101 struct tok_state *tok = (struct tok_state *)PyMem_MALLOC(
292854c43b99 Fix the code in Parser/ to also compile with C++. This was mostly casts for
Anthony Baxter <anthonybaxter@gmail.com>
parents: 37557
diff changeset
102 sizeof(struct tok_state));
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
103 if (tok == NULL)
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
104 return NULL;
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
105 tok->buf = tok->cur = tok->end = tok->inp = tok->start = NULL;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
106 tok->done = E_OK;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
107 tok->fp = NULL;
57204
89f42938d5cc fix several compile() issues by translating newlines in the tokenizer
Benjamin Peterson <benjamin@python.org>
parents: 57131
diff changeset
108 tok->input = NULL;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
109 tok->tabsize = TABSIZE;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
110 tok->indent = 0;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
111 tok->indstack[0] = 0;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
112 tok->atbol = 1;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
113 tok->pendin = 0;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
114 tok->prompt = tok->nextprompt = NULL;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
115 tok->lineno = 0;
1284
dbce72afe261 * selectmodule.c: fix (another!) two memory leaks -- this time in list2set
Guido van Rossum <guido@python.org>
parents: 1282
diff changeset
116 tok->level = 0;
7945
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
117 tok->filename = NULL;
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
118 tok->altwarning = 0;
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
119 tok->alterror = 0;
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
120 tok->alttabsize = 1;
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
121 tok->altindstack[0] = 0;
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
122 tok->decoding_state = 0;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
123 tok->decoding_erred = 0;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
124 tok->read_coding_spec = 0;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
125 tok->encoding = NULL;
25098
c02a668488d0 Ignore encoding declarations inside strings. Fixes #603509.
Martin v. Löwis <martin@v.loewis.de>
parents: 24867
diff changeset
126 tok->cont_line = 0;
24544
0bd521eb714e Make pgen compile with pydebug. Duplicate normalized names, as it may
Martin v. Löwis <martin@v.loewis.de>
parents: 24543
diff changeset
127 #ifndef PGEN
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
128 tok->decoding_readline = NULL;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
129 tok->decoding_buffer = NULL;
24544
0bd521eb714e Make pgen compile with pydebug. Duplicate normalized names, as it may
Martin v. Löwis <martin@v.loewis.de>
parents: 24543
diff changeset
130 #endif
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
131 return tok;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
132 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
133
57204
89f42938d5cc fix several compile() issues by translating newlines in the tokenizer
Benjamin Peterson <benjamin@python.org>
parents: 57131
diff changeset
134 static char *
89f42938d5cc fix several compile() issues by translating newlines in the tokenizer
Benjamin Peterson <benjamin@python.org>
parents: 57131
diff changeset
135 new_string(const char *s, Py_ssize_t len)
89f42938d5cc fix several compile() issues by translating newlines in the tokenizer
Benjamin Peterson <benjamin@python.org>
parents: 57131
diff changeset
136 {
89f42938d5cc fix several compile() issues by translating newlines in the tokenizer
Benjamin Peterson <benjamin@python.org>
parents: 57131
diff changeset
137 char* result = (char *)PyMem_MALLOC(len + 1);
89f42938d5cc fix several compile() issues by translating newlines in the tokenizer
Benjamin Peterson <benjamin@python.org>
parents: 57131
diff changeset
138 if (result != NULL) {
89f42938d5cc fix several compile() issues by translating newlines in the tokenizer
Benjamin Peterson <benjamin@python.org>
parents: 57131
diff changeset
139 memcpy(result, s, len);
89f42938d5cc fix several compile() issues by translating newlines in the tokenizer
Benjamin Peterson <benjamin@python.org>
parents: 57131
diff changeset
140 result[len] = '\0';
89f42938d5cc fix several compile() issues by translating newlines in the tokenizer
Benjamin Peterson <benjamin@python.org>
parents: 57131
diff changeset
141 }
89f42938d5cc fix several compile() issues by translating newlines in the tokenizer
Benjamin Peterson <benjamin@python.org>
parents: 57131
diff changeset
142 return result;
89f42938d5cc fix several compile() issues by translating newlines in the tokenizer
Benjamin Peterson <benjamin@python.org>
parents: 57131
diff changeset
143 }
89f42938d5cc fix several compile() issues by translating newlines in the tokenizer
Benjamin Peterson <benjamin@python.org>
parents: 57131
diff changeset
144
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
145 #ifdef PGEN
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
146
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
147 static char *
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
148 decoding_fgets(char *s, int size, struct tok_state *tok)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
149 {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
150 return fgets(s, size, tok->fp);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
151 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
152
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
153 static int
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
154 decoding_feof(struct tok_state *tok)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
155 {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
156 return feof(tok->fp);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
157 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
158
57204
89f42938d5cc fix several compile() issues by translating newlines in the tokenizer
Benjamin Peterson <benjamin@python.org>
parents: 57131
diff changeset
159 static char *
89f42938d5cc fix several compile() issues by translating newlines in the tokenizer
Benjamin Peterson <benjamin@python.org>
parents: 57131
diff changeset
160 decode_str(const char *str, int exec_input, struct tok_state *tok)
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
161 {
57204
89f42938d5cc fix several compile() issues by translating newlines in the tokenizer
Benjamin Peterson <benjamin@python.org>
parents: 57131
diff changeset
162 return new_string(str, strlen(str));
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
163 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
164
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
165 #else /* PGEN */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
166
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
167 static char *
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
168 error_ret(struct tok_state *tok) /* XXX */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
169 {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
170 tok->decoding_erred = 1;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
171 if (tok->fp != NULL && tok->buf != NULL) /* see PyTokenizer_Free */
37600
c4c5eeb07739 As discussed on python-dev, really fix the PyMem_*/PyObject_* memory API
Neal Norwitz <nnorwitz@gmail.com>
parents: 37585
diff changeset
172 PyMem_FREE(tok->buf);
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
173 tok->buf = NULL;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
174 return NULL; /* as if it were EOF */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
175 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
176
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
177
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
178 static char *
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
179 get_normal_name(char *s) /* for utf-8 and latin-1 */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
180 {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
181 char buf[13];
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
182 int i;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
183 for (i = 0; i < 12; i++) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
184 int c = s[i];
56432
896a4fb13f35 fix some coding style
Benjamin Peterson <benjamin@python.org>
parents: 54760
diff changeset
185 if (c == '\0')
896a4fb13f35 fix some coding style
Benjamin Peterson <benjamin@python.org>
parents: 54760
diff changeset
186 break;
896a4fb13f35 fix some coding style
Benjamin Peterson <benjamin@python.org>
parents: 54760
diff changeset
187 else if (c == '_')
896a4fb13f35 fix some coding style
Benjamin Peterson <benjamin@python.org>
parents: 54760
diff changeset
188 buf[i] = '-';
896a4fb13f35 fix some coding style
Benjamin Peterson <benjamin@python.org>
parents: 54760
diff changeset
189 else
896a4fb13f35 fix some coding style
Benjamin Peterson <benjamin@python.org>
parents: 54760
diff changeset
190 buf[i] = tolower(c);
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
191 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
192 buf[i] = '\0';
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
193 if (strcmp(buf, "utf-8") == 0 ||
56432
896a4fb13f35 fix some coding style
Benjamin Peterson <benjamin@python.org>
parents: 54760
diff changeset
194 strncmp(buf, "utf-8-", 6) == 0)
896a4fb13f35 fix some coding style
Benjamin Peterson <benjamin@python.org>
parents: 54760
diff changeset
195 return "utf-8";
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
196 else if (strcmp(buf, "latin-1") == 0 ||
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
197 strcmp(buf, "iso-8859-1") == 0 ||
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
198 strcmp(buf, "iso-latin-1") == 0 ||
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
199 strncmp(buf, "latin-1-", 8) == 0 ||
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
200 strncmp(buf, "iso-8859-1-", 11) == 0 ||
56432
896a4fb13f35 fix some coding style
Benjamin Peterson <benjamin@python.org>
parents: 54760
diff changeset
201 strncmp(buf, "iso-latin-1-", 12) == 0)
896a4fb13f35 fix some coding style
Benjamin Peterson <benjamin@python.org>
parents: 54760
diff changeset
202 return "iso-8859-1";
896a4fb13f35 fix some coding style
Benjamin Peterson <benjamin@python.org>
parents: 54760
diff changeset
203 else
896a4fb13f35 fix some coding style
Benjamin Peterson <benjamin@python.org>
parents: 54760
diff changeset
204 return s;
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
205 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
206
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
207 /* Return the coding spec in S, or NULL if none is found. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
208
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
209 static char *
36501
7fdc639bc5b4 Merge ssize_t branch.
Martin v. Löwis <martin@v.loewis.de>
parents: 36090
diff changeset
210 get_coding_spec(const char *s, Py_ssize_t size)
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
211 {
36501
7fdc639bc5b4 Merge ssize_t branch.
Martin v. Löwis <martin@v.loewis.de>
parents: 36090
diff changeset
212 Py_ssize_t i;
25098
c02a668488d0 Ignore encoding declarations inside strings. Fixes #603509.
Martin v. Löwis <martin@v.loewis.de>
parents: 24867
diff changeset
213 /* Coding spec must be in a comment, and that comment must be
c02a668488d0 Ignore encoding declarations inside strings. Fixes #603509.
Martin v. Löwis <martin@v.loewis.de>
parents: 24867
diff changeset
214 * the only statement on the source code line. */
c02a668488d0 Ignore encoding declarations inside strings. Fixes #603509.
Martin v. Löwis <martin@v.loewis.de>
parents: 24867
diff changeset
215 for (i = 0; i < size - 6; i++) {
c02a668488d0 Ignore encoding declarations inside strings. Fixes #603509.
Martin v. Löwis <martin@v.loewis.de>
parents: 24867
diff changeset
216 if (s[i] == '#')
c02a668488d0 Ignore encoding declarations inside strings. Fixes #603509.
Martin v. Löwis <martin@v.loewis.de>
parents: 24867
diff changeset
217 break;
c02a668488d0 Ignore encoding declarations inside strings. Fixes #603509.
Martin v. Löwis <martin@v.loewis.de>
parents: 24867
diff changeset
218 if (s[i] != ' ' && s[i] != '\t' && s[i] != '\014')
c02a668488d0 Ignore encoding declarations inside strings. Fixes #603509.
Martin v. Löwis <martin@v.loewis.de>
parents: 24867
diff changeset
219 return NULL;
c02a668488d0 Ignore encoding declarations inside strings. Fixes #603509.
Martin v. Löwis <martin@v.loewis.de>
parents: 24867
diff changeset
220 }
c02a668488d0 Ignore encoding declarations inside strings. Fixes #603509.
Martin v. Löwis <martin@v.loewis.de>
parents: 24867
diff changeset
221 for (; i < size - 6; i++) { /* XXX inefficient search */
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
222 const char* t = s + i;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
223 if (strncmp(t, "coding", 6) == 0) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
224 const char* begin = NULL;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
225 t += 6;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
226 if (t[0] != ':' && t[0] != '=')
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
227 continue;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
228 do {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
229 t++;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
230 } while (t[0] == '\x20' || t[0] == '\t');
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
231
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
232 begin = t;
36090
8cb686dd6e64 Fix SF bug #1072182, problems with signed characters.
Neal Norwitz <nnorwitz@gmail.com>
parents: 36078
diff changeset
233 while (isalnum(Py_CHARMASK(t[0])) ||
25995
1aaa12c08b35 Fix compiler warning on HP-UX.
Neal Norwitz <nnorwitz@gmail.com>
parents: 25924
diff changeset
234 t[0] == '-' || t[0] == '_' || t[0] == '.')
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
235 t++;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
236
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
237 if (begin < t) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
238 char* r = new_string(begin, t - begin);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
239 char* q = get_normal_name(r);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
240 if (r != q) {
37600
c4c5eeb07739 As discussed on python-dev, really fix the PyMem_*/PyObject_* memory API
Neal Norwitz <nnorwitz@gmail.com>
parents: 37585
diff changeset
241 PyMem_FREE(r);
24544
0bd521eb714e Make pgen compile with pydebug. Duplicate normalized names, as it may
Martin v. Löwis <martin@v.loewis.de>
parents: 24543
diff changeset
242 r = new_string(q, strlen(q));
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
243 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
244 return r;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
245 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
246 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
247 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
248 return NULL;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
249 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
250
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
251 /* Check whether the line contains a coding spec. If it does,
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
252 invoke the set_readline function for the new encoding.
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
253 This function receives the tok_state and the new encoding.
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
254 Return 1 on success, 0 on failure. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
255
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
256 static int
36501
7fdc639bc5b4 Merge ssize_t branch.
Martin v. Löwis <martin@v.loewis.de>
parents: 36090
diff changeset
257 check_coding_spec(const char* line, Py_ssize_t size, struct tok_state *tok,
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
258 int set_readline(struct tok_state *, const char *))
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
259 {
25101
32331d26d7cf Removed reliance on gcc/C99 extension.
Tim Peters <tim.peters@gmail.com>
parents: 25098
diff changeset
260 char * cs;
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
261 int r = 1;
25101
32331d26d7cf Removed reliance on gcc/C99 extension.
Tim Peters <tim.peters@gmail.com>
parents: 25098
diff changeset
262
25098
c02a668488d0 Ignore encoding declarations inside strings. Fixes #603509.
Martin v. Löwis <martin@v.loewis.de>
parents: 24867
diff changeset
263 if (tok->cont_line)
c02a668488d0 Ignore encoding declarations inside strings. Fixes #603509.
Martin v. Löwis <martin@v.loewis.de>
parents: 24867
diff changeset
264 /* It's a continuation line, so it can't be a coding spec. */
c02a668488d0 Ignore encoding declarations inside strings. Fixes #603509.
Martin v. Löwis <martin@v.loewis.de>
parents: 24867
diff changeset
265 return 1;
25101
32331d26d7cf Removed reliance on gcc/C99 extension.
Tim Peters <tim.peters@gmail.com>
parents: 25098
diff changeset
266 cs = get_coding_spec(line, size);
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
267 if (cs != NULL) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
268 tok->read_coding_spec = 1;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
269 if (tok->encoding == NULL) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
270 assert(tok->decoding_state == 1); /* raw */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
271 if (strcmp(cs, "utf-8") == 0 ||
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
272 strcmp(cs, "iso-8859-1") == 0) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
273 tok->encoding = cs;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
274 } else {
24624
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
275 #ifdef Py_USING_UNICODE
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
276 r = set_readline(tok, cs);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
277 if (r) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
278 tok->encoding = cs;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
279 tok->decoding_state = -1;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
280 }
35751
e56144704808 Free coding spec (cs) if there was an error to prevent mem leak. Maybe backport candidate
Neal Norwitz <nnorwitz@gmail.com>
parents: 35666
diff changeset
281 else
37600
c4c5eeb07739 As discussed on python-dev, really fix the PyMem_*/PyObject_* memory API
Neal Norwitz <nnorwitz@gmail.com>
parents: 37585
diff changeset
282 PyMem_FREE(cs);
24624
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
283 #else
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
284 /* Without Unicode support, we cannot
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
285 process the coding spec. Since there
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
286 won't be any Unicode literals, that
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
287 won't matter. */
37600
c4c5eeb07739 As discussed on python-dev, really fix the PyMem_*/PyObject_* memory API
Neal Norwitz <nnorwitz@gmail.com>
parents: 37585
diff changeset
288 PyMem_FREE(cs);
24624
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
289 #endif
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
290 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
291 } else { /* then, compare cs with BOM */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
292 r = (strcmp(tok->encoding, cs) == 0);
37600
c4c5eeb07739 As discussed on python-dev, really fix the PyMem_*/PyObject_* memory API
Neal Norwitz <nnorwitz@gmail.com>
parents: 37585
diff changeset
293 PyMem_FREE(cs);
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
294 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
295 }
36078
1ff4790b7605 Fix Bug #1378022, UTF-8 files with a leading BOM crashed the interpreter.
Neal Norwitz <nnorwitz@gmail.com>
parents: 35878
diff changeset
296 if (!r) {
1ff4790b7605 Fix Bug #1378022, UTF-8 files with a leading BOM crashed the interpreter.
Neal Norwitz <nnorwitz@gmail.com>
parents: 35878
diff changeset
297 cs = tok->encoding;
1ff4790b7605 Fix Bug #1378022, UTF-8 files with a leading BOM crashed the interpreter.
Neal Norwitz <nnorwitz@gmail.com>
parents: 35878
diff changeset
298 if (!cs)
1ff4790b7605 Fix Bug #1378022, UTF-8 files with a leading BOM crashed the interpreter.
Neal Norwitz <nnorwitz@gmail.com>
parents: 35878
diff changeset
299 cs = "with BOM";
1ff4790b7605 Fix Bug #1378022, UTF-8 files with a leading BOM crashed the interpreter.
Neal Norwitz <nnorwitz@gmail.com>
parents: 35878
diff changeset
300 PyErr_Format(PyExc_SyntaxError, "encoding problem: %s", cs);
1ff4790b7605 Fix Bug #1378022, UTF-8 files with a leading BOM crashed the interpreter.
Neal Norwitz <nnorwitz@gmail.com>
parents: 35878
diff changeset
301 }
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
302 return r;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
303 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
304
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
305 /* See whether the file starts with a BOM. If it does,
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
306 invoke the set_readline function with the new encoding.
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
307 Return 1 on success, 0 on failure. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
308
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
309 static int
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
310 check_bom(int get_char(struct tok_state *),
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
311 void unget_char(int, struct tok_state *),
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
312 int set_readline(struct tok_state *, const char *),
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
313 struct tok_state *tok)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
314 {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
315 int ch = get_char(tok);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
316 tok->decoding_state = 1;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
317 if (ch == EOF) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
318 return 1;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
319 } else if (ch == 0xEF) {
56432
896a4fb13f35 fix some coding style
Benjamin Peterson <benjamin@python.org>
parents: 54760
diff changeset
320 ch = get_char(tok);
896a4fb13f35 fix some coding style
Benjamin Peterson <benjamin@python.org>
parents: 54760
diff changeset
321 if (ch != 0xBB)
896a4fb13f35 fix some coding style
Benjamin Peterson <benjamin@python.org>
parents: 54760
diff changeset
322 goto NON_BOM;
896a4fb13f35 fix some coding style
Benjamin Peterson <benjamin@python.org>
parents: 54760
diff changeset
323 ch = get_char(tok);
896a4fb13f35 fix some coding style
Benjamin Peterson <benjamin@python.org>
parents: 54760
diff changeset
324 if (ch != 0xBF)
896a4fb13f35 fix some coding style
Benjamin Peterson <benjamin@python.org>
parents: 54760
diff changeset
325 goto NON_BOM;
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
326 #if 0
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
327 /* Disable support for UTF-16 BOMs until a decision
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
328 is made whether this needs to be supported. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
329 } else if (ch == 0xFE) {
56432
896a4fb13f35 fix some coding style
Benjamin Peterson <benjamin@python.org>
parents: 54760
diff changeset
330 ch = get_char(tok);
896a4fb13f35 fix some coding style
Benjamin Peterson <benjamin@python.org>
parents: 54760
diff changeset
331 if (ch != 0xFF)
896a4fb13f35 fix some coding style
Benjamin Peterson <benjamin@python.org>
parents: 54760
diff changeset
332 goto NON_BOM;
896a4fb13f35 fix some coding style
Benjamin Peterson <benjamin@python.org>
parents: 54760
diff changeset
333 if (!set_readline(tok, "utf-16-be"))
896a4fb13f35 fix some coding style
Benjamin Peterson <benjamin@python.org>
parents: 54760
diff changeset
334 return 0;
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
335 tok->decoding_state = -1;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
336 } else if (ch == 0xFF) {
56432
896a4fb13f35 fix some coding style
Benjamin Peterson <benjamin@python.org>
parents: 54760
diff changeset
337 ch = get_char(tok);
896a4fb13f35 fix some coding style
Benjamin Peterson <benjamin@python.org>
parents: 54760
diff changeset
338 if (ch != 0xFE)
896a4fb13f35 fix some coding style
Benjamin Peterson <benjamin@python.org>
parents: 54760
diff changeset
339 goto NON_BOM;
896a4fb13f35 fix some coding style
Benjamin Peterson <benjamin@python.org>
parents: 54760
diff changeset
340 if (!set_readline(tok, "utf-16-le"))
896a4fb13f35 fix some coding style
Benjamin Peterson <benjamin@python.org>
parents: 54760
diff changeset
341 return 0;
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
342 tok->decoding_state = -1;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
343 #endif
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
344 } else {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
345 unget_char(ch, tok);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
346 return 1;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
347 }
35878
94a409da4bd4 Fix some more memory leaks.
Neal Norwitz <nnorwitz@gmail.com>
parents: 35751
diff changeset
348 if (tok->encoding != NULL)
37600
c4c5eeb07739 As discussed on python-dev, really fix the PyMem_*/PyObject_* memory API
Neal Norwitz <nnorwitz@gmail.com>
parents: 37585
diff changeset
349 PyMem_FREE(tok->encoding);
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
350 tok->encoding = new_string("utf-8", 5); /* resulting is in utf-8 */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
351 return 1;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
352 NON_BOM:
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
353 /* any token beginning with '\xEF', '\xFE', '\xFF' is a bad token */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
354 unget_char(0xFF, tok); /* XXX this will cause a syntax error */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
355 return 1;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
356 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
357
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
358 /* Read a line of text from TOK into S, using the stream in TOK.
35231
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
359 Return NULL on failure, else S.
37229
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
360
35231
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
361 On entry, tok->decoding_buffer will be one of:
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
362 1) NULL: need to call tok->decoding_readline to get a new line
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
363 2) PyUnicodeObject *: decoding_feof has called tok->decoding_readline and
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
364 stored the result in tok->decoding_buffer
47527
52294af4b663 This reverts r63675 based on the discussion in this thread:
Gregory P. Smith <greg@mad-scientist.com>
parents: 47294
diff changeset
365 3) PyStringObject *: previous call to fp_readl did not have enough room
35231
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
366 (in the s buffer) to copy entire contents of the line read
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
367 by tok->decoding_readline. tok->decoding_buffer has the overflow.
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
368 In this case, fp_readl is called in a loop (with an expanded buffer)
37229
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
369 until the buffer ends with a '\n' (or until the end of the file is
35231
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
370 reached): see tok_nextc and its calls to decoding_fgets.
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
371 */
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
372
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
373 static char *
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
374 fp_readl(char *s, int size, struct tok_state *tok)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
375 {
24624
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
376 #ifndef Py_USING_UNICODE
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
377 /* In a non-Unicode built, this should never be called. */
24630
43caa212b959 Use Py_FatalError instead of abort.
Martin v. Löwis <martin@v.loewis.de>
parents: 24624
diff changeset
378 Py_FatalError("fp_readl should not be called in this build.");
24867
255d1d3e66a3 Squash a few calls to the hideously expensive PyObject_CallObject(o,a)
Guido van Rossum <guido@python.org>
parents: 24839
diff changeset
379 return NULL; /* Keep compiler happy (not reachable) */
24624
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
380 #else
35231
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
381 PyObject* utf8 = NULL;
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
382 PyObject* buf = tok->decoding_buffer;
35231
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
383 char *str;
36520
26470cdeea3d Use Py_ssize_t to count the length.
Martin v. Löwis <martin@v.loewis.de>
parents: 36501
diff changeset
384 Py_ssize_t utf8len;
35231
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
385
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
386 /* Ask for one less byte so we can terminate it */
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
387 assert(size > 0);
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
388 size--;
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
389
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
390 if (buf == NULL) {
35231
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
391 buf = PyObject_CallObject(tok->decoding_readline, NULL);
24543
c00df23e856e Group statements properly.
Martin v. Löwis <martin@v.loewis.de>
parents: 24542
diff changeset
392 if (buf == NULL)
c00df23e856e Group statements properly.
Martin v. Löwis <martin@v.loewis.de>
parents: 24542
diff changeset
393 return error_ret(tok);
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
394 } else {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
395 tok->decoding_buffer = NULL;
47527
52294af4b663 This reverts r63675 based on the discussion in this thread:
Gregory P. Smith <greg@mad-scientist.com>
parents: 47294
diff changeset
396 if (PyString_CheckExact(buf))
35231
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
397 utf8 = buf;
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
398 }
35231
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
399 if (utf8 == NULL) {
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
400 utf8 = PyUnicode_AsUTF8String(buf);
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
401 Py_DECREF(buf);
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
402 if (utf8 == NULL)
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
403 return error_ret(tok);
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
404 }
47527
52294af4b663 This reverts r63675 based on the discussion in this thread:
Gregory P. Smith <greg@mad-scientist.com>
parents: 47294
diff changeset
405 str = PyString_AsString(utf8);
52294af4b663 This reverts r63675 based on the discussion in this thread:
Gregory P. Smith <greg@mad-scientist.com>
parents: 47294
diff changeset
406 utf8len = PyString_GET_SIZE(utf8);
35231
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
407 if (utf8len > size) {
47527
52294af4b663 This reverts r63675 based on the discussion in this thread:
Gregory P. Smith <greg@mad-scientist.com>
parents: 47294
diff changeset
408 tok->decoding_buffer = PyString_FromStringAndSize(str+size, utf8len-size);
35231
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
409 if (tok->decoding_buffer == NULL) {
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
410 Py_DECREF(utf8);
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
411 return error_ret(tok);
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
412 }
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
413 utf8len = size;
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
414 }
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
415 memcpy(s, str, utf8len);
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
416 s[utf8len] = '\0';
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
417 Py_DECREF(utf8);
56432
896a4fb13f35 fix some coding style
Benjamin Peterson <benjamin@python.org>
parents: 54760
diff changeset
418 if (utf8len == 0)
896a4fb13f35 fix some coding style
Benjamin Peterson <benjamin@python.org>
parents: 54760
diff changeset
419 return NULL; /* EOF */
35231
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
420 return s;
24624
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
421 #endif
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
422 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
423
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
424 /* Set the readline function for TOK to a StreamReader's
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
425 readline function. The StreamReader is named ENC.
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
426
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
427 This function is called from check_bom and check_coding_spec.
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
428
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
429 ENC is usually identical to the future value of tok->encoding,
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
430 except for the (currently unsupported) case of UTF-16.
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
431
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
432 Return 1 on success, 0 on failure. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
433
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
434 static int
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
435 fp_setreadl(struct tok_state *tok, const char* enc)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
436 {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
437 PyObject *reader, *stream, *readline;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
438
26601
cc05c2c8660e Constify filenames and scripts. Fixes #651362.
Martin v. Löwis <martin@v.loewis.de>
parents: 25995
diff changeset
439 /* XXX: constify filename argument. */
cc05c2c8660e Constify filenames and scripts. Fixes #651362.
Martin v. Löwis <martin@v.loewis.de>
parents: 25995
diff changeset
440 stream = PyFile_FromFile(tok->fp, (char*)tok->filename, "rb", NULL);
24543
c00df23e856e Group statements properly.
Martin v. Löwis <martin@v.loewis.de>
parents: 24542
diff changeset
441 if (stream == NULL)
c00df23e856e Group statements properly.
Martin v. Löwis <martin@v.loewis.de>
parents: 24542
diff changeset
442 return 0;
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
443
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
444 reader = PyCodec_StreamReader(enc, stream, NULL);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
445 Py_DECREF(stream);
24543
c00df23e856e Group statements properly.
Martin v. Löwis <martin@v.loewis.de>
parents: 24542
diff changeset
446 if (reader == NULL)
c00df23e856e Group statements properly.
Martin v. Löwis <martin@v.loewis.de>
parents: 24542
diff changeset
447 return 0;
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
448
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
449 readline = PyObject_GetAttrString(reader, "readline");
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
450 Py_DECREF(reader);
24543
c00df23e856e Group statements properly.
Martin v. Löwis <martin@v.loewis.de>
parents: 24542
diff changeset
451 if (readline == NULL)
c00df23e856e Group statements properly.
Martin v. Löwis <martin@v.loewis.de>
parents: 24542
diff changeset
452 return 0;
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
453
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
454 tok->decoding_readline = readline;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
455 return 1;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
456 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
457
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
458 /* Fetch the next byte from TOK. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
459
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
460 static int fp_getc(struct tok_state *tok) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
461 return getc(tok->fp);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
462 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
463
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
464 /* Unfetch the last byte back into TOK. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
465
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
466 static void fp_ungetc(int c, struct tok_state *tok) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
467 ungetc(c, tok->fp);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
468 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
469
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
470 /* Read a line of input from TOK. Determine encoding
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
471 if necessary. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
472
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
473 static char *
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
474 decoding_fgets(char *s, int size, struct tok_state *tok)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
475 {
24630
43caa212b959 Use Py_FatalError instead of abort.
Martin v. Löwis <martin@v.loewis.de>
parents: 24624
diff changeset
476 char *line = NULL;
36741
c0982b8155c6 Change non-ASCII warning into a SyntaxError.
Martin v. Löwis <martin@v.loewis.de>
parents: 36520
diff changeset
477 int badchar = 0;
24543
c00df23e856e Group statements properly.
Martin v. Löwis <martin@v.loewis.de>
parents: 24542
diff changeset
478 for (;;) {
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
479 if (tok->decoding_state < 0) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
480 /* We already have a codec associated with
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
481 this input. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
482 line = fp_readl(s, size, tok);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
483 break;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
484 } else if (tok->decoding_state > 0) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
485 /* We want a 'raw' read. */
37229
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
486 line = Py_UniversalNewlineFgets(s, size,
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
487 tok->fp, NULL);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
488 break;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
489 } else {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
490 /* We have not yet determined the encoding.
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
491 If an encoding is found, use the file-pointer
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
492 reader functions from now on. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
493 if (!check_bom(fp_getc, fp_ungetc, fp_setreadl, tok))
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
494 return error_ret(tok);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
495 assert(tok->decoding_state != 0);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
496 }
24543
c00df23e856e Group statements properly.
Martin v. Löwis <martin@v.loewis.de>
parents: 24542
diff changeset
497 }
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
498 if (line != NULL && tok->lineno < 2 && !tok->read_coding_spec) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
499 if (!check_coding_spec(line, strlen(line), tok, fp_setreadl)) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
500 return error_ret(tok);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
501 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
502 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
503 #ifndef PGEN
36741
c0982b8155c6 Change non-ASCII warning into a SyntaxError.
Martin v. Löwis <martin@v.loewis.de>
parents: 36520
diff changeset
504 /* The default encoding is ASCII, so make sure we don't have any
c0982b8155c6 Change non-ASCII warning into a SyntaxError.
Martin v. Löwis <martin@v.loewis.de>
parents: 36520
diff changeset
505 non-ASCII bytes in it. */
c0982b8155c6 Change non-ASCII warning into a SyntaxError.
Martin v. Löwis <martin@v.loewis.de>
parents: 36520
diff changeset
506 if (line && !tok->encoding) {
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
507 unsigned char *c;
24565
c8d694b641f1 Added a cast to shut up a compiler warning.
Jack Jansen <jack.jansen@cwi.nl>
parents: 24558
diff changeset
508 for (c = (unsigned char *)line; *c; c++)
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
509 if (*c > 127) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
510 badchar = *c;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
511 break;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
512 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
513 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
514 if (badchar) {
27918
be371ffc4cd0 Add URL for PEP to the source code encoding warning.
Marc-André Lemburg <mal@egenix.com>
parents: 27772
diff changeset
515 char buf[500];
24558
d26b62f0c6db Add 1 to lineno in deprecation warning. Fixes #590888.
Martin v. Löwis <martin@v.loewis.de>
parents: 24544
diff changeset
516 /* Need to add 1 to the line number, since this line
d26b62f0c6db Add 1 to lineno in deprecation warning. Fixes #590888.
Martin v. Löwis <martin@v.loewis.de>
parents: 24544
diff changeset
517 has not been counted, yet. */
37229
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
518 sprintf(buf,
27918
be371ffc4cd0 Add URL for PEP to the source code encoding warning.
Marc-André Lemburg <mal@egenix.com>
parents: 27772
diff changeset
519 "Non-ASCII character '\\x%.2x' "
be371ffc4cd0 Add URL for PEP to the source code encoding warning.
Marc-André Lemburg <mal@egenix.com>
parents: 27772
diff changeset
520 "in file %.200s on line %i, "
be371ffc4cd0 Add URL for PEP to the source code encoding warning.
Marc-André Lemburg <mal@egenix.com>
parents: 27772
diff changeset
521 "but no encoding declared; "
37229
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
522 "see http://www.python.org/peps/pep-0263.html for details",
27918
be371ffc4cd0 Add URL for PEP to the source code encoding warning.
Marc-André Lemburg <mal@egenix.com>
parents: 27772
diff changeset
523 badchar, tok->filename, tok->lineno + 1);
36741
c0982b8155c6 Change non-ASCII warning into a SyntaxError.
Martin v. Löwis <martin@v.loewis.de>
parents: 36520
diff changeset
524 PyErr_SetString(PyExc_SyntaxError, buf);
c0982b8155c6 Change non-ASCII warning into a SyntaxError.
Martin v. Löwis <martin@v.loewis.de>
parents: 36520
diff changeset
525 return error_ret(tok);
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
526 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
527 #endif
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
528 return line;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
529 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
530
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
531 static int
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
532 decoding_feof(struct tok_state *tok)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
533 {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
534 if (tok->decoding_state >= 0) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
535 return feof(tok->fp);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
536 } else {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
537 PyObject* buf = tok->decoding_buffer;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
538 if (buf == NULL) {
35231
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
539 buf = PyObject_CallObject(tok->decoding_readline, NULL);
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
540 if (buf == NULL) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
541 error_ret(tok);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
542 return 1;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
543 } else {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
544 tok->decoding_buffer = buf;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
545 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
546 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
547 return PyObject_Length(buf) == 0;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
548 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
549 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
550
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
551 /* Fetch a byte from TOK, using the string buffer. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
552
37229
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
553 static int
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
554 buf_getc(struct tok_state *tok) {
27772
3566b84e66b9 patch 680474 that fixes bug 679880: compile/eval/exec refused utf-8 bom
Just van Rossum <just@letterror.com>
parents: 27247
diff changeset
555 return Py_CHARMASK(*tok->str++);
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
556 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
557
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
558 /* Unfetch a byte from TOK, using the string buffer. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
559
37229
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
560 static void
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
561 buf_ungetc(int c, struct tok_state *tok) {
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
562 tok->str--;
27772
3566b84e66b9 patch 680474 that fixes bug 679880: compile/eval/exec refused utf-8 bom
Just van Rossum <just@letterror.com>
parents: 27247
diff changeset
563 assert(Py_CHARMASK(*tok->str) == c); /* tok->cur may point to read-only segment */
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
564 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
565
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
566 /* Set the readline function for TOK to ENC. For the string-based
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
567 tokenizer, this means to just record the encoding. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
568
37229
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
569 static int
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
570 buf_setreadl(struct tok_state *tok, const char* enc) {
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
571 tok->enc = enc;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
572 return 1;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
573 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
574
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
575 /* Return a UTF-8 encoding Python string object from the
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
576 C byte string STR, which is encoded with ENC. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
577
24624
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
578 #ifdef Py_USING_UNICODE
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
579 static PyObject *
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
580 translate_into_utf8(const char* str, const char* enc) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
581 PyObject *utf8;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
582 PyObject* buf = PyUnicode_Decode(str, strlen(str), enc, NULL);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
583 if (buf == NULL)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
584 return NULL;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
585 utf8 = PyUnicode_AsUTF8String(buf);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
586 Py_DECREF(buf);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
587 return utf8;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
588 }
24624
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
589 #endif
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
590
57204
89f42938d5cc fix several compile() issues by translating newlines in the tokenizer
Benjamin Peterson <benjamin@python.org>
parents: 57131
diff changeset
591
89f42938d5cc fix several compile() issues by translating newlines in the tokenizer
Benjamin Peterson <benjamin@python.org>
parents: 57131
diff changeset
592 static char *
89f42938d5cc fix several compile() issues by translating newlines in the tokenizer
Benjamin Peterson <benjamin@python.org>
parents: 57131
diff changeset
593 translate_newlines(const char *s, int exec_input, struct tok_state *tok) {
57590
eea803d235d3 rewrite translate_newlines for clarity
Benjamin Peterson <benjamin@python.org>
parents: 57204
diff changeset
594 int skip_next_lf = 0, needed_length = strlen(s) + 2, final_length;
57204
89f42938d5cc fix several compile() issues by translating newlines in the tokenizer
Benjamin Peterson <benjamin@python.org>
parents: 57131
diff changeset
595 char *buf, *current;
57590
eea803d235d3 rewrite translate_newlines for clarity
Benjamin Peterson <benjamin@python.org>
parents: 57204
diff changeset
596 char c = '\0';
eea803d235d3 rewrite translate_newlines for clarity
Benjamin Peterson <benjamin@python.org>
parents: 57204
diff changeset
597 buf = PyMem_MALLOC(needed_length);
57204
89f42938d5cc fix several compile() issues by translating newlines in the tokenizer
Benjamin Peterson <benjamin@python.org>
parents: 57131
diff changeset
598 if (buf == NULL) {
89f42938d5cc fix several compile() issues by translating newlines in the tokenizer
Benjamin Peterson <benjamin@python.org>
parents: 57131
diff changeset
599 tok->done = E_NOMEM;
89f42938d5cc fix several compile() issues by translating newlines in the tokenizer
Benjamin Peterson <benjamin@python.org>
parents: 57131
diff changeset
600 return NULL;
89f42938d5cc fix several compile() issues by translating newlines in the tokenizer
Benjamin Peterson <benjamin@python.org>
parents: 57131
diff changeset
601 }
57590
eea803d235d3 rewrite translate_newlines for clarity
Benjamin Peterson <benjamin@python.org>
parents: 57204
diff changeset
602 for (current = buf; *s; s++, current++) {
eea803d235d3 rewrite translate_newlines for clarity
Benjamin Peterson <benjamin@python.org>
parents: 57204
diff changeset
603 c = *s;
57204
89f42938d5cc fix several compile() issues by translating newlines in the tokenizer
Benjamin Peterson <benjamin@python.org>
parents: 57131
diff changeset
604 if (skip_next_lf) {
89f42938d5cc fix several compile() issues by translating newlines in the tokenizer
Benjamin Peterson <benjamin@python.org>
parents: 57131
diff changeset
605 skip_next_lf = 0;
89f42938d5cc fix several compile() issues by translating newlines in the tokenizer
Benjamin Peterson <benjamin@python.org>
parents: 57131
diff changeset
606 if (c == '\n') {
57590
eea803d235d3 rewrite translate_newlines for clarity
Benjamin Peterson <benjamin@python.org>
parents: 57204
diff changeset
607 c = *++s;
57204
89f42938d5cc fix several compile() issues by translating newlines in the tokenizer
Benjamin Peterson <benjamin@python.org>
parents: 57131
diff changeset
608 if (!c)
89f42938d5cc fix several compile() issues by translating newlines in the tokenizer
Benjamin Peterson <benjamin@python.org>
parents: 57131
diff changeset
609 break;
89f42938d5cc fix several compile() issues by translating newlines in the tokenizer
Benjamin Peterson <benjamin@python.org>
parents: 57131
diff changeset
610 }
89f42938d5cc fix several compile() issues by translating newlines in the tokenizer
Benjamin Peterson <benjamin@python.org>
parents: 57131
diff changeset
611 }
89f42938d5cc fix several compile() issues by translating newlines in the tokenizer
Benjamin Peterson <benjamin@python.org>
parents: 57131
diff changeset
612 if (c == '\r') {
89f42938d5cc fix several compile() issues by translating newlines in the tokenizer
Benjamin Peterson <benjamin@python.org>
parents: 57131
diff changeset
613 skip_next_lf = 1;
89f42938d5cc fix several compile() issues by translating newlines in the tokenizer
Benjamin Peterson <benjamin@python.org>
parents: 57131
diff changeset
614 c = '\n';
89f42938d5cc fix several compile() issues by translating newlines in the tokenizer
Benjamin Peterson <benjamin@python.org>
parents: 57131
diff changeset
615 }
89f42938d5cc fix several compile() issues by translating newlines in the tokenizer
Benjamin Peterson <benjamin@python.org>
parents: 57131
diff changeset
616 *current = c;
89f42938d5cc fix several compile() issues by translating newlines in the tokenizer
Benjamin Peterson <benjamin@python.org>
parents: 57131
diff changeset
617 }
57590
eea803d235d3 rewrite translate_newlines for clarity
Benjamin Peterson <benjamin@python.org>
parents: 57204
diff changeset
618 /* If this is exec input, add a newline to the end of the string if
57204
89f42938d5cc fix several compile() issues by translating newlines in the tokenizer
Benjamin Peterson <benjamin@python.org>
parents: 57131
diff changeset
619 there isn't one already. */
57590
eea803d235d3 rewrite translate_newlines for clarity
Benjamin Peterson <benjamin@python.org>
parents: 57204
diff changeset
620 if (exec_input && c != '\n') {
57204
89f42938d5cc fix several compile() issues by translating newlines in the tokenizer
Benjamin Peterson <benjamin@python.org>
parents: 57131
diff changeset
621 *current = '\n';
89f42938d5cc fix several compile() issues by translating newlines in the tokenizer
Benjamin Peterson <benjamin@python.org>
parents: 57131
diff changeset
622 current++;
89f42938d5cc fix several compile() issues by translating newlines in the tokenizer
Benjamin Peterson <benjamin@python.org>
parents: 57131
diff changeset
623 }
89f42938d5cc fix several compile() issues by translating newlines in the tokenizer
Benjamin Peterson <benjamin@python.org>
parents: 57131
diff changeset
624 *current = '\0';
57590
eea803d235d3 rewrite translate_newlines for clarity
Benjamin Peterson <benjamin@python.org>
parents: 57204
diff changeset
625 final_length = current - buf + 1;
eea803d235d3 rewrite translate_newlines for clarity
Benjamin Peterson <benjamin@python.org>
parents: 57204
diff changeset
626 if (final_length < needed_length && final_length)
57204
89f42938d5cc fix several compile() issues by translating newlines in the tokenizer
Benjamin Peterson <benjamin@python.org>
parents: 57131
diff changeset
627 /* should never fail */
57590
eea803d235d3 rewrite translate_newlines for clarity
Benjamin Peterson <benjamin@python.org>
parents: 57204
diff changeset
628 buf = PyMem_REALLOC(buf, final_length);
57204
89f42938d5cc fix several compile() issues by translating newlines in the tokenizer
Benjamin Peterson <benjamin@python.org>
parents: 57131
diff changeset
629 return buf;
89f42938d5cc fix several compile() issues by translating newlines in the tokenizer
Benjamin Peterson <benjamin@python.org>
parents: 57131
diff changeset
630 }
89f42938d5cc fix several compile() issues by translating newlines in the tokenizer
Benjamin Peterson <benjamin@python.org>
parents: 57131
diff changeset
631
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
632 /* Decode a byte string STR for use as the buffer of TOK.
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
633 Look for encoding declarations inside STR, and record them
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
634 inside TOK. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
635
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
636 static const char *
57204
89f42938d5cc fix several compile() issues by translating newlines in the tokenizer
Benjamin Peterson <benjamin@python.org>
parents: 57131
diff changeset
637 decode_str(const char *input, int single, struct tok_state *tok)
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
638 {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
639 PyObject* utf8 = NULL;
57204
89f42938d5cc fix several compile() issues by translating newlines in the tokenizer
Benjamin Peterson <benjamin@python.org>
parents: 57131
diff changeset
640 const char *str;
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
641 const char *s;
44861
5bea4592cd7a Add a "const" to make gcc happy.
Georg Brandl <georg@python.org>
parents: 44852
diff changeset
642 const char *newl[2] = {NULL, NULL};
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
643 int lineno = 0;
57204
89f42938d5cc fix several compile() issues by translating newlines in the tokenizer
Benjamin Peterson <benjamin@python.org>
parents: 57131
diff changeset
644 tok->input = str = translate_newlines(input, single, tok);
89f42938d5cc fix several compile() issues by translating newlines in the tokenizer
Benjamin Peterson <benjamin@python.org>
parents: 57131
diff changeset
645 if (str == NULL)
89f42938d5cc fix several compile() issues by translating newlines in the tokenizer
Benjamin Peterson <benjamin@python.org>
parents: 57131
diff changeset
646 return NULL;
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
647 tok->enc = NULL;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
648 tok->str = str;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
649 if (!check_bom(buf_getc, buf_ungetc, buf_setreadl, tok))
35878
94a409da4bd4 Fix some more memory leaks.
Neal Norwitz <nnorwitz@gmail.com>
parents: 35751
diff changeset
650 return error_ret(tok);
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
651 str = tok->str; /* string after BOM if any */
24542
d0d3f760d067 Repaired a fatal compiler error in the debug build: it's not clear what
Tim Peters <tim.peters@gmail.com>
parents: 24541
diff changeset
652 assert(str);
24624
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
653 #ifdef Py_USING_UNICODE
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
654 if (tok->enc != NULL) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
655 utf8 = translate_into_utf8(str, tok->enc);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
656 if (utf8 == NULL)
35878
94a409da4bd4 Fix some more memory leaks.
Neal Norwitz <nnorwitz@gmail.com>
parents: 35751
diff changeset
657 return error_ret(tok);
47527
52294af4b663 This reverts r63675 based on the discussion in this thread:
Gregory P. Smith <greg@mad-scientist.com>
parents: 47294
diff changeset
658 str = PyString_AsString(utf8);
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
659 }
24624
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
660 #endif
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
661 for (s = str;; s++) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
662 if (*s == '\0') break;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
663 else if (*s == '\n') {
45005
707dd9f384f3 Add assertion that we do not blow out newl
Neal Norwitz <nnorwitz@gmail.com>
parents: 44889
diff changeset
664 assert(lineno < 2);
44852
8a9b3b0be573 Issue #1882: when compiling code from a string, encoding cookies in the
Georg Brandl <georg@python.org>
parents: 44780
diff changeset
665 newl[lineno] = s;
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
666 lineno++;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
667 if (lineno == 2) break;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
668 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
669 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
670 tok->enc = NULL;
44852
8a9b3b0be573 Issue #1882: when compiling code from a string, encoding cookies in the
Georg Brandl <georg@python.org>
parents: 44780
diff changeset
671 /* need to check line 1 and 2 separately since check_coding_spec
8a9b3b0be573 Issue #1882: when compiling code from a string, encoding cookies in the
Georg Brandl <georg@python.org>
parents: 44780
diff changeset
672 assumes a single line as input */
8a9b3b0be573 Issue #1882: when compiling code from a string, encoding cookies in the
Georg Brandl <georg@python.org>
parents: 44780
diff changeset
673 if (newl[0]) {
8a9b3b0be573 Issue #1882: when compiling code from a string, encoding cookies in the
Georg Brandl <georg@python.org>
parents: 44780
diff changeset
674 if (!check_coding_spec(str, newl[0] - str, tok, buf_setreadl))
8a9b3b0be573 Issue #1882: when compiling code from a string, encoding cookies in the
Georg Brandl <georg@python.org>
parents: 44780
diff changeset
675 return error_ret(tok);
8a9b3b0be573 Issue #1882: when compiling code from a string, encoding cookies in the
Georg Brandl <georg@python.org>
parents: 44780
diff changeset
676 if (tok->enc == NULL && newl[1]) {
8a9b3b0be573 Issue #1882: when compiling code from a string, encoding cookies in the
Georg Brandl <georg@python.org>
parents: 44780
diff changeset
677 if (!check_coding_spec(newl[0]+1, newl[1] - newl[0],
8a9b3b0be573 Issue #1882: when compiling code from a string, encoding cookies in the
Georg Brandl <georg@python.org>
parents: 44780
diff changeset
678 tok, buf_setreadl))
8a9b3b0be573 Issue #1882: when compiling code from a string, encoding cookies in the
Georg Brandl <georg@python.org>
parents: 44780
diff changeset
679 return error_ret(tok);
8a9b3b0be573 Issue #1882: when compiling code from a string, encoding cookies in the
Georg Brandl <georg@python.org>
parents: 44780
diff changeset
680 }
8a9b3b0be573 Issue #1882: when compiling code from a string, encoding cookies in the
Georg Brandl <georg@python.org>
parents: 44780
diff changeset
681 }
24624
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
682 #ifdef Py_USING_UNICODE
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
683 if (tok->enc != NULL) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
684 assert(utf8 == NULL);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
685 utf8 = translate_into_utf8(str, tok->enc);
54760
4a50f0bdc806 don't mask encoding errors when decoding a string #6289
Benjamin Peterson <benjamin@python.org>
parents: 48184
diff changeset
686 if (utf8 == NULL)
35878
94a409da4bd4 Fix some more memory leaks.
Neal Norwitz <nnorwitz@gmail.com>
parents: 35751
diff changeset
687 return error_ret(tok);
47527
52294af4b663 This reverts r63675 based on the discussion in this thread:
Gregory P. Smith <greg@mad-scientist.com>
parents: 47294
diff changeset
688 str = PyString_AsString(utf8);
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
689 }
24624
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
690 #endif
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
691 assert(tok->decoding_buffer == NULL);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
692 tok->decoding_buffer = utf8; /* CAUTION */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
693 return str;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
694 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
695
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
696 #endif /* PGEN */
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
697
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
698 /* Set up tokenizer for string */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
699
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
700 struct tok_state *
57204
89f42938d5cc fix several compile() issues by translating newlines in the tokenizer
Benjamin Peterson <benjamin@python.org>
parents: 57131
diff changeset
701 PyTokenizer_FromString(const char *str, int exec_input)
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
702 {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
703 struct tok_state *tok = tok_new();
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
704 if (tok == NULL)
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
705 return NULL;
57204
89f42938d5cc fix several compile() issues by translating newlines in the tokenizer
Benjamin Peterson <benjamin@python.org>
parents: 57131
diff changeset
706 str = (char *)decode_str(str, exec_input, tok);
35878
94a409da4bd4 Fix some more memory leaks.
Neal Norwitz <nnorwitz@gmail.com>
parents: 35751
diff changeset
707 if (str == NULL) {
94a409da4bd4 Fix some more memory leaks.
Neal Norwitz <nnorwitz@gmail.com>
parents: 35751
diff changeset
708 PyTokenizer_Free(tok);
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
709 return NULL;
35878
94a409da4bd4 Fix some more memory leaks.
Neal Norwitz <nnorwitz@gmail.com>
parents: 35751
diff changeset
710 }
94a409da4bd4 Fix some more memory leaks.
Neal Norwitz <nnorwitz@gmail.com>
parents: 35751
diff changeset
711
26601
cc05c2c8660e Constify filenames and scripts. Fixes #651362.
Martin v. Löwis <martin@v.loewis.de>
parents: 25995
diff changeset
712 /* XXX: constify members. */
cc05c2c8660e Constify filenames and scripts. Fixes #651362.
Martin v. Löwis <martin@v.loewis.de>
parents: 25995
diff changeset
713 tok->buf = tok->cur = tok->end = tok->inp = (char*)str;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
714 return tok;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
715 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
716
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
717
326
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
718 /* Set up tokenizer for file */
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
719
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
720 struct tok_state *
13659
e78153e179fb Mass ANSIfication.
Thomas Wouters <thomas@python.org>
parents: 13539
diff changeset
721 PyTokenizer_FromFile(FILE *fp, char *ps1, char *ps2)
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
722 {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
723 struct tok_state *tok = tok_new();
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
724 if (tok == NULL)
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
725 return NULL;
37600
c4c5eeb07739 As discussed on python-dev, really fix the PyMem_*/PyObject_* memory API
Neal Norwitz <nnorwitz@gmail.com>
parents: 37585
diff changeset
726 if ((tok->buf = (char *)PyMem_MALLOC(BUFSIZ)) == NULL) {
35878
94a409da4bd4 Fix some more memory leaks.
Neal Norwitz <nnorwitz@gmail.com>
parents: 35751
diff changeset
727 PyTokenizer_Free(tok);
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
728 return NULL;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
729 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
730 tok->cur = tok->inp = tok->buf;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
731 tok->end = tok->buf + BUFSIZ;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
732 tok->fp = fp;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
733 tok->prompt = ps1;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
734 tok->nextprompt = ps2;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
735 return tok;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
736 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
737
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
738
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
739 /* Free a tok_state structure */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
740
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
741 void
13659
e78153e179fb Mass ANSIfication.
Thomas Wouters <thomas@python.org>
parents: 13539
diff changeset
742 PyTokenizer_Free(struct tok_state *tok)
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
743 {
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
744 if (tok->encoding != NULL)
37600
c4c5eeb07739 As discussed on python-dev, really fix the PyMem_*/PyObject_* memory API
Neal Norwitz <nnorwitz@gmail.com>
parents: 37585
diff changeset
745 PyMem_FREE(tok->encoding);
24544
0bd521eb714e Make pgen compile with pydebug. Duplicate normalized names, as it may
Martin v. Löwis <martin@v.loewis.de>
parents: 24543
diff changeset
746 #ifndef PGEN
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
747 Py_XDECREF(tok->decoding_readline);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
748 Py_XDECREF(tok->decoding_buffer);
24544
0bd521eb714e Make pgen compile with pydebug. Duplicate normalized names, as it may
Martin v. Löwis <martin@v.loewis.de>
parents: 24543
diff changeset
749 #endif
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
750 if (tok->fp != NULL && tok->buf != NULL)
37600
c4c5eeb07739 As discussed on python-dev, really fix the PyMem_*/PyObject_* memory API
Neal Norwitz <nnorwitz@gmail.com>
parents: 37585
diff changeset
751 PyMem_FREE(tok->buf);
57204
89f42938d5cc fix several compile() issues by translating newlines in the tokenizer
Benjamin Peterson <benjamin@python.org>
parents: 57131
diff changeset
752 if (tok->input)
89f42938d5cc fix several compile() issues by translating newlines in the tokenizer
Benjamin Peterson <benjamin@python.org>
parents: 57131
diff changeset
753 PyMem_FREE((char *)tok->input);
37229
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
754 PyMem_FREE(tok);
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
755 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
756
32961
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
757 #if !defined(PGEN) && defined(Py_USING_UNICODE)
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
758 static int
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
759 tok_stdin_decode(struct tok_state *tok, char **inp)
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
760 {
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
761 PyObject *enc, *sysstdin, *decoded, *utf8;
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
762 const char *encoding;
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
763 char *converted;
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
764
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
765 if (PySys_GetFile((char *)"stdin", NULL) != stdin)
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
766 return 0;
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
767 sysstdin = PySys_GetObject("stdin");
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
768 if (sysstdin == NULL || !PyFile_Check(sysstdin))
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
769 return 0;
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
770
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
771 enc = ((PyFileObject *)sysstdin)->f_encoding;
47527
52294af4b663 This reverts r63675 based on the discussion in this thread:
Gregory P. Smith <greg@mad-scientist.com>
parents: 47294
diff changeset
772 if (enc == NULL || !PyString_Check(enc))
32961
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
773 return 0;
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
774 Py_INCREF(enc);
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
775
47527
52294af4b663 This reverts r63675 based on the discussion in this thread:
Gregory P. Smith <greg@mad-scientist.com>
parents: 47294
diff changeset
776 encoding = PyString_AsString(enc);
32961
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
777 decoded = PyUnicode_Decode(*inp, strlen(*inp), encoding, NULL);
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
778 if (decoded == NULL)
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
779 goto error_clear;
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
780
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
781 utf8 = PyUnicode_AsEncodedString(decoded, "utf-8", NULL);
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
782 Py_DECREF(decoded);
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
783 if (utf8 == NULL)
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
784 goto error_clear;
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
785
47527
52294af4b663 This reverts r63675 based on the discussion in this thread:
Gregory P. Smith <greg@mad-scientist.com>
parents: 47294
diff changeset
786 assert(PyString_Check(utf8));
52294af4b663 This reverts r63675 based on the discussion in this thread:
Gregory P. Smith <greg@mad-scientist.com>
parents: 47294
diff changeset
787 converted = new_string(PyString_AS_STRING(utf8),
52294af4b663 This reverts r63675 based on the discussion in this thread:
Gregory P. Smith <greg@mad-scientist.com>
parents: 47294
diff changeset
788 PyString_GET_SIZE(utf8));
32961
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
789 Py_DECREF(utf8);
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
790 if (converted == NULL)
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
791 goto error_nomem;
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
792
37600
c4c5eeb07739 As discussed on python-dev, really fix the PyMem_*/PyObject_* memory API
Neal Norwitz <nnorwitz@gmail.com>
parents: 37585
diff changeset
793 PyMem_FREE(*inp);
32961
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
794 *inp = converted;
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
795 if (tok->encoding != NULL)
37600
c4c5eeb07739 As discussed on python-dev, really fix the PyMem_*/PyObject_* memory API
Neal Norwitz <nnorwitz@gmail.com>
parents: 37585
diff changeset
796 PyMem_FREE(tok->encoding);
32961
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
797 tok->encoding = new_string(encoding, strlen(encoding));
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
798 if (tok->encoding == NULL)
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
799 goto error_nomem;
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
800
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
801 Py_DECREF(enc);
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
802 return 0;
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
803
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
804 error_nomem:
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
805 Py_DECREF(enc);
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
806 tok->done = E_NOMEM;
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
807 return -1;
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
808
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
809 error_clear:
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
810 /* Fallback to iso-8859-1: for backward compatibility */
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
811 Py_DECREF(enc);
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
812 PyErr_Clear();
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
813 return 0;
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
814 }
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
815 #endif
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
816
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
817 /* Get next char, updating state; error code goes into tok->done */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
818
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
819 static int
13659
e78153e179fb Mass ANSIfication.
Thomas Wouters <thomas@python.org>
parents: 13539
diff changeset
820 tok_nextc(register struct tok_state *tok)
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
821 {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
822 for (;;) {
1897
3fbda8768855 * Parser/tokenizer.c (tok_nextc): count line numbers when parsing
Guido van Rossum <guido@python.org>
parents: 1835
diff changeset
823 if (tok->cur != tok->inp) {
7118
e9108e97a5a0 tok_nextc() should return unsigned characters, to avoid mistaking
Guido van Rossum <guido@python.org>
parents: 5379
diff changeset
824 return Py_CHARMASK(*tok->cur++); /* Fast path */
1897
3fbda8768855 * Parser/tokenizer.c (tok_nextc): count line numbers when parsing
Guido van Rossum <guido@python.org>
parents: 1835
diff changeset
825 }
1282
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
826 if (tok->done != E_OK)
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
827 return EOF;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
828 if (tok->fp == NULL) {
1897
3fbda8768855 * Parser/tokenizer.c (tok_nextc): count line numbers when parsing
Guido van Rossum <guido@python.org>
parents: 1835
diff changeset
829 char *end = strchr(tok->inp, '\n');
3fbda8768855 * Parser/tokenizer.c (tok_nextc): count line numbers when parsing
Guido van Rossum <guido@python.org>
parents: 1835
diff changeset
830 if (end != NULL)
3fbda8768855 * Parser/tokenizer.c (tok_nextc): count line numbers when parsing
Guido van Rossum <guido@python.org>
parents: 1835
diff changeset
831 end++;
3fbda8768855 * Parser/tokenizer.c (tok_nextc): count line numbers when parsing
Guido van Rossum <guido@python.org>
parents: 1835
diff changeset
832 else {
3fbda8768855 * Parser/tokenizer.c (tok_nextc): count line numbers when parsing
Guido van Rossum <guido@python.org>
parents: 1835
diff changeset
833 end = strchr(tok->inp, '\0');
3fbda8768855 * Parser/tokenizer.c (tok_nextc): count line numbers when parsing
Guido van Rossum <guido@python.org>
parents: 1835
diff changeset
834 if (end == tok->inp) {
3fbda8768855 * Parser/tokenizer.c (tok_nextc): count line numbers when parsing
Guido van Rossum <guido@python.org>
parents: 1835
diff changeset
835 tok->done = E_EOF;
3fbda8768855 * Parser/tokenizer.c (tok_nextc): count line numbers when parsing
Guido van Rossum <guido@python.org>
parents: 1835
diff changeset
836 return EOF;
3fbda8768855 * Parser/tokenizer.c (tok_nextc): count line numbers when parsing
Guido van Rossum <guido@python.org>
parents: 1835
diff changeset
837 }
3fbda8768855 * Parser/tokenizer.c (tok_nextc): count line numbers when parsing
Guido van Rossum <guido@python.org>
parents: 1835
diff changeset
838 }
3fbda8768855 * Parser/tokenizer.c (tok_nextc): count line numbers when parsing
Guido van Rossum <guido@python.org>
parents: 1835
diff changeset
839 if (tok->start == NULL)
3fbda8768855 * Parser/tokenizer.c (tok_nextc): count line numbers when parsing
Guido van Rossum <guido@python.org>
parents: 1835
diff changeset
840 tok->buf = tok->cur;
36786
57322cb2ca11 Patch #1440601: Add col_offset attribute to AST nodes.
Martin v. Löwis <martin@v.loewis.de>
parents: 36741
diff changeset
841 tok->line_start = tok->cur;
1897
3fbda8768855 * Parser/tokenizer.c (tok_nextc): count line numbers when parsing
Guido van Rossum <guido@python.org>
parents: 1835
diff changeset
842 tok->lineno++;
3fbda8768855 * Parser/tokenizer.c (tok_nextc): count line numbers when parsing
Guido van Rossum <guido@python.org>
parents: 1835
diff changeset
843 tok->inp = end;
7118
e9108e97a5a0 tok_nextc() should return unsigned characters, to avoid mistaking
Guido van Rossum <guido@python.org>
parents: 5379
diff changeset
844 return Py_CHARMASK(*tok->cur++);
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
845 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
846 if (tok->prompt != NULL) {
37585
292854c43b99 Fix the code in Parser/ to also compile with C++. This was mostly casts for
Anthony Baxter <anthonybaxter@gmail.com>
parents: 37557
diff changeset
847 char *newtok = PyOS_Readline(stdin, stdout, tok->prompt);
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
848 if (tok->nextprompt != NULL)
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
849 tok->prompt = tok->nextprompt;
37585
292854c43b99 Fix the code in Parser/ to also compile with C++. This was mostly casts for
Anthony Baxter <anthonybaxter@gmail.com>
parents: 37557
diff changeset
850 if (newtok == NULL)
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
851 tok->done = E_INTR;
37585
292854c43b99 Fix the code in Parser/ to also compile with C++. This was mostly casts for
Anthony Baxter <anthonybaxter@gmail.com>
parents: 37557
diff changeset
852 else if (*newtok == '\0') {
37600
c4c5eeb07739 As discussed on python-dev, really fix the PyMem_*/PyObject_* memory API
Neal Norwitz <nnorwitz@gmail.com>
parents: 37585
diff changeset
853 PyMem_FREE(newtok);
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
854 tok->done = E_EOF;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
855 }
32961
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
856 #if !defined(PGEN) && defined(Py_USING_UNICODE)
37585
292854c43b99 Fix the code in Parser/ to also compile with C++. This was mostly casts for
Anthony Baxter <anthonybaxter@gmail.com>
parents: 37557
diff changeset
857 else if (tok_stdin_decode(tok, &newtok) != 0)
37600
c4c5eeb07739 As discussed on python-dev, really fix the PyMem_*/PyObject_* memory API
Neal Norwitz <nnorwitz@gmail.com>
parents: 37585
diff changeset
858 PyMem_FREE(newtok);
32961
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
859 #endif
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
860 else if (tok->start != NULL) {
13093
cbe02b8aee07 Trent Mick: familiar simple Win64 patches
Guido van Rossum <guido@python.org>
parents: 12576
diff changeset
861 size_t start = tok->start - tok->buf;
cbe02b8aee07 Trent Mick: familiar simple Win64 patches
Guido van Rossum <guido@python.org>
parents: 12576
diff changeset
862 size_t oldlen = tok->cur - tok->buf;
37585
292854c43b99 Fix the code in Parser/ to also compile with C++. This was mostly casts for
Anthony Baxter <anthonybaxter@gmail.com>
parents: 37557
diff changeset
863 size_t newlen = oldlen + strlen(newtok);
12576
1c7c2dd1beb1 Vladimir Marangozov's long-awaited malloc restructuring.
Guido van Rossum <guido@python.org>
parents: 12224
diff changeset
864 char *buf = tok->buf;
37600
c4c5eeb07739 As discussed on python-dev, really fix the PyMem_*/PyObject_* memory API
Neal Norwitz <nnorwitz@gmail.com>
parents: 37585
diff changeset
865 buf = (char *)PyMem_REALLOC(buf, newlen+1);
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
866 tok->lineno++;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
867 if (buf == NULL) {
37600
c4c5eeb07739 As discussed on python-dev, really fix the PyMem_*/PyObject_* memory API
Neal Norwitz <nnorwitz@gmail.com>
parents: 37585
diff changeset
868 PyMem_FREE(tok->buf);
2044
281e40652e23 Parser/tokenizer.c (tok_nextc): zap tok->buf after freeing;
Guido van Rossum <guido@python.org>
parents: 1897
diff changeset
869 tok->buf = NULL;
37600
c4c5eeb07739 As discussed on python-dev, really fix the PyMem_*/PyObject_* memory API
Neal Norwitz <nnorwitz@gmail.com>
parents: 37585
diff changeset
870 PyMem_FREE(newtok);
1282
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
871 tok->done = E_NOMEM;
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
872 return EOF;
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
873 }
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
874 tok->buf = buf;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
875 tok->cur = tok->buf + oldlen;
36786
57322cb2ca11 Patch #1440601: Add col_offset attribute to AST nodes.
Martin v. Löwis <martin@v.loewis.de>
parents: 36741
diff changeset
876 tok->line_start = tok->cur;
37585
292854c43b99 Fix the code in Parser/ to also compile with C++. This was mostly casts for
Anthony Baxter <anthonybaxter@gmail.com>
parents: 37557
diff changeset
877 strcpy(tok->buf + oldlen, newtok);
37600
c4c5eeb07739 As discussed on python-dev, really fix the PyMem_*/PyObject_* memory API
Neal Norwitz <nnorwitz@gmail.com>
parents: 37585
diff changeset
878 PyMem_FREE(newtok);
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
879 tok->inp = tok->buf + newlen;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
880 tok->end = tok->inp + 1;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
881 tok->start = tok->buf + start;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
882 }
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
883 else {
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
884 tok->lineno++;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
885 if (tok->buf != NULL)
37600
c4c5eeb07739 As discussed on python-dev, really fix the PyMem_*/PyObject_* memory API
Neal Norwitz <nnorwitz@gmail.com>
parents: 37585
diff changeset
886 PyMem_FREE(tok->buf);
37585
292854c43b99 Fix the code in Parser/ to also compile with C++. This was mostly casts for
Anthony Baxter <anthonybaxter@gmail.com>
parents: 37557
diff changeset
887 tok->buf = newtok;
36786
57322cb2ca11 Patch #1440601: Add col_offset attribute to AST nodes.
Martin v. Löwis <martin@v.loewis.de>
parents: 36741
diff changeset
888 tok->line_start = tok->buf;
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
889 tok->cur = tok->buf;
36786
57322cb2ca11 Patch #1440601: Add col_offset attribute to AST nodes.
Martin v. Löwis <martin@v.loewis.de>
parents: 36741
diff changeset
890 tok->line_start = tok->buf;
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
891 tok->inp = strchr(tok->buf, '\0');
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
892 tok->end = tok->inp + 1;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
893 }
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
894 }
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
895 else {
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
896 int done = 0;
36501
7fdc639bc5b4 Merge ssize_t branch.
Martin v. Löwis <martin@v.loewis.de>
parents: 36090
diff changeset
897 Py_ssize_t cur = 0;
2732
daa150b3a6e4 replace "\r\n" with "\n" at line end (Jim Ahlstrom)
Guido van Rossum <guido@python.org>
parents: 2149
diff changeset
898 char *pt;
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
899 if (tok->start == NULL) {
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
900 if (tok->buf == NULL) {
37229
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
901 tok->buf = (char *)
37600
c4c5eeb07739 As discussed on python-dev, really fix the PyMem_*/PyObject_* memory API
Neal Norwitz <nnorwitz@gmail.com>
parents: 37585
diff changeset
902 PyMem_MALLOC(BUFSIZ);
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
903 if (tok->buf == NULL) {
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
904 tok->done = E_NOMEM;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
905 return EOF;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
906 }
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
907 tok->end = tok->buf + BUFSIZ;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
908 }
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
909 if (decoding_fgets(tok->buf, (int)(tok->end - tok->buf),
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
910 tok) == NULL) {
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
911 tok->done = E_EOF;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
912 done = 1;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
913 }
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
914 else {
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
915 tok->done = E_OK;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
916 tok->inp = strchr(tok->buf, '\0');
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
917 done = tok->inp[-1] == '\n';
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
918 }
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
919 }
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
920 else {
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
921 cur = tok->cur - tok->buf;
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
922 if (decoding_feof(tok)) {
2149
6060d7fb07e3 fix loop on unterminated triple quotes
Guido van Rossum <guido@python.org>
parents: 2060
diff changeset
923 tok->done = E_EOF;
6060d7fb07e3 fix loop on unterminated triple quotes
Guido van Rossum <guido@python.org>
parents: 2060
diff changeset
924 done = 1;
6060d7fb07e3 fix loop on unterminated triple quotes
Guido van Rossum <guido@python.org>
parents: 2060
diff changeset
925 }
6060d7fb07e3 fix loop on unterminated triple quotes
Guido van Rossum <guido@python.org>
parents: 2060
diff changeset
926 else
6060d7fb07e3 fix loop on unterminated triple quotes
Guido van Rossum <guido@python.org>
parents: 2060
diff changeset
927 tok->done = E_OK;
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
928 }
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
929 tok->lineno++;
1282
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
930 /* Read until '\n' or EOF */
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
931 while (!done) {
36501
7fdc639bc5b4 Merge ssize_t branch.
Martin v. Löwis <martin@v.loewis.de>
parents: 36090
diff changeset
932 Py_ssize_t curstart = tok->start == NULL ? -1 :
7fdc639bc5b4 Merge ssize_t branch.
Martin v. Löwis <martin@v.loewis.de>
parents: 36090
diff changeset
933 tok->start - tok->buf;
7fdc639bc5b4 Merge ssize_t branch.
Martin v. Löwis <martin@v.loewis.de>
parents: 36090
diff changeset
934 Py_ssize_t curvalid = tok->inp - tok->buf;
7fdc639bc5b4 Merge ssize_t branch.
Martin v. Löwis <martin@v.loewis.de>
parents: 36090
diff changeset
935 Py_ssize_t newsize = curvalid + BUFSIZ;
1282
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
936 char *newbuf = tok->buf;
37600
c4c5eeb07739 As discussed on python-dev, really fix the PyMem_*/PyObject_* memory API
Neal Norwitz <nnorwitz@gmail.com>
parents: 37585
diff changeset
937 newbuf = (char *)PyMem_REALLOC(newbuf,
c4c5eeb07739 As discussed on python-dev, really fix the PyMem_*/PyObject_* memory API
Neal Norwitz <nnorwitz@gmail.com>
parents: 37585
diff changeset
938 newsize);
1282
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
939 if (newbuf == NULL) {
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
940 tok->done = E_NOMEM;
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
941 tok->cur = tok->inp;
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
942 return EOF;
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
943 }
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
944 tok->buf = newbuf;
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
945 tok->inp = tok->buf + curvalid;
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
946 tok->end = tok->buf + newsize;
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
947 tok->start = curstart < 0 ? NULL :
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
948 tok->buf + curstart;
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
949 if (decoding_fgets(tok->inp,
1282
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
950 (int)(tok->end - tok->inp),
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
951 tok) == NULL) {
36809
2c2dffe070df Fix crashing bug in tokenizer, when tokenizing files with non-ASCII bytes
Thomas Wouters <thomas@python.org>
parents: 36786
diff changeset
952 /* Break out early on decoding
2c2dffe070df Fix crashing bug in tokenizer, when tokenizing files with non-ASCII bytes
Thomas Wouters <thomas@python.org>
parents: 36786
diff changeset
953 errors, as tok->buf will be NULL
2c2dffe070df Fix crashing bug in tokenizer, when tokenizing files with non-ASCII bytes
Thomas Wouters <thomas@python.org>
parents: 36786
diff changeset
954 */
2c2dffe070df Fix crashing bug in tokenizer, when tokenizing files with non-ASCII bytes
Thomas Wouters <thomas@python.org>
parents: 36786
diff changeset
955 if (tok->decoding_erred)
2c2dffe070df Fix crashing bug in tokenizer, when tokenizing files with non-ASCII bytes
Thomas Wouters <thomas@python.org>
parents: 36786
diff changeset
956 return EOF;
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
957 /* Last line does not end in \n,
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
958 fake one */
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
959 strcpy(tok->inp, "\n");
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
960 }
1282
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
961 tok->inp = strchr(tok->inp, '\0');
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
962 done = tok->inp[-1] == '\n';
1282
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
963 }
38544
697dd848e1f1 Patch #1357836:
Neal Norwitz <nnorwitz@gmail.com>
parents: 37817
diff changeset
964 if (tok->buf != NULL) {
697dd848e1f1 Patch #1357836:
Neal Norwitz <nnorwitz@gmail.com>
parents: 37817
diff changeset
965 tok->cur = tok->buf + cur;
697dd848e1f1 Patch #1357836:
Neal Norwitz <nnorwitz@gmail.com>
parents: 37817
diff changeset
966 tok->line_start = tok->cur;
697dd848e1f1 Patch #1357836:
Neal Norwitz <nnorwitz@gmail.com>
parents: 37817
diff changeset
967 /* replace "\r\n" with "\n" */
40036
5d83994cd8cd Comment grammar
Andrew M. Kuchling <amk@amk.ca>
parents: 38762
diff changeset
968 /* For Mac leave the \r, giving a syntax error */
38544
697dd848e1f1 Patch #1357836:
Neal Norwitz <nnorwitz@gmail.com>
parents: 37817
diff changeset
969 pt = tok->inp - 2;
697dd848e1f1 Patch #1357836:
Neal Norwitz <nnorwitz@gmail.com>
parents: 37817
diff changeset
970 if (pt >= tok->buf && *pt == '\r') {
697dd848e1f1 Patch #1357836:
Neal Norwitz <nnorwitz@gmail.com>
parents: 37817
diff changeset
971 *pt++ = '\n';
697dd848e1f1 Patch #1357836:
Neal Norwitz <nnorwitz@gmail.com>
parents: 37817
diff changeset
972 *pt = '\0';
697dd848e1f1 Patch #1357836:
Neal Norwitz <nnorwitz@gmail.com>
parents: 37817
diff changeset
973 tok->inp = pt;
697dd848e1f1 Patch #1357836:
Neal Norwitz <nnorwitz@gmail.com>
parents: 37817
diff changeset
974 }
2732
daa150b3a6e4 replace "\r\n" with "\n" at line end (Jim Ahlstrom)
Guido van Rossum <guido@python.org>
parents: 2149
diff changeset
975 }
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
976 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
977 if (tok->done != E_OK) {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
978 if (tok->prompt != NULL)
9047
6d2eb58f0418 Replace all calls to fprintf(stderr, ...) with PySys_WriteStderr(...).
Guido van Rossum <guido@python.org>
parents: 7945
diff changeset
979 PySys_WriteStderr("\n");
1282
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
980 tok->cur = tok->inp;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
981 return EOF;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
982 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
983 }
1282
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
984 /*NOTREACHED*/
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
985 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
986
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
987
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
988 /* Back-up one character */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
989
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
990 static void
13659
e78153e179fb Mass ANSIfication.
Thomas Wouters <thomas@python.org>
parents: 13539
diff changeset
991 tok_backup(register struct tok_state *tok, register int c)
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
992 {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
993 if (c != EOF) {
2044
281e40652e23 Parser/tokenizer.c (tok_nextc): zap tok->buf after freeing;
Guido van Rossum <guido@python.org>
parents: 1897
diff changeset
994 if (--tok->cur < tok->buf)
57131
9309daaf32d1 spelling
Benjamin Peterson <benjamin@python.org>
parents: 56432
diff changeset
995 Py_FatalError("tok_backup: beginning of buffer");
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
996 if (*tok->cur != c)
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
997 *tok->cur = c;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
998 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
999 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1000
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1001
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1002 /* Return the token corresponding to a single character */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1003
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1004 int
13659
e78153e179fb Mass ANSIfication.
Thomas Wouters <thomas@python.org>
parents: 13539
diff changeset
1005 PyToken_OneChar(int c)
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1006 {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1007 switch (c) {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1008 case '(': return LPAR;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1009 case ')': return RPAR;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1010 case '[': return LSQB;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1011 case ']': return RSQB;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1012 case ':': return COLON;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1013 case ',': return COMMA;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1014 case ';': return SEMI;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1015 case '+': return PLUS;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1016 case '-': return MINUS;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1017 case '*': return STAR;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1018 case '/': return SLASH;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1019 case '|': return VBAR;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1020 case '&': return AMPER;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1021 case '<': return LESS;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1022 case '>': return GREATER;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1023 case '=': return EQUAL;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1024 case '.': return DOT;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1025 case '%': return PERCENT;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1026 case '`': return BACKQUOTE;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1027 case '{': return LBRACE;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1028 case '}': return RBRACE;
410
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
1029 case '^': return CIRCUMFLEX;
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
1030 case '~': return TILDE;
32888
c7e03cf8333e PEP-0318, @decorator-style. In Guido's words:
Anthony Baxter <anthonybaxter@gmail.com>
parents: 30979
diff changeset
1031 case '@': return AT;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1032 default: return OP;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1033 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1034 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1035
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1036
400
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1037 int
13659
e78153e179fb Mass ANSIfication.
Thomas Wouters <thomas@python.org>
parents: 13539
diff changeset
1038 PyToken_TwoChars(int c1, int c2)
400
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1039 {
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1040 switch (c1) {
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1041 case '=':
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1042 switch (c2) {
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1043 case '=': return EQEQUAL;
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1044 }
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1045 break;
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1046 case '!':
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1047 switch (c2) {
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1048 case '=': return NOTEQUAL;
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1049 }
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1050 break;
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1051 case '<':
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1052 switch (c2) {
43983
e725166c8c81 Fixed problems in the last commit. Filenames and line numbers weren't reported correctly.
Christian Heimes <christian@cheimes.de>
parents: 43982
diff changeset
1053 case '>': return NOTEQUAL;
400
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1054 case '=': return LESSEQUAL;
410
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
1055 case '<': return LEFTSHIFT;
400
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1056 }
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1057 break;
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1058 case '>':
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1059 switch (c2) {
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1060 case '=': return GREATEREQUAL;
410
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
1061 case '>': return RIGHTSHIFT;
400
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1062 }
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1063 break;
14163
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1064 case '+':
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1065 switch (c2) {
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1066 case '=': return PLUSEQUAL;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1067 }
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1068 break;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1069 case '-':
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1070 switch (c2) {
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1071 case '=': return MINEQUAL;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1072 }
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1073 break;
3333
4b5bff95804e changes for pow(**) and complex
Guido van Rossum <guido@python.org>
parents: 3116
diff changeset
1074 case '*':
4b5bff95804e changes for pow(**) and complex
Guido van Rossum <guido@python.org>
parents: 3116
diff changeset
1075 switch (c2) {
4b5bff95804e changes for pow(**) and complex
Guido van Rossum <guido@python.org>
parents: 3116
diff changeset
1076 case '*': return DOUBLESTAR;
14163
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1077 case '=': return STAREQUAL;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1078 }
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1079 break;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1080 case '/':
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1081 switch (c2) {
19051
f20727cc3d0f Implement PEP 238 in its (almost) full glory.
Guido van Rossum <guido@python.org>
parents: 17600
diff changeset
1082 case '/': return DOUBLESLASH;
14163
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1083 case '=': return SLASHEQUAL;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1084 }
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1085 break;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1086 case '|':
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1087 switch (c2) {
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1088 case '=': return VBAREQUAL;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1089 }
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1090 break;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1091 case '%':
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1092 switch (c2) {
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1093 case '=': return PERCENTEQUAL;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1094 }
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1095 break;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1096 case '&':
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1097 switch (c2) {
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1098 case '=': return AMPEREQUAL;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1099 }
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1100 break;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1101 case '^':
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1102 switch (c2) {
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1103 case '=': return CIRCUMFLEXEQUAL;
3333
4b5bff95804e changes for pow(**) and complex
Guido van Rossum <guido@python.org>
parents: 3116
diff changeset
1104 }
4b5bff95804e changes for pow(**) and complex
Guido van Rossum <guido@python.org>
parents: 3116
diff changeset
1105 break;
400
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1106 }
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1107 return OP;
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1108 }
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1109
14163
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1110 int
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1111 PyToken_ThreeChars(int c1, int c2, int c3)
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1112 {
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1113 switch (c1) {
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1114 case '<':
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1115 switch (c2) {
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1116 case '<':
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1117 switch (c3) {
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1118 case '=':
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1119 return LEFTSHIFTEQUAL;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1120 }
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1121 break;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1122 }
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1123 break;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1124 case '>':
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1125 switch (c2) {
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1126 case '>':
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1127 switch (c3) {
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1128 case '=':
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1129 return RIGHTSHIFTEQUAL;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1130 }
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1131 break;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1132 }
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1133 break;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1134 case '*':
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1135 switch (c2) {
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1136 case '*':
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1137 switch (c3) {
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1138 case '=':
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1139 return DOUBLESTAREQUAL;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1140 }
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1141 break;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1142 }
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1143 break;
19051
f20727cc3d0f Implement PEP 238 in its (almost) full glory.
Guido van Rossum <guido@python.org>
parents: 17600
diff changeset
1144 case '/':
f20727cc3d0f Implement PEP 238 in its (almost) full glory.
Guido van Rossum <guido@python.org>
parents: 17600
diff changeset
1145 switch (c2) {
f20727cc3d0f Implement PEP 238 in its (almost) full glory.
Guido van Rossum <guido@python.org>
parents: 17600
diff changeset
1146 case '/':
f20727cc3d0f Implement PEP 238 in its (almost) full glory.
Guido van Rossum <guido@python.org>
parents: 17600
diff changeset
1147 switch (c3) {
f20727cc3d0f Implement PEP 238 in its (almost) full glory.
Guido van Rossum <guido@python.org>
parents: 17600
diff changeset
1148 case '=':
f20727cc3d0f Implement PEP 238 in its (almost) full glory.
Guido van Rossum <guido@python.org>
parents: 17600
diff changeset
1149 return DOUBLESLASHEQUAL;
f20727cc3d0f Implement PEP 238 in its (almost) full glory.
Guido van Rossum <guido@python.org>
parents: 17600
diff changeset
1150 }
f20727cc3d0f Implement PEP 238 in its (almost) full glory.
Guido van Rossum <guido@python.org>
parents: 17600
diff changeset
1151 break;
f20727cc3d0f Implement PEP 238 in its (almost) full glory.
Guido van Rossum <guido@python.org>
parents: 17600
diff changeset
1152 }
f20727cc3d0f Implement PEP 238 in its (almost) full glory.
Guido van Rossum <guido@python.org>
parents: 17600
diff changeset
1153 break;
14163
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1154 }
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1155 return OP;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1156 }
400
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1157
7945
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1158 static int
13659
e78153e179fb Mass ANSIfication.
Thomas Wouters <thomas@python.org>
parents: 13539
diff changeset
1159 indenterror(struct tok_state *tok)
7945
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1160 {
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1161 if (tok->alterror) {
13539
feab677269e0 Create two new exceptions: IndentationError and TabError. These are
Fred Drake <fdrake@acm.org>
parents: 13447
diff changeset
1162 tok->done = E_TABSPACE;
7945
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1163 tok->cur = tok->inp;
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1164 return 1;
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1165 }
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1166 if (tok->altwarning) {
13539
feab677269e0 Create two new exceptions: IndentationError and TabError. These are
Fred Drake <fdrake@acm.org>
parents: 13447
diff changeset
1167 PySys_WriteStderr("%s: inconsistent use of tabs and spaces "
feab677269e0 Create two new exceptions: IndentationError and TabError. These are
Fred Drake <fdrake@acm.org>
parents: 13447
diff changeset
1168 "in indentation\n", tok->filename);
7945
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1169 tok->altwarning = 0;
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1170 }
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1171 return 0;
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1172 }
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1173
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1174
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1175 /* Get next token, after space stripping etc. */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1176
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
1177 static int
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
1178 tok_get(register struct tok_state *tok, char **p_start, char **p_end)
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1179 {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1180 register int c;
326
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1181 int blankline;
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1182
48184
b654d14cb9d9 #3367: revert rev. 65539: this change causes test_parser to fail
Andrew M. Kuchling <amk@amk.ca>
parents: 48183
diff changeset
1183 *p_start = *p_end = NULL;
326
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1184 nextline:
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1185 tok->start = NULL;
326
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1186 blankline = 0;
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1187
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1188 /* Get indentation level */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1189 if (tok->atbol) {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1190 register int col = 0;
7945
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1191 register int altcol = 0;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1192 tok->atbol = 0;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1193 for (;;) {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1194 c = tok_nextc(tok);
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1195 if (c == ' ')
7945
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1196 col++, altcol++;
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1197 else if (c == '\t') {
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1198 col = (col/tok->tabsize + 1) * tok->tabsize;
7945
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1199 altcol = (altcol/tok->alttabsize + 1)
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1200 * tok->alttabsize;
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1201 }
2801
e911f2fb179c ignore control-l in whitespace
Guido van Rossum <guido@python.org>
parents: 2732
diff changeset
1202 else if (c == '\014') /* Control-L (formfeed) */
7945
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1203 col = altcol = 0; /* For Emacs users */
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1204 else
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1205 break;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1206 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1207 tok_backup(tok, c);
326
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1208 if (c == '#' || c == '\n') {
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1209 /* Lines with only whitespace and/or comments
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1210 shouldn't affect the indentation and are
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1211 not passed to the parser as NEWLINE tokens,
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1212 except *totally* empty lines in interactive
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1213 mode, which signal the end of a command group. */
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1214 if (col == 0 && c == '\n' && tok->prompt != NULL)
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1215 blankline = 0; /* Let it through */
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1216 else
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1217 blankline = 1; /* Ignore completely */
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1218 /* We can't jump back right here since we still
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1219 may need to skip to the end of a comment */
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1220 }
1284
dbce72afe261 * selectmodule.c: fix (another!) two memory leaks -- this time in list2set
Guido van Rossum <guido@python.org>
parents: 1282
diff changeset
1221 if (!blankline && tok->level == 0) {
326
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1222 if (col == tok->indstack[tok->indent]) {
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1223 /* No change */
7945
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1224 if (altcol != tok->altindstack[tok->indent]) {
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1225 if (indenterror(tok))
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1226 return ERRORTOKEN;
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1227 }
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1228 }
326
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1229 else if (col > tok->indstack[tok->indent]) {
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1230 /* Indent -- always one */
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1231 if (tok->indent+1 >= MAXINDENT) {
13539
feab677269e0 Create two new exceptions: IndentationError and TabError. These are
Fred Drake <fdrake@acm.org>
parents: 13447
diff changeset
1232 tok->done = E_TOODEEP;
1282
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
1233 tok->cur = tok->inp;
326
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1234 return ERRORTOKEN;
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1235 }
7945
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1236 if (altcol <= tok->altindstack[tok->indent]) {
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1237 if (indenterror(tok))
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1238 return ERRORTOKEN;
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1239 }
326
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1240 tok->pendin++;
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1241 tok->indstack[++tok->indent] = col;
7945
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1242 tok->altindstack[tok->indent] = altcol;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1243 }
326
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1244 else /* col < tok->indstack[tok->indent] */ {
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1245 /* Dedent -- any number, must be consistent */
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1246 while (tok->indent > 0 &&
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1247 col < tok->indstack[tok->indent]) {
7324
55125b1144fa Swap two statements in the dedent check loop. This makes absolutely
Guido van Rossum <guido@python.org>
parents: 7315
diff changeset
1248 tok->pendin--;
326
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1249 tok->indent--;
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1250 }
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1251 if (col != tok->indstack[tok->indent]) {
13539
feab677269e0 Create two new exceptions: IndentationError and TabError. These are
Fred Drake <fdrake@acm.org>
parents: 13447
diff changeset
1252 tok->done = E_DEDENT;
1282
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
1253 tok->cur = tok->inp;
326
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1254 return ERRORTOKEN;
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1255 }
7945
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1256 if (altcol != tok->altindstack[tok->indent]) {
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1257 if (indenterror(tok))
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1258 return ERRORTOKEN;
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1259 }
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1260 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1261 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1262 }
37229
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
1263
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1264 tok->start = tok->cur;
37229
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
1265
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1266 /* Return pending indents/dedents */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1267 if (tok->pendin != 0) {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1268 if (tok->pendin < 0) {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1269 tok->pendin++;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1270 return DEDENT;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1271 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1272 else {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1273 tok->pendin--;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1274 return INDENT;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1275 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1276 }
37229
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
1277
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1278 again:
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1279 tok->start = NULL;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1280 /* Skip spaces */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1281 do {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1282 c = tok_nextc(tok);
2801
e911f2fb179c ignore control-l in whitespace
Guido van Rossum <guido@python.org>
parents: 2732
diff changeset
1283 } while (c == ' ' || c == '\t' || c == '\014');
37229
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
1284
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1285 /* Set start of current token */
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1286 tok->start = tok->cur - 1;
37229
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
1287
12142
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1288 /* Skip comment, while looking for tab-setting magic */
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1289 if (c == '#') {
12142
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1290 static char *tabforms[] = {
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1291 "tab-width:", /* Emacs */
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1292 ":tabstop=", /* vim, full form */
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1293 ":ts=", /* vim, abbreviated form */
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1294 "set tabsize=", /* will vi never die? */
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1295 /* more templates can be added here to support other editors */
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1296 };
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1297 char cbuf[80];
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1298 char *tp, **cp;
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1299 tp = cbuf;
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1300 do {
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1301 *tp++ = c = tok_nextc(tok);
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1302 } while (c != EOF && c != '\n' &&
38762
92d8cdc4b90c Don't truncate if size_t is bigger than uint
Neal Norwitz <nnorwitz@gmail.com>
parents: 38544
diff changeset
1303 (size_t)(tp - cbuf + 1) < sizeof(cbuf));
12142
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1304 *tp = '\0';
37229
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
1305 for (cp = tabforms;
12142
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1306 cp < tabforms + sizeof(tabforms)/sizeof(tabforms[0]);
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1307 cp++) {
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1308 if ((tp = strstr(cbuf, *cp))) {
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1309 int newsize = atoi(tp + strlen(*cp));
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1310
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1311 if (newsize >= 1 && newsize <= 40) {
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1312 tok->tabsize = newsize;
12224
2dc1bdb4088e Only write message about changed Tab size with -v.
Guido van Rossum <guido@python.org>
parents: 12142
diff changeset
1313 if (Py_VerboseFlag)
2dc1bdb4088e Only write message about changed Tab size with -v.
Guido van Rossum <guido@python.org>
parents: 12142
diff changeset
1314 PySys_WriteStderr(
12142
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1315 "Tab size set to %d\n",
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1316 newsize);
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1317 }
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1318 }
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1319 }
12142
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1320 while (c != EOF && c != '\n')
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1321 c = tok_nextc(tok);
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1322 }
37229
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
1323
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1324 /* Check for EOF and errors now */
410
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
1325 if (c == EOF) {
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1326 return tok->done == E_EOF ? ENDMARKER : ERRORTOKEN;
410
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
1327 }
37229
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
1328
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1329 /* Identifier (most frequent token!) */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1330 if (isalpha(c) || c == '_') {
11971
574283db2b3d Marc-Andre Lemburg: add new string token types u"..." and ur"..."
Guido van Rossum <guido@python.org>
parents: 9590
diff changeset
1331 /* Process r"", u"" and ur"" */
5341
67e5237ce141 Now that the string-sig has settled on r"obin" strings, restrict the
Guido van Rossum <guido@python.org>
parents: 5311
diff changeset
1332 switch (c) {
44747
8da9ebf73026 Added bytes and b'' as aliases for str and ''
Christian Heimes <christian@cheimes.de>
parents: 44575
diff changeset
1333 case 'b':
8da9ebf73026 Added bytes and b'' as aliases for str and ''
Christian Heimes <christian@cheimes.de>
parents: 44575
diff changeset
1334 case 'B':
8da9ebf73026 Added bytes and b'' as aliases for str and ''
Christian Heimes <christian@cheimes.de>
parents: 44575
diff changeset
1335 c = tok_nextc(tok);
8da9ebf73026 Added bytes and b'' as aliases for str and ''
Christian Heimes <christian@cheimes.de>
parents: 44575
diff changeset
1336 if (c == 'r' || c == 'R')
8da9ebf73026 Added bytes and b'' as aliases for str and ''
Christian Heimes <christian@cheimes.de>
parents: 44575
diff changeset
1337 c = tok_nextc(tok);
8da9ebf73026 Added bytes and b'' as aliases for str and ''
Christian Heimes <christian@cheimes.de>
parents: 44575
diff changeset
1338 if (c == '"' || c == '\'')
8da9ebf73026 Added bytes and b'' as aliases for str and ''
Christian Heimes <christian@cheimes.de>
parents: 44575
diff changeset
1339 goto letter_quote;
8da9ebf73026 Added bytes and b'' as aliases for str and ''
Christian Heimes <christian@cheimes.de>
parents: 44575
diff changeset
1340 break;
5341
67e5237ce141 Now that the string-sig has settled on r"obin" strings, restrict the
Guido van Rossum <guido@python.org>
parents: 5311
diff changeset
1341 case 'r':
67e5237ce141 Now that the string-sig has settled on r"obin" strings, restrict the
Guido van Rossum <guido@python.org>
parents: 5311
diff changeset
1342 case 'R':
67e5237ce141 Now that the string-sig has settled on r"obin" strings, restrict the
Guido van Rossum <guido@python.org>
parents: 5311
diff changeset
1343 c = tok_nextc(tok);
67e5237ce141 Now that the string-sig has settled on r"obin" strings, restrict the
Guido van Rossum <guido@python.org>
parents: 5311
diff changeset
1344 if (c == '"' || c == '\'')
67e5237ce141 Now that the string-sig has settled on r"obin" strings, restrict the
Guido van Rossum <guido@python.org>
parents: 5311
diff changeset
1345 goto letter_quote;
11971
574283db2b3d Marc-Andre Lemburg: add new string token types u"..." and ur"..."
Guido van Rossum <guido@python.org>
parents: 9590
diff changeset
1346 break;
574283db2b3d Marc-Andre Lemburg: add new string token types u"..." and ur"..."
Guido van Rossum <guido@python.org>
parents: 9590
diff changeset
1347 case 'u':
574283db2b3d Marc-Andre Lemburg: add new string token types u"..." and ur"..."
Guido van Rossum <guido@python.org>
parents: 9590
diff changeset
1348 case 'U':
574283db2b3d Marc-Andre Lemburg: add new string token types u"..." and ur"..."
Guido van Rossum <guido@python.org>
parents: 9590
diff changeset
1349 c = tok_nextc(tok);
574283db2b3d Marc-Andre Lemburg: add new string token types u"..." and ur"..."
Guido van Rossum <guido@python.org>
parents: 9590
diff changeset
1350 if (c == 'r' || c == 'R')
574283db2b3d Marc-Andre Lemburg: add new string token types u"..." and ur"..."
Guido van Rossum <guido@python.org>
parents: 9590
diff changeset
1351 c = tok_nextc(tok);
574283db2b3d Marc-Andre Lemburg: add new string token types u"..." and ur"..."
Guido van Rossum <guido@python.org>
parents: 9590
diff changeset
1352 if (c == '"' || c == '\'')
574283db2b3d Marc-Andre Lemburg: add new string token types u"..." and ur"..."
Guido van Rossum <guido@python.org>
parents: 9590
diff changeset
1353 goto letter_quote;
574283db2b3d Marc-Andre Lemburg: add new string token types u"..." and ur"..."
Guido van Rossum <guido@python.org>
parents: 9590
diff changeset
1354 break;
5341
67e5237ce141 Now that the string-sig has settled on r"obin" strings, restrict the
Guido van Rossum <guido@python.org>
parents: 5311
diff changeset
1355 }
5265
3c1788ab9761 Support for alternative string quotes (a"xx", b"xx", c"xx", ...).
Guido van Rossum <guido@python.org>
parents: 4839
diff changeset
1356 while (isalnum(c) || c == '_') {
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1357 c = tok_nextc(tok);
5265
3c1788ab9761 Support for alternative string quotes (a"xx", b"xx", c"xx", ...).
Guido van Rossum <guido@python.org>
parents: 4839
diff changeset
1358 }
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1359 tok_backup(tok, c);
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1360 *p_start = tok->start;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1361 *p_end = tok->cur;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1362 return NAME;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1363 }
37229
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
1364
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1365 /* Newline */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1366 if (c == '\n') {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1367 tok->atbol = 1;
1284
dbce72afe261 * selectmodule.c: fix (another!) two memory leaks -- this time in list2set
Guido van Rossum <guido@python.org>
parents: 1282
diff changeset
1368 if (blankline || tok->level > 0)
326
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1369 goto nextline;
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1370 *p_start = tok->start;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1371 *p_end = tok->cur - 1; /* Leave '\n' out of the string */
25098
c02a668488d0 Ignore encoding declarations inside strings. Fixes #603509.
Martin v. Löwis <martin@v.loewis.de>
parents: 24867
diff changeset
1372 tok->cont_line = 0;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1373 return NEWLINE;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1374 }
37229
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
1375
410
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
1376 /* Period or number starting with period? */
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
1377 if (c == '.') {
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
1378 c = tok_nextc(tok);
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
1379 if (isdigit(c)) {
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
1380 goto fraction;
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
1381 }
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
1382 else {
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
1383 tok_backup(tok, c);
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1384 *p_start = tok->start;
410
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
1385 *p_end = tok->cur;
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
1386 return DOT;
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
1387 }
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
1388 }
3333
4b5bff95804e changes for pow(**) and complex
Guido van Rossum <guido@python.org>
parents: 3116
diff changeset
1389
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1390 /* Number */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1391 if (isdigit(c)) {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1392 if (c == '0') {
45791
62ef0ea034f4 Finished backporting PEP 3127, Integer Literal Support and Syntax.
Eric Smith <eric@trueblade.com>
parents: 45005
diff changeset
1393 /* Hex, octal or binary -- maybe. */
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1394 c = tok_nextc(tok);
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1395 if (c == '.')
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1396 goto fraction;
3333
4b5bff95804e changes for pow(**) and complex
Guido van Rossum <guido@python.org>
parents: 3116
diff changeset
1397 #ifndef WITHOUT_COMPLEX
3364
bc17e8313ea0 use only j for imaginary constants
Guido van Rossum <guido@python.org>
parents: 3333
diff changeset
1398 if (c == 'j' || c == 'J')
3333
4b5bff95804e changes for pow(**) and complex
Guido van Rossum <guido@python.org>
parents: 3116
diff changeset
1399 goto imaginary;
4b5bff95804e changes for pow(**) and complex
Guido van Rossum <guido@python.org>
parents: 3116
diff changeset
1400 #endif
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1401 if (c == 'x' || c == 'X') {
44780
61a11cc95632 Fix #1679: "0x" was taken as a valid integer literal.
Georg Brandl <georg@python.org>
parents: 44747
diff changeset
1402
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1403 /* Hex */
44780
61a11cc95632 Fix #1679: "0x" was taken as a valid integer literal.
Georg Brandl <georg@python.org>
parents: 44747
diff changeset
1404 c = tok_nextc(tok);
61a11cc95632 Fix #1679: "0x" was taken as a valid integer literal.
Georg Brandl <georg@python.org>
parents: 44747
diff changeset
1405 if (!isxdigit(c)) {
61a11cc95632 Fix #1679: "0x" was taken as a valid integer literal.
Georg Brandl <georg@python.org>
parents: 44747
diff changeset
1406 tok->done = E_TOKEN;
61a11cc95632 Fix #1679: "0x" was taken as a valid integer literal.
Georg Brandl <georg@python.org>
parents: 44747
diff changeset
1407 tok_backup(tok, c);
61a11cc95632 Fix #1679: "0x" was taken as a valid integer literal.
Georg Brandl <georg@python.org>
parents: 44747
diff changeset
1408 return ERRORTOKEN;
61a11cc95632 Fix #1679: "0x" was taken as a valid integer literal.
Georg Brandl <georg@python.org>
parents: 44747
diff changeset
1409 }
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1410 do {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1411 c = tok_nextc(tok);
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1412 } while (isxdigit(c));
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1413 }
45791
62ef0ea034f4 Finished backporting PEP 3127, Integer Literal Support and Syntax.
Eric Smith <eric@trueblade.com>
parents: 45005
diff changeset
1414 else if (c == 'o' || c == 'O') {
62ef0ea034f4 Finished backporting PEP 3127, Integer Literal Support and Syntax.
Eric Smith <eric@trueblade.com>
parents: 45005
diff changeset
1415 /* Octal */
62ef0ea034f4 Finished backporting PEP 3127, Integer Literal Support and Syntax.
Eric Smith <eric@trueblade.com>
parents: 45005
diff changeset
1416 c = tok_nextc(tok);
46545
459083c6be15 Issue2681: the literal 0o8 was wrongly accepted, and evaluated as float(0.0).
Amaury Forgeot d'Arc <amauryfa@gmail.com>
parents: 46149
diff changeset
1417 if (c < '0' || c >= '8') {
45791
62ef0ea034f4 Finished backporting PEP 3127, Integer Literal Support and Syntax.
Eric Smith <eric@trueblade.com>
parents: 45005
diff changeset
1418 tok->done = E_TOKEN;
62ef0ea034f4 Finished backporting PEP 3127, Integer Literal Support and Syntax.
Eric Smith <eric@trueblade.com>
parents: 45005
diff changeset
1419 tok_backup(tok, c);
62ef0ea034f4 Finished backporting PEP 3127, Integer Literal Support and Syntax.
Eric Smith <eric@trueblade.com>
parents: 45005
diff changeset
1420 return ERRORTOKEN;
62ef0ea034f4 Finished backporting PEP 3127, Integer Literal Support and Syntax.
Eric Smith <eric@trueblade.com>
parents: 45005
diff changeset
1421 }
62ef0ea034f4 Finished backporting PEP 3127, Integer Literal Support and Syntax.
Eric Smith <eric@trueblade.com>
parents: 45005
diff changeset
1422 do {
62ef0ea034f4 Finished backporting PEP 3127, Integer Literal Support and Syntax.
Eric Smith <eric@trueblade.com>
parents: 45005
diff changeset
1423 c = tok_nextc(tok);
62ef0ea034f4 Finished backporting PEP 3127, Integer Literal Support and Syntax.
Eric Smith <eric@trueblade.com>
parents: 45005
diff changeset
1424 } while ('0' <= c && c < '8');
62ef0ea034f4 Finished backporting PEP 3127, Integer Literal Support and Syntax.
Eric Smith <eric@trueblade.com>
parents: 45005
diff changeset
1425 }
62ef0ea034f4 Finished backporting PEP 3127, Integer Literal Support and Syntax.
Eric Smith <eric@trueblade.com>
parents: 45005
diff changeset
1426 else if (c == 'b' || c == 'B') {
62ef0ea034f4 Finished backporting PEP 3127, Integer Literal Support and Syntax.
Eric Smith <eric@trueblade.com>
parents: 45005
diff changeset
1427 /* Binary */
62ef0ea034f4 Finished backporting PEP 3127, Integer Literal Support and Syntax.
Eric Smith <eric@trueblade.com>
parents: 45005
diff changeset
1428 c = tok_nextc(tok);
62ef0ea034f4 Finished backporting PEP 3127, Integer Literal Support and Syntax.
Eric Smith <eric@trueblade.com>
parents: 45005
diff changeset
1429 if (c != '0' && c != '1') {
62ef0ea034f4 Finished backporting PEP 3127, Integer Literal Support and Syntax.
Eric Smith <eric@trueblade.com>
parents: 45005
diff changeset
1430 tok->done = E_TOKEN;
62ef0ea034f4 Finished backporting PEP 3127, Integer Literal Support and Syntax.
Eric Smith <eric@trueblade.com>
parents: 45005
diff changeset
1431 tok_backup(tok, c);
62ef0ea034f4 Finished backporting PEP 3127, Integer Literal Support and Syntax.
Eric Smith <eric@trueblade.com>
parents: 45005
diff changeset
1432 return ERRORTOKEN;
62ef0ea034f4 Finished backporting PEP 3127, Integer Literal Support and Syntax.
Eric Smith <eric@trueblade.com>
parents: 45005
diff changeset
1433 }
62ef0ea034f4 Finished backporting PEP 3127, Integer Literal Support and Syntax.
Eric Smith <eric@trueblade.com>
parents: 45005
diff changeset
1434 do {
62ef0ea034f4 Finished backporting PEP 3127, Integer Literal Support and Syntax.
Eric Smith <eric@trueblade.com>
parents: 45005
diff changeset
1435 c = tok_nextc(tok);
62ef0ea034f4 Finished backporting PEP 3127, Integer Literal Support and Syntax.
Eric Smith <eric@trueblade.com>
parents: 45005
diff changeset
1436 } while (c == '0' || c == '1');
62ef0ea034f4 Finished backporting PEP 3127, Integer Literal Support and Syntax.
Eric Smith <eric@trueblade.com>
parents: 45005
diff changeset
1437 }
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1438 else {
19621
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1439 int found_decimal = 0;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1440 /* Octal; c is first char of it */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1441 /* There's no 'isoctdigit' macro, sigh */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1442 while ('0' <= c && c < '8') {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1443 c = tok_nextc(tok);
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1444 }
19621
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1445 if (isdigit(c)) {
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1446 found_decimal = 1;
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1447 do {
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1448 c = tok_nextc(tok);
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1449 } while (isdigit(c));
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1450 }
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1451 if (c == '.')
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1452 goto fraction;
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1453 else if (c == 'e' || c == 'E')
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1454 goto exponent;
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1455 #ifndef WITHOUT_COMPLEX
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1456 else if (c == 'j' || c == 'J')
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1457 goto imaginary;
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1458 #endif
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1459 else if (found_decimal) {
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1460 tok->done = E_TOKEN;
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1461 tok_backup(tok, c);
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1462 return ERRORTOKEN;
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1463 }
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1464 }
236
0a92b6a1b9c0 Added recognition of 'l' or 'L' as long integer suffix
Guido van Rossum <guido@python.org>
parents: 145
diff changeset
1465 if (c == 'l' || c == 'L')
0a92b6a1b9c0 Added recognition of 'l' or 'L' as long integer suffix
Guido van Rossum <guido@python.org>
parents: 145
diff changeset
1466 c = tok_nextc(tok);
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1467 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1468 else {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1469 /* Decimal */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1470 do {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1471 c = tok_nextc(tok);
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1472 } while (isdigit(c));
236
0a92b6a1b9c0 Added recognition of 'l' or 'L' as long integer suffix
Guido van Rossum <guido@python.org>
parents: 145
diff changeset
1473 if (c == 'l' || c == 'L')
0a92b6a1b9c0 Added recognition of 'l' or 'L' as long integer suffix
Guido van Rossum <guido@python.org>
parents: 145
diff changeset
1474 c = tok_nextc(tok);
0a92b6a1b9c0 Added recognition of 'l' or 'L' as long integer suffix
Guido van Rossum <guido@python.org>
parents: 145
diff changeset
1475 else {
19501
ce4f1e98ba7e SF bug [#455775] float parsing discrepancy.
Tim Peters <tim.peters@gmail.com>
parents: 19051
diff changeset
1476 /* Accept floating point numbers. */
236
0a92b6a1b9c0 Added recognition of 'l' or 'L' as long integer suffix
Guido van Rossum <guido@python.org>
parents: 145
diff changeset
1477 if (c == '.') {
0a92b6a1b9c0 Added recognition of 'l' or 'L' as long integer suffix
Guido van Rossum <guido@python.org>
parents: 145
diff changeset
1478 fraction:
0a92b6a1b9c0 Added recognition of 'l' or 'L' as long integer suffix
Guido van Rossum <guido@python.org>
parents: 145
diff changeset
1479 /* Fraction */
0a92b6a1b9c0 Added recognition of 'l' or 'L' as long integer suffix
Guido van Rossum <guido@python.org>
parents: 145
diff changeset
1480 do {
0a92b6a1b9c0 Added recognition of 'l' or 'L' as long integer suffix
Guido van Rossum <guido@python.org>
parents: 145
diff changeset
1481 c = tok_nextc(tok);
0a92b6a1b9c0 Added recognition of 'l' or 'L' as long integer suffix
Guido van Rossum <guido@python.org>
parents: 145
diff changeset
1482 } while (isdigit(c));
0a92b6a1b9c0 Added recognition of 'l' or 'L' as long integer suffix
Guido van Rossum <guido@python.org>
parents: 145
diff changeset
1483 }
0a92b6a1b9c0 Added recognition of 'l' or 'L' as long integer suffix
Guido van Rossum <guido@python.org>
parents: 145
diff changeset
1484 if (c == 'e' || c == 'E') {
19621
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1485 exponent:
236
0a92b6a1b9c0 Added recognition of 'l' or 'L' as long integer suffix
Guido van Rossum <guido@python.org>
parents: 145
diff changeset
1486 /* Exponent part */
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1487 c = tok_nextc(tok);
236
0a92b6a1b9c0 Added recognition of 'l' or 'L' as long integer suffix
Guido van Rossum <guido@python.org>
parents: 145
diff changeset
1488 if (c == '+' || c == '-')
0a92b6a1b9c0 Added recognition of 'l' or 'L' as long integer suffix
Guido van Rossum <guido@python.org>
parents: 145
diff changeset
1489 c = tok_nextc(tok);
19501
ce4f1e98ba7e SF bug [#455775] float parsing discrepancy.
Tim Peters <tim.peters@gmail.com>
parents: 19051
diff changeset
1490 if (!isdigit(c)) {
ce4f1e98ba7e SF bug [#455775] float parsing discrepancy.
Tim Peters <tim.peters@gmail.com>
parents: 19051
diff changeset
1491 tok->done = E_TOKEN;
ce4f1e98ba7e SF bug [#455775] float parsing discrepancy.
Tim Peters <tim.peters@gmail.com>
parents: 19051
diff changeset
1492 tok_backup(tok, c);
ce4f1e98ba7e SF bug [#455775] float parsing discrepancy.
Tim Peters <tim.peters@gmail.com>
parents: 19051
diff changeset
1493 return ERRORTOKEN;
ce4f1e98ba7e SF bug [#455775] float parsing discrepancy.
Tim Peters <tim.peters@gmail.com>
parents: 19051
diff changeset
1494 }
ce4f1e98ba7e SF bug [#455775] float parsing discrepancy.
Tim Peters <tim.peters@gmail.com>
parents: 19051
diff changeset
1495 do {
236
0a92b6a1b9c0 Added recognition of 'l' or 'L' as long integer suffix
Guido van Rossum <guido@python.org>
parents: 145
diff changeset
1496 c = tok_nextc(tok);
19501
ce4f1e98ba7e SF bug [#455775] float parsing discrepancy.
Tim Peters <tim.peters@gmail.com>
parents: 19051
diff changeset
1497 } while (isdigit(c));
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1498 }
3333
4b5bff95804e changes for pow(**) and complex
Guido van Rossum <guido@python.org>
parents: 3116
diff changeset
1499 #ifndef WITHOUT_COMPLEX
3364
bc17e8313ea0 use only j for imaginary constants
Guido van Rossum <guido@python.org>
parents: 3333
diff changeset
1500 if (c == 'j' || c == 'J')
3333
4b5bff95804e changes for pow(**) and complex
Guido van Rossum <guido@python.org>
parents: 3116
diff changeset
1501 /* Imaginary part */
4b5bff95804e changes for pow(**) and complex
Guido van Rossum <guido@python.org>
parents: 3116
diff changeset
1502 imaginary:
4b5bff95804e changes for pow(**) and complex
Guido van Rossum <guido@python.org>
parents: 3116
diff changeset
1503 c = tok_nextc(tok);
4b5bff95804e changes for pow(**) and complex
Guido van Rossum <guido@python.org>
parents: 3116
diff changeset
1504 #endif
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1505 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1506 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1507 tok_backup(tok, c);
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1508 *p_start = tok->start;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1509 *p_end = tok->cur;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1510 return NUMBER;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1511 }
5265
3c1788ab9761 Support for alternative string quotes (a"xx", b"xx", c"xx", ...).
Guido van Rossum <guido@python.org>
parents: 4839
diff changeset
1512
3c1788ab9761 Support for alternative string quotes (a"xx", b"xx", c"xx", ...).
Guido van Rossum <guido@python.org>
parents: 4839
diff changeset
1513 letter_quote:
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1514 /* String */
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1515 if (c == '\'' || c == '"') {
36501
7fdc639bc5b4 Merge ssize_t branch.
Martin v. Löwis <martin@v.loewis.de>
parents: 36090
diff changeset
1516 Py_ssize_t quote2 = tok->cur - tok->start + 1;
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1517 int quote = c;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1518 int triple = 0;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1519 int tripcount = 0;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1520 for (;;) {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1521 c = tok_nextc(tok);
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1522 if (c == '\n') {
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1523 if (!triple) {
24839
17e95cc06f69 provide less mysterious error messages when seeing end-of-line in
Skip Montanaro <skip@pobox.com>
parents: 24630
diff changeset
1524 tok->done = E_EOLS;
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1525 tok_backup(tok, c);
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1526 return ERRORTOKEN;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1527 }
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1528 tripcount = 0;
25098
c02a668488d0 Ignore encoding declarations inside strings. Fixes #603509.
Martin v. Löwis <martin@v.loewis.de>
parents: 24867
diff changeset
1529 tok->cont_line = 1; /* multiline string. */
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1530 }
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1531 else if (c == EOF) {
24839
17e95cc06f69 provide less mysterious error messages when seeing end-of-line in
Skip Montanaro <skip@pobox.com>
parents: 24630
diff changeset
1532 if (triple)
17e95cc06f69 provide less mysterious error messages when seeing end-of-line in
Skip Montanaro <skip@pobox.com>
parents: 24630
diff changeset
1533 tok->done = E_EOFS;
17e95cc06f69 provide less mysterious error messages when seeing end-of-line in
Skip Montanaro <skip@pobox.com>
parents: 24630
diff changeset
1534 else
17e95cc06f69 provide less mysterious error messages when seeing end-of-line in
Skip Montanaro <skip@pobox.com>
parents: 24630
diff changeset
1535 tok->done = E_EOLS;
1282
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
1536 tok->cur = tok->inp;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1537 return ERRORTOKEN;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1538 }
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1539 else if (c == quote) {
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1540 tripcount++;
7315
3a55fbb99f20 Fixed the bug in searching for triple quotes -- change the 'quote2'
Guido van Rossum <guido@python.org>
parents: 7118
diff changeset
1541 if (tok->cur - tok->start == quote2) {
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1542 c = tok_nextc(tok);
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1543 if (c == quote) {
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1544 triple = 1;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1545 tripcount = 0;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1546 continue;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1547 }
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1548 tok_backup(tok, c);
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1549 }
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1550 if (!triple || tripcount == 3)
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1551 break;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1552 }
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1553 else if (c == '\\') {
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1554 tripcount = 0;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1555 c = tok_nextc(tok);
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1556 if (c == EOF) {
24839
17e95cc06f69 provide less mysterious error messages when seeing end-of-line in
Skip Montanaro <skip@pobox.com>
parents: 24630
diff changeset
1557 tok->done = E_EOLS;
1282
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
1558 tok->cur = tok->inp;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1559 return ERRORTOKEN;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1560 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1561 }
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1562 else
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1563 tripcount = 0;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1564 }
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1565 *p_start = tok->start;
1364
132602d854d8 Changes to accept double-quoted strings on input.
Guido van Rossum <guido@python.org>
parents: 1284
diff changeset
1566 *p_end = tok->cur;
132602d854d8 Changes to accept double-quoted strings on input.
Guido van Rossum <guido@python.org>
parents: 1284
diff changeset
1567 return STRING;
132602d854d8 Changes to accept double-quoted strings on input.
Guido van Rossum <guido@python.org>
parents: 1284
diff changeset
1568 }
37229
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
1569
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1570 /* Line continuation */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1571 if (c == '\\') {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1572 c = tok_nextc(tok);
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1573 if (c != '\n') {
34664
3785610ff763 Patch #802188: better parser error message for non-EOL following line cont.
Martin v. Löwis <martin@v.loewis.de>
parents: 32961
diff changeset
1574 tok->done = E_LINECONT;
1282
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
1575 tok->cur = tok->inp;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1576 return ERRORTOKEN;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1577 }
25098
c02a668488d0 Ignore encoding declarations inside strings. Fixes #603509.
Martin v. Löwis <martin@v.loewis.de>
parents: 24867
diff changeset
1578 tok->cont_line = 1;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1579 goto again; /* Read next line */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1580 }
37229
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
1581
400
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1582 /* Check for two-character token */
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1583 {
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1584 int c2 = tok_nextc(tok);
5379
1c9a84e0981d Another directory quickly renamed.
Guido van Rossum <guido@python.org>
parents: 5341
diff changeset
1585 int token = PyToken_TwoChars(c, c2);
43983
e725166c8c81 Fixed problems in the last commit. Filenames and line numbers weren't reported correctly.
Christian Heimes <christian@cheimes.de>
parents: 43982
diff changeset
1586 #ifndef PGEN
44009
2068bf77891d Warning "<> not supported in 3.x" should be enabled only when the -3 option is set.
Amaury Forgeot d'Arc <amauryfa@gmail.com>
parents: 43983
diff changeset
1587 if (Py_Py3kWarningFlag && token == NOTEQUAL && c == '<') {
43983
e725166c8c81 Fixed problems in the last commit. Filenames and line numbers weren't reported correctly.
Christian Heimes <christian@cheimes.de>
parents: 43982
diff changeset
1588 if (PyErr_WarnExplicit(PyExc_DeprecationWarning,
46075
84bb7701efba Make Py3k warnings consistent w.r.t. punctuation; also respect the
Georg Brandl <georg@python.org>
parents: 45791
diff changeset
1589 "<> not supported in 3.x; use !=",
43983
e725166c8c81 Fixed problems in the last commit. Filenames and line numbers weren't reported correctly.
Christian Heimes <christian@cheimes.de>
parents: 43982
diff changeset
1590 tok->filename, tok->lineno,
e725166c8c81 Fixed problems in the last commit. Filenames and line numbers weren't reported correctly.
Christian Heimes <christian@cheimes.de>
parents: 43982
diff changeset
1591 NULL, NULL)) {
e725166c8c81 Fixed problems in the last commit. Filenames and line numbers weren't reported correctly.
Christian Heimes <christian@cheimes.de>
parents: 43982
diff changeset
1592 return ERRORTOKEN;
e725166c8c81 Fixed problems in the last commit. Filenames and line numbers weren't reported correctly.
Christian Heimes <christian@cheimes.de>
parents: 43982
diff changeset
1593 }
e725166c8c81 Fixed problems in the last commit. Filenames and line numbers weren't reported correctly.
Christian Heimes <christian@cheimes.de>
parents: 43982
diff changeset
1594 }
e725166c8c81 Fixed problems in the last commit. Filenames and line numbers weren't reported correctly.
Christian Heimes <christian@cheimes.de>
parents: 43982
diff changeset
1595 #endif
400
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1596 if (token != OP) {
14163
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1597 int c3 = tok_nextc(tok);
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1598 int token3 = PyToken_ThreeChars(c, c2, c3);
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1599 if (token3 != OP) {
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1600 token = token3;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1601 } else {
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1602 tok_backup(tok, c3);
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1603 }
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1604 *p_start = tok->start;
400
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1605 *p_end = tok->cur;
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1606 return token;
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1607 }
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1608 tok_backup(tok, c2);
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1609 }
37229
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
1610
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1611 /* Keep track of parentheses nesting level */
1284
dbce72afe261 * selectmodule.c: fix (another!) two memory leaks -- this time in list2set
Guido van Rossum <guido@python.org>
parents: 1282
diff changeset
1612 switch (c) {
dbce72afe261 * selectmodule.c: fix (another!) two memory leaks -- this time in list2set
Guido van Rossum <guido@python.org>
parents: 1282
diff changeset
1613 case '(':
dbce72afe261 * selectmodule.c: fix (another!) two memory leaks -- this time in list2set
Guido van Rossum <guido@python.org>
parents: 1282
diff changeset
1614 case '[':
dbce72afe261 * selectmodule.c: fix (another!) two memory leaks -- this time in list2set
Guido van Rossum <guido@python.org>
parents: 1282
diff changeset
1615 case '{':
dbce72afe261 * selectmodule.c: fix (another!) two memory leaks -- this time in list2set
Guido van Rossum <guido@python.org>
parents: 1282
diff changeset
1616 tok->level++;
dbce72afe261 * selectmodule.c: fix (another!) two memory leaks -- this time in list2set
Guido van Rossum <guido@python.org>
parents: 1282
diff changeset
1617 break;
dbce72afe261 * selectmodule.c: fix (another!) two memory leaks -- this time in list2set
Guido van Rossum <guido@python.org>
parents: 1282
diff changeset
1618 case ')':
dbce72afe261 * selectmodule.c: fix (another!) two memory leaks -- this time in list2set
Guido van Rossum <guido@python.org>
parents: 1282
diff changeset
1619 case ']':
dbce72afe261 * selectmodule.c: fix (another!) two memory leaks -- this time in list2set
Guido van Rossum <guido@python.org>
parents: 1282
diff changeset
1620 case '}':
dbce72afe261 * selectmodule.c: fix (another!) two memory leaks -- this time in list2set
Guido van Rossum <guido@python.org>
parents: 1282
diff changeset
1621 tok->level--;
dbce72afe261 * selectmodule.c: fix (another!) two memory leaks -- this time in list2set
Guido van Rossum <guido@python.org>
parents: 1282
diff changeset
1622 break;
dbce72afe261 * selectmodule.c: fix (another!) two memory leaks -- this time in list2set
Guido van Rossum <guido@python.org>
parents: 1282
diff changeset
1623 }
37229
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
1624
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1625 /* Punctuation character */
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1626 *p_start = tok->start;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1627 *p_end = tok->cur;
5379
1c9a84e0981d Another directory quickly renamed.
Guido van Rossum <guido@python.org>
parents: 5341
diff changeset
1628 return PyToken_OneChar(c);
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1629 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1630
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
1631 int
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
1632 PyTokenizer_Get(struct tok_state *tok, char **p_start, char **p_end)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
1633 {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
1634 int result = tok_get(tok, p_start, p_end);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
1635 if (tok->decoding_erred) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
1636 result = ERRORTOKEN;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
1637 tok->done = E_DECODE;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
1638 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
1639 return result;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
1640 }
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1641
43229
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1642 /* This function is only called from parsetok. However, it cannot live
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1643 there, as it must be empty for PGEN, and we can check for PGEN only
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1644 in this file. */
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1645
44889
9a1211c77e02 Fixed bug #1915: Python compiles with --enable-unicode=no again. However several extension methods and modules do not work without unicode support.
Christian Heimes <christian@cheimes.de>
parents: 44861
diff changeset
1646 #if defined(PGEN) || !defined(Py_USING_UNICODE)
43229
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1647 char*
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1648 PyTokenizer_RestoreEncoding(struct tok_state* tok, int len, int* offset)
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1649 {
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1650 return NULL;
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1651 }
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1652 #else
44575
cf36e01e83f2 Fix #define ordering.
Georg Brandl <georg@python.org>
parents: 44572
diff changeset
1653 #ifdef Py_USING_UNICODE
43229
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1654 static PyObject *
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1655 dec_utf8(const char *enc, const char *text, size_t len) {
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1656 PyObject *ret = NULL;
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1657 PyObject *unicode_text = PyUnicode_DecodeUTF8(text, len, "replace");
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1658 if (unicode_text) {
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1659 ret = PyUnicode_AsEncodedString(unicode_text, enc, "replace");
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1660 Py_DECREF(unicode_text);
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1661 }
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1662 if (!ret) {
43572
697400fce7c5 Change a PyErr_Print() into a PyErr_Clear(),
Guido van Rossum <guido@python.org>
parents: 43229
diff changeset
1663 PyErr_Clear();
43229
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1664 }
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1665 return ret;
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1666 }
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1667 char *
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1668 PyTokenizer_RestoreEncoding(struct tok_state* tok, int len, int *offset)
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1669 {
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1670 char *text = NULL;
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1671 if (tok->encoding) {
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1672 /* convert source to original encondig */
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1673 PyObject *lineobj = dec_utf8(tok->encoding, tok->buf, len);
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1674 if (lineobj != NULL) {
47527
52294af4b663 This reverts r63675 based on the discussion in this thread:
Gregory P. Smith <greg@mad-scientist.com>
parents: 47294
diff changeset
1675 int linelen = PyString_Size(lineobj);
52294af4b663 This reverts r63675 based on the discussion in this thread:
Gregory P. Smith <greg@mad-scientist.com>
parents: 47294
diff changeset
1676 const char *line = PyString_AsString(lineobj);
43229
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1677 text = PyObject_MALLOC(linelen + 1);
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1678 if (text != NULL && line != NULL) {
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1679 if (linelen)
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1680 strncpy(text, line, linelen);
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1681 text[linelen] = '\0';
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1682 }
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1683 Py_DECREF(lineobj);
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1684
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1685 /* adjust error offset */
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1686 if (*offset > 1) {
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1687 PyObject *offsetobj = dec_utf8(tok->encoding,
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1688 tok->buf, *offset-1);
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1689 if (offsetobj) {
47527
52294af4b663 This reverts r63675 based on the discussion in this thread:
Gregory P. Smith <greg@mad-scientist.com>
parents: 47294
diff changeset
1690 *offset = PyString_Size(offsetobj) + 1;
43229
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1691 Py_DECREF(offsetobj);
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1692 }
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1693 }
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1694
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1695 }
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1696 }
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1697 return text;
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1698
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1699 }
44575
cf36e01e83f2 Fix #define ordering.
Georg Brandl <georg@python.org>
parents: 44572
diff changeset
1700 #endif /* defined(Py_USING_UNICODE) */
43229
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1701 #endif
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1702
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1703
4839
4567e158d20c Rename DEBUG macro to Py_DEBUG
Guido van Rossum <guido@python.org>
parents: 4577
diff changeset
1704 #ifdef Py_DEBUG
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1705
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1706 void
13659
e78153e179fb Mass ANSIfication.
Thomas Wouters <thomas@python.org>
parents: 13539
diff changeset
1707 tok_dump(int type, char *start, char *end)
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1708 {
5379
1c9a84e0981d Another directory quickly renamed.
Guido van Rossum <guido@python.org>
parents: 5341
diff changeset
1709 printf("%s", _PyParser_TokenNames[type]);
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1710 if (type == NAME || type == NUMBER || type == STRING || type == OP)
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1711 printf("(%.*s)", (int)(end - start), start);
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1712 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1713
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1714 #endif