Mercurial > cpython-lru_cache-weakref

annotate Parser/tokenizer.c @ 58925:eecf1bef052d 2.6

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Merged revisions 78207 via svnmerge from svn+ssh://pythondev@svn.python.org/python/trunk ........ r78207 | ezio.melotti | 2010-02-17 01:26:09 +0200 (Wed, 17 Feb 2010) | 1 line #7930: fix stripid ........
author Ezio Melotti <ezio.melotti@gmail.com>
date Tue, 16 Feb 2010 23:29:44 +0000
parents b654d14cb9d9
children b25d1024f394 4a50f0bdc806
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
145
853b736a6a9e Added copyright notice.
Guido van Rossum <guido@python.org>
parents: 96
diff changeset
1
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
2 /* Tokenizer implementation */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
3
23029
8291473fde3e Mass checkin of universal newline support.
Jack Jansen <jack.jansen@cwi.nl>
parents: 19621
diff changeset
4 #include "Python.h"
86
a6934380c6e7 "Compiling" version
Guido van Rossum <guido@python.org>
parents: 67
diff changeset
5 #include "pgenheaders.h"
a6934380c6e7 "Compiling" version
Guido van Rossum <guido@python.org>
parents: 67
diff changeset
6
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
7 #include <ctype.h>
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
8 #include <assert.h>
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
9
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
10 #include "tokenizer.h"
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
11 #include "errcode.h"
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
12
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
13 #ifndef PGEN
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
14 #include "unicodeobject.h"
47527
52294af4b663 This reverts r63675 based on the discussion in this thread:
Gregory P. Smith <greg@mad-scientist.com>
parents: 47294
diff changeset
15 #include "stringobject.h"
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
16 #include "fileobject.h"
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
17 #include "codecs.h"
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
18 #include "abstract.h"
43982
5f6a8e2b9e9c Applied patch #1754273 and #1754271 from Thomas Glee
Christian Heimes <christian@cheimes.de>
parents: 43572
diff changeset
19 #include "pydebug.h"
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
20 #endif /* PGEN */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
21
25924
dc4a0336a2a3 Patch #512981: Update readline input stream on sys.stdin/out change.
Martin v. Löwis <martin@v.loewis.de>
parents: 25101
diff changeset
22 extern char *PyOS_Readline(FILE *, FILE *, char *);
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
23 /* Return malloc'ed string including trailing \n;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
24 empty malloc'ed string for EOF;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
25 NULL if interrupted */
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
26
678
205b947b1d3c Make tabs always 8 spaces wide -- it's more portable.
Guido van Rossum <guido@python.org>
parents: 477
diff changeset
27 /* Don't ever change this -- it would break the portability of Python code */
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
28 #define TABSIZE 8
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
29
86
a6934380c6e7 "Compiling" version
Guido van Rossum <guido@python.org>
parents: 67
diff changeset
30 /* Forward */
13447
a89f7f268de4 Nuke all remaining occurrences of Py_PROTO and Py_FPROTO.
Tim Peters <tim.peters@gmail.com>
parents: 13256
diff changeset
31 static struct tok_state *tok_new(void);
a89f7f268de4 Nuke all remaining occurrences of Py_PROTO and Py_FPROTO.
Tim Peters <tim.peters@gmail.com>
parents: 13256
diff changeset
32 static int tok_nextc(struct tok_state *tok);
a89f7f268de4 Nuke all remaining occurrences of Py_PROTO and Py_FPROTO.
Tim Peters <tim.peters@gmail.com>
parents: 13256
diff changeset
33 static void tok_backup(struct tok_state *tok, int c);
86
a6934380c6e7 "Compiling" version
Guido van Rossum <guido@python.org>
parents: 67
diff changeset
34
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
35 /* Token names */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
36
5379
1c9a84e0981d Another directory quickly renamed.
Guido van Rossum <guido@python.org>
parents: 5341
diff changeset
37 char *_PyParser_TokenNames[] = {
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
38 "ENDMARKER",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
39 "NAME",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
40 "NUMBER",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
41 "STRING",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
42 "NEWLINE",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
43 "INDENT",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
44 "DEDENT",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
45 "LPAR",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
46 "RPAR",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
47 "LSQB",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
48 "RSQB",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
49 "COLON",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
50 "COMMA",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
51 "SEMI",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
52 "PLUS",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
53 "MINUS",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
54 "STAR",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
55 "SLASH",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
56 "VBAR",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
57 "AMPER",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
58 "LESS",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
59 "GREATER",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
60 "EQUAL",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
61 "DOT",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
62 "PERCENT",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
63 "BACKQUOTE",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
64 "LBRACE",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
65 "RBRACE",
400
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
66 "EQEQUAL",
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
67 "NOTEQUAL",
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
68 "LESSEQUAL",
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
69 "GREATEREQUAL",
410
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
70 "TILDE",
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
71 "CIRCUMFLEX",
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
72 "LEFTSHIFT",
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
73 "RIGHTSHIFT",
3333
4b5bff95804e changes for pow(**) and complex
Guido van Rossum <guido@python.org>
parents: 3116
diff changeset
74 "DOUBLESTAR",
14163
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
75 "PLUSEQUAL",
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
76 "MINEQUAL",
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
77 "STAREQUAL",
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
78 "SLASHEQUAL",
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
79 "PERCENTEQUAL",
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
80 "AMPEREQUAL",
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
81 "VBAREQUAL",
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
82 "CIRCUMFLEXEQUAL",
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
83 "LEFTSHIFTEQUAL",
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
84 "RIGHTSHIFTEQUAL",
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
85 "DOUBLESTAREQUAL",
19051
f20727cc3d0f Implement PEP 238 in its (almost) full glory.
Guido van Rossum <guido@python.org>
parents: 17600
diff changeset
86 "DOUBLESLASH",
f20727cc3d0f Implement PEP 238 in its (almost) full glory.
Guido van Rossum <guido@python.org>
parents: 17600
diff changeset
87 "DOUBLESLASHEQUAL",
32888
c7e03cf8333e PEP-0318, @decorator-style. In Guido's words:
Anthony Baxter <anthonybaxter@gmail.com>
parents: 30979
diff changeset
88 "AT",
400
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
89 /* This table must match the #defines in token.h! */
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
90 "OP",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
91 "<ERRORTOKEN>",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
92 "<N_TOKENS>"
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
93 };
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
94
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
95
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
96 /* Create and initialize a new tok_state structure */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
97
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
98 static struct tok_state *
13659
e78153e179fb Mass ANSIfication.
Thomas Wouters <thomas@python.org>
parents: 13539
diff changeset
99 tok_new(void)
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
100 {
37585
292854c43b99 Fix the code in Parser/ to also compile with C++. This was mostly casts for
Anthony Baxter <anthonybaxter@gmail.com>
parents: 37557
diff changeset
101 struct tok_state *tok = (struct tok_state *)PyMem_MALLOC(
292854c43b99 Fix the code in Parser/ to also compile with C++. This was mostly casts for
Anthony Baxter <anthonybaxter@gmail.com>
parents: 37557
diff changeset
102 sizeof(struct tok_state));
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
103 if (tok == NULL)
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
104 return NULL;
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
105 tok->buf = tok->cur = tok->end = tok->inp = tok->start = NULL;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
106 tok->done = E_OK;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
107 tok->fp = NULL;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
108 tok->tabsize = TABSIZE;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
109 tok->indent = 0;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
110 tok->indstack[0] = 0;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
111 tok->atbol = 1;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
112 tok->pendin = 0;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
113 tok->prompt = tok->nextprompt = NULL;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
114 tok->lineno = 0;
1284
dbce72afe261 * selectmodule.c: fix (another!) two memory leaks -- this time in list2set
Guido van Rossum <guido@python.org>
parents: 1282
diff changeset
115 tok->level = 0;
7945
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
116 tok->filename = NULL;
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
117 tok->altwarning = 0;
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
118 tok->alterror = 0;
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
119 tok->alttabsize = 1;
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
120 tok->altindstack[0] = 0;
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
121 tok->decoding_state = 0;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
122 tok->decoding_erred = 0;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
123 tok->read_coding_spec = 0;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
124 tok->encoding = NULL;
25098
c02a668488d0 Ignore encoding declarations inside strings. Fixes #603509.
Martin v. Löwis <martin@v.loewis.de>
parents: 24867
diff changeset
125 tok->cont_line = 0;
24544
0bd521eb714e Make pgen compile with pydebug. Duplicate normalized names, as it may
Martin v. Löwis <martin@v.loewis.de>
parents: 24543
diff changeset
126 #ifndef PGEN
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
127 tok->decoding_readline = NULL;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
128 tok->decoding_buffer = NULL;
24544
0bd521eb714e Make pgen compile with pydebug. Duplicate normalized names, as it may
Martin v. Löwis <martin@v.loewis.de>
parents: 24543
diff changeset
129 #endif
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
130 return tok;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
131 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
132
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
133 #ifdef PGEN
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
134
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
135 static char *
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
136 decoding_fgets(char *s, int size, struct tok_state *tok)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
137 {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
138 return fgets(s, size, tok->fp);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
139 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
140
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
141 static int
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
142 decoding_feof(struct tok_state *tok)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
143 {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
144 return feof(tok->fp);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
145 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
146
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
147 static const char *
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
148 decode_str(const char *str, struct tok_state *tok)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
149 {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
150 return str;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
151 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
152
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
153 #else /* PGEN */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
154
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
155 static char *
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
156 error_ret(struct tok_state *tok) /* XXX */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
157 {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
158 tok->decoding_erred = 1;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
159 if (tok->fp != NULL && tok->buf != NULL) /* see PyTokenizer_Free */
37600
c4c5eeb07739 As discussed on python-dev, really fix the PyMem_*/PyObject_* memory API
Neal Norwitz <nnorwitz@gmail.com>
parents: 37585
diff changeset
160 PyMem_FREE(tok->buf);
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
161 tok->buf = NULL;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
162 return NULL; /* as if it were EOF */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
163 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
164
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
165 static char *
36501
7fdc639bc5b4 Merge ssize_t branch.
Martin v. Löwis <martin@v.loewis.de>
parents: 36090
diff changeset
166 new_string(const char *s, Py_ssize_t len)
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
167 {
37600
c4c5eeb07739 As discussed on python-dev, really fix the PyMem_*/PyObject_* memory API
Neal Norwitz <nnorwitz@gmail.com>
parents: 37585
diff changeset
168 char* result = (char *)PyMem_MALLOC(len + 1);
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
169 if (result != NULL) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
170 memcpy(result, s, len);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
171 result[len] = '\0';
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
172 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
173 return result;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
174 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
175
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
176 static char *
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
177 get_normal_name(char *s) /* for utf-8 and latin-1 */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
178 {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
179 char buf[13];
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
180 int i;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
181 for (i = 0; i < 12; i++) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
182 int c = s[i];
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
183 if (c == '\0') break;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
184 else if (c == '_') buf[i] = '-';
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
185 else buf[i] = tolower(c);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
186 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
187 buf[i] = '\0';
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
188 if (strcmp(buf, "utf-8") == 0 ||
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
189 strncmp(buf, "utf-8-", 6) == 0) return "utf-8";
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
190 else if (strcmp(buf, "latin-1") == 0 ||
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
191 strcmp(buf, "iso-8859-1") == 0 ||
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
192 strcmp(buf, "iso-latin-1") == 0 ||
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
193 strncmp(buf, "latin-1-", 8) == 0 ||
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
194 strncmp(buf, "iso-8859-1-", 11) == 0 ||
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
195 strncmp(buf, "iso-latin-1-", 12) == 0) return "iso-8859-1";
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
196 else return s;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
197 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
198
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
199 /* Return the coding spec in S, or NULL if none is found. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
200
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
201 static char *
36501
7fdc639bc5b4 Merge ssize_t branch.
Martin v. Löwis <martin@v.loewis.de>
parents: 36090
diff changeset
202 get_coding_spec(const char *s, Py_ssize_t size)
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
203 {
36501
7fdc639bc5b4 Merge ssize_t branch.
Martin v. Löwis <martin@v.loewis.de>
parents: 36090
diff changeset
204 Py_ssize_t i;
25098
c02a668488d0 Ignore encoding declarations inside strings. Fixes #603509.
Martin v. Löwis <martin@v.loewis.de>
parents: 24867
diff changeset
205 /* Coding spec must be in a comment, and that comment must be
c02a668488d0 Ignore encoding declarations inside strings. Fixes #603509.
Martin v. Löwis <martin@v.loewis.de>
parents: 24867
diff changeset
206 * the only statement on the source code line. */
c02a668488d0 Ignore encoding declarations inside strings. Fixes #603509.
Martin v. Löwis <martin@v.loewis.de>
parents: 24867
diff changeset
207 for (i = 0; i < size - 6; i++) {
c02a668488d0 Ignore encoding declarations inside strings. Fixes #603509.
Martin v. Löwis <martin@v.loewis.de>
parents: 24867
diff changeset
208 if (s[i] == '#')
c02a668488d0 Ignore encoding declarations inside strings. Fixes #603509.
Martin v. Löwis <martin@v.loewis.de>
parents: 24867
diff changeset
209 break;
c02a668488d0 Ignore encoding declarations inside strings. Fixes #603509.
Martin v. Löwis <martin@v.loewis.de>
parents: 24867
diff changeset
210 if (s[i] != ' ' && s[i] != '\t' && s[i] != '\014')
c02a668488d0 Ignore encoding declarations inside strings. Fixes #603509.
Martin v. Löwis <martin@v.loewis.de>
parents: 24867
diff changeset
211 return NULL;
c02a668488d0 Ignore encoding declarations inside strings. Fixes #603509.
Martin v. Löwis <martin@v.loewis.de>
parents: 24867
diff changeset
212 }
c02a668488d0 Ignore encoding declarations inside strings. Fixes #603509.
Martin v. Löwis <martin@v.loewis.de>
parents: 24867
diff changeset
213 for (; i < size - 6; i++) { /* XXX inefficient search */
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
214 const char* t = s + i;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
215 if (strncmp(t, "coding", 6) == 0) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
216 const char* begin = NULL;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
217 t += 6;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
218 if (t[0] != ':' && t[0] != '=')
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
219 continue;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
220 do {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
221 t++;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
222 } while (t[0] == '\x20' || t[0] == '\t');
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
223
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
224 begin = t;
36090
8cb686dd6e64 Fix SF bug #1072182, problems with signed characters.
Neal Norwitz <nnorwitz@gmail.com>
parents: 36078
diff changeset
225 while (isalnum(Py_CHARMASK(t[0])) ||
25995
1aaa12c08b35 Fix compiler warning on HP-UX.
Neal Norwitz <nnorwitz@gmail.com>
parents: 25924
diff changeset
226 t[0] == '-' || t[0] == '_' || t[0] == '.')
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
227 t++;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
228
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
229 if (begin < t) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
230 char* r = new_string(begin, t - begin);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
231 char* q = get_normal_name(r);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
232 if (r != q) {
37600
c4c5eeb07739 As discussed on python-dev, really fix the PyMem_*/PyObject_* memory API
Neal Norwitz <nnorwitz@gmail.com>
parents: 37585
diff changeset
233 PyMem_FREE(r);
24544
0bd521eb714e Make pgen compile with pydebug. Duplicate normalized names, as it may
Martin v. Löwis <martin@v.loewis.de>
parents: 24543
diff changeset
234 r = new_string(q, strlen(q));
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
235 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
236 return r;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
237 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
238 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
239 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
240 return NULL;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
241 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
242
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
243 /* Check whether the line contains a coding spec. If it does,
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
244 invoke the set_readline function for the new encoding.
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
245 This function receives the tok_state and the new encoding.
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
246 Return 1 on success, 0 on failure. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
247
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
248 static int
36501
7fdc639bc5b4 Merge ssize_t branch.
Martin v. Löwis <martin@v.loewis.de>
parents: 36090
diff changeset
249 check_coding_spec(const char* line, Py_ssize_t size, struct tok_state *tok,
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
250 int set_readline(struct tok_state *, const char *))
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
251 {
25101
32331d26d7cf Removed reliance on gcc/C99 extension.
Tim Peters <tim.peters@gmail.com>
parents: 25098
diff changeset
252 char * cs;
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
253 int r = 1;
25101
32331d26d7cf Removed reliance on gcc/C99 extension.
Tim Peters <tim.peters@gmail.com>
parents: 25098
diff changeset
254
25098
c02a668488d0 Ignore encoding declarations inside strings. Fixes #603509.
Martin v. Löwis <martin@v.loewis.de>
parents: 24867
diff changeset
255 if (tok->cont_line)
c02a668488d0 Ignore encoding declarations inside strings. Fixes #603509.
Martin v. Löwis <martin@v.loewis.de>
parents: 24867
diff changeset
256 /* It's a continuation line, so it can't be a coding spec. */
c02a668488d0 Ignore encoding declarations inside strings. Fixes #603509.
Martin v. Löwis <martin@v.loewis.de>
parents: 24867
diff changeset
257 return 1;
25101
32331d26d7cf Removed reliance on gcc/C99 extension.
Tim Peters <tim.peters@gmail.com>
parents: 25098
diff changeset
258 cs = get_coding_spec(line, size);
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
259 if (cs != NULL) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
260 tok->read_coding_spec = 1;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
261 if (tok->encoding == NULL) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
262 assert(tok->decoding_state == 1); /* raw */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
263 if (strcmp(cs, "utf-8") == 0 ||
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
264 strcmp(cs, "iso-8859-1") == 0) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
265 tok->encoding = cs;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
266 } else {
24624
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
267 #ifdef Py_USING_UNICODE
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
268 r = set_readline(tok, cs);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
269 if (r) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
270 tok->encoding = cs;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
271 tok->decoding_state = -1;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
272 }
35751
e56144704808 Free coding spec (cs) if there was an error to prevent mem leak. Maybe backport candidate
Neal Norwitz <nnorwitz@gmail.com>
parents: 35666
diff changeset
273 else
37600
c4c5eeb07739 As discussed on python-dev, really fix the PyMem_*/PyObject_* memory API
Neal Norwitz <nnorwitz@gmail.com>
parents: 37585
diff changeset
274 PyMem_FREE(cs);
24624
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
275 #else
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
276 /* Without Unicode support, we cannot
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
277 process the coding spec. Since there
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
278 won't be any Unicode literals, that
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
279 won't matter. */
37600
c4c5eeb07739 As discussed on python-dev, really fix the PyMem_*/PyObject_* memory API
Neal Norwitz <nnorwitz@gmail.com>
parents: 37585
diff changeset
280 PyMem_FREE(cs);
24624
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
281 #endif
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
282 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
283 } else { /* then, compare cs with BOM */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
284 r = (strcmp(tok->encoding, cs) == 0);
37600
c4c5eeb07739 As discussed on python-dev, really fix the PyMem_*/PyObject_* memory API
Neal Norwitz <nnorwitz@gmail.com>
parents: 37585
diff changeset
285 PyMem_FREE(cs);
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
286 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
287 }
36078
1ff4790b7605 Fix Bug #1378022, UTF-8 files with a leading BOM crashed the interpreter.
Neal Norwitz <nnorwitz@gmail.com>
parents: 35878
diff changeset
288 if (!r) {
1ff4790b7605 Fix Bug #1378022, UTF-8 files with a leading BOM crashed the interpreter.
Neal Norwitz <nnorwitz@gmail.com>
parents: 35878
diff changeset
289 cs = tok->encoding;
1ff4790b7605 Fix Bug #1378022, UTF-8 files with a leading BOM crashed the interpreter.
Neal Norwitz <nnorwitz@gmail.com>
parents: 35878
diff changeset
290 if (!cs)
1ff4790b7605 Fix Bug #1378022, UTF-8 files with a leading BOM crashed the interpreter.
Neal Norwitz <nnorwitz@gmail.com>
parents: 35878
diff changeset
291 cs = "with BOM";
1ff4790b7605 Fix Bug #1378022, UTF-8 files with a leading BOM crashed the interpreter.
Neal Norwitz <nnorwitz@gmail.com>
parents: 35878
diff changeset
292 PyErr_Format(PyExc_SyntaxError, "encoding problem: %s", cs);
1ff4790b7605 Fix Bug #1378022, UTF-8 files with a leading BOM crashed the interpreter.
Neal Norwitz <nnorwitz@gmail.com>
parents: 35878
diff changeset
293 }
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
294 return r;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
295 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
296
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
297 /* See whether the file starts with a BOM. If it does,
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
298 invoke the set_readline function with the new encoding.
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
299 Return 1 on success, 0 on failure. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
300
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
301 static int
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
302 check_bom(int get_char(struct tok_state *),
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
303 void unget_char(int, struct tok_state *),
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
304 int set_readline(struct tok_state *, const char *),
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
305 struct tok_state *tok)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
306 {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
307 int ch = get_char(tok);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
308 tok->decoding_state = 1;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
309 if (ch == EOF) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
310 return 1;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
311 } else if (ch == 0xEF) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
312 ch = get_char(tok); if (ch != 0xBB) goto NON_BOM;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
313 ch = get_char(tok); if (ch != 0xBF) goto NON_BOM;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
314 #if 0
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
315 /* Disable support for UTF-16 BOMs until a decision
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
316 is made whether this needs to be supported. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
317 } else if (ch == 0xFE) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
318 ch = get_char(tok); if (ch != 0xFF) goto NON_BOM;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
319 if (!set_readline(tok, "utf-16-be")) return 0;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
320 tok->decoding_state = -1;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
321 } else if (ch == 0xFF) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
322 ch = get_char(tok); if (ch != 0xFE) goto NON_BOM;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
323 if (!set_readline(tok, "utf-16-le")) return 0;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
324 tok->decoding_state = -1;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
325 #endif
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
326 } else {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
327 unget_char(ch, tok);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
328 return 1;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
329 }
35878
94a409da4bd4 Fix some more memory leaks.
Neal Norwitz <nnorwitz@gmail.com>
parents: 35751
diff changeset
330 if (tok->encoding != NULL)
37600
c4c5eeb07739 As discussed on python-dev, really fix the PyMem_*/PyObject_* memory API
Neal Norwitz <nnorwitz@gmail.com>
parents: 37585
diff changeset
331 PyMem_FREE(tok->encoding);
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
332 tok->encoding = new_string("utf-8", 5); /* resulting is in utf-8 */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
333 return 1;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
334 NON_BOM:
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
335 /* any token beginning with '\xEF', '\xFE', '\xFF' is a bad token */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
336 unget_char(0xFF, tok); /* XXX this will cause a syntax error */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
337 return 1;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
338 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
339
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
340 /* Read a line of text from TOK into S, using the stream in TOK.
35231
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
341 Return NULL on failure, else S.
37229
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
342
35231
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
343 On entry, tok->decoding_buffer will be one of:
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
344 1) NULL: need to call tok->decoding_readline to get a new line
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
345 2) PyUnicodeObject *: decoding_feof has called tok->decoding_readline and
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
346 stored the result in tok->decoding_buffer
47527
52294af4b663 This reverts r63675 based on the discussion in this thread:
Gregory P. Smith <greg@mad-scientist.com>
parents: 47294
diff changeset
347 3) PyStringObject *: previous call to fp_readl did not have enough room
35231
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
348 (in the s buffer) to copy entire contents of the line read
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
349 by tok->decoding_readline. tok->decoding_buffer has the overflow.
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
350 In this case, fp_readl is called in a loop (with an expanded buffer)
37229
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
351 until the buffer ends with a '\n' (or until the end of the file is
35231
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
352 reached): see tok_nextc and its calls to decoding_fgets.
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
353 */
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
354
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
355 static char *
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
356 fp_readl(char *s, int size, struct tok_state *tok)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
357 {
24624
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
358 #ifndef Py_USING_UNICODE
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
359 /* In a non-Unicode built, this should never be called. */
24630
43caa212b959 Use Py_FatalError instead of abort.
Martin v. Löwis <martin@v.loewis.de>
parents: 24624
diff changeset
360 Py_FatalError("fp_readl should not be called in this build.");
24867
255d1d3e66a3 Squash a few calls to the hideously expensive PyObject_CallObject(o,a)
Guido van Rossum <guido@python.org>
parents: 24839
diff changeset
361 return NULL; /* Keep compiler happy (not reachable) */
24624
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
362 #else
35231
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
363 PyObject* utf8 = NULL;
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
364 PyObject* buf = tok->decoding_buffer;
35231
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
365 char *str;
36520
26470cdeea3d Use Py_ssize_t to count the length.
Martin v. Löwis <martin@v.loewis.de>
parents: 36501
diff changeset
366 Py_ssize_t utf8len;
35231
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
367
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
368 /* Ask for one less byte so we can terminate it */
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
369 assert(size > 0);
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
370 size--;
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
371
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
372 if (buf == NULL) {
35231
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
373 buf = PyObject_CallObject(tok->decoding_readline, NULL);
24543
c00df23e856e Group statements properly.
Martin v. Löwis <martin@v.loewis.de>
parents: 24542
diff changeset
374 if (buf == NULL)
c00df23e856e Group statements properly.
Martin v. Löwis <martin@v.loewis.de>
parents: 24542
diff changeset
375 return error_ret(tok);
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
376 } else {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
377 tok->decoding_buffer = NULL;
47527
52294af4b663 This reverts r63675 based on the discussion in this thread:
Gregory P. Smith <greg@mad-scientist.com>
parents: 47294
diff changeset
378 if (PyString_CheckExact(buf))
35231
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
379 utf8 = buf;
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
380 }
35231
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
381 if (utf8 == NULL) {
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
382 utf8 = PyUnicode_AsUTF8String(buf);
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
383 Py_DECREF(buf);
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
384 if (utf8 == NULL)
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
385 return error_ret(tok);
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
386 }
47527
52294af4b663 This reverts r63675 based on the discussion in this thread:
Gregory P. Smith <greg@mad-scientist.com>
parents: 47294
diff changeset
387 str = PyString_AsString(utf8);
52294af4b663 This reverts r63675 based on the discussion in this thread:
Gregory P. Smith <greg@mad-scientist.com>
parents: 47294
diff changeset
388 utf8len = PyString_GET_SIZE(utf8);
35231
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
389 if (utf8len > size) {
47527
52294af4b663 This reverts r63675 based on the discussion in this thread:
Gregory P. Smith <greg@mad-scientist.com>
parents: 47294
diff changeset
390 tok->decoding_buffer = PyString_FromStringAndSize(str+size, utf8len-size);
35231
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
391 if (tok->decoding_buffer == NULL) {
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
392 Py_DECREF(utf8);
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
393 return error_ret(tok);
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
394 }
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
395 utf8len = size;
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
396 }
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
397 memcpy(s, str, utf8len);
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
398 s[utf8len] = '\0';
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
399 Py_DECREF(utf8);
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
400 if (utf8len == 0) return NULL; /* EOF */
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
401 return s;
24624
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
402 #endif
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
403 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
404
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
405 /* Set the readline function for TOK to a StreamReader's
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
406 readline function. The StreamReader is named ENC.
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
407
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
408 This function is called from check_bom and check_coding_spec.
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
409
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
410 ENC is usually identical to the future value of tok->encoding,
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
411 except for the (currently unsupported) case of UTF-16.
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
412
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
413 Return 1 on success, 0 on failure. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
414
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
415 static int
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
416 fp_setreadl(struct tok_state *tok, const char* enc)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
417 {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
418 PyObject *reader, *stream, *readline;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
419
26601
cc05c2c8660e Constify filenames and scripts. Fixes #651362.
Martin v. Löwis <martin@v.loewis.de>
parents: 25995
diff changeset
420 /* XXX: constify filename argument. */
cc05c2c8660e Constify filenames and scripts. Fixes #651362.
Martin v. Löwis <martin@v.loewis.de>
parents: 25995
diff changeset
421 stream = PyFile_FromFile(tok->fp, (char*)tok->filename, "rb", NULL);
24543
c00df23e856e Group statements properly.
Martin v. Löwis <martin@v.loewis.de>
parents: 24542
diff changeset
422 if (stream == NULL)
c00df23e856e Group statements properly.
Martin v. Löwis <martin@v.loewis.de>
parents: 24542
diff changeset
423 return 0;
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
424
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
425 reader = PyCodec_StreamReader(enc, stream, NULL);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
426 Py_DECREF(stream);
24543
c00df23e856e Group statements properly.
Martin v. Löwis <martin@v.loewis.de>
parents: 24542
diff changeset
427 if (reader == NULL)
c00df23e856e Group statements properly.
Martin v. Löwis <martin@v.loewis.de>
parents: 24542
diff changeset
428 return 0;
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
429
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
430 readline = PyObject_GetAttrString(reader, "readline");
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
431 Py_DECREF(reader);
24543
c00df23e856e Group statements properly.
Martin v. Löwis <martin@v.loewis.de>
parents: 24542
diff changeset
432 if (readline == NULL)
c00df23e856e Group statements properly.
Martin v. Löwis <martin@v.loewis.de>
parents: 24542
diff changeset
433 return 0;
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
434
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
435 tok->decoding_readline = readline;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
436 return 1;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
437 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
438
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
439 /* Fetch the next byte from TOK. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
440
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
441 static int fp_getc(struct tok_state *tok) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
442 return getc(tok->fp);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
443 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
444
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
445 /* Unfetch the last byte back into TOK. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
446
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
447 static void fp_ungetc(int c, struct tok_state *tok) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
448 ungetc(c, tok->fp);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
449 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
450
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
451 /* Read a line of input from TOK. Determine encoding
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
452 if necessary. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
453
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
454 static char *
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
455 decoding_fgets(char *s, int size, struct tok_state *tok)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
456 {
24630
43caa212b959 Use Py_FatalError instead of abort.
Martin v. Löwis <martin@v.loewis.de>
parents: 24624
diff changeset
457 char *line = NULL;
36741
c0982b8155c6 Change non-ASCII warning into a SyntaxError.
Martin v. Löwis <martin@v.loewis.de>
parents: 36520
diff changeset
458 int badchar = 0;
24543
c00df23e856e Group statements properly.
Martin v. Löwis <martin@v.loewis.de>
parents: 24542
diff changeset
459 for (;;) {
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
460 if (tok->decoding_state < 0) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
461 /* We already have a codec associated with
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
462 this input. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
463 line = fp_readl(s, size, tok);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
464 break;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
465 } else if (tok->decoding_state > 0) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
466 /* We want a 'raw' read. */
37229
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
467 line = Py_UniversalNewlineFgets(s, size,
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
468 tok->fp, NULL);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
469 break;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
470 } else {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
471 /* We have not yet determined the encoding.
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
472 If an encoding is found, use the file-pointer
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
473 reader functions from now on. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
474 if (!check_bom(fp_getc, fp_ungetc, fp_setreadl, tok))
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
475 return error_ret(tok);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
476 assert(tok->decoding_state != 0);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
477 }
24543
c00df23e856e Group statements properly.
Martin v. Löwis <martin@v.loewis.de>
parents: 24542
diff changeset
478 }
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
479 if (line != NULL && tok->lineno < 2 && !tok->read_coding_spec) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
480 if (!check_coding_spec(line, strlen(line), tok, fp_setreadl)) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
481 return error_ret(tok);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
482 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
483 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
484 #ifndef PGEN
36741
c0982b8155c6 Change non-ASCII warning into a SyntaxError.
Martin v. Löwis <martin@v.loewis.de>
parents: 36520
diff changeset
485 /* The default encoding is ASCII, so make sure we don't have any
c0982b8155c6 Change non-ASCII warning into a SyntaxError.
Martin v. Löwis <martin@v.loewis.de>
parents: 36520
diff changeset
486 non-ASCII bytes in it. */
c0982b8155c6 Change non-ASCII warning into a SyntaxError.
Martin v. Löwis <martin@v.loewis.de>
parents: 36520
diff changeset
487 if (line && !tok->encoding) {
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
488 unsigned char *c;
24565
c8d694b641f1 Added a cast to shut up a compiler warning.
Jack Jansen <jack.jansen@cwi.nl>
parents: 24558
diff changeset
489 for (c = (unsigned char *)line; *c; c++)
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
490 if (*c > 127) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
491 badchar = *c;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
492 break;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
493 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
494 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
495 if (badchar) {
27918
be371ffc4cd0 Add URL for PEP to the source code encoding warning.
Marc-André Lemburg <mal@egenix.com>
parents: 27772
diff changeset
496 char buf[500];
24558
d26b62f0c6db Add 1 to lineno in deprecation warning. Fixes #590888.
Martin v. Löwis <martin@v.loewis.de>
parents: 24544
diff changeset
497 /* Need to add 1 to the line number, since this line
d26b62f0c6db Add 1 to lineno in deprecation warning. Fixes #590888.
Martin v. Löwis <martin@v.loewis.de>
parents: 24544
diff changeset
498 has not been counted, yet. */
37229
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
499 sprintf(buf,
27918
be371ffc4cd0 Add URL for PEP to the source code encoding warning.
Marc-André Lemburg <mal@egenix.com>
parents: 27772
diff changeset
500 "Non-ASCII character '\\x%.2x' "
be371ffc4cd0 Add URL for PEP to the source code encoding warning.
Marc-André Lemburg <mal@egenix.com>
parents: 27772
diff changeset
501 "in file %.200s on line %i, "
be371ffc4cd0 Add URL for PEP to the source code encoding warning.
Marc-André Lemburg <mal@egenix.com>
parents: 27772
diff changeset
502 "but no encoding declared; "
37229
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
503 "see http://www.python.org/peps/pep-0263.html for details",
27918
be371ffc4cd0 Add URL for PEP to the source code encoding warning.
Marc-André Lemburg <mal@egenix.com>
parents: 27772
diff changeset
504 badchar, tok->filename, tok->lineno + 1);
36741
c0982b8155c6 Change non-ASCII warning into a SyntaxError.
Martin v. Löwis <martin@v.loewis.de>
parents: 36520
diff changeset
505 PyErr_SetString(PyExc_SyntaxError, buf);
c0982b8155c6 Change non-ASCII warning into a SyntaxError.
Martin v. Löwis <martin@v.loewis.de>
parents: 36520
diff changeset
506 return error_ret(tok);
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
507 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
508 #endif
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
509 return line;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
510 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
511
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
512 static int
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
513 decoding_feof(struct tok_state *tok)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
514 {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
515 if (tok->decoding_state >= 0) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
516 return feof(tok->fp);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
517 } else {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
518 PyObject* buf = tok->decoding_buffer;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
519 if (buf == NULL) {
35231
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
520 buf = PyObject_CallObject(tok->decoding_readline, NULL);
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
521 if (buf == NULL) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
522 error_ret(tok);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
523 return 1;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
524 } else {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
525 tok->decoding_buffer = buf;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
526 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
527 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
528 return PyObject_Length(buf) == 0;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
529 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
530 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
531
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
532 /* Fetch a byte from TOK, using the string buffer. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
533
37229
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
534 static int
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
535 buf_getc(struct tok_state *tok) {
27772
3566b84e66b9 patch 680474 that fixes bug 679880: compile/eval/exec refused utf-8 bom
Just van Rossum <just@letterror.com>
parents: 27247
diff changeset
536 return Py_CHARMASK(*tok->str++);
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
537 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
538
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
539 /* Unfetch a byte from TOK, using the string buffer. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
540
37229
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
541 static void
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
542 buf_ungetc(int c, struct tok_state *tok) {
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
543 tok->str--;
27772
3566b84e66b9 patch 680474 that fixes bug 679880: compile/eval/exec refused utf-8 bom
Just van Rossum <just@letterror.com>
parents: 27247
diff changeset
544 assert(Py_CHARMASK(*tok->str) == c); /* tok->cur may point to read-only segment */
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
545 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
546
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
547 /* Set the readline function for TOK to ENC. For the string-based
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
548 tokenizer, this means to just record the encoding. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
549
37229
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
550 static int
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
551 buf_setreadl(struct tok_state *tok, const char* enc) {
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
552 tok->enc = enc;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
553 return 1;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
554 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
555
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
556 /* Return a UTF-8 encoding Python string object from the
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
557 C byte string STR, which is encoded with ENC. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
558
24624
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
559 #ifdef Py_USING_UNICODE
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
560 static PyObject *
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
561 translate_into_utf8(const char* str, const char* enc) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
562 PyObject *utf8;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
563 PyObject* buf = PyUnicode_Decode(str, strlen(str), enc, NULL);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
564 if (buf == NULL)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
565 return NULL;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
566 utf8 = PyUnicode_AsUTF8String(buf);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
567 Py_DECREF(buf);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
568 return utf8;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
569 }
24624
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
570 #endif
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
571
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
572 /* Decode a byte string STR for use as the buffer of TOK.
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
573 Look for encoding declarations inside STR, and record them
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
574 inside TOK. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
575
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
576 static const char *
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
577 decode_str(const char *str, struct tok_state *tok)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
578 {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
579 PyObject* utf8 = NULL;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
580 const char *s;
44861
5bea4592cd7a Add a "const" to make gcc happy.
Georg Brandl <georg@python.org>
parents: 44852
diff changeset
581 const char *newl[2] = {NULL, NULL};
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
582 int lineno = 0;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
583 tok->enc = NULL;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
584 tok->str = str;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
585 if (!check_bom(buf_getc, buf_ungetc, buf_setreadl, tok))
35878
94a409da4bd4 Fix some more memory leaks.
Neal Norwitz <nnorwitz@gmail.com>
parents: 35751
diff changeset
586 return error_ret(tok);
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
587 str = tok->str; /* string after BOM if any */
24542
d0d3f760d067 Repaired a fatal compiler error in the debug build: it's not clear what
Tim Peters <tim.peters@gmail.com>
parents: 24541
diff changeset
588 assert(str);
24624
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
589 #ifdef Py_USING_UNICODE
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
590 if (tok->enc != NULL) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
591 utf8 = translate_into_utf8(str, tok->enc);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
592 if (utf8 == NULL)
35878
94a409da4bd4 Fix some more memory leaks.
Neal Norwitz <nnorwitz@gmail.com>
parents: 35751
diff changeset
593 return error_ret(tok);
47527
52294af4b663 This reverts r63675 based on the discussion in this thread:
Gregory P. Smith <greg@mad-scientist.com>
parents: 47294
diff changeset
594 str = PyString_AsString(utf8);
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
595 }
24624
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
596 #endif
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
597 for (s = str;; s++) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
598 if (*s == '\0') break;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
599 else if (*s == '\n') {
45005
707dd9f384f3 Add assertion that we do not blow out newl
Neal Norwitz <nnorwitz@gmail.com>
parents: 44889
diff changeset
600 assert(lineno < 2);
44852
8a9b3b0be573 Issue #1882: when compiling code from a string, encoding cookies in the
Georg Brandl <georg@python.org>
parents: 44780
diff changeset
601 newl[lineno] = s;
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
602 lineno++;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
603 if (lineno == 2) break;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
604 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
605 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
606 tok->enc = NULL;
44852
8a9b3b0be573 Issue #1882: when compiling code from a string, encoding cookies in the
Georg Brandl <georg@python.org>
parents: 44780
diff changeset
607 /* need to check line 1 and 2 separately since check_coding_spec
8a9b3b0be573 Issue #1882: when compiling code from a string, encoding cookies in the
Georg Brandl <georg@python.org>
parents: 44780
diff changeset
608 assumes a single line as input */
8a9b3b0be573 Issue #1882: when compiling code from a string, encoding cookies in the
Georg Brandl <georg@python.org>
parents: 44780
diff changeset
609 if (newl[0]) {
8a9b3b0be573 Issue #1882: when compiling code from a string, encoding cookies in the
Georg Brandl <georg@python.org>
parents: 44780
diff changeset
610 if (!check_coding_spec(str, newl[0] - str, tok, buf_setreadl))
8a9b3b0be573 Issue #1882: when compiling code from a string, encoding cookies in the
Georg Brandl <georg@python.org>
parents: 44780
diff changeset
611 return error_ret(tok);
8a9b3b0be573 Issue #1882: when compiling code from a string, encoding cookies in the
Georg Brandl <georg@python.org>
parents: 44780
diff changeset
612 if (tok->enc == NULL && newl[1]) {
8a9b3b0be573 Issue #1882: when compiling code from a string, encoding cookies in the
Georg Brandl <georg@python.org>
parents: 44780
diff changeset
613 if (!check_coding_spec(newl[0]+1, newl[1] - newl[0],
8a9b3b0be573 Issue #1882: when compiling code from a string, encoding cookies in the
Georg Brandl <georg@python.org>
parents: 44780
diff changeset
614 tok, buf_setreadl))
8a9b3b0be573 Issue #1882: when compiling code from a string, encoding cookies in the
Georg Brandl <georg@python.org>
parents: 44780
diff changeset
615 return error_ret(tok);
8a9b3b0be573 Issue #1882: when compiling code from a string, encoding cookies in the
Georg Brandl <georg@python.org>
parents: 44780
diff changeset
616 }
8a9b3b0be573 Issue #1882: when compiling code from a string, encoding cookies in the
Georg Brandl <georg@python.org>
parents: 44780
diff changeset
617 }
24624
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
618 #ifdef Py_USING_UNICODE
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
619 if (tok->enc != NULL) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
620 assert(utf8 == NULL);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
621 utf8 = translate_into_utf8(str, tok->enc);
35666
b8e571e6de2e - Fix segfault with invalid coding.
Neal Norwitz <nnorwitz@gmail.com>
parents: 35231
diff changeset
622 if (utf8 == NULL) {
b8e571e6de2e - Fix segfault with invalid coding.
Neal Norwitz <nnorwitz@gmail.com>
parents: 35231
diff changeset
623 PyErr_Format(PyExc_SyntaxError,
b8e571e6de2e - Fix segfault with invalid coding.
Neal Norwitz <nnorwitz@gmail.com>
parents: 35231
diff changeset
624 "unknown encoding: %s", tok->enc);
35878
94a409da4bd4 Fix some more memory leaks.
Neal Norwitz <nnorwitz@gmail.com>
parents: 35751
diff changeset
625 return error_ret(tok);
35666
b8e571e6de2e - Fix segfault with invalid coding.
Neal Norwitz <nnorwitz@gmail.com>
parents: 35231
diff changeset
626 }
47527
52294af4b663 This reverts r63675 based on the discussion in this thread:
Gregory P. Smith <greg@mad-scientist.com>
parents: 47294
diff changeset
627 str = PyString_AsString(utf8);
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
628 }
24624
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
629 #endif
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
630 assert(tok->decoding_buffer == NULL);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
631 tok->decoding_buffer = utf8; /* CAUTION */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
632 return str;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
633 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
634
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
635 #endif /* PGEN */
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
636
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
637 /* Set up tokenizer for string */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
638
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
639 struct tok_state *
26601
cc05c2c8660e Constify filenames and scripts. Fixes #651362.
Martin v. Löwis <martin@v.loewis.de>
parents: 25995
diff changeset
640 PyTokenizer_FromString(const char *str)
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
641 {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
642 struct tok_state *tok = tok_new();
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
643 if (tok == NULL)
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
644 return NULL;
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
645 str = (char *)decode_str(str, tok);
35878
94a409da4bd4 Fix some more memory leaks.
Neal Norwitz <nnorwitz@gmail.com>
parents: 35751
diff changeset
646 if (str == NULL) {
94a409da4bd4 Fix some more memory leaks.
Neal Norwitz <nnorwitz@gmail.com>
parents: 35751
diff changeset
647 PyTokenizer_Free(tok);
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
648 return NULL;
35878
94a409da4bd4 Fix some more memory leaks.
Neal Norwitz <nnorwitz@gmail.com>
parents: 35751
diff changeset
649 }
94a409da4bd4 Fix some more memory leaks.
Neal Norwitz <nnorwitz@gmail.com>
parents: 35751
diff changeset
650
26601
cc05c2c8660e Constify filenames and scripts. Fixes #651362.
Martin v. Löwis <martin@v.loewis.de>
parents: 25995
diff changeset
651 /* XXX: constify members. */
cc05c2c8660e Constify filenames and scripts. Fixes #651362.
Martin v. Löwis <martin@v.loewis.de>
parents: 25995
diff changeset
652 tok->buf = tok->cur = tok->end = tok->inp = (char*)str;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
653 return tok;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
654 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
655
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
656
326
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
657 /* Set up tokenizer for file */
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
658
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
659 struct tok_state *
13659
e78153e179fb Mass ANSIfication.
Thomas Wouters <thomas@python.org>
parents: 13539
diff changeset
660 PyTokenizer_FromFile(FILE *fp, char *ps1, char *ps2)
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
661 {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
662 struct tok_state *tok = tok_new();
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
663 if (tok == NULL)
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
664 return NULL;
37600
c4c5eeb07739 As discussed on python-dev, really fix the PyMem_*/PyObject_* memory API
Neal Norwitz <nnorwitz@gmail.com>
parents: 37585
diff changeset
665 if ((tok->buf = (char *)PyMem_MALLOC(BUFSIZ)) == NULL) {
35878
94a409da4bd4 Fix some more memory leaks.
Neal Norwitz <nnorwitz@gmail.com>
parents: 35751
diff changeset
666 PyTokenizer_Free(tok);
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
667 return NULL;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
668 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
669 tok->cur = tok->inp = tok->buf;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
670 tok->end = tok->buf + BUFSIZ;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
671 tok->fp = fp;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
672 tok->prompt = ps1;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
673 tok->nextprompt = ps2;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
674 return tok;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
675 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
676
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
677
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
678 /* Free a tok_state structure */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
679
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
680 void
13659
e78153e179fb Mass ANSIfication.
Thomas Wouters <thomas@python.org>
parents: 13539
diff changeset
681 PyTokenizer_Free(struct tok_state *tok)
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
682 {
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
683 if (tok->encoding != NULL)
37600
c4c5eeb07739 As discussed on python-dev, really fix the PyMem_*/PyObject_* memory API
Neal Norwitz <nnorwitz@gmail.com>
parents: 37585
diff changeset
684 PyMem_FREE(tok->encoding);
24544
0bd521eb714e Make pgen compile with pydebug. Duplicate normalized names, as it may
Martin v. Löwis <martin@v.loewis.de>
parents: 24543
diff changeset
685 #ifndef PGEN
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
686 Py_XDECREF(tok->decoding_readline);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
687 Py_XDECREF(tok->decoding_buffer);
24544
0bd521eb714e Make pgen compile with pydebug. Duplicate normalized names, as it may
Martin v. Löwis <martin@v.loewis.de>
parents: 24543
diff changeset
688 #endif
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
689 if (tok->fp != NULL && tok->buf != NULL)
37600
c4c5eeb07739 As discussed on python-dev, really fix the PyMem_*/PyObject_* memory API
Neal Norwitz <nnorwitz@gmail.com>
parents: 37585
diff changeset
690 PyMem_FREE(tok->buf);
37229
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
691 PyMem_FREE(tok);
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
692 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
693
32961
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
694 #if !defined(PGEN) && defined(Py_USING_UNICODE)
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
695 static int
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
696 tok_stdin_decode(struct tok_state *tok, char **inp)
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
697 {
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
698 PyObject *enc, *sysstdin, *decoded, *utf8;
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
699 const char *encoding;
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
700 char *converted;
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
701
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
702 if (PySys_GetFile((char *)"stdin", NULL) != stdin)
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
703 return 0;
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
704 sysstdin = PySys_GetObject("stdin");
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
705 if (sysstdin == NULL || !PyFile_Check(sysstdin))
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
706 return 0;
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
707
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
708 enc = ((PyFileObject *)sysstdin)->f_encoding;
47527
52294af4b663 This reverts r63675 based on the discussion in this thread:
Gregory P. Smith <greg@mad-scientist.com>
parents: 47294
diff changeset
709 if (enc == NULL || !PyString_Check(enc))
32961
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
710 return 0;
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
711 Py_INCREF(enc);
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
712
47527
52294af4b663 This reverts r63675 based on the discussion in this thread:
Gregory P. Smith <greg@mad-scientist.com>
parents: 47294
diff changeset
713 encoding = PyString_AsString(enc);
32961
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
714 decoded = PyUnicode_Decode(*inp, strlen(*inp), encoding, NULL);
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
715 if (decoded == NULL)
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
716 goto error_clear;
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
717
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
718 utf8 = PyUnicode_AsEncodedString(decoded, "utf-8", NULL);
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
719 Py_DECREF(decoded);
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
720 if (utf8 == NULL)
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
721 goto error_clear;
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
722
47527
52294af4b663 This reverts r63675 based on the discussion in this thread:
Gregory P. Smith <greg@mad-scientist.com>
parents: 47294
diff changeset
723 assert(PyString_Check(utf8));
52294af4b663 This reverts r63675 based on the discussion in this thread:
Gregory P. Smith <greg@mad-scientist.com>
parents: 47294
diff changeset
724 converted = new_string(PyString_AS_STRING(utf8),
52294af4b663 This reverts r63675 based on the discussion in this thread:
Gregory P. Smith <greg@mad-scientist.com>
parents: 47294
diff changeset
725 PyString_GET_SIZE(utf8));
32961
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
726 Py_DECREF(utf8);
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
727 if (converted == NULL)
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
728 goto error_nomem;
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
729
37600
c4c5eeb07739 As discussed on python-dev, really fix the PyMem_*/PyObject_* memory API
Neal Norwitz <nnorwitz@gmail.com>
parents: 37585
diff changeset
730 PyMem_FREE(*inp);
32961
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
731 *inp = converted;
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
732 if (tok->encoding != NULL)
37600
c4c5eeb07739 As discussed on python-dev, really fix the PyMem_*/PyObject_* memory API
Neal Norwitz <nnorwitz@gmail.com>
parents: 37585
diff changeset
733 PyMem_FREE(tok->encoding);
32961
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
734 tok->encoding = new_string(encoding, strlen(encoding));
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
735 if (tok->encoding == NULL)
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
736 goto error_nomem;
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
737
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
738 Py_DECREF(enc);
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
739 return 0;
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
740
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
741 error_nomem:
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
742 Py_DECREF(enc);
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
743 tok->done = E_NOMEM;
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
744 return -1;
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
745
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
746 error_clear:
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
747 /* Fallback to iso-8859-1: for backward compatibility */
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
748 Py_DECREF(enc);
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
749 PyErr_Clear();
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
750 return 0;
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
751 }
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
752 #endif
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
753
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
754 /* Get next char, updating state; error code goes into tok->done */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
755
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
756 static int
13659
e78153e179fb Mass ANSIfication.
Thomas Wouters <thomas@python.org>
parents: 13539
diff changeset
757 tok_nextc(register struct tok_state *tok)
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
758 {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
759 for (;;) {
1897
3fbda8768855 * Parser/tokenizer.c (tok_nextc): count line numbers when parsing
Guido van Rossum <guido@python.org>
parents: 1835
diff changeset
760 if (tok->cur != tok->inp) {
7118
e9108e97a5a0 tok_nextc() should return unsigned characters, to avoid mistaking
Guido van Rossum <guido@python.org>
parents: 5379
diff changeset
761 return Py_CHARMASK(*tok->cur++); /* Fast path */
1897
3fbda8768855 * Parser/tokenizer.c (tok_nextc): count line numbers when parsing
Guido van Rossum <guido@python.org>
parents: 1835
diff changeset
762 }
1282
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
763 if (tok->done != E_OK)
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
764 return EOF;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
765 if (tok->fp == NULL) {
1897
3fbda8768855 * Parser/tokenizer.c (tok_nextc): count line numbers when parsing
Guido van Rossum <guido@python.org>
parents: 1835
diff changeset
766 char *end = strchr(tok->inp, '\n');
3fbda8768855 * Parser/tokenizer.c (tok_nextc): count line numbers when parsing
Guido van Rossum <guido@python.org>
parents: 1835
diff changeset
767 if (end != NULL)
3fbda8768855 * Parser/tokenizer.c (tok_nextc): count line numbers when parsing
Guido van Rossum <guido@python.org>
parents: 1835
diff changeset
768 end++;
3fbda8768855 * Parser/tokenizer.c (tok_nextc): count line numbers when parsing
Guido van Rossum <guido@python.org>
parents: 1835
diff changeset
769 else {
3fbda8768855 * Parser/tokenizer.c (tok_nextc): count line numbers when parsing
Guido van Rossum <guido@python.org>
parents: 1835
diff changeset
770 end = strchr(tok->inp, '\0');
3fbda8768855 * Parser/tokenizer.c (tok_nextc): count line numbers when parsing
Guido van Rossum <guido@python.org>
parents: 1835
diff changeset
771 if (end == tok->inp) {
3fbda8768855 * Parser/tokenizer.c (tok_nextc): count line numbers when parsing
Guido van Rossum <guido@python.org>
parents: 1835
diff changeset
772 tok->done = E_EOF;
3fbda8768855 * Parser/tokenizer.c (tok_nextc): count line numbers when parsing
Guido van Rossum <guido@python.org>
parents: 1835
diff changeset
773 return EOF;
3fbda8768855 * Parser/tokenizer.c (tok_nextc): count line numbers when parsing
Guido van Rossum <guido@python.org>
parents: 1835
diff changeset
774 }
3fbda8768855 * Parser/tokenizer.c (tok_nextc): count line numbers when parsing
Guido van Rossum <guido@python.org>
parents: 1835
diff changeset
775 }
3fbda8768855 * Parser/tokenizer.c (tok_nextc): count line numbers when parsing
Guido van Rossum <guido@python.org>
parents: 1835
diff changeset
776 if (tok->start == NULL)
3fbda8768855 * Parser/tokenizer.c (tok_nextc): count line numbers when parsing
Guido van Rossum <guido@python.org>
parents: 1835
diff changeset
777 tok->buf = tok->cur;
36786
57322cb2ca11 Patch #1440601: Add col_offset attribute to AST nodes.
Martin v. Löwis <martin@v.loewis.de>
parents: 36741
diff changeset
778 tok->line_start = tok->cur;
1897
3fbda8768855 * Parser/tokenizer.c (tok_nextc): count line numbers when parsing
Guido van Rossum <guido@python.org>
parents: 1835
diff changeset
779 tok->lineno++;
3fbda8768855 * Parser/tokenizer.c (tok_nextc): count line numbers when parsing
Guido van Rossum <guido@python.org>
parents: 1835
diff changeset
780 tok->inp = end;
7118
e9108e97a5a0 tok_nextc() should return unsigned characters, to avoid mistaking
Guido van Rossum <guido@python.org>
parents: 5379
diff changeset
781 return Py_CHARMASK(*tok->cur++);
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
782 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
783 if (tok->prompt != NULL) {
37585
292854c43b99 Fix the code in Parser/ to also compile with C++. This was mostly casts for
Anthony Baxter <anthonybaxter@gmail.com>
parents: 37557
diff changeset
784 char *newtok = PyOS_Readline(stdin, stdout, tok->prompt);
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
785 if (tok->nextprompt != NULL)
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
786 tok->prompt = tok->nextprompt;
37585
292854c43b99 Fix the code in Parser/ to also compile with C++. This was mostly casts for
Anthony Baxter <anthonybaxter@gmail.com>
parents: 37557
diff changeset
787 if (newtok == NULL)
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
788 tok->done = E_INTR;
37585
292854c43b99 Fix the code in Parser/ to also compile with C++. This was mostly casts for
Anthony Baxter <anthonybaxter@gmail.com>
parents: 37557
diff changeset
789 else if (*newtok == '\0') {
37600
c4c5eeb07739 As discussed on python-dev, really fix the PyMem_*/PyObject_* memory API
Neal Norwitz <nnorwitz@gmail.com>
parents: 37585
diff changeset
790 PyMem_FREE(newtok);
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
791 tok->done = E_EOF;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
792 }
32961
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
793 #if !defined(PGEN) && defined(Py_USING_UNICODE)
37585
292854c43b99 Fix the code in Parser/ to also compile with C++. This was mostly casts for
Anthony Baxter <anthonybaxter@gmail.com>
parents: 37557
diff changeset
794 else if (tok_stdin_decode(tok, &newtok) != 0)
37600
c4c5eeb07739 As discussed on python-dev, really fix the PyMem_*/PyObject_* memory API
Neal Norwitz <nnorwitz@gmail.com>
parents: 37585
diff changeset
795 PyMem_FREE(newtok);
32961
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
796 #endif
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
797 else if (tok->start != NULL) {
13093
cbe02b8aee07 Trent Mick: familiar simple Win64 patches
Guido van Rossum <guido@python.org>
parents: 12576
diff changeset
798 size_t start = tok->start - tok->buf;
cbe02b8aee07 Trent Mick: familiar simple Win64 patches
Guido van Rossum <guido@python.org>
parents: 12576
diff changeset
799 size_t oldlen = tok->cur - tok->buf;
37585
292854c43b99 Fix the code in Parser/ to also compile with C++. This was mostly casts for
Anthony Baxter <anthonybaxter@gmail.com>
parents: 37557
diff changeset
800 size_t newlen = oldlen + strlen(newtok);
12576
1c7c2dd1beb1 Vladimir Marangozov's long-awaited malloc restructuring.
Guido van Rossum <guido@python.org>
parents: 12224
diff changeset
801 char *buf = tok->buf;
37600
c4c5eeb07739 As discussed on python-dev, really fix the PyMem_*/PyObject_* memory API
Neal Norwitz <nnorwitz@gmail.com>
parents: 37585
diff changeset
802 buf = (char *)PyMem_REALLOC(buf, newlen+1);
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
803 tok->lineno++;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
804 if (buf == NULL) {
37600
c4c5eeb07739 As discussed on python-dev, really fix the PyMem_*/PyObject_* memory API
Neal Norwitz <nnorwitz@gmail.com>
parents: 37585
diff changeset
805 PyMem_FREE(tok->buf);
2044
281e40652e23 Parser/tokenizer.c (tok_nextc): zap tok->buf after freeing;
Guido van Rossum <guido@python.org>
parents: 1897
diff changeset
806 tok->buf = NULL;
37600
c4c5eeb07739 As discussed on python-dev, really fix the PyMem_*/PyObject_* memory API
Neal Norwitz <nnorwitz@gmail.com>
parents: 37585
diff changeset
807 PyMem_FREE(newtok);
1282
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
808 tok->done = E_NOMEM;
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
809 return EOF;
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
810 }
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
811 tok->buf = buf;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
812 tok->cur = tok->buf + oldlen;
36786
57322cb2ca11 Patch #1440601: Add col_offset attribute to AST nodes.
Martin v. Löwis <martin@v.loewis.de>
parents: 36741
diff changeset
813 tok->line_start = tok->cur;
37585
292854c43b99 Fix the code in Parser/ to also compile with C++. This was mostly casts for
Anthony Baxter <anthonybaxter@gmail.com>
parents: 37557
diff changeset
814 strcpy(tok->buf + oldlen, newtok);
37600
c4c5eeb07739 As discussed on python-dev, really fix the PyMem_*/PyObject_* memory API
Neal Norwitz <nnorwitz@gmail.com>
parents: 37585
diff changeset
815 PyMem_FREE(newtok);
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
816 tok->inp = tok->buf + newlen;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
817 tok->end = tok->inp + 1;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
818 tok->start = tok->buf + start;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
819 }
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
820 else {
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
821 tok->lineno++;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
822 if (tok->buf != NULL)
37600
c4c5eeb07739 As discussed on python-dev, really fix the PyMem_*/PyObject_* memory API
Neal Norwitz <nnorwitz@gmail.com>
parents: 37585
diff changeset
823 PyMem_FREE(tok->buf);
37585
292854c43b99 Fix the code in Parser/ to also compile with C++. This was mostly casts for
Anthony Baxter <anthonybaxter@gmail.com>
parents: 37557
diff changeset
824 tok->buf = newtok;
36786
57322cb2ca11 Patch #1440601: Add col_offset attribute to AST nodes.
Martin v. Löwis <martin@v.loewis.de>
parents: 36741
diff changeset
825 tok->line_start = tok->buf;
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
826 tok->cur = tok->buf;
36786
57322cb2ca11 Patch #1440601: Add col_offset attribute to AST nodes.
Martin v. Löwis <martin@v.loewis.de>
parents: 36741
diff changeset
827 tok->line_start = tok->buf;
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
828 tok->inp = strchr(tok->buf, '\0');
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
829 tok->end = tok->inp + 1;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
830 }
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
831 }
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
832 else {
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
833 int done = 0;
36501
7fdc639bc5b4 Merge ssize_t branch.
Martin v. Löwis <martin@v.loewis.de>
parents: 36090
diff changeset
834 Py_ssize_t cur = 0;
2732
daa150b3a6e4 replace "\r\n" with "\n" at line end (Jim Ahlstrom)
Guido van Rossum <guido@python.org>
parents: 2149
diff changeset
835 char *pt;
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
836 if (tok->start == NULL) {
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
837 if (tok->buf == NULL) {
37229
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
838 tok->buf = (char *)
37600
c4c5eeb07739 As discussed on python-dev, really fix the PyMem_*/PyObject_* memory API
Neal Norwitz <nnorwitz@gmail.com>
parents: 37585
diff changeset
839 PyMem_MALLOC(BUFSIZ);
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
840 if (tok->buf == NULL) {
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
841 tok->done = E_NOMEM;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
842 return EOF;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
843 }
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
844 tok->end = tok->buf + BUFSIZ;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
845 }
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
846 if (decoding_fgets(tok->buf, (int)(tok->end - tok->buf),
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
847 tok) == NULL) {
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
848 tok->done = E_EOF;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
849 done = 1;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
850 }
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
851 else {
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
852 tok->done = E_OK;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
853 tok->inp = strchr(tok->buf, '\0');
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
854 done = tok->inp[-1] == '\n';
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
855 }
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
856 }
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
857 else {
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
858 cur = tok->cur - tok->buf;
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
859 if (decoding_feof(tok)) {
2149
6060d7fb07e3 fix loop on unterminated triple quotes
Guido van Rossum <guido@python.org>
parents: 2060
diff changeset
860 tok->done = E_EOF;
6060d7fb07e3 fix loop on unterminated triple quotes
Guido van Rossum <guido@python.org>
parents: 2060
diff changeset
861 done = 1;
6060d7fb07e3 fix loop on unterminated triple quotes
Guido van Rossum <guido@python.org>
parents: 2060
diff changeset
862 }
6060d7fb07e3 fix loop on unterminated triple quotes
Guido van Rossum <guido@python.org>
parents: 2060
diff changeset
863 else
6060d7fb07e3 fix loop on unterminated triple quotes
Guido van Rossum <guido@python.org>
parents: 2060
diff changeset
864 tok->done = E_OK;
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
865 }
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
866 tok->lineno++;
1282
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
867 /* Read until '\n' or EOF */
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
868 while (!done) {
36501
7fdc639bc5b4 Merge ssize_t branch.
Martin v. Löwis <martin@v.loewis.de>
parents: 36090
diff changeset
869 Py_ssize_t curstart = tok->start == NULL ? -1 :
7fdc639bc5b4 Merge ssize_t branch.
Martin v. Löwis <martin@v.loewis.de>
parents: 36090
diff changeset
870 tok->start - tok->buf;
7fdc639bc5b4 Merge ssize_t branch.
Martin v. Löwis <martin@v.loewis.de>
parents: 36090
diff changeset
871 Py_ssize_t curvalid = tok->inp - tok->buf;
7fdc639bc5b4 Merge ssize_t branch.
Martin v. Löwis <martin@v.loewis.de>
parents: 36090
diff changeset
872 Py_ssize_t newsize = curvalid + BUFSIZ;
1282
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
873 char *newbuf = tok->buf;
37600
c4c5eeb07739 As discussed on python-dev, really fix the PyMem_*/PyObject_* memory API
Neal Norwitz <nnorwitz@gmail.com>
parents: 37585
diff changeset
874 newbuf = (char *)PyMem_REALLOC(newbuf,
c4c5eeb07739 As discussed on python-dev, really fix the PyMem_*/PyObject_* memory API
Neal Norwitz <nnorwitz@gmail.com>
parents: 37585
diff changeset
875 newsize);
1282
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
876 if (newbuf == NULL) {
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
877 tok->done = E_NOMEM;
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
878 tok->cur = tok->inp;
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
879 return EOF;
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
880 }
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
881 tok->buf = newbuf;
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
882 tok->inp = tok->buf + curvalid;
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
883 tok->end = tok->buf + newsize;
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
884 tok->start = curstart < 0 ? NULL :
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
885 tok->buf + curstart;
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
886 if (decoding_fgets(tok->inp,
1282
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
887 (int)(tok->end - tok->inp),
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
888 tok) == NULL) {
36809
2c2dffe070df Fix crashing bug in tokenizer, when tokenizing files with non-ASCII bytes
Thomas Wouters <thomas@python.org>
parents: 36786
diff changeset
889 /* Break out early on decoding
2c2dffe070df Fix crashing bug in tokenizer, when tokenizing files with non-ASCII bytes
Thomas Wouters <thomas@python.org>
parents: 36786
diff changeset
890 errors, as tok->buf will be NULL
2c2dffe070df Fix crashing bug in tokenizer, when tokenizing files with non-ASCII bytes
Thomas Wouters <thomas@python.org>
parents: 36786
diff changeset
891 */
2c2dffe070df Fix crashing bug in tokenizer, when tokenizing files with non-ASCII bytes
Thomas Wouters <thomas@python.org>
parents: 36786
diff changeset
892 if (tok->decoding_erred)
2c2dffe070df Fix crashing bug in tokenizer, when tokenizing files with non-ASCII bytes
Thomas Wouters <thomas@python.org>
parents: 36786
diff changeset
893 return EOF;
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
894 /* Last line does not end in \n,
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
895 fake one */
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
896 strcpy(tok->inp, "\n");
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
897 }
1282
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
898 tok->inp = strchr(tok->inp, '\0');
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
899 done = tok->inp[-1] == '\n';
1282
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
900 }
38544
697dd848e1f1 Patch #1357836:
Neal Norwitz <nnorwitz@gmail.com>
parents: 37817
diff changeset
901 if (tok->buf != NULL) {
697dd848e1f1 Patch #1357836:
Neal Norwitz <nnorwitz@gmail.com>
parents: 37817
diff changeset
902 tok->cur = tok->buf + cur;
697dd848e1f1 Patch #1357836:
Neal Norwitz <nnorwitz@gmail.com>
parents: 37817
diff changeset
903 tok->line_start = tok->cur;
697dd848e1f1 Patch #1357836:
Neal Norwitz <nnorwitz@gmail.com>
parents: 37817
diff changeset
904 /* replace "\r\n" with "\n" */
40036
5d83994cd8cd Comment grammar
Andrew M. Kuchling <amk@amk.ca>
parents: 38762
diff changeset
905 /* For Mac leave the \r, giving a syntax error */
38544
697dd848e1f1 Patch #1357836:
Neal Norwitz <nnorwitz@gmail.com>
parents: 37817
diff changeset
906 pt = tok->inp - 2;
697dd848e1f1 Patch #1357836:
Neal Norwitz <nnorwitz@gmail.com>
parents: 37817
diff changeset
907 if (pt >= tok->buf && *pt == '\r') {
697dd848e1f1 Patch #1357836:
Neal Norwitz <nnorwitz@gmail.com>
parents: 37817
diff changeset
908 *pt++ = '\n';
697dd848e1f1 Patch #1357836:
Neal Norwitz <nnorwitz@gmail.com>
parents: 37817
diff changeset
909 *pt = '\0';
697dd848e1f1 Patch #1357836:
Neal Norwitz <nnorwitz@gmail.com>
parents: 37817
diff changeset
910 tok->inp = pt;
697dd848e1f1 Patch #1357836:
Neal Norwitz <nnorwitz@gmail.com>
parents: 37817
diff changeset
911 }
2732
daa150b3a6e4 replace "\r\n" with "\n" at line end (Jim Ahlstrom)
Guido van Rossum <guido@python.org>
parents: 2149
diff changeset
912 }
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
913 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
914 if (tok->done != E_OK) {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
915 if (tok->prompt != NULL)
9047
6d2eb58f0418 Replace all calls to fprintf(stderr, ...) with PySys_WriteStderr(...).
Guido van Rossum <guido@python.org>
parents: 7945
diff changeset
916 PySys_WriteStderr("\n");
1282
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
917 tok->cur = tok->inp;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
918 return EOF;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
919 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
920 }
1282
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
921 /*NOTREACHED*/
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
922 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
923
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
924
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
925 /* Back-up one character */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
926
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
927 static void
13659
e78153e179fb Mass ANSIfication.
Thomas Wouters <thomas@python.org>
parents: 13539
diff changeset
928 tok_backup(register struct tok_state *tok, register int c)
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
929 {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
930 if (c != EOF) {
2044
281e40652e23 Parser/tokenizer.c (tok_nextc): zap tok->buf after freeing;
Guido van Rossum <guido@python.org>
parents: 1897
diff changeset
931 if (--tok->cur < tok->buf)
5379
1c9a84e0981d Another directory quickly renamed.
Guido van Rossum <guido@python.org>
parents: 5341
diff changeset
932 Py_FatalError("tok_backup: begin of buffer");
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
933 if (*tok->cur != c)
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
934 *tok->cur = c;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
935 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
936 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
937
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
938
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
939 /* Return the token corresponding to a single character */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
940
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
941 int
13659
e78153e179fb Mass ANSIfication.
Thomas Wouters <thomas@python.org>
parents: 13539
diff changeset
942 PyToken_OneChar(int c)
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
943 {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
944 switch (c) {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
945 case '(': return LPAR;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
946 case ')': return RPAR;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
947 case '[': return LSQB;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
948 case ']': return RSQB;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
949 case ':': return COLON;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
950 case ',': return COMMA;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
951 case ';': return SEMI;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
952 case '+': return PLUS;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
953 case '-': return MINUS;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
954 case '*': return STAR;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
955 case '/': return SLASH;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
956 case '|': return VBAR;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
957 case '&': return AMPER;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
958 case '<': return LESS;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
959 case '>': return GREATER;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
960 case '=': return EQUAL;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
961 case '.': return DOT;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
962 case '%': return PERCENT;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
963 case '`': return BACKQUOTE;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
964 case '{': return LBRACE;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
965 case '}': return RBRACE;
410
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
966 case '^': return CIRCUMFLEX;
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
967 case '~': return TILDE;
32888
c7e03cf8333e PEP-0318, @decorator-style. In Guido's words:
Anthony Baxter <anthonybaxter@gmail.com>
parents: 30979
diff changeset
968 case '@': return AT;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
969 default: return OP;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
970 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
971 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
972
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
973
400
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
974 int
13659
e78153e179fb Mass ANSIfication.
Thomas Wouters <thomas@python.org>
parents: 13539
diff changeset
975 PyToken_TwoChars(int c1, int c2)
400
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
976 {
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
977 switch (c1) {
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
978 case '=':
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
979 switch (c2) {
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
980 case '=': return EQEQUAL;
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
981 }
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
982 break;
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
983 case '!':
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
984 switch (c2) {
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
985 case '=': return NOTEQUAL;
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
986 }
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
987 break;
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
988 case '<':
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
989 switch (c2) {
43983
e725166c8c81 Fixed problems in the last commit. Filenames and line numbers weren't reported correctly.
Christian Heimes <christian@cheimes.de>
parents: 43982
diff changeset
990 case '>': return NOTEQUAL;
400
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
991 case '=': return LESSEQUAL;
410
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
992 case '<': return LEFTSHIFT;
400
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
993 }
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
994 break;
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
995 case '>':
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
996 switch (c2) {
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
997 case '=': return GREATEREQUAL;
410
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
998 case '>': return RIGHTSHIFT;
400
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
999 }
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1000 break;
14163
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1001 case '+':
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1002 switch (c2) {
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1003 case '=': return PLUSEQUAL;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1004 }
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1005 break;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1006 case '-':
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1007 switch (c2) {
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1008 case '=': return MINEQUAL;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1009 }
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1010 break;
3333
4b5bff95804e changes for pow(**) and complex
Guido van Rossum <guido@python.org>
parents: 3116
diff changeset
1011 case '*':
4b5bff95804e changes for pow(**) and complex
Guido van Rossum <guido@python.org>
parents: 3116
diff changeset
1012 switch (c2) {
4b5bff95804e changes for pow(**) and complex
Guido van Rossum <guido@python.org>
parents: 3116
diff changeset
1013 case '*': return DOUBLESTAR;
14163
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1014 case '=': return STAREQUAL;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1015 }
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1016 break;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1017 case '/':
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1018 switch (c2) {
19051
f20727cc3d0f Implement PEP 238 in its (almost) full glory.
Guido van Rossum <guido@python.org>
parents: 17600
diff changeset
1019 case '/': return DOUBLESLASH;
14163
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1020 case '=': return SLASHEQUAL;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1021 }
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1022 break;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1023 case '|':
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1024 switch (c2) {
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1025 case '=': return VBAREQUAL;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1026 }
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1027 break;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1028 case '%':
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1029 switch (c2) {
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1030 case '=': return PERCENTEQUAL;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1031 }
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1032 break;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1033 case '&':
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1034 switch (c2) {
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1035 case '=': return AMPEREQUAL;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1036 }
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1037 break;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1038 case '^':
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1039 switch (c2) {
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1040 case '=': return CIRCUMFLEXEQUAL;
3333
4b5bff95804e changes for pow(**) and complex
Guido van Rossum <guido@python.org>
parents: 3116
diff changeset
1041 }
4b5bff95804e changes for pow(**) and complex
Guido van Rossum <guido@python.org>
parents: 3116
diff changeset
1042 break;
400
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1043 }
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1044 return OP;
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1045 }
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1046
14163
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1047 int
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1048 PyToken_ThreeChars(int c1, int c2, int c3)
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1049 {
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1050 switch (c1) {
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1051 case '<':
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1052 switch (c2) {
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1053 case '<':
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1054 switch (c3) {
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1055 case '=':
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1056 return LEFTSHIFTEQUAL;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1057 }
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1058 break;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1059 }
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1060 break;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1061 case '>':
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1062 switch (c2) {
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1063 case '>':
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1064 switch (c3) {
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1065 case '=':
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1066 return RIGHTSHIFTEQUAL;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1067 }
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1068 break;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1069 }
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1070 break;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1071 case '*':
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1072 switch (c2) {
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1073 case '*':
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1074 switch (c3) {
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1075 case '=':
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1076 return DOUBLESTAREQUAL;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1077 }
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1078 break;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1079 }
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1080 break;
19051
f20727cc3d0f Implement PEP 238 in its (almost) full glory.
Guido van Rossum <guido@python.org>
parents: 17600
diff changeset
1081 case '/':
f20727cc3d0f Implement PEP 238 in its (almost) full glory.
Guido van Rossum <guido@python.org>
parents: 17600
diff changeset
1082 switch (c2) {
f20727cc3d0f Implement PEP 238 in its (almost) full glory.
Guido van Rossum <guido@python.org>
parents: 17600
diff changeset
1083 case '/':
f20727cc3d0f Implement PEP 238 in its (almost) full glory.
Guido van Rossum <guido@python.org>
parents: 17600
diff changeset
1084 switch (c3) {
f20727cc3d0f Implement PEP 238 in its (almost) full glory.
Guido van Rossum <guido@python.org>
parents: 17600
diff changeset
1085 case '=':
f20727cc3d0f Implement PEP 238 in its (almost) full glory.
Guido van Rossum <guido@python.org>
parents: 17600
diff changeset
1086 return DOUBLESLASHEQUAL;
f20727cc3d0f Implement PEP 238 in its (almost) full glory.
Guido van Rossum <guido@python.org>
parents: 17600
diff changeset
1087 }
f20727cc3d0f Implement PEP 238 in its (almost) full glory.
Guido van Rossum <guido@python.org>
parents: 17600
diff changeset
1088 break;
f20727cc3d0f Implement PEP 238 in its (almost) full glory.
Guido van Rossum <guido@python.org>
parents: 17600
diff changeset
1089 }
f20727cc3d0f Implement PEP 238 in its (almost) full glory.
Guido van Rossum <guido@python.org>
parents: 17600
diff changeset
1090 break;
14163
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1091 }
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1092 return OP;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1093 }
400
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1094
7945
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1095 static int
13659
e78153e179fb Mass ANSIfication.
Thomas Wouters <thomas@python.org>
parents: 13539
diff changeset
1096 indenterror(struct tok_state *tok)
7945
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1097 {
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1098 if (tok->alterror) {
13539
feab677269e0 Create two new exceptions: IndentationError and TabError. These are
Fred Drake <fdrake@acm.org>
parents: 13447
diff changeset
1099 tok->done = E_TABSPACE;
7945
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1100 tok->cur = tok->inp;
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1101 return 1;
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1102 }
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1103 if (tok->altwarning) {
13539
feab677269e0 Create two new exceptions: IndentationError and TabError. These are
Fred Drake <fdrake@acm.org>
parents: 13447
diff changeset
1104 PySys_WriteStderr("%s: inconsistent use of tabs and spaces "
feab677269e0 Create two new exceptions: IndentationError and TabError. These are
Fred Drake <fdrake@acm.org>
parents: 13447
diff changeset
1105 "in indentation\n", tok->filename);
7945
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1106 tok->altwarning = 0;
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1107 }
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1108 return 0;
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1109 }
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1110
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1111
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1112 /* Get next token, after space stripping etc. */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1113
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
1114 static int
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
1115 tok_get(register struct tok_state *tok, char **p_start, char **p_end)
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1116 {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1117 register int c;
326
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1118 int blankline;
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1119
48184
b654d14cb9d9 #3367: revert rev. 65539: this change causes test_parser to fail
Andrew M. Kuchling <amk@amk.ca>
parents: 48183
diff changeset
1120 *p_start = *p_end = NULL;
326
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1121 nextline:
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1122 tok->start = NULL;
326
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1123 blankline = 0;
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1124
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1125 /* Get indentation level */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1126 if (tok->atbol) {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1127 register int col = 0;
7945
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1128 register int altcol = 0;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1129 tok->atbol = 0;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1130 for (;;) {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1131 c = tok_nextc(tok);
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1132 if (c == ' ')
7945
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1133 col++, altcol++;
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1134 else if (c == '\t') {
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1135 col = (col/tok->tabsize + 1) * tok->tabsize;
7945
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1136 altcol = (altcol/tok->alttabsize + 1)
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1137 * tok->alttabsize;
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1138 }
2801
e911f2fb179c ignore control-l in whitespace
Guido van Rossum <guido@python.org>
parents: 2732
diff changeset
1139 else if (c == '\014') /* Control-L (formfeed) */
7945
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1140 col = altcol = 0; /* For Emacs users */
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1141 else
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1142 break;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1143 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1144 tok_backup(tok, c);
326
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1145 if (c == '#' || c == '\n') {
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1146 /* Lines with only whitespace and/or comments
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1147 shouldn't affect the indentation and are
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1148 not passed to the parser as NEWLINE tokens,
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1149 except *totally* empty lines in interactive
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1150 mode, which signal the end of a command group. */
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1151 if (col == 0 && c == '\n' && tok->prompt != NULL)
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1152 blankline = 0; /* Let it through */
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1153 else
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1154 blankline = 1; /* Ignore completely */
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1155 /* We can't jump back right here since we still
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1156 may need to skip to the end of a comment */
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1157 }
1284
dbce72afe261 * selectmodule.c: fix (another!) two memory leaks -- this time in list2set
Guido van Rossum <guido@python.org>
parents: 1282
diff changeset
1158 if (!blankline && tok->level == 0) {
326
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1159 if (col == tok->indstack[tok->indent]) {
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1160 /* No change */
7945
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1161 if (altcol != tok->altindstack[tok->indent]) {
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1162 if (indenterror(tok))
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1163 return ERRORTOKEN;
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1164 }
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1165 }
326
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1166 else if (col > tok->indstack[tok->indent]) {
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1167 /* Indent -- always one */
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1168 if (tok->indent+1 >= MAXINDENT) {
13539
feab677269e0 Create two new exceptions: IndentationError and TabError. These are
Fred Drake <fdrake@acm.org>
parents: 13447
diff changeset
1169 tok->done = E_TOODEEP;
1282
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
1170 tok->cur = tok->inp;
326
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1171 return ERRORTOKEN;
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1172 }
7945
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1173 if (altcol <= tok->altindstack[tok->indent]) {
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1174 if (indenterror(tok))
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1175 return ERRORTOKEN;
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1176 }
326
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1177 tok->pendin++;
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1178 tok->indstack[++tok->indent] = col;
7945
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1179 tok->altindstack[tok->indent] = altcol;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1180 }
326
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1181 else /* col < tok->indstack[tok->indent] */ {
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1182 /* Dedent -- any number, must be consistent */
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1183 while (tok->indent > 0 &&
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1184 col < tok->indstack[tok->indent]) {
7324
55125b1144fa Swap two statements in the dedent check loop. This makes absolutely
Guido van Rossum <guido@python.org>
parents: 7315
diff changeset
1185 tok->pendin--;
326
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1186 tok->indent--;
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1187 }
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1188 if (col != tok->indstack[tok->indent]) {
13539
feab677269e0 Create two new exceptions: IndentationError and TabError. These are
Fred Drake <fdrake@acm.org>
parents: 13447
diff changeset
1189 tok->done = E_DEDENT;
1282
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
1190 tok->cur = tok->inp;
326
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1191 return ERRORTOKEN;
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1192 }
7945
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1193 if (altcol != tok->altindstack[tok->indent]) {
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1194 if (indenterror(tok))
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1195 return ERRORTOKEN;
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1196 }
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1197 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1198 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1199 }
37229
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
1200
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1201 tok->start = tok->cur;
37229
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
1202
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1203 /* Return pending indents/dedents */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1204 if (tok->pendin != 0) {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1205 if (tok->pendin < 0) {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1206 tok->pendin++;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1207 return DEDENT;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1208 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1209 else {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1210 tok->pendin--;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1211 return INDENT;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1212 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1213 }
37229
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
1214
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1215 again:
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1216 tok->start = NULL;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1217 /* Skip spaces */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1218 do {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1219 c = tok_nextc(tok);
2801
e911f2fb179c ignore control-l in whitespace
Guido van Rossum <guido@python.org>
parents: 2732
diff changeset
1220 } while (c == ' ' || c == '\t' || c == '\014');
37229
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
1221
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1222 /* Set start of current token */
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1223 tok->start = tok->cur - 1;
37229
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
1224
12142
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1225 /* Skip comment, while looking for tab-setting magic */
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1226 if (c == '#') {
12142
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1227 static char *tabforms[] = {
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1228 "tab-width:", /* Emacs */
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1229 ":tabstop=", /* vim, full form */
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1230 ":ts=", /* vim, abbreviated form */
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1231 "set tabsize=", /* will vi never die? */
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1232 /* more templates can be added here to support other editors */
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1233 };
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1234 char cbuf[80];
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1235 char *tp, **cp;
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1236 tp = cbuf;
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1237 do {
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1238 *tp++ = c = tok_nextc(tok);
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1239 } while (c != EOF && c != '\n' &&
38762
92d8cdc4b90c Don't truncate if size_t is bigger than uint
Neal Norwitz <nnorwitz@gmail.com>
parents: 38544
diff changeset
1240 (size_t)(tp - cbuf + 1) < sizeof(cbuf));
12142
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1241 *tp = '\0';
37229
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
1242 for (cp = tabforms;
12142
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1243 cp < tabforms + sizeof(tabforms)/sizeof(tabforms[0]);
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1244 cp++) {
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1245 if ((tp = strstr(cbuf, *cp))) {
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1246 int newsize = atoi(tp + strlen(*cp));
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1247
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1248 if (newsize >= 1 && newsize <= 40) {
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1249 tok->tabsize = newsize;
12224
2dc1bdb4088e Only write message about changed Tab size with -v.
Guido van Rossum <guido@python.org>
parents: 12142
diff changeset
1250 if (Py_VerboseFlag)
2dc1bdb4088e Only write message about changed Tab size with -v.
Guido van Rossum <guido@python.org>
parents: 12142
diff changeset
1251 PySys_WriteStderr(
12142
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1252 "Tab size set to %d\n",
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1253 newsize);
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1254 }
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1255 }
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1256 }
12142
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1257 while (c != EOF && c != '\n')
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1258 c = tok_nextc(tok);
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1259 }
37229
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
1260
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1261 /* Check for EOF and errors now */
410
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
1262 if (c == EOF) {
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1263 return tok->done == E_EOF ? ENDMARKER : ERRORTOKEN;
410
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
1264 }
37229
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
1265
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1266 /* Identifier (most frequent token!) */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1267 if (isalpha(c) || c == '_') {
11971
574283db2b3d Marc-Andre Lemburg: add new string token types u"..." and ur"..."
Guido van Rossum <guido@python.org>
parents: 9590
diff changeset
1268 /* Process r"", u"" and ur"" */
5341
67e5237ce141 Now that the string-sig has settled on r"obin" strings, restrict the
Guido van Rossum <guido@python.org>
parents: 5311
diff changeset
1269 switch (c) {
44747
8da9ebf73026 Added bytes and b'' as aliases for str and ''
Christian Heimes <christian@cheimes.de>
parents: 44575
diff changeset
1270 case 'b':
8da9ebf73026 Added bytes and b'' as aliases for str and ''
Christian Heimes <christian@cheimes.de>
parents: 44575
diff changeset
1271 case 'B':
8da9ebf73026 Added bytes and b'' as aliases for str and ''
Christian Heimes <christian@cheimes.de>
parents: 44575
diff changeset
1272 c = tok_nextc(tok);
8da9ebf73026 Added bytes and b'' as aliases for str and ''
Christian Heimes <christian@cheimes.de>
parents: 44575
diff changeset
1273 if (c == 'r' || c == 'R')
8da9ebf73026 Added bytes and b'' as aliases for str and ''
Christian Heimes <christian@cheimes.de>
parents: 44575
diff changeset
1274 c = tok_nextc(tok);
8da9ebf73026 Added bytes and b'' as aliases for str and ''
Christian Heimes <christian@cheimes.de>
parents: 44575
diff changeset
1275 if (c == '"' || c == '\'')
8da9ebf73026 Added bytes and b'' as aliases for str and ''
Christian Heimes <christian@cheimes.de>
parents: 44575
diff changeset
1276 goto letter_quote;
8da9ebf73026 Added bytes and b'' as aliases for str and ''
Christian Heimes <christian@cheimes.de>
parents: 44575
diff changeset
1277 break;
5341
67e5237ce141 Now that the string-sig has settled on r"obin" strings, restrict the
Guido van Rossum <guido@python.org>
parents: 5311
diff changeset
1278 case 'r':
67e5237ce141 Now that the string-sig has settled on r"obin" strings, restrict the
Guido van Rossum <guido@python.org>
parents: 5311
diff changeset
1279 case 'R':
67e5237ce141 Now that the string-sig has settled on r"obin" strings, restrict the
Guido van Rossum <guido@python.org>
parents: 5311
diff changeset
1280 c = tok_nextc(tok);
67e5237ce141 Now that the string-sig has settled on r"obin" strings, restrict the
Guido van Rossum <guido@python.org>
parents: 5311
diff changeset
1281 if (c == '"' || c == '\'')
67e5237ce141 Now that the string-sig has settled on r"obin" strings, restrict the
Guido van Rossum <guido@python.org>
parents: 5311
diff changeset
1282 goto letter_quote;
11971
574283db2b3d Marc-Andre Lemburg: add new string token types u"..." and ur"..."
Guido van Rossum <guido@python.org>
parents: 9590
diff changeset
1283 break;
574283db2b3d Marc-Andre Lemburg: add new string token types u"..." and ur"..."
Guido van Rossum <guido@python.org>
parents: 9590
diff changeset
1284 case 'u':
574283db2b3d Marc-Andre Lemburg: add new string token types u"..." and ur"..."
Guido van Rossum <guido@python.org>
parents: 9590
diff changeset
1285 case 'U':
574283db2b3d Marc-Andre Lemburg: add new string token types u"..." and ur"..."
Guido van Rossum <guido@python.org>
parents: 9590
diff changeset
1286 c = tok_nextc(tok);
574283db2b3d Marc-Andre Lemburg: add new string token types u"..." and ur"..."
Guido van Rossum <guido@python.org>
parents: 9590
diff changeset
1287 if (c == 'r' || c == 'R')
574283db2b3d Marc-Andre Lemburg: add new string token types u"..." and ur"..."
Guido van Rossum <guido@python.org>
parents: 9590
diff changeset
1288 c = tok_nextc(tok);
574283db2b3d Marc-Andre Lemburg: add new string token types u"..." and ur"..."
Guido van Rossum <guido@python.org>
parents: 9590
diff changeset
1289 if (c == '"' || c == '\'')
574283db2b3d Marc-Andre Lemburg: add new string token types u"..." and ur"..."
Guido van Rossum <guido@python.org>
parents: 9590
diff changeset
1290 goto letter_quote;
574283db2b3d Marc-Andre Lemburg: add new string token types u"..." and ur"..."
Guido van Rossum <guido@python.org>
parents: 9590
diff changeset
1291 break;
5341
67e5237ce141 Now that the string-sig has settled on r"obin" strings, restrict the
Guido van Rossum <guido@python.org>
parents: 5311
diff changeset
1292 }
5265
3c1788ab9761 Support for alternative string quotes (a"xx", b"xx", c"xx", ...).
Guido van Rossum <guido@python.org>
parents: 4839
diff changeset
1293 while (isalnum(c) || c == '_') {
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1294 c = tok_nextc(tok);
5265
3c1788ab9761 Support for alternative string quotes (a"xx", b"xx", c"xx", ...).
Guido van Rossum <guido@python.org>
parents: 4839
diff changeset
1295 }
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1296 tok_backup(tok, c);
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1297 *p_start = tok->start;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1298 *p_end = tok->cur;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1299 return NAME;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1300 }
37229
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
1301
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1302 /* Newline */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1303 if (c == '\n') {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1304 tok->atbol = 1;
1284
dbce72afe261 * selectmodule.c: fix (another!) two memory leaks -- this time in list2set
Guido van Rossum <guido@python.org>
parents: 1282
diff changeset
1305 if (blankline || tok->level > 0)
326
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1306 goto nextline;
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1307 *p_start = tok->start;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1308 *p_end = tok->cur - 1; /* Leave '\n' out of the string */
25098
c02a668488d0 Ignore encoding declarations inside strings. Fixes #603509.
Martin v. Löwis <martin@v.loewis.de>
parents: 24867
diff changeset
1309 tok->cont_line = 0;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1310 return NEWLINE;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1311 }
37229
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
1312
410
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
1313 /* Period or number starting with period? */
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
1314 if (c == '.') {
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
1315 c = tok_nextc(tok);
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
1316 if (isdigit(c)) {
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
1317 goto fraction;
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
1318 }
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
1319 else {
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
1320 tok_backup(tok, c);
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1321 *p_start = tok->start;
410
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
1322 *p_end = tok->cur;
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
1323 return DOT;
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
1324 }
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
1325 }
3333
4b5bff95804e changes for pow(**) and complex
Guido van Rossum <guido@python.org>
parents: 3116
diff changeset
1326
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1327 /* Number */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1328 if (isdigit(c)) {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1329 if (c == '0') {
45791
62ef0ea034f4 Finished backporting PEP 3127, Integer Literal Support and Syntax.
Eric Smith <eric@trueblade.com>
parents: 45005
diff changeset
1330 /* Hex, octal or binary -- maybe. */
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1331 c = tok_nextc(tok);
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1332 if (c == '.')
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1333 goto fraction;
3333
4b5bff95804e changes for pow(**) and complex
Guido van Rossum <guido@python.org>
parents: 3116
diff changeset
1334 #ifndef WITHOUT_COMPLEX
3364
bc17e8313ea0 use only j for imaginary constants
Guido van Rossum <guido@python.org>
parents: 3333
diff changeset
1335 if (c == 'j' || c == 'J')
3333
4b5bff95804e changes for pow(**) and complex
Guido van Rossum <guido@python.org>
parents: 3116
diff changeset
1336 goto imaginary;
4b5bff95804e changes for pow(**) and complex
Guido van Rossum <guido@python.org>
parents: 3116
diff changeset
1337 #endif
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1338 if (c == 'x' || c == 'X') {
44780
61a11cc95632 Fix #1679: "0x" was taken as a valid integer literal.
Georg Brandl <georg@python.org>
parents: 44747
diff changeset
1339
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1340 /* Hex */
44780
61a11cc95632 Fix #1679: "0x" was taken as a valid integer literal.
Georg Brandl <georg@python.org>
parents: 44747
diff changeset
1341 c = tok_nextc(tok);
61a11cc95632 Fix #1679: "0x" was taken as a valid integer literal.
Georg Brandl <georg@python.org>
parents: 44747
diff changeset
1342 if (!isxdigit(c)) {
61a11cc95632 Fix #1679: "0x" was taken as a valid integer literal.
Georg Brandl <georg@python.org>
parents: 44747
diff changeset
1343 tok->done = E_TOKEN;
61a11cc95632 Fix #1679: "0x" was taken as a valid integer literal.
Georg Brandl <georg@python.org>
parents: 44747
diff changeset
1344 tok_backup(tok, c);
61a11cc95632 Fix #1679: "0x" was taken as a valid integer literal.
Georg Brandl <georg@python.org>
parents: 44747
diff changeset
1345 return ERRORTOKEN;
61a11cc95632 Fix #1679: "0x" was taken as a valid integer literal.
Georg Brandl <georg@python.org>
parents: 44747
diff changeset
1346 }
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1347 do {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1348 c = tok_nextc(tok);
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1349 } while (isxdigit(c));
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1350 }
45791
62ef0ea034f4 Finished backporting PEP 3127, Integer Literal Support and Syntax.
Eric Smith <eric@trueblade.com>
parents: 45005
diff changeset
1351 else if (c == 'o' || c == 'O') {
62ef0ea034f4 Finished backporting PEP 3127, Integer Literal Support and Syntax.
Eric Smith <eric@trueblade.com>
parents: 45005
diff changeset
1352 /* Octal */
62ef0ea034f4 Finished backporting PEP 3127, Integer Literal Support and Syntax.
Eric Smith <eric@trueblade.com>
parents: 45005
diff changeset
1353 c = tok_nextc(tok);
46545
459083c6be15 Issue2681: the literal 0o8 was wrongly accepted, and evaluated as float(0.0).
Amaury Forgeot d'Arc <amauryfa@gmail.com>
parents: 46149
diff changeset
1354 if (c < '0' || c >= '8') {
45791
62ef0ea034f4 Finished backporting PEP 3127, Integer Literal Support and Syntax.
Eric Smith <eric@trueblade.com>
parents: 45005
diff changeset
1355 tok->done = E_TOKEN;
62ef0ea034f4 Finished backporting PEP 3127, Integer Literal Support and Syntax.
Eric Smith <eric@trueblade.com>
parents: 45005
diff changeset
1356 tok_backup(tok, c);
62ef0ea034f4 Finished backporting PEP 3127, Integer Literal Support and Syntax.
Eric Smith <eric@trueblade.com>
parents: 45005
diff changeset
1357 return ERRORTOKEN;
62ef0ea034f4 Finished backporting PEP 3127, Integer Literal Support and Syntax.
Eric Smith <eric@trueblade.com>
parents: 45005
diff changeset
1358 }
62ef0ea034f4 Finished backporting PEP 3127, Integer Literal Support and Syntax.
Eric Smith <eric@trueblade.com>
parents: 45005
diff changeset
1359 do {
62ef0ea034f4 Finished backporting PEP 3127, Integer Literal Support and Syntax.
Eric Smith <eric@trueblade.com>
parents: 45005
diff changeset
1360 c = tok_nextc(tok);
62ef0ea034f4 Finished backporting PEP 3127, Integer Literal Support and Syntax.
Eric Smith <eric@trueblade.com>
parents: 45005
diff changeset
1361 } while ('0' <= c && c < '8');
62ef0ea034f4 Finished backporting PEP 3127, Integer Literal Support and Syntax.
Eric Smith <eric@trueblade.com>
parents: 45005
diff changeset
1362 }
62ef0ea034f4 Finished backporting PEP 3127, Integer Literal Support and Syntax.
Eric Smith <eric@trueblade.com>
parents: 45005
diff changeset
1363 else if (c == 'b' || c == 'B') {
62ef0ea034f4 Finished backporting PEP 3127, Integer Literal Support and Syntax.
Eric Smith <eric@trueblade.com>
parents: 45005
diff changeset
1364 /* Binary */
62ef0ea034f4 Finished backporting PEP 3127, Integer Literal Support and Syntax.
Eric Smith <eric@trueblade.com>
parents: 45005
diff changeset
1365 c = tok_nextc(tok);
62ef0ea034f4 Finished backporting PEP 3127, Integer Literal Support and Syntax.
Eric Smith <eric@trueblade.com>
parents: 45005
diff changeset
1366 if (c != '0' && c != '1') {
62ef0ea034f4 Finished backporting PEP 3127, Integer Literal Support and Syntax.
Eric Smith <eric@trueblade.com>
parents: 45005
diff changeset
1367 tok->done = E_TOKEN;
62ef0ea034f4 Finished backporting PEP 3127, Integer Literal Support and Syntax.
Eric Smith <eric@trueblade.com>
parents: 45005
diff changeset
1368 tok_backup(tok, c);
62ef0ea034f4 Finished backporting PEP 3127, Integer Literal Support and Syntax.
Eric Smith <eric@trueblade.com>
parents: 45005
diff changeset
1369 return ERRORTOKEN;
62ef0ea034f4 Finished backporting PEP 3127, Integer Literal Support and Syntax.
Eric Smith <eric@trueblade.com>
parents: 45005
diff changeset
1370 }
62ef0ea034f4 Finished backporting PEP 3127, Integer Literal Support and Syntax.
Eric Smith <eric@trueblade.com>
parents: 45005
diff changeset
1371 do {
62ef0ea034f4 Finished backporting PEP 3127, Integer Literal Support and Syntax.
Eric Smith <eric@trueblade.com>
parents: 45005
diff changeset
1372 c = tok_nextc(tok);
62ef0ea034f4 Finished backporting PEP 3127, Integer Literal Support and Syntax.
Eric Smith <eric@trueblade.com>
parents: 45005
diff changeset
1373 } while (c == '0' || c == '1');
62ef0ea034f4 Finished backporting PEP 3127, Integer Literal Support and Syntax.
Eric Smith <eric@trueblade.com>
parents: 45005
diff changeset
1374 }
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1375 else {
19621
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1376 int found_decimal = 0;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1377 /* Octal; c is first char of it */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1378 /* There's no 'isoctdigit' macro, sigh */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1379 while ('0' <= c && c < '8') {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1380 c = tok_nextc(tok);
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1381 }
19621
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1382 if (isdigit(c)) {
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1383 found_decimal = 1;
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1384 do {
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1385 c = tok_nextc(tok);
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1386 } while (isdigit(c));
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1387 }
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1388 if (c == '.')
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1389 goto fraction;
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1390 else if (c == 'e' || c == 'E')
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1391 goto exponent;
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1392 #ifndef WITHOUT_COMPLEX
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1393 else if (c == 'j' || c == 'J')
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1394 goto imaginary;
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1395 #endif
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1396 else if (found_decimal) {
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1397 tok->done = E_TOKEN;
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1398 tok_backup(tok, c);
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1399 return ERRORTOKEN;
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1400 }
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1401 }
236
0a92b6a1b9c0 Added recognition of 'l' or 'L' as long integer suffix
Guido van Rossum <guido@python.org>
parents: 145
diff changeset
1402 if (c == 'l' || c == 'L')
0a92b6a1b9c0 Added recognition of 'l' or 'L' as long integer suffix
Guido van Rossum <guido@python.org>
parents: 145
diff changeset
1403 c = tok_nextc(tok);
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1404 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1405 else {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1406 /* Decimal */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1407 do {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1408 c = tok_nextc(tok);
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1409 } while (isdigit(c));
236
0a92b6a1b9c0 Added recognition of 'l' or 'L' as long integer suffix
Guido van Rossum <guido@python.org>
parents: 145
diff changeset
1410 if (c == 'l' || c == 'L')
0a92b6a1b9c0 Added recognition of 'l' or 'L' as long integer suffix
Guido van Rossum <guido@python.org>
parents: 145
diff changeset
1411 c = tok_nextc(tok);
0a92b6a1b9c0 Added recognition of 'l' or 'L' as long integer suffix
Guido van Rossum <guido@python.org>
parents: 145
diff changeset
1412 else {
19501
ce4f1e98ba7e SF bug [#455775] float parsing discrepancy.
Tim Peters <tim.peters@gmail.com>
parents: 19051
diff changeset
1413 /* Accept floating point numbers. */
236
0a92b6a1b9c0 Added recognition of 'l' or 'L' as long integer suffix
Guido van Rossum <guido@python.org>
parents: 145
diff changeset
1414 if (c == '.') {
0a92b6a1b9c0 Added recognition of 'l' or 'L' as long integer suffix
Guido van Rossum <guido@python.org>
parents: 145
diff changeset
1415 fraction:
0a92b6a1b9c0 Added recognition of 'l' or 'L' as long integer suffix
Guido van Rossum <guido@python.org>
parents: 145
diff changeset
1416 /* Fraction */
0a92b6a1b9c0 Added recognition of 'l' or 'L' as long integer suffix
Guido van Rossum <guido@python.org>
parents: 145
diff changeset
1417 do {
0a92b6a1b9c0 Added recognition of 'l' or 'L' as long integer suffix
Guido van Rossum <guido@python.org>
parents: 145
diff changeset
1418 c = tok_nextc(tok);
0a92b6a1b9c0 Added recognition of 'l' or 'L' as long integer suffix
Guido van Rossum <guido@python.org>
parents: 145
diff changeset
1419 } while (isdigit(c));
0a92b6a1b9c0 Added recognition of 'l' or 'L' as long integer suffix
Guido van Rossum <guido@python.org>
parents: 145
diff changeset
1420 }
0a92b6a1b9c0 Added recognition of 'l' or 'L' as long integer suffix
Guido van Rossum <guido@python.org>
parents: 145
diff changeset
1421 if (c == 'e' || c == 'E') {
19621
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1422 exponent:
236
0a92b6a1b9c0 Added recognition of 'l' or 'L' as long integer suffix
Guido van Rossum <guido@python.org>
parents: 145
diff changeset
1423 /* Exponent part */
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1424 c = tok_nextc(tok);
236
0a92b6a1b9c0 Added recognition of 'l' or 'L' as long integer suffix
Guido van Rossum <guido@python.org>
parents: 145
diff changeset
1425 if (c == '+' || c == '-')
0a92b6a1b9c0 Added recognition of 'l' or 'L' as long integer suffix
Guido van Rossum <guido@python.org>
parents: 145
diff changeset
1426 c = tok_nextc(tok);
19501
ce4f1e98ba7e SF bug [#455775] float parsing discrepancy.
Tim Peters <tim.peters@gmail.com>
parents: 19051
diff changeset
1427 if (!isdigit(c)) {
ce4f1e98ba7e SF bug [#455775] float parsing discrepancy.
Tim Peters <tim.peters@gmail.com>
parents: 19051
diff changeset
1428 tok->done = E_TOKEN;
ce4f1e98ba7e SF bug [#455775] float parsing discrepancy.
Tim Peters <tim.peters@gmail.com>
parents: 19051
diff changeset
1429 tok_backup(tok, c);
ce4f1e98ba7e SF bug [#455775] float parsing discrepancy.
Tim Peters <tim.peters@gmail.com>
parents: 19051
diff changeset
1430 return ERRORTOKEN;
ce4f1e98ba7e SF bug [#455775] float parsing discrepancy.
Tim Peters <tim.peters@gmail.com>
parents: 19051
diff changeset
1431 }
ce4f1e98ba7e SF bug [#455775] float parsing discrepancy.
Tim Peters <tim.peters@gmail.com>
parents: 19051
diff changeset
1432 do {
236
0a92b6a1b9c0 Added recognition of 'l' or 'L' as long integer suffix
Guido van Rossum <guido@python.org>
parents: 145
diff changeset
1433 c = tok_nextc(tok);
19501
ce4f1e98ba7e SF bug [#455775] float parsing discrepancy.
Tim Peters <tim.peters@gmail.com>
parents: 19051
diff changeset
1434 } while (isdigit(c));
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1435 }
3333
4b5bff95804e changes for pow(**) and complex
Guido van Rossum <guido@python.org>
parents: 3116
diff changeset
1436 #ifndef WITHOUT_COMPLEX
3364
bc17e8313ea0 use only j for imaginary constants
Guido van Rossum <guido@python.org>
parents: 3333
diff changeset
1437 if (c == 'j' || c == 'J')
3333
4b5bff95804e changes for pow(**) and complex
Guido van Rossum <guido@python.org>
parents: 3116
diff changeset
1438 /* Imaginary part */
4b5bff95804e changes for pow(**) and complex
Guido van Rossum <guido@python.org>
parents: 3116
diff changeset
1439 imaginary:
4b5bff95804e changes for pow(**) and complex
Guido van Rossum <guido@python.org>
parents: 3116
diff changeset
1440 c = tok_nextc(tok);
4b5bff95804e changes for pow(**) and complex
Guido van Rossum <guido@python.org>
parents: 3116
diff changeset
1441 #endif
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1442 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1443 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1444 tok_backup(tok, c);
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1445 *p_start = tok->start;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1446 *p_end = tok->cur;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1447 return NUMBER;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1448 }
5265
3c1788ab9761 Support for alternative string quotes (a"xx", b"xx", c"xx", ...).
Guido van Rossum <guido@python.org>
parents: 4839
diff changeset
1449
3c1788ab9761 Support for alternative string quotes (a"xx", b"xx", c"xx", ...).
Guido van Rossum <guido@python.org>
parents: 4839
diff changeset
1450 letter_quote:
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1451 /* String */
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1452 if (c == '\'' || c == '"') {
36501
7fdc639bc5b4 Merge ssize_t branch.
Martin v. Löwis <martin@v.loewis.de>
parents: 36090
diff changeset
1453 Py_ssize_t quote2 = tok->cur - tok->start + 1;
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1454 int quote = c;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1455 int triple = 0;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1456 int tripcount = 0;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1457 for (;;) {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1458 c = tok_nextc(tok);
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1459 if (c == '\n') {
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1460 if (!triple) {
24839
17e95cc06f69 provide less mysterious error messages when seeing end-of-line in
Skip Montanaro <skip@pobox.com>
parents: 24630
diff changeset
1461 tok->done = E_EOLS;
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1462 tok_backup(tok, c);
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1463 return ERRORTOKEN;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1464 }
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1465 tripcount = 0;
25098
c02a668488d0 Ignore encoding declarations inside strings. Fixes #603509.
Martin v. Löwis <martin@v.loewis.de>
parents: 24867
diff changeset
1466 tok->cont_line = 1; /* multiline string. */
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1467 }
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1468 else if (c == EOF) {
24839
17e95cc06f69 provide less mysterious error messages when seeing end-of-line in
Skip Montanaro <skip@pobox.com>
parents: 24630
diff changeset
1469 if (triple)
17e95cc06f69 provide less mysterious error messages when seeing end-of-line in
Skip Montanaro <skip@pobox.com>
parents: 24630
diff changeset
1470 tok->done = E_EOFS;
17e95cc06f69 provide less mysterious error messages when seeing end-of-line in
Skip Montanaro <skip@pobox.com>
parents: 24630
diff changeset
1471 else
17e95cc06f69 provide less mysterious error messages when seeing end-of-line in
Skip Montanaro <skip@pobox.com>
parents: 24630
diff changeset
1472 tok->done = E_EOLS;
1282
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
1473 tok->cur = tok->inp;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1474 return ERRORTOKEN;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1475 }
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1476 else if (c == quote) {
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1477 tripcount++;
7315
3a55fbb99f20 Fixed the bug in searching for triple quotes -- change the 'quote2'
Guido van Rossum <guido@python.org>
parents: 7118
diff changeset
1478 if (tok->cur - tok->start == quote2) {
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1479 c = tok_nextc(tok);
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1480 if (c == quote) {
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1481 triple = 1;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1482 tripcount = 0;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1483 continue;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1484 }
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1485 tok_backup(tok, c);
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1486 }
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1487 if (!triple || tripcount == 3)
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1488 break;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1489 }
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1490 else if (c == '\\') {
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1491 tripcount = 0;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1492 c = tok_nextc(tok);
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1493 if (c == EOF) {
24839
17e95cc06f69 provide less mysterious error messages when seeing end-of-line in
Skip Montanaro <skip@pobox.com>
parents: 24630
diff changeset
1494 tok->done = E_EOLS;
1282
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
1495 tok->cur = tok->inp;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1496 return ERRORTOKEN;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1497 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1498 }
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1499 else
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1500 tripcount = 0;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1501 }
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1502 *p_start = tok->start;
1364
132602d854d8 Changes to accept double-quoted strings on input.
Guido van Rossum <guido@python.org>
parents: 1284
diff changeset
1503 *p_end = tok->cur;
132602d854d8 Changes to accept double-quoted strings on input.
Guido van Rossum <guido@python.org>
parents: 1284
diff changeset
1504 return STRING;
132602d854d8 Changes to accept double-quoted strings on input.
Guido van Rossum <guido@python.org>
parents: 1284
diff changeset
1505 }
37229
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
1506
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1507 /* Line continuation */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1508 if (c == '\\') {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1509 c = tok_nextc(tok);
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1510 if (c != '\n') {
34664
3785610ff763 Patch #802188: better parser error message for non-EOL following line cont.
Martin v. Löwis <martin@v.loewis.de>
parents: 32961
diff changeset
1511 tok->done = E_LINECONT;
1282
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
1512 tok->cur = tok->inp;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1513 return ERRORTOKEN;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1514 }
25098
c02a668488d0 Ignore encoding declarations inside strings. Fixes #603509.
Martin v. Löwis <martin@v.loewis.de>
parents: 24867
diff changeset
1515 tok->cont_line = 1;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1516 goto again; /* Read next line */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1517 }
37229
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
1518
400
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1519 /* Check for two-character token */
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1520 {
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1521 int c2 = tok_nextc(tok);
5379
1c9a84e0981d Another directory quickly renamed.
Guido van Rossum <guido@python.org>
parents: 5341
diff changeset
1522 int token = PyToken_TwoChars(c, c2);
43983
e725166c8c81 Fixed problems in the last commit. Filenames and line numbers weren't reported correctly.
Christian Heimes <christian@cheimes.de>
parents: 43982
diff changeset
1523 #ifndef PGEN
44009
2068bf77891d Warning "<> not supported in 3.x" should be enabled only when the -3 option is set.
Amaury Forgeot d'Arc <amauryfa@gmail.com>
parents: 43983
diff changeset
1524 if (Py_Py3kWarningFlag && token == NOTEQUAL && c == '<') {
43983
e725166c8c81 Fixed problems in the last commit. Filenames and line numbers weren't reported correctly.
Christian Heimes <christian@cheimes.de>
parents: 43982
diff changeset
1525 if (PyErr_WarnExplicit(PyExc_DeprecationWarning,
46075
84bb7701efba Make Py3k warnings consistent w.r.t. punctuation; also respect the
Georg Brandl <georg@python.org>
parents: 45791
diff changeset
1526 "<> not supported in 3.x; use !=",
43983
e725166c8c81 Fixed problems in the last commit. Filenames and line numbers weren't reported correctly.
Christian Heimes <christian@cheimes.de>
parents: 43982
diff changeset
1527 tok->filename, tok->lineno,
e725166c8c81 Fixed problems in the last commit. Filenames and line numbers weren't reported correctly.
Christian Heimes <christian@cheimes.de>
parents: 43982
diff changeset
1528 NULL, NULL)) {
e725166c8c81 Fixed problems in the last commit. Filenames and line numbers weren't reported correctly.
Christian Heimes <christian@cheimes.de>
parents: 43982
diff changeset
1529 return ERRORTOKEN;
e725166c8c81 Fixed problems in the last commit. Filenames and line numbers weren't reported correctly.
Christian Heimes <christian@cheimes.de>
parents: 43982
diff changeset
1530 }
e725166c8c81 Fixed problems in the last commit. Filenames and line numbers weren't reported correctly.
Christian Heimes <christian@cheimes.de>
parents: 43982
diff changeset
1531 }
e725166c8c81 Fixed problems in the last commit. Filenames and line numbers weren't reported correctly.
Christian Heimes <christian@cheimes.de>
parents: 43982
diff changeset
1532 #endif
400
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1533 if (token != OP) {
14163
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1534 int c3 = tok_nextc(tok);
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1535 int token3 = PyToken_ThreeChars(c, c2, c3);
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1536 if (token3 != OP) {
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1537 token = token3;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1538 } else {
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1539 tok_backup(tok, c3);
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1540 }
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1541 *p_start = tok->start;
400
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1542 *p_end = tok->cur;
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1543 return token;
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1544 }
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1545 tok_backup(tok, c2);
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1546 }
37229
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
1547
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1548 /* Keep track of parentheses nesting level */
1284
dbce72afe261 * selectmodule.c: fix (another!) two memory leaks -- this time in list2set
Guido van Rossum <guido@python.org>
parents: 1282
diff changeset
1549 switch (c) {
dbce72afe261 * selectmodule.c: fix (another!) two memory leaks -- this time in list2set
Guido van Rossum <guido@python.org>
parents: 1282
diff changeset
1550 case '(':
dbce72afe261 * selectmodule.c: fix (another!) two memory leaks -- this time in list2set
Guido van Rossum <guido@python.org>
parents: 1282
diff changeset
1551 case '[':
dbce72afe261 * selectmodule.c: fix (another!) two memory leaks -- this time in list2set
Guido van Rossum <guido@python.org>
parents: 1282
diff changeset
1552 case '{':
dbce72afe261 * selectmodule.c: fix (another!) two memory leaks -- this time in list2set
Guido van Rossum <guido@python.org>
parents: 1282
diff changeset
1553 tok->level++;
dbce72afe261 * selectmodule.c: fix (another!) two memory leaks -- this time in list2set
Guido van Rossum <guido@python.org>
parents: 1282
diff changeset
1554 break;
dbce72afe261 * selectmodule.c: fix (another!) two memory leaks -- this time in list2set
Guido van Rossum <guido@python.org>
parents: 1282
diff changeset
1555 case ')':
dbce72afe261 * selectmodule.c: fix (another!) two memory leaks -- this time in list2set
Guido van Rossum <guido@python.org>
parents: 1282
diff changeset
1556 case ']':
dbce72afe261 * selectmodule.c: fix (another!) two memory leaks -- this time in list2set
Guido van Rossum <guido@python.org>
parents: 1282
diff changeset
1557 case '}':
dbce72afe261 * selectmodule.c: fix (another!) two memory leaks -- this time in list2set
Guido van Rossum <guido@python.org>
parents: 1282
diff changeset
1558 tok->level--;
dbce72afe261 * selectmodule.c: fix (another!) two memory leaks -- this time in list2set
Guido van Rossum <guido@python.org>
parents: 1282
diff changeset
1559 break;
dbce72afe261 * selectmodule.c: fix (another!) two memory leaks -- this time in list2set
Guido van Rossum <guido@python.org>
parents: 1282
diff changeset
1560 }
37229
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
1561
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1562 /* Punctuation character */
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1563 *p_start = tok->start;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1564 *p_end = tok->cur;
5379
1c9a84e0981d Another directory quickly renamed.
Guido van Rossum <guido@python.org>
parents: 5341
diff changeset
1565 return PyToken_OneChar(c);
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1566 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1567
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
1568 int
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
1569 PyTokenizer_Get(struct tok_state *tok, char **p_start, char **p_end)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
1570 {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
1571 int result = tok_get(tok, p_start, p_end);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
1572 if (tok->decoding_erred) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
1573 result = ERRORTOKEN;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
1574 tok->done = E_DECODE;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
1575 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
1576 return result;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
1577 }
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1578
43229
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1579 /* This function is only called from parsetok. However, it cannot live
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1580 there, as it must be empty for PGEN, and we can check for PGEN only
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1581 in this file. */
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1582
44889
9a1211c77e02 Fixed bug #1915: Python compiles with --enable-unicode=no again. However several extension methods and modules do not work without unicode support.
Christian Heimes <christian@cheimes.de>
parents: 44861
diff changeset
1583 #if defined(PGEN) || !defined(Py_USING_UNICODE)
43229
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1584 char*
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1585 PyTokenizer_RestoreEncoding(struct tok_state* tok, int len, int* offset)
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1586 {
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1587 return NULL;
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1588 }
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1589 #else
44575
cf36e01e83f2 Fix #define ordering.
Georg Brandl <georg@python.org>
parents: 44572
diff changeset
1590 #ifdef Py_USING_UNICODE
43229
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1591 static PyObject *
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1592 dec_utf8(const char *enc, const char *text, size_t len) {
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1593 PyObject *ret = NULL;
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1594 PyObject *unicode_text = PyUnicode_DecodeUTF8(text, len, "replace");
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1595 if (unicode_text) {
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1596 ret = PyUnicode_AsEncodedString(unicode_text, enc, "replace");
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1597 Py_DECREF(unicode_text);
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1598 }
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1599 if (!ret) {
43572
697400fce7c5 Change a PyErr_Print() into a PyErr_Clear(),
Guido van Rossum <guido@python.org>
parents: 43229
diff changeset
1600 PyErr_Clear();
43229
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1601 }
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1602 return ret;
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1603 }
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1604 char *
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1605 PyTokenizer_RestoreEncoding(struct tok_state* tok, int len, int *offset)
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1606 {
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1607 char *text = NULL;
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1608 if (tok->encoding) {
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1609 /* convert source to original encondig */
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1610 PyObject *lineobj = dec_utf8(tok->encoding, tok->buf, len);
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1611 if (lineobj != NULL) {
47527
52294af4b663 This reverts r63675 based on the discussion in this thread:
Gregory P. Smith <greg@mad-scientist.com>
parents: 47294
diff changeset
1612 int linelen = PyString_Size(lineobj);
52294af4b663 This reverts r63675 based on the discussion in this thread:
Gregory P. Smith <greg@mad-scientist.com>
parents: 47294
diff changeset
1613 const char *line = PyString_AsString(lineobj);
43229
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1614 text = PyObject_MALLOC(linelen + 1);
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1615 if (text != NULL && line != NULL) {
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1616 if (linelen)
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1617 strncpy(text, line, linelen);
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1618 text[linelen] = '\0';
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1619 }
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1620 Py_DECREF(lineobj);
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1621
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1622 /* adjust error offset */
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1623 if (*offset > 1) {
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1624 PyObject *offsetobj = dec_utf8(tok->encoding,
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1625 tok->buf, *offset-1);
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1626 if (offsetobj) {
47527
52294af4b663 This reverts r63675 based on the discussion in this thread:
Gregory P. Smith <greg@mad-scientist.com>
parents: 47294
diff changeset
1627 *offset = PyString_Size(offsetobj) + 1;
43229
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1628 Py_DECREF(offsetobj);
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1629 }
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1630 }
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1631
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1632 }
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1633 }
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1634 return text;
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1635
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1636 }
44575
cf36e01e83f2 Fix #define ordering.
Georg Brandl <georg@python.org>
parents: 44572
diff changeset
1637 #endif /* defined(Py_USING_UNICODE) */
43229
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1638 #endif
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1639
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 40036
diff changeset
1640
4839
4567e158d20c Rename DEBUG macro to Py_DEBUG
Guido van Rossum <guido@python.org>
parents: 4577
diff changeset
1641 #ifdef Py_DEBUG
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1642
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1643 void
13659
e78153e179fb Mass ANSIfication.
Thomas Wouters <thomas@python.org>
parents: 13539
diff changeset
1644 tok_dump(int type, char *start, char *end)
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1645 {
5379
1c9a84e0981d Another directory quickly renamed.
Guido van Rossum <guido@python.org>
parents: 5341
diff changeset
1646 printf("%s", _PyParser_TokenNames[type]);
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1647 if (type == NAME || type == NUMBER || type == STRING || type == OP)
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1648 printf("(%.*s)", (int)(end - start), start);
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1649 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1650
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1651 #endif