Mercurial > cpython-lru_cache-weakref

annotate Parser/tokenizer.c @ 28646:193b8aec00ce legacy-trunk

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Changes from Jonathan Riehl to allow his pgen extension (PEP 269) to work. This includes some more code that used to be part of pgen in the main parser; I'm okay with that. I'll see if the Windows build needs work next.
author Guido van Rossum <guido@python.org>
date Thu, 17 Apr 2003 14:55:42 +0000
parents be371ffc4cd0
children 86fdf8fe7f17
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
145
853b736a6a9e Added copyright notice.
Guido van Rossum <guido@python.org>
parents: 96
diff changeset
1
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
2 /* Tokenizer implementation */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
3
23029
8291473fde3e Mass checkin of universal newline support.
Jack Jansen <jack.jansen@cwi.nl>
parents: 19621
diff changeset
4 #include "Python.h"
86
a6934380c6e7 "Compiling" version
Guido van Rossum <guido@python.org>
parents: 67
diff changeset
5 #include "pgenheaders.h"
a6934380c6e7 "Compiling" version
Guido van Rossum <guido@python.org>
parents: 67
diff changeset
6
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
7 #include <ctype.h>
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
8 #include <assert.h>
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
9
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
10 #include "tokenizer.h"
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
11 #include "errcode.h"
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
12
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
13 #ifndef PGEN
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
14 #include "unicodeobject.h"
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
15 #include "stringobject.h"
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
16 #include "fileobject.h"
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
17 #include "codecs.h"
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
18 #include "abstract.h"
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
19 #endif /* PGEN */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
20
25924
dc4a0336a2a3 Patch #512981: Update readline input stream on sys.stdin/out change.
Martin v. Löwis <martin@v.loewis.de>
parents: 25101
diff changeset
21 extern char *PyOS_Readline(FILE *, FILE *, char *);
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
22 /* Return malloc'ed string including trailing \n;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
23 empty malloc'ed string for EOF;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
24 NULL if interrupted */
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
25
678
205b947b1d3c Make tabs always 8 spaces wide -- it's more portable.
Guido van Rossum <guido@python.org>
parents: 477
diff changeset
26 /* Don't ever change this -- it would break the portability of Python code */
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
27 #define TABSIZE 8
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
28
7118
e9108e97a5a0 tok_nextc() should return unsigned characters, to avoid mistaking
Guido van Rossum <guido@python.org>
parents: 5379
diff changeset
29 /* Convert a possibly signed character to a nonnegative int */
e9108e97a5a0 tok_nextc() should return unsigned characters, to avoid mistaking
Guido van Rossum <guido@python.org>
parents: 5379
diff changeset
30 /* XXX This assumes characters are 8 bits wide */
e9108e97a5a0 tok_nextc() should return unsigned characters, to avoid mistaking
Guido van Rossum <guido@python.org>
parents: 5379
diff changeset
31 #ifdef __CHAR_UNSIGNED__
e9108e97a5a0 tok_nextc() should return unsigned characters, to avoid mistaking
Guido van Rossum <guido@python.org>
parents: 5379
diff changeset
32 #define Py_CHARMASK(c) (c)
e9108e97a5a0 tok_nextc() should return unsigned characters, to avoid mistaking
Guido van Rossum <guido@python.org>
parents: 5379
diff changeset
33 #else
e9108e97a5a0 tok_nextc() should return unsigned characters, to avoid mistaking
Guido van Rossum <guido@python.org>
parents: 5379
diff changeset
34 #define Py_CHARMASK(c) ((c) & 0xff)
e9108e97a5a0 tok_nextc() should return unsigned characters, to avoid mistaking
Guido van Rossum <guido@python.org>
parents: 5379
diff changeset
35 #endif
e9108e97a5a0 tok_nextc() should return unsigned characters, to avoid mistaking
Guido van Rossum <guido@python.org>
parents: 5379
diff changeset
36
86
a6934380c6e7 "Compiling" version
Guido van Rossum <guido@python.org>
parents: 67
diff changeset
37 /* Forward */
13447
a89f7f268de4 Nuke all remaining occurrences of Py_PROTO and Py_FPROTO.
Tim Peters <tim.peters@gmail.com>
parents: 13256
diff changeset
38 static struct tok_state *tok_new(void);
a89f7f268de4 Nuke all remaining occurrences of Py_PROTO and Py_FPROTO.
Tim Peters <tim.peters@gmail.com>
parents: 13256
diff changeset
39 static int tok_nextc(struct tok_state *tok);
a89f7f268de4 Nuke all remaining occurrences of Py_PROTO and Py_FPROTO.
Tim Peters <tim.peters@gmail.com>
parents: 13256
diff changeset
40 static void tok_backup(struct tok_state *tok, int c);
86
a6934380c6e7 "Compiling" version
Guido van Rossum <guido@python.org>
parents: 67
diff changeset
41
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
42 /* Token names */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
43
5379
1c9a84e0981d Another directory quickly renamed.
Guido van Rossum <guido@python.org>
parents: 5341
diff changeset
44 char *_PyParser_TokenNames[] = {
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
45 "ENDMARKER",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
46 "NAME",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
47 "NUMBER",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
48 "STRING",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
49 "NEWLINE",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
50 "INDENT",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
51 "DEDENT",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
52 "LPAR",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
53 "RPAR",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
54 "LSQB",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
55 "RSQB",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
56 "COLON",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
57 "COMMA",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
58 "SEMI",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
59 "PLUS",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
60 "MINUS",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
61 "STAR",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
62 "SLASH",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
63 "VBAR",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
64 "AMPER",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
65 "LESS",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
66 "GREATER",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
67 "EQUAL",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
68 "DOT",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
69 "PERCENT",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
70 "BACKQUOTE",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
71 "LBRACE",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
72 "RBRACE",
400
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
73 "EQEQUAL",
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
74 "NOTEQUAL",
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
75 "LESSEQUAL",
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
76 "GREATEREQUAL",
410
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
77 "TILDE",
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
78 "CIRCUMFLEX",
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
79 "LEFTSHIFT",
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
80 "RIGHTSHIFT",
3333
4b5bff95804e changes for pow(**) and complex
Guido van Rossum <guido@python.org>
parents: 3116
diff changeset
81 "DOUBLESTAR",
14163
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
82 "PLUSEQUAL",
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
83 "MINEQUAL",
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
84 "STAREQUAL",
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
85 "SLASHEQUAL",
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
86 "PERCENTEQUAL",
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
87 "AMPEREQUAL",
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
88 "VBAREQUAL",
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
89 "CIRCUMFLEXEQUAL",
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
90 "LEFTSHIFTEQUAL",
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
91 "RIGHTSHIFTEQUAL",
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
92 "DOUBLESTAREQUAL",
19051
f20727cc3d0f Implement PEP 238 in its (almost) full glory.
Guido van Rossum <guido@python.org>
parents: 17600
diff changeset
93 "DOUBLESLASH",
f20727cc3d0f Implement PEP 238 in its (almost) full glory.
Guido van Rossum <guido@python.org>
parents: 17600
diff changeset
94 "DOUBLESLASHEQUAL",
400
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
95 /* This table must match the #defines in token.h! */
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
96 "OP",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
97 "<ERRORTOKEN>",
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
98 "<N_TOKENS>"
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
99 };
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
100
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
101
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
102 /* Create and initialize a new tok_state structure */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
103
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
104 static struct tok_state *
13659
e78153e179fb Mass ANSIfication.
Thomas Wouters <thomas@python.org>
parents: 13539
diff changeset
105 tok_new(void)
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
106 {
5379
1c9a84e0981d Another directory quickly renamed.
Guido van Rossum <guido@python.org>
parents: 5341
diff changeset
107 struct tok_state *tok = PyMem_NEW(struct tok_state, 1);
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
108 if (tok == NULL)
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
109 return NULL;
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
110 tok->buf = tok->cur = tok->end = tok->inp = tok->start = NULL;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
111 tok->done = E_OK;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
112 tok->fp = NULL;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
113 tok->tabsize = TABSIZE;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
114 tok->indent = 0;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
115 tok->indstack[0] = 0;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
116 tok->atbol = 1;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
117 tok->pendin = 0;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
118 tok->prompt = tok->nextprompt = NULL;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
119 tok->lineno = 0;
1284
dbce72afe261 * selectmodule.c: fix (another!) two memory leaks -- this time in list2set
Guido van Rossum <guido@python.org>
parents: 1282
diff changeset
120 tok->level = 0;
7945
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
121 tok->filename = NULL;
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
122 tok->altwarning = 0;
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
123 tok->alterror = 0;
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
124 tok->alttabsize = 1;
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
125 tok->altindstack[0] = 0;
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
126 tok->decoding_state = 0;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
127 tok->decoding_erred = 0;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
128 tok->read_coding_spec = 0;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
129 tok->issued_encoding_warning = 0;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
130 tok->encoding = NULL;
25098
c02a668488d0 Ignore encoding declarations inside strings. Fixes #603509.
Martin v. Löwis <martin@v.loewis.de>
parents: 24867
diff changeset
131 tok->cont_line = 0;
24544
0bd521eb714e Make pgen compile with pydebug. Duplicate normalized names, as it may
Martin v. Löwis <martin@v.loewis.de>
parents: 24543
diff changeset
132 #ifndef PGEN
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
133 tok->decoding_readline = NULL;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
134 tok->decoding_buffer = NULL;
24544
0bd521eb714e Make pgen compile with pydebug. Duplicate normalized names, as it may
Martin v. Löwis <martin@v.loewis.de>
parents: 24543
diff changeset
135 #endif
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
136 return tok;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
137 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
138
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
139 #ifdef PGEN
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
140
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
141 static char *
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
142 decoding_fgets(char *s, int size, struct tok_state *tok)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
143 {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
144 return fgets(s, size, tok->fp);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
145 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
146
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
147 static int
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
148 decoding_feof(struct tok_state *tok)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
149 {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
150 return feof(tok->fp);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
151 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
152
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
153 static const char *
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
154 decode_str(const char *str, struct tok_state *tok)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
155 {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
156 return str;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
157 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
158
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
159 #else /* PGEN */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
160
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
161 static char *
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
162 error_ret(struct tok_state *tok) /* XXX */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
163 {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
164 tok->decoding_erred = 1;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
165 if (tok->fp != NULL && tok->buf != NULL) /* see PyTokenizer_Free */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
166 PyMem_DEL(tok->buf);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
167 tok->buf = NULL;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
168 return NULL; /* as if it were EOF */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
169 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
170
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
171 static char *
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
172 new_string(const char *s, int len)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
173 {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
174 char* result = PyMem_NEW(char, len + 1);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
175 if (result != NULL) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
176 memcpy(result, s, len);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
177 result[len] = '\0';
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
178 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
179 return result;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
180 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
181
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
182 static char *
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
183 get_normal_name(char *s) /* for utf-8 and latin-1 */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
184 {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
185 char buf[13];
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
186 int i;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
187 for (i = 0; i < 12; i++) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
188 int c = s[i];
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
189 if (c == '\0') break;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
190 else if (c == '_') buf[i] = '-';
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
191 else buf[i] = tolower(c);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
192 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
193 buf[i] = '\0';
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
194 if (strcmp(buf, "utf-8") == 0 ||
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
195 strncmp(buf, "utf-8-", 6) == 0) return "utf-8";
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
196 else if (strcmp(buf, "latin-1") == 0 ||
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
197 strcmp(buf, "iso-8859-1") == 0 ||
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
198 strcmp(buf, "iso-latin-1") == 0 ||
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
199 strncmp(buf, "latin-1-", 8) == 0 ||
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
200 strncmp(buf, "iso-8859-1-", 11) == 0 ||
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
201 strncmp(buf, "iso-latin-1-", 12) == 0) return "iso-8859-1";
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
202 else return s;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
203 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
204
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
205 /* Return the coding spec in S, or NULL if none is found. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
206
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
207 static char *
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
208 get_coding_spec(const char *s, int size)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
209 {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
210 int i;
25098
c02a668488d0 Ignore encoding declarations inside strings. Fixes #603509.
Martin v. Löwis <martin@v.loewis.de>
parents: 24867
diff changeset
211 /* Coding spec must be in a comment, and that comment must be
c02a668488d0 Ignore encoding declarations inside strings. Fixes #603509.
Martin v. Löwis <martin@v.loewis.de>
parents: 24867
diff changeset
212 * the only statement on the source code line. */
c02a668488d0 Ignore encoding declarations inside strings. Fixes #603509.
Martin v. Löwis <martin@v.loewis.de>
parents: 24867
diff changeset
213 for (i = 0; i < size - 6; i++) {
c02a668488d0 Ignore encoding declarations inside strings. Fixes #603509.
Martin v. Löwis <martin@v.loewis.de>
parents: 24867
diff changeset
214 if (s[i] == '#')
c02a668488d0 Ignore encoding declarations inside strings. Fixes #603509.
Martin v. Löwis <martin@v.loewis.de>
parents: 24867
diff changeset
215 break;
c02a668488d0 Ignore encoding declarations inside strings. Fixes #603509.
Martin v. Löwis <martin@v.loewis.de>
parents: 24867
diff changeset
216 if (s[i] != ' ' && s[i] != '\t' && s[i] != '\014')
c02a668488d0 Ignore encoding declarations inside strings. Fixes #603509.
Martin v. Löwis <martin@v.loewis.de>
parents: 24867
diff changeset
217 return NULL;
c02a668488d0 Ignore encoding declarations inside strings. Fixes #603509.
Martin v. Löwis <martin@v.loewis.de>
parents: 24867
diff changeset
218 }
c02a668488d0 Ignore encoding declarations inside strings. Fixes #603509.
Martin v. Löwis <martin@v.loewis.de>
parents: 24867
diff changeset
219 for (; i < size - 6; i++) { /* XXX inefficient search */
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
220 const char* t = s + i;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
221 if (strncmp(t, "coding", 6) == 0) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
222 const char* begin = NULL;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
223 t += 6;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
224 if (t[0] != ':' && t[0] != '=')
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
225 continue;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
226 do {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
227 t++;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
228 } while (t[0] == '\x20' || t[0] == '\t');
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
229
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
230 begin = t;
25995
1aaa12c08b35 Fix compiler warning on HP-UX.
Neal Norwitz <nnorwitz@gmail.com>
parents: 25924
diff changeset
231 while (isalnum((int)t[0]) ||
1aaa12c08b35 Fix compiler warning on HP-UX.
Neal Norwitz <nnorwitz@gmail.com>
parents: 25924
diff changeset
232 t[0] == '-' || t[0] == '_' || t[0] == '.')
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
233 t++;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
234
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
235 if (begin < t) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
236 char* r = new_string(begin, t - begin);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
237 char* q = get_normal_name(r);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
238 if (r != q) {
24544
0bd521eb714e Make pgen compile with pydebug. Duplicate normalized names, as it may
Martin v. Löwis <martin@v.loewis.de>
parents: 24543
diff changeset
239 PyMem_DEL(r);
0bd521eb714e Make pgen compile with pydebug. Duplicate normalized names, as it may
Martin v. Löwis <martin@v.loewis.de>
parents: 24543
diff changeset
240 r = new_string(q, strlen(q));
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
241 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
242 return r;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
243 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
244 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
245 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
246 return NULL;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
247 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
248
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
249 /* Check whether the line contains a coding spec. If it does,
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
250 invoke the set_readline function for the new encoding.
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
251 This function receives the tok_state and the new encoding.
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
252 Return 1 on success, 0 on failure. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
253
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
254 static int
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
255 check_coding_spec(const char* line, int size, struct tok_state *tok,
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
256 int set_readline(struct tok_state *, const char *))
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
257 {
25101
32331d26d7cf Removed reliance on gcc/C99 extension.
Tim Peters <tim.peters@gmail.com>
parents: 25098
diff changeset
258 char * cs;
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
259 int r = 1;
25101
32331d26d7cf Removed reliance on gcc/C99 extension.
Tim Peters <tim.peters@gmail.com>
parents: 25098
diff changeset
260
25098
c02a668488d0 Ignore encoding declarations inside strings. Fixes #603509.
Martin v. Löwis <martin@v.loewis.de>
parents: 24867
diff changeset
261 if (tok->cont_line)
c02a668488d0 Ignore encoding declarations inside strings. Fixes #603509.
Martin v. Löwis <martin@v.loewis.de>
parents: 24867
diff changeset
262 /* It's a continuation line, so it can't be a coding spec. */
c02a668488d0 Ignore encoding declarations inside strings. Fixes #603509.
Martin v. Löwis <martin@v.loewis.de>
parents: 24867
diff changeset
263 return 1;
25101
32331d26d7cf Removed reliance on gcc/C99 extension.
Tim Peters <tim.peters@gmail.com>
parents: 25098
diff changeset
264 cs = get_coding_spec(line, size);
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
265 if (cs != NULL) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
266 tok->read_coding_spec = 1;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
267 if (tok->encoding == NULL) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
268 assert(tok->decoding_state == 1); /* raw */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
269 if (strcmp(cs, "utf-8") == 0 ||
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
270 strcmp(cs, "iso-8859-1") == 0) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
271 tok->encoding = cs;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
272 } else {
24624
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
273 #ifdef Py_USING_UNICODE
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
274 r = set_readline(tok, cs);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
275 if (r) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
276 tok->encoding = cs;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
277 tok->decoding_state = -1;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
278 }
24624
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
279 #else
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
280 /* Without Unicode support, we cannot
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
281 process the coding spec. Since there
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
282 won't be any Unicode literals, that
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
283 won't matter. */
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
284 #endif
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
285 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
286 } else { /* then, compare cs with BOM */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
287 r = (strcmp(tok->encoding, cs) == 0);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
288 PyMem_DEL(cs);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
289 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
290 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
291 return r;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
292 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
293
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
294 /* See whether the file starts with a BOM. If it does,
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
295 invoke the set_readline function with the new encoding.
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
296 Return 1 on success, 0 on failure. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
297
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
298 static int
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
299 check_bom(int get_char(struct tok_state *),
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
300 void unget_char(int, struct tok_state *),
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
301 int set_readline(struct tok_state *, const char *),
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
302 struct tok_state *tok)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
303 {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
304 int ch = get_char(tok);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
305 tok->decoding_state = 1;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
306 if (ch == EOF) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
307 return 1;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
308 } else if (ch == 0xEF) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
309 ch = get_char(tok); if (ch != 0xBB) goto NON_BOM;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
310 ch = get_char(tok); if (ch != 0xBF) goto NON_BOM;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
311 #if 0
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
312 /* Disable support for UTF-16 BOMs until a decision
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
313 is made whether this needs to be supported. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
314 } else if (ch == 0xFE) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
315 ch = get_char(tok); if (ch != 0xFF) goto NON_BOM;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
316 if (!set_readline(tok, "utf-16-be")) return 0;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
317 tok->decoding_state = -1;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
318 } else if (ch == 0xFF) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
319 ch = get_char(tok); if (ch != 0xFE) goto NON_BOM;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
320 if (!set_readline(tok, "utf-16-le")) return 0;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
321 tok->decoding_state = -1;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
322 #endif
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
323 } else {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
324 unget_char(ch, tok);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
325 return 1;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
326 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
327 tok->encoding = new_string("utf-8", 5); /* resulting is in utf-8 */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
328 return 1;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
329 NON_BOM:
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
330 /* any token beginning with '\xEF', '\xFE', '\xFF' is a bad token */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
331 unget_char(0xFF, tok); /* XXX this will cause a syntax error */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
332 return 1;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
333 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
334
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
335 /* Read a line of text from TOK into S, using the stream in TOK.
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
336 Return NULL on failure, else S. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
337
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
338 static char *
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
339 fp_readl(char *s, int size, struct tok_state *tok)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
340 {
24624
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
341 #ifndef Py_USING_UNICODE
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
342 /* In a non-Unicode built, this should never be called. */
24630
43caa212b959 Use Py_FatalError instead of abort.
Martin v. Löwis <martin@v.loewis.de>
parents: 24624
diff changeset
343 Py_FatalError("fp_readl should not be called in this build.");
24867
255d1d3e66a3 Squash a few calls to the hideously expensive PyObject_CallObject(o,a)
Guido van Rossum <guido@python.org>
parents: 24839
diff changeset
344 return NULL; /* Keep compiler happy (not reachable) */
24624
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
345 #else
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
346 PyObject* utf8;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
347 PyObject* buf = tok->decoding_buffer;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
348 if (buf == NULL) {
27247
5ddb90f8169a Fix [ 665014 ] files with long lines and an encoding crash.
Mark Hammond <mhammond@skippinet.com.au>
parents: 26601
diff changeset
349 /* Ask for one less byte so we can terminate it */
5ddb90f8169a Fix [ 665014 ] files with long lines and an encoding crash.
Mark Hammond <mhammond@skippinet.com.au>
parents: 26601
diff changeset
350 PyObject *args = Py_BuildValue("(i)", size-1);
24867
255d1d3e66a3 Squash a few calls to the hideously expensive PyObject_CallObject(o,a)
Guido van Rossum <guido@python.org>
parents: 24839
diff changeset
351 if (args == NULL)
255d1d3e66a3 Squash a few calls to the hideously expensive PyObject_CallObject(o,a)
Guido van Rossum <guido@python.org>
parents: 24839
diff changeset
352 return error_ret(tok);
255d1d3e66a3 Squash a few calls to the hideously expensive PyObject_CallObject(o,a)
Guido van Rossum <guido@python.org>
parents: 24839
diff changeset
353 buf = PyObject_Call(tok->decoding_readline, args, NULL);
255d1d3e66a3 Squash a few calls to the hideously expensive PyObject_CallObject(o,a)
Guido van Rossum <guido@python.org>
parents: 24839
diff changeset
354 Py_DECREF(args);
24543
c00df23e856e Group statements properly.
Martin v. Löwis <martin@v.loewis.de>
parents: 24542
diff changeset
355 if (buf == NULL)
c00df23e856e Group statements properly.
Martin v. Löwis <martin@v.loewis.de>
parents: 24542
diff changeset
356 return error_ret(tok);
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
357 } else {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
358 tok->decoding_buffer = NULL;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
359 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
360 utf8 = PyUnicode_AsUTF8String(buf);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
361 Py_DECREF(buf);
24543
c00df23e856e Group statements properly.
Martin v. Löwis <martin@v.loewis.de>
parents: 24542
diff changeset
362 if (utf8 == NULL)
c00df23e856e Group statements properly.
Martin v. Löwis <martin@v.loewis.de>
parents: 24542
diff changeset
363 return error_ret(tok);
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
364 else {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
365 const char* str = PyString_AsString(utf8);
24541
ff9efa692594 Squash compiler wng about signed-vs-unsigned mismatch.
Tim Peters <tim.peters@gmail.com>
parents: 24539
diff changeset
366 assert(strlen(str) < (size_t)size); /* XXX */
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
367 strcpy(s, str);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
368 Py_DECREF(utf8);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
369 if (s[0] == '\0') return NULL; /* EOF */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
370 return s;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
371 }
24624
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
372 #endif
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
373 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
374
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
375 /* Set the readline function for TOK to a StreamReader's
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
376 readline function. The StreamReader is named ENC.
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
377
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
378 This function is called from check_bom and check_coding_spec.
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
379
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
380 ENC is usually identical to the future value of tok->encoding,
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
381 except for the (currently unsupported) case of UTF-16.
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
382
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
383 Return 1 on success, 0 on failure. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
384
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
385 static int
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
386 fp_setreadl(struct tok_state *tok, const char* enc)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
387 {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
388 PyObject *reader, *stream, *readline;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
389
26601
cc05c2c8660e Constify filenames and scripts. Fixes #651362.
Martin v. Löwis <martin@v.loewis.de>
parents: 25995
diff changeset
390 /* XXX: constify filename argument. */
cc05c2c8660e Constify filenames and scripts. Fixes #651362.
Martin v. Löwis <martin@v.loewis.de>
parents: 25995
diff changeset
391 stream = PyFile_FromFile(tok->fp, (char*)tok->filename, "rb", NULL);
24543
c00df23e856e Group statements properly.
Martin v. Löwis <martin@v.loewis.de>
parents: 24542
diff changeset
392 if (stream == NULL)
c00df23e856e Group statements properly.
Martin v. Löwis <martin@v.loewis.de>
parents: 24542
diff changeset
393 return 0;
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
394
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
395 reader = PyCodec_StreamReader(enc, stream, NULL);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
396 Py_DECREF(stream);
24543
c00df23e856e Group statements properly.
Martin v. Löwis <martin@v.loewis.de>
parents: 24542
diff changeset
397 if (reader == NULL)
c00df23e856e Group statements properly.
Martin v. Löwis <martin@v.loewis.de>
parents: 24542
diff changeset
398 return 0;
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
399
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
400 readline = PyObject_GetAttrString(reader, "readline");
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
401 Py_DECREF(reader);
24543
c00df23e856e Group statements properly.
Martin v. Löwis <martin@v.loewis.de>
parents: 24542
diff changeset
402 if (readline == NULL)
c00df23e856e Group statements properly.
Martin v. Löwis <martin@v.loewis.de>
parents: 24542
diff changeset
403 return 0;
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
404
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
405 tok->decoding_readline = readline;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
406 return 1;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
407 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
408
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
409 /* Fetch the next byte from TOK. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
410
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
411 static int fp_getc(struct tok_state *tok) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
412 return getc(tok->fp);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
413 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
414
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
415 /* Unfetch the last byte back into TOK. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
416
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
417 static void fp_ungetc(int c, struct tok_state *tok) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
418 ungetc(c, tok->fp);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
419 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
420
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
421 /* Read a line of input from TOK. Determine encoding
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
422 if necessary. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
423
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
424 static char *
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
425 decoding_fgets(char *s, int size, struct tok_state *tok)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
426 {
24630
43caa212b959 Use Py_FatalError instead of abort.
Martin v. Löwis <martin@v.loewis.de>
parents: 24624
diff changeset
427 char *line = NULL;
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
428 int warn = 0, badchar = 0;
24543
c00df23e856e Group statements properly.
Martin v. Löwis <martin@v.loewis.de>
parents: 24542
diff changeset
429 for (;;) {
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
430 if (tok->decoding_state < 0) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
431 /* We already have a codec associated with
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
432 this input. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
433 line = fp_readl(s, size, tok);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
434 break;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
435 } else if (tok->decoding_state > 0) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
436 /* We want a 'raw' read. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
437 line = Py_UniversalNewlineFgets(s, size,
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
438 tok->fp, NULL);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
439 warn = 1;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
440 break;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
441 } else {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
442 /* We have not yet determined the encoding.
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
443 If an encoding is found, use the file-pointer
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
444 reader functions from now on. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
445 if (!check_bom(fp_getc, fp_ungetc, fp_setreadl, tok))
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
446 return error_ret(tok);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
447 assert(tok->decoding_state != 0);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
448 }
24543
c00df23e856e Group statements properly.
Martin v. Löwis <martin@v.loewis.de>
parents: 24542
diff changeset
449 }
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
450 if (line != NULL && tok->lineno < 2 && !tok->read_coding_spec) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
451 if (!check_coding_spec(line, strlen(line), tok, fp_setreadl)) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
452 return error_ret(tok);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
453 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
454 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
455 #ifndef PGEN
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
456 if (warn && line && !tok->issued_encoding_warning && !tok->encoding) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
457 unsigned char *c;
24565
c8d694b641f1 Added a cast to shut up a compiler warning.
Jack Jansen <jack.jansen@cwi.nl>
parents: 24558
diff changeset
458 for (c = (unsigned char *)line; *c; c++)
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
459 if (*c > 127) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
460 badchar = *c;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
461 break;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
462 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
463 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
464 if (badchar) {
27918
be371ffc4cd0 Add URL for PEP to the source code encoding warning.
Marc-André Lemburg <mal@egenix.com>
parents: 27772
diff changeset
465 char buf[500];
24558
d26b62f0c6db Add 1 to lineno in deprecation warning. Fixes #590888.
Martin v. Löwis <martin@v.loewis.de>
parents: 24544
diff changeset
466 /* Need to add 1 to the line number, since this line
d26b62f0c6db Add 1 to lineno in deprecation warning. Fixes #590888.
Martin v. Löwis <martin@v.loewis.de>
parents: 24544
diff changeset
467 has not been counted, yet. */
27918
be371ffc4cd0 Add URL for PEP to the source code encoding warning.
Marc-André Lemburg <mal@egenix.com>
parents: 27772
diff changeset
468 sprintf(buf,
be371ffc4cd0 Add URL for PEP to the source code encoding warning.
Marc-André Lemburg <mal@egenix.com>
parents: 27772
diff changeset
469 "Non-ASCII character '\\x%.2x' "
be371ffc4cd0 Add URL for PEP to the source code encoding warning.
Marc-André Lemburg <mal@egenix.com>
parents: 27772
diff changeset
470 "in file %.200s on line %i, "
be371ffc4cd0 Add URL for PEP to the source code encoding warning.
Marc-André Lemburg <mal@egenix.com>
parents: 27772
diff changeset
471 "but no encoding declared; "
be371ffc4cd0 Add URL for PEP to the source code encoding warning.
Marc-André Lemburg <mal@egenix.com>
parents: 27772
diff changeset
472 "see http://www.python.org/peps/pep-0263.html for details",
be371ffc4cd0 Add URL for PEP to the source code encoding warning.
Marc-André Lemburg <mal@egenix.com>
parents: 27772
diff changeset
473 badchar, tok->filename, tok->lineno + 1);
be371ffc4cd0 Add URL for PEP to the source code encoding warning.
Marc-André Lemburg <mal@egenix.com>
parents: 27772
diff changeset
474 /* We don't use PyErr_WarnExplicit() here because
be371ffc4cd0 Add URL for PEP to the source code encoding warning.
Marc-André Lemburg <mal@egenix.com>
parents: 27772
diff changeset
475 printing the line in question to e.g. a log file
be371ffc4cd0 Add URL for PEP to the source code encoding warning.
Marc-André Lemburg <mal@egenix.com>
parents: 27772
diff changeset
476 could result in sensitive information being
be371ffc4cd0 Add URL for PEP to the source code encoding warning.
Marc-André Lemburg <mal@egenix.com>
parents: 27772
diff changeset
477 exposed. */
be371ffc4cd0 Add URL for PEP to the source code encoding warning.
Marc-André Lemburg <mal@egenix.com>
parents: 27772
diff changeset
478 PyErr_Warn(PyExc_DeprecationWarning, buf);
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
479 tok->issued_encoding_warning = 1;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
480 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
481 #endif
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
482 return line;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
483 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
484
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
485 static int
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
486 decoding_feof(struct tok_state *tok)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
487 {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
488 if (tok->decoding_state >= 0) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
489 return feof(tok->fp);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
490 } else {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
491 PyObject* buf = tok->decoding_buffer;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
492 if (buf == NULL) {
24867
255d1d3e66a3 Squash a few calls to the hideously expensive PyObject_CallObject(o,a)
Guido van Rossum <guido@python.org>
parents: 24839
diff changeset
493 PyObject *args = PyTuple_New(0);
255d1d3e66a3 Squash a few calls to the hideously expensive PyObject_CallObject(o,a)
Guido van Rossum <guido@python.org>
parents: 24839
diff changeset
494 if (args == NULL) {
255d1d3e66a3 Squash a few calls to the hideously expensive PyObject_CallObject(o,a)
Guido van Rossum <guido@python.org>
parents: 24839
diff changeset
495 error_ret(tok);
255d1d3e66a3 Squash a few calls to the hideously expensive PyObject_CallObject(o,a)
Guido van Rossum <guido@python.org>
parents: 24839
diff changeset
496 return 1;
255d1d3e66a3 Squash a few calls to the hideously expensive PyObject_CallObject(o,a)
Guido van Rossum <guido@python.org>
parents: 24839
diff changeset
497 }
255d1d3e66a3 Squash a few calls to the hideously expensive PyObject_CallObject(o,a)
Guido van Rossum <guido@python.org>
parents: 24839
diff changeset
498 buf = PyObject_Call(tok->decoding_readline,
255d1d3e66a3 Squash a few calls to the hideously expensive PyObject_CallObject(o,a)
Guido van Rossum <guido@python.org>
parents: 24839
diff changeset
499 args, NULL);
255d1d3e66a3 Squash a few calls to the hideously expensive PyObject_CallObject(o,a)
Guido van Rossum <guido@python.org>
parents: 24839
diff changeset
500 Py_DECREF(args);
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
501 if (buf == NULL) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
502 error_ret(tok);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
503 return 1;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
504 } else {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
505 tok->decoding_buffer = buf;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
506 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
507 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
508 return PyObject_Length(buf) == 0;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
509 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
510 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
511
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
512 /* Fetch a byte from TOK, using the string buffer. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
513
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
514 static int buf_getc(struct tok_state *tok) {
27772
3566b84e66b9 patch 680474 that fixes bug 679880: compile/eval/exec refused utf-8 bom
Just van Rossum <just@letterror.com>
parents: 27247
diff changeset
515 return Py_CHARMASK(*tok->str++);
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
516 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
517
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
518 /* Unfetch a byte from TOK, using the string buffer. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
519
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
520 static void buf_ungetc(int c, struct tok_state *tok) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
521 tok->str--;
27772
3566b84e66b9 patch 680474 that fixes bug 679880: compile/eval/exec refused utf-8 bom
Just van Rossum <just@letterror.com>
parents: 27247
diff changeset
522 assert(Py_CHARMASK(*tok->str) == c); /* tok->cur may point to read-only segment */
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
523 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
524
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
525 /* Set the readline function for TOK to ENC. For the string-based
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
526 tokenizer, this means to just record the encoding. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
527
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
528 static int buf_setreadl(struct tok_state *tok, const char* enc) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
529 tok->enc = enc;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
530 return 1;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
531 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
532
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
533 /* Return a UTF-8 encoding Python string object from the
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
534 C byte string STR, which is encoded with ENC. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
535
24624
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
536 #ifdef Py_USING_UNICODE
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
537 static PyObject *
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
538 translate_into_utf8(const char* str, const char* enc) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
539 PyObject *utf8;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
540 PyObject* buf = PyUnicode_Decode(str, strlen(str), enc, NULL);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
541 if (buf == NULL)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
542 return NULL;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
543 utf8 = PyUnicode_AsUTF8String(buf);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
544 Py_DECREF(buf);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
545 return utf8;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
546 }
24624
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
547 #endif
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
548
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
549 /* Decode a byte string STR for use as the buffer of TOK.
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
550 Look for encoding declarations inside STR, and record them
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
551 inside TOK. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
552
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
553 static const char *
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
554 decode_str(const char *str, struct tok_state *tok)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
555 {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
556 PyObject* utf8 = NULL;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
557 const char *s;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
558 int lineno = 0;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
559 tok->enc = NULL;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
560 tok->str = str;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
561 if (!check_bom(buf_getc, buf_ungetc, buf_setreadl, tok))
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
562 return NULL;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
563 str = tok->str; /* string after BOM if any */
24542
d0d3f760d067 Repaired a fatal compiler error in the debug build: it's not clear what
Tim Peters <tim.peters@gmail.com>
parents: 24541
diff changeset
564 assert(str);
24624
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
565 #ifdef Py_USING_UNICODE
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
566 if (tok->enc != NULL) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
567 utf8 = translate_into_utf8(str, tok->enc);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
568 if (utf8 == NULL)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
569 return NULL;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
570 str = PyString_AsString(utf8);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
571 }
24624
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
572 #endif
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
573 for (s = str;; s++) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
574 if (*s == '\0') break;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
575 else if (*s == '\n') {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
576 lineno++;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
577 if (lineno == 2) break;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
578 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
579 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
580 tok->enc = NULL;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
581 if (!check_coding_spec(str, s - str, tok, buf_setreadl))
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
582 return NULL;
24624
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
583 #ifdef Py_USING_UNICODE
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
584 if (tok->enc != NULL) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
585 assert(utf8 == NULL);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
586 utf8 = translate_into_utf8(str, tok->enc);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
587 if (utf8 == NULL)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
588 return NULL;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
589 str = PyString_AsString(utf8);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
590 }
24624
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
591 #endif
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
592 assert(tok->decoding_buffer == NULL);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
593 tok->decoding_buffer = utf8; /* CAUTION */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
594 return str;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
595 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
596
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
597 #endif /* PGEN */
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
598
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
599 /* Set up tokenizer for string */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
600
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
601 struct tok_state *
26601
cc05c2c8660e Constify filenames and scripts. Fixes #651362.
Martin v. Löwis <martin@v.loewis.de>
parents: 25995
diff changeset
602 PyTokenizer_FromString(const char *str)
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
603 {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
604 struct tok_state *tok = tok_new();
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
605 if (tok == NULL)
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
606 return NULL;
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
607 str = (char *)decode_str(str, tok);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
608 if (str == NULL)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
609 return NULL;
26601
cc05c2c8660e Constify filenames and scripts. Fixes #651362.
Martin v. Löwis <martin@v.loewis.de>
parents: 25995
diff changeset
610 /* XXX: constify members. */
cc05c2c8660e Constify filenames and scripts. Fixes #651362.
Martin v. Löwis <martin@v.loewis.de>
parents: 25995
diff changeset
611 tok->buf = tok->cur = tok->end = tok->inp = (char*)str;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
612 return tok;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
613 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
614
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
615
326
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
616 /* Set up tokenizer for file */
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
617
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
618 struct tok_state *
13659
e78153e179fb Mass ANSIfication.
Thomas Wouters <thomas@python.org>
parents: 13539
diff changeset
619 PyTokenizer_FromFile(FILE *fp, char *ps1, char *ps2)
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
620 {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
621 struct tok_state *tok = tok_new();
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
622 if (tok == NULL)
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
623 return NULL;
5379
1c9a84e0981d Another directory quickly renamed.
Guido van Rossum <guido@python.org>
parents: 5341
diff changeset
624 if ((tok->buf = PyMem_NEW(char, BUFSIZ)) == NULL) {
1c9a84e0981d Another directory quickly renamed.
Guido van Rossum <guido@python.org>
parents: 5341
diff changeset
625 PyMem_DEL(tok);
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
626 return NULL;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
627 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
628 tok->cur = tok->inp = tok->buf;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
629 tok->end = tok->buf + BUFSIZ;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
630 tok->fp = fp;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
631 tok->prompt = ps1;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
632 tok->nextprompt = ps2;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
633 return tok;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
634 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
635
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
636
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
637 /* Free a tok_state structure */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
638
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
639 void
13659
e78153e179fb Mass ANSIfication.
Thomas Wouters <thomas@python.org>
parents: 13539
diff changeset
640 PyTokenizer_Free(struct tok_state *tok)
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
641 {
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
642 if (tok->encoding != NULL)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
643 PyMem_DEL(tok->encoding);
24544
0bd521eb714e Make pgen compile with pydebug. Duplicate normalized names, as it may
Martin v. Löwis <martin@v.loewis.de>
parents: 24543
diff changeset
644 #ifndef PGEN
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
645 Py_XDECREF(tok->decoding_readline);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
646 Py_XDECREF(tok->decoding_buffer);
24544
0bd521eb714e Make pgen compile with pydebug. Duplicate normalized names, as it may
Martin v. Löwis <martin@v.loewis.de>
parents: 24543
diff changeset
647 #endif
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
648 if (tok->fp != NULL && tok->buf != NULL)
5379
1c9a84e0981d Another directory quickly renamed.
Guido van Rossum <guido@python.org>
parents: 5341
diff changeset
649 PyMem_DEL(tok->buf);
1c9a84e0981d Another directory quickly renamed.
Guido van Rossum <guido@python.org>
parents: 5341
diff changeset
650 PyMem_DEL(tok);
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
651 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
652
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
653
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
654 /* Get next char, updating state; error code goes into tok->done */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
655
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
656 static int
13659
e78153e179fb Mass ANSIfication.
Thomas Wouters <thomas@python.org>
parents: 13539
diff changeset
657 tok_nextc(register struct tok_state *tok)
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
658 {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
659 for (;;) {
1897
3fbda8768855 * Parser/tokenizer.c (tok_nextc): count line numbers when parsing
Guido van Rossum <guido@python.org>
parents: 1835
diff changeset
660 if (tok->cur != tok->inp) {
7118
e9108e97a5a0 tok_nextc() should return unsigned characters, to avoid mistaking
Guido van Rossum <guido@python.org>
parents: 5379
diff changeset
661 return Py_CHARMASK(*tok->cur++); /* Fast path */
1897
3fbda8768855 * Parser/tokenizer.c (tok_nextc): count line numbers when parsing
Guido van Rossum <guido@python.org>
parents: 1835
diff changeset
662 }
1282
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
663 if (tok->done != E_OK)
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
664 return EOF;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
665 if (tok->fp == NULL) {
1897
3fbda8768855 * Parser/tokenizer.c (tok_nextc): count line numbers when parsing
Guido van Rossum <guido@python.org>
parents: 1835
diff changeset
666 char *end = strchr(tok->inp, '\n');
3fbda8768855 * Parser/tokenizer.c (tok_nextc): count line numbers when parsing
Guido van Rossum <guido@python.org>
parents: 1835
diff changeset
667 if (end != NULL)
3fbda8768855 * Parser/tokenizer.c (tok_nextc): count line numbers when parsing
Guido van Rossum <guido@python.org>
parents: 1835
diff changeset
668 end++;
3fbda8768855 * Parser/tokenizer.c (tok_nextc): count line numbers when parsing
Guido van Rossum <guido@python.org>
parents: 1835
diff changeset
669 else {
3fbda8768855 * Parser/tokenizer.c (tok_nextc): count line numbers when parsing
Guido van Rossum <guido@python.org>
parents: 1835
diff changeset
670 end = strchr(tok->inp, '\0');
3fbda8768855 * Parser/tokenizer.c (tok_nextc): count line numbers when parsing
Guido van Rossum <guido@python.org>
parents: 1835
diff changeset
671 if (end == tok->inp) {
3fbda8768855 * Parser/tokenizer.c (tok_nextc): count line numbers when parsing
Guido van Rossum <guido@python.org>
parents: 1835
diff changeset
672 tok->done = E_EOF;
3fbda8768855 * Parser/tokenizer.c (tok_nextc): count line numbers when parsing
Guido van Rossum <guido@python.org>
parents: 1835
diff changeset
673 return EOF;
3fbda8768855 * Parser/tokenizer.c (tok_nextc): count line numbers when parsing
Guido van Rossum <guido@python.org>
parents: 1835
diff changeset
674 }
3fbda8768855 * Parser/tokenizer.c (tok_nextc): count line numbers when parsing
Guido van Rossum <guido@python.org>
parents: 1835
diff changeset
675 }
3fbda8768855 * Parser/tokenizer.c (tok_nextc): count line numbers when parsing
Guido van Rossum <guido@python.org>
parents: 1835
diff changeset
676 if (tok->start == NULL)
3fbda8768855 * Parser/tokenizer.c (tok_nextc): count line numbers when parsing
Guido van Rossum <guido@python.org>
parents: 1835
diff changeset
677 tok->buf = tok->cur;
3fbda8768855 * Parser/tokenizer.c (tok_nextc): count line numbers when parsing
Guido van Rossum <guido@python.org>
parents: 1835
diff changeset
678 tok->lineno++;
3fbda8768855 * Parser/tokenizer.c (tok_nextc): count line numbers when parsing
Guido van Rossum <guido@python.org>
parents: 1835
diff changeset
679 tok->inp = end;
7118
e9108e97a5a0 tok_nextc() should return unsigned characters, to avoid mistaking
Guido van Rossum <guido@python.org>
parents: 5379
diff changeset
680 return Py_CHARMASK(*tok->cur++);
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
681 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
682 if (tok->prompt != NULL) {
25924
dc4a0336a2a3 Patch #512981: Update readline input stream on sys.stdin/out change.
Martin v. Löwis <martin@v.loewis.de>
parents: 25101
diff changeset
683 char *new = PyOS_Readline(stdin, stdout, tok->prompt);
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
684 if (tok->nextprompt != NULL)
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
685 tok->prompt = tok->nextprompt;
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
686 if (new == NULL)
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
687 tok->done = E_INTR;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
688 else if (*new == '\0') {
12576
1c7c2dd1beb1 Vladimir Marangozov's long-awaited malloc restructuring.
Guido van Rossum <guido@python.org>
parents: 12224
diff changeset
689 PyMem_FREE(new);
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
690 tok->done = E_EOF;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
691 }
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
692 else if (tok->start != NULL) {
13093
cbe02b8aee07 Trent Mick: familiar simple Win64 patches
Guido van Rossum <guido@python.org>
parents: 12576
diff changeset
693 size_t start = tok->start - tok->buf;
cbe02b8aee07 Trent Mick: familiar simple Win64 patches
Guido van Rossum <guido@python.org>
parents: 12576
diff changeset
694 size_t oldlen = tok->cur - tok->buf;
cbe02b8aee07 Trent Mick: familiar simple Win64 patches
Guido van Rossum <guido@python.org>
parents: 12576
diff changeset
695 size_t newlen = oldlen + strlen(new);
12576
1c7c2dd1beb1 Vladimir Marangozov's long-awaited malloc restructuring.
Guido van Rossum <guido@python.org>
parents: 12224
diff changeset
696 char *buf = tok->buf;
1c7c2dd1beb1 Vladimir Marangozov's long-awaited malloc restructuring.
Guido van Rossum <guido@python.org>
parents: 12224
diff changeset
697 PyMem_RESIZE(buf, char, newlen+1);
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
698 tok->lineno++;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
699 if (buf == NULL) {
12576
1c7c2dd1beb1 Vladimir Marangozov's long-awaited malloc restructuring.
Guido van Rossum <guido@python.org>
parents: 12224
diff changeset
700 PyMem_DEL(tok->buf);
2044
281e40652e23 Parser/tokenizer.c (tok_nextc): zap tok->buf after freeing;
Guido van Rossum <guido@python.org>
parents: 1897
diff changeset
701 tok->buf = NULL;
12576
1c7c2dd1beb1 Vladimir Marangozov's long-awaited malloc restructuring.
Guido van Rossum <guido@python.org>
parents: 12224
diff changeset
702 PyMem_FREE(new);
1282
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
703 tok->done = E_NOMEM;
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
704 return EOF;
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
705 }
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
706 tok->buf = buf;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
707 tok->cur = tok->buf + oldlen;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
708 strcpy(tok->buf + oldlen, new);
12576
1c7c2dd1beb1 Vladimir Marangozov's long-awaited malloc restructuring.
Guido van Rossum <guido@python.org>
parents: 12224
diff changeset
709 PyMem_FREE(new);
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
710 tok->inp = tok->buf + newlen;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
711 tok->end = tok->inp + 1;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
712 tok->start = tok->buf + start;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
713 }
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
714 else {
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
715 tok->lineno++;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
716 if (tok->buf != NULL)
12576
1c7c2dd1beb1 Vladimir Marangozov's long-awaited malloc restructuring.
Guido van Rossum <guido@python.org>
parents: 12224
diff changeset
717 PyMem_DEL(tok->buf);
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
718 tok->buf = new;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
719 tok->cur = tok->buf;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
720 tok->inp = strchr(tok->buf, '\0');
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
721 tok->end = tok->inp + 1;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
722 }
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
723 }
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
724 else {
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
725 int done = 0;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
726 int cur = 0;
2732
daa150b3a6e4 replace "\r\n" with "\n" at line end (Jim Ahlstrom)
Guido van Rossum <guido@python.org>
parents: 2149
diff changeset
727 char *pt;
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
728 if (tok->start == NULL) {
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
729 if (tok->buf == NULL) {
5379
1c9a84e0981d Another directory quickly renamed.
Guido van Rossum <guido@python.org>
parents: 5341
diff changeset
730 tok->buf = PyMem_NEW(char, BUFSIZ);
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
731 if (tok->buf == NULL) {
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
732 tok->done = E_NOMEM;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
733 return EOF;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
734 }
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
735 tok->end = tok->buf + BUFSIZ;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
736 }
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
737 if (decoding_fgets(tok->buf, (int)(tok->end - tok->buf),
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
738 tok) == NULL) {
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
739 tok->done = E_EOF;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
740 done = 1;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
741 }
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
742 else {
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
743 tok->done = E_OK;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
744 tok->inp = strchr(tok->buf, '\0');
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
745 done = tok->inp[-1] == '\n';
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
746 }
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
747 }
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
748 else {
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
749 cur = tok->cur - tok->buf;
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
750 if (decoding_feof(tok)) {
2149
6060d7fb07e3 fix loop on unterminated triple quotes
Guido van Rossum <guido@python.org>
parents: 2060
diff changeset
751 tok->done = E_EOF;
6060d7fb07e3 fix loop on unterminated triple quotes
Guido van Rossum <guido@python.org>
parents: 2060
diff changeset
752 done = 1;
6060d7fb07e3 fix loop on unterminated triple quotes
Guido van Rossum <guido@python.org>
parents: 2060
diff changeset
753 }
6060d7fb07e3 fix loop on unterminated triple quotes
Guido van Rossum <guido@python.org>
parents: 2060
diff changeset
754 else
6060d7fb07e3 fix loop on unterminated triple quotes
Guido van Rossum <guido@python.org>
parents: 2060
diff changeset
755 tok->done = E_OK;
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
756 }
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
757 tok->lineno++;
1282
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
758 /* Read until '\n' or EOF */
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
759 while (!done) {
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
760 int curstart = tok->start == NULL ? -1 :
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
761 tok->start - tok->buf;
1282
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
762 int curvalid = tok->inp - tok->buf;
3116
db139a4112f3 fix bogus resize length in nextc
Guido van Rossum <guido@python.org>
parents: 2801
diff changeset
763 int newsize = curvalid + BUFSIZ;
1282
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
764 char *newbuf = tok->buf;
5379
1c9a84e0981d Another directory quickly renamed.
Guido van Rossum <guido@python.org>
parents: 5341
diff changeset
765 PyMem_RESIZE(newbuf, char, newsize);
1282
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
766 if (newbuf == NULL) {
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
767 tok->done = E_NOMEM;
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
768 tok->cur = tok->inp;
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
769 return EOF;
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
770 }
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
771 tok->buf = newbuf;
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
772 tok->inp = tok->buf + curvalid;
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
773 tok->end = tok->buf + newsize;
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
774 tok->start = curstart < 0 ? NULL :
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
775 tok->buf + curstart;
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
776 if (decoding_fgets(tok->inp,
1282
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
777 (int)(tok->end - tok->inp),
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
778 tok) == NULL) {
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
779 /* Last line does not end in \n,
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
780 fake one */
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
781 strcpy(tok->inp, "\n");
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
782 }
1282
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
783 tok->inp = strchr(tok->inp, '\0');
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
784 done = tok->inp[-1] == '\n';
1282
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
785 }
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
786 tok->cur = tok->buf + cur;
5311
782597cf6151 (Jack:) On the Mac, give syntax error on \r.
Guido van Rossum <guido@python.org>
parents: 5265
diff changeset
787 #ifndef macintosh
2732
daa150b3a6e4 replace "\r\n" with "\n" at line end (Jim Ahlstrom)
Guido van Rossum <guido@python.org>
parents: 2149
diff changeset
788 /* replace "\r\n" with "\n" */
5311
782597cf6151 (Jack:) On the Mac, give syntax error on \r.
Guido van Rossum <guido@python.org>
parents: 5265
diff changeset
789 /* For Mac we leave the \r, giving a syntax error */
2732
daa150b3a6e4 replace "\r\n" with "\n" at line end (Jim Ahlstrom)
Guido van Rossum <guido@python.org>
parents: 2149
diff changeset
790 pt = tok->inp - 2;
daa150b3a6e4 replace "\r\n" with "\n" at line end (Jim Ahlstrom)
Guido van Rossum <guido@python.org>
parents: 2149
diff changeset
791 if (pt >= tok->buf && *pt == '\r') {
daa150b3a6e4 replace "\r\n" with "\n" at line end (Jim Ahlstrom)
Guido van Rossum <guido@python.org>
parents: 2149
diff changeset
792 *pt++ = '\n';
daa150b3a6e4 replace "\r\n" with "\n" at line end (Jim Ahlstrom)
Guido van Rossum <guido@python.org>
parents: 2149
diff changeset
793 *pt = '\0';
daa150b3a6e4 replace "\r\n" with "\n" at line end (Jim Ahlstrom)
Guido van Rossum <guido@python.org>
parents: 2149
diff changeset
794 tok->inp = pt;
daa150b3a6e4 replace "\r\n" with "\n" at line end (Jim Ahlstrom)
Guido van Rossum <guido@python.org>
parents: 2149
diff changeset
795 }
5311
782597cf6151 (Jack:) On the Mac, give syntax error on \r.
Guido van Rossum <guido@python.org>
parents: 5265
diff changeset
796 #endif
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
797 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
798 if (tok->done != E_OK) {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
799 if (tok->prompt != NULL)
9047
6d2eb58f0418 Replace all calls to fprintf(stderr, ...) with PySys_WriteStderr(...).
Guido van Rossum <guido@python.org>
parents: 7945
diff changeset
800 PySys_WriteStderr("\n");
1282
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
801 tok->cur = tok->inp;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
802 return EOF;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
803 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
804 }
1282
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
805 /*NOTREACHED*/
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
806 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
807
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
808
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
809 /* Back-up one character */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
810
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
811 static void
13659
e78153e179fb Mass ANSIfication.
Thomas Wouters <thomas@python.org>
parents: 13539
diff changeset
812 tok_backup(register struct tok_state *tok, register int c)
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
813 {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
814 if (c != EOF) {
2044
281e40652e23 Parser/tokenizer.c (tok_nextc): zap tok->buf after freeing;
Guido van Rossum <guido@python.org>
parents: 1897
diff changeset
815 if (--tok->cur < tok->buf)
5379
1c9a84e0981d Another directory quickly renamed.
Guido van Rossum <guido@python.org>
parents: 5341
diff changeset
816 Py_FatalError("tok_backup: begin of buffer");
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
817 if (*tok->cur != c)
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
818 *tok->cur = c;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
819 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
820 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
821
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
822
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
823 /* Return the token corresponding to a single character */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
824
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
825 int
13659
e78153e179fb Mass ANSIfication.
Thomas Wouters <thomas@python.org>
parents: 13539
diff changeset
826 PyToken_OneChar(int c)
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
827 {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
828 switch (c) {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
829 case '(': return LPAR;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
830 case ')': return RPAR;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
831 case '[': return LSQB;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
832 case ']': return RSQB;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
833 case ':': return COLON;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
834 case ',': return COMMA;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
835 case ';': return SEMI;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
836 case '+': return PLUS;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
837 case '-': return MINUS;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
838 case '*': return STAR;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
839 case '/': return SLASH;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
840 case '|': return VBAR;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
841 case '&': return AMPER;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
842 case '<': return LESS;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
843 case '>': return GREATER;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
844 case '=': return EQUAL;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
845 case '.': return DOT;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
846 case '%': return PERCENT;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
847 case '`': return BACKQUOTE;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
848 case '{': return LBRACE;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
849 case '}': return RBRACE;
410
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
850 case '^': return CIRCUMFLEX;
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
851 case '~': return TILDE;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
852 default: return OP;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
853 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
854 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
855
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
856
400
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
857 int
13659
e78153e179fb Mass ANSIfication.
Thomas Wouters <thomas@python.org>
parents: 13539
diff changeset
858 PyToken_TwoChars(int c1, int c2)
400
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
859 {
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
860 switch (c1) {
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
861 case '=':
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
862 switch (c2) {
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
863 case '=': return EQEQUAL;
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
864 }
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
865 break;
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
866 case '!':
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
867 switch (c2) {
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
868 case '=': return NOTEQUAL;
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
869 }
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
870 break;
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
871 case '<':
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
872 switch (c2) {
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
873 case '>': return NOTEQUAL;
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
874 case '=': return LESSEQUAL;
410
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
875 case '<': return LEFTSHIFT;
400
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
876 }
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
877 break;
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
878 case '>':
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
879 switch (c2) {
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
880 case '=': return GREATEREQUAL;
410
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
881 case '>': return RIGHTSHIFT;
400
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
882 }
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
883 break;
14163
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
884 case '+':
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
885 switch (c2) {
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
886 case '=': return PLUSEQUAL;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
887 }
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
888 break;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
889 case '-':
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
890 switch (c2) {
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
891 case '=': return MINEQUAL;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
892 }
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
893 break;
3333
4b5bff95804e changes for pow(**) and complex
Guido van Rossum <guido@python.org>
parents: 3116
diff changeset
894 case '*':
4b5bff95804e changes for pow(**) and complex
Guido van Rossum <guido@python.org>
parents: 3116
diff changeset
895 switch (c2) {
4b5bff95804e changes for pow(**) and complex
Guido van Rossum <guido@python.org>
parents: 3116
diff changeset
896 case '*': return DOUBLESTAR;
14163
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
897 case '=': return STAREQUAL;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
898 }
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
899 break;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
900 case '/':
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
901 switch (c2) {
19051
f20727cc3d0f Implement PEP 238 in its (almost) full glory.
Guido van Rossum <guido@python.org>
parents: 17600
diff changeset
902 case '/': return DOUBLESLASH;
14163
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
903 case '=': return SLASHEQUAL;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
904 }
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
905 break;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
906 case '|':
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
907 switch (c2) {
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
908 case '=': return VBAREQUAL;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
909 }
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
910 break;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
911 case '%':
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
912 switch (c2) {
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
913 case '=': return PERCENTEQUAL;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
914 }
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
915 break;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
916 case '&':
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
917 switch (c2) {
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
918 case '=': return AMPEREQUAL;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
919 }
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
920 break;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
921 case '^':
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
922 switch (c2) {
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
923 case '=': return CIRCUMFLEXEQUAL;
3333
4b5bff95804e changes for pow(**) and complex
Guido van Rossum <guido@python.org>
parents: 3116
diff changeset
924 }
4b5bff95804e changes for pow(**) and complex
Guido van Rossum <guido@python.org>
parents: 3116
diff changeset
925 break;
400
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
926 }
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
927 return OP;
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
928 }
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
929
14163
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
930 int
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
931 PyToken_ThreeChars(int c1, int c2, int c3)
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
932 {
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
933 switch (c1) {
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
934 case '<':
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
935 switch (c2) {
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
936 case '<':
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
937 switch (c3) {
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
938 case '=':
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
939 return LEFTSHIFTEQUAL;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
940 }
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
941 break;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
942 }
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
943 break;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
944 case '>':
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
945 switch (c2) {
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
946 case '>':
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
947 switch (c3) {
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
948 case '=':
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
949 return RIGHTSHIFTEQUAL;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
950 }
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
951 break;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
952 }
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
953 break;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
954 case '*':
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
955 switch (c2) {
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
956 case '*':
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
957 switch (c3) {
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
958 case '=':
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
959 return DOUBLESTAREQUAL;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
960 }
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
961 break;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
962 }
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
963 break;
19051
f20727cc3d0f Implement PEP 238 in its (almost) full glory.
Guido van Rossum <guido@python.org>
parents: 17600
diff changeset
964 case '/':
f20727cc3d0f Implement PEP 238 in its (almost) full glory.
Guido van Rossum <guido@python.org>
parents: 17600
diff changeset
965 switch (c2) {
f20727cc3d0f Implement PEP 238 in its (almost) full glory.
Guido van Rossum <guido@python.org>
parents: 17600
diff changeset
966 case '/':
f20727cc3d0f Implement PEP 238 in its (almost) full glory.
Guido van Rossum <guido@python.org>
parents: 17600
diff changeset
967 switch (c3) {
f20727cc3d0f Implement PEP 238 in its (almost) full glory.
Guido van Rossum <guido@python.org>
parents: 17600
diff changeset
968 case '=':
f20727cc3d0f Implement PEP 238 in its (almost) full glory.
Guido van Rossum <guido@python.org>
parents: 17600
diff changeset
969 return DOUBLESLASHEQUAL;
f20727cc3d0f Implement PEP 238 in its (almost) full glory.
Guido van Rossum <guido@python.org>
parents: 17600
diff changeset
970 }
f20727cc3d0f Implement PEP 238 in its (almost) full glory.
Guido van Rossum <guido@python.org>
parents: 17600
diff changeset
971 break;
f20727cc3d0f Implement PEP 238 in its (almost) full glory.
Guido van Rossum <guido@python.org>
parents: 17600
diff changeset
972 }
f20727cc3d0f Implement PEP 238 in its (almost) full glory.
Guido van Rossum <guido@python.org>
parents: 17600
diff changeset
973 break;
14163
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
974 }
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
975 return OP;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
976 }
400
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
977
7945
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
978 static int
13659
e78153e179fb Mass ANSIfication.
Thomas Wouters <thomas@python.org>
parents: 13539
diff changeset
979 indenterror(struct tok_state *tok)
7945
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
980 {
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
981 if (tok->alterror) {
13539
feab677269e0 Create two new exceptions: IndentationError and TabError. These are
Fred Drake <fdrake@acm.org>
parents: 13447
diff changeset
982 tok->done = E_TABSPACE;
7945
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
983 tok->cur = tok->inp;
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
984 return 1;
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
985 }
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
986 if (tok->altwarning) {
13539
feab677269e0 Create two new exceptions: IndentationError and TabError. These are
Fred Drake <fdrake@acm.org>
parents: 13447
diff changeset
987 PySys_WriteStderr("%s: inconsistent use of tabs and spaces "
feab677269e0 Create two new exceptions: IndentationError and TabError. These are
Fred Drake <fdrake@acm.org>
parents: 13447
diff changeset
988 "in indentation\n", tok->filename);
7945
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
989 tok->altwarning = 0;
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
990 }
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
991 return 0;
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
992 }
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
993
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
994
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
995 /* Get next token, after space stripping etc. */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
996
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
997 static int
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
998 tok_get(register struct tok_state *tok, char **p_start, char **p_end)
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
999 {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1000 register int c;
326
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1001 int blankline;
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1002
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1003 *p_start = *p_end = NULL;
326
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1004 nextline:
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1005 tok->start = NULL;
326
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1006 blankline = 0;
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1007
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1008 /* Get indentation level */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1009 if (tok->atbol) {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1010 register int col = 0;
7945
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1011 register int altcol = 0;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1012 tok->atbol = 0;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1013 for (;;) {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1014 c = tok_nextc(tok);
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1015 if (c == ' ')
7945
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1016 col++, altcol++;
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1017 else if (c == '\t') {
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1018 col = (col/tok->tabsize + 1) * tok->tabsize;
7945
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1019 altcol = (altcol/tok->alttabsize + 1)
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1020 * tok->alttabsize;
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1021 }
2801
e911f2fb179c ignore control-l in whitespace
Guido van Rossum <guido@python.org>
parents: 2732
diff changeset
1022 else if (c == '\014') /* Control-L (formfeed) */
7945
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1023 col = altcol = 0; /* For Emacs users */
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1024 else
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1025 break;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1026 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1027 tok_backup(tok, c);
326
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1028 if (c == '#' || c == '\n') {
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1029 /* Lines with only whitespace and/or comments
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1030 shouldn't affect the indentation and are
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1031 not passed to the parser as NEWLINE tokens,
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1032 except *totally* empty lines in interactive
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1033 mode, which signal the end of a command group. */
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1034 if (col == 0 && c == '\n' && tok->prompt != NULL)
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1035 blankline = 0; /* Let it through */
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1036 else
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1037 blankline = 1; /* Ignore completely */
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1038 /* We can't jump back right here since we still
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1039 may need to skip to the end of a comment */
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1040 }
1284
dbce72afe261 * selectmodule.c: fix (another!) two memory leaks -- this time in list2set
Guido van Rossum <guido@python.org>
parents: 1282
diff changeset
1041 if (!blankline && tok->level == 0) {
326
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1042 if (col == tok->indstack[tok->indent]) {
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1043 /* No change */
7945
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1044 if (altcol != tok->altindstack[tok->indent]) {
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1045 if (indenterror(tok))
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1046 return ERRORTOKEN;
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1047 }
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1048 }
326
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1049 else if (col > tok->indstack[tok->indent]) {
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1050 /* Indent -- always one */
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1051 if (tok->indent+1 >= MAXINDENT) {
13539
feab677269e0 Create two new exceptions: IndentationError and TabError. These are
Fred Drake <fdrake@acm.org>
parents: 13447
diff changeset
1052 tok->done = E_TOODEEP;
1282
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
1053 tok->cur = tok->inp;
326
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1054 return ERRORTOKEN;
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1055 }
7945
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1056 if (altcol <= tok->altindstack[tok->indent]) {
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1057 if (indenterror(tok))
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1058 return ERRORTOKEN;
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1059 }
326
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1060 tok->pendin++;
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1061 tok->indstack[++tok->indent] = col;
7945
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1062 tok->altindstack[tok->indent] = altcol;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1063 }
326
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1064 else /* col < tok->indstack[tok->indent] */ {
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1065 /* Dedent -- any number, must be consistent */
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1066 while (tok->indent > 0 &&
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1067 col < tok->indstack[tok->indent]) {
7324
55125b1144fa Swap two statements in the dedent check loop. This makes absolutely
Guido van Rossum <guido@python.org>
parents: 7315
diff changeset
1068 tok->pendin--;
326
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1069 tok->indent--;
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1070 }
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1071 if (col != tok->indstack[tok->indent]) {
13539
feab677269e0 Create two new exceptions: IndentationError and TabError. These are
Fred Drake <fdrake@acm.org>
parents: 13447
diff changeset
1072 tok->done = E_DEDENT;
1282
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
1073 tok->cur = tok->inp;
326
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1074 return ERRORTOKEN;
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1075 }
7945
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1076 if (altcol != tok->altindstack[tok->indent]) {
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1077 if (indenterror(tok))
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1078 return ERRORTOKEN;
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1079 }
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1080 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1081 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1082 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1083
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1084 tok->start = tok->cur;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1085
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1086 /* Return pending indents/dedents */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1087 if (tok->pendin != 0) {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1088 if (tok->pendin < 0) {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1089 tok->pendin++;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1090 return DEDENT;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1091 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1092 else {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1093 tok->pendin--;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1094 return INDENT;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1095 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1096 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1097
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1098 again:
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1099 tok->start = NULL;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1100 /* Skip spaces */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1101 do {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1102 c = tok_nextc(tok);
2801
e911f2fb179c ignore control-l in whitespace
Guido van Rossum <guido@python.org>
parents: 2732
diff changeset
1103 } while (c == ' ' || c == '\t' || c == '\014');
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1104
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1105 /* Set start of current token */
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1106 tok->start = tok->cur - 1;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1107
12142
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1108 /* Skip comment, while looking for tab-setting magic */
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1109 if (c == '#') {
12142
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1110 static char *tabforms[] = {
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1111 "tab-width:", /* Emacs */
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1112 ":tabstop=", /* vim, full form */
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1113 ":ts=", /* vim, abbreviated form */
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1114 "set tabsize=", /* will vi never die? */
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1115 /* more templates can be added here to support other editors */
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1116 };
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1117 char cbuf[80];
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1118 char *tp, **cp;
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1119 tp = cbuf;
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1120 do {
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1121 *tp++ = c = tok_nextc(tok);
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1122 } while (c != EOF && c != '\n' &&
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1123 tp - cbuf + 1 < sizeof(cbuf));
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1124 *tp = '\0';
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1125 for (cp = tabforms;
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1126 cp < tabforms + sizeof(tabforms)/sizeof(tabforms[0]);
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1127 cp++) {
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1128 if ((tp = strstr(cbuf, *cp))) {
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1129 int newsize = atoi(tp + strlen(*cp));
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1130
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1131 if (newsize >= 1 && newsize <= 40) {
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1132 tok->tabsize = newsize;
12224
2dc1bdb4088e Only write message about changed Tab size with -v.
Guido van Rossum <guido@python.org>
parents: 12142
diff changeset
1133 if (Py_VerboseFlag)
2dc1bdb4088e Only write message about changed Tab size with -v.
Guido van Rossum <guido@python.org>
parents: 12142
diff changeset
1134 PySys_WriteStderr(
12142
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1135 "Tab size set to %d\n",
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1136 newsize);
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1137 }
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1138 }
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1139 }
12142
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1140 while (c != EOF && c != '\n')
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1141 c = tok_nextc(tok);
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1142 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1143
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1144 /* Check for EOF and errors now */
410
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
1145 if (c == EOF) {
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1146 return tok->done == E_EOF ? ENDMARKER : ERRORTOKEN;
410
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
1147 }
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1148
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1149 /* Identifier (most frequent token!) */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1150 if (isalpha(c) || c == '_') {
11971
574283db2b3d Marc-Andre Lemburg: add new string token types u"..." and ur"..."
Guido van Rossum <guido@python.org>
parents: 9590
diff changeset
1151 /* Process r"", u"" and ur"" */
5341
67e5237ce141 Now that the string-sig has settled on r"obin" strings, restrict the
Guido van Rossum <guido@python.org>
parents: 5311
diff changeset
1152 switch (c) {
67e5237ce141 Now that the string-sig has settled on r"obin" strings, restrict the
Guido van Rossum <guido@python.org>
parents: 5311
diff changeset
1153 case 'r':
67e5237ce141 Now that the string-sig has settled on r"obin" strings, restrict the
Guido van Rossum <guido@python.org>
parents: 5311
diff changeset
1154 case 'R':
67e5237ce141 Now that the string-sig has settled on r"obin" strings, restrict the
Guido van Rossum <guido@python.org>
parents: 5311
diff changeset
1155 c = tok_nextc(tok);
67e5237ce141 Now that the string-sig has settled on r"obin" strings, restrict the
Guido van Rossum <guido@python.org>
parents: 5311
diff changeset
1156 if (c == '"' || c == '\'')
67e5237ce141 Now that the string-sig has settled on r"obin" strings, restrict the
Guido van Rossum <guido@python.org>
parents: 5311
diff changeset
1157 goto letter_quote;
11971
574283db2b3d Marc-Andre Lemburg: add new string token types u"..." and ur"..."
Guido van Rossum <guido@python.org>
parents: 9590
diff changeset
1158 break;
574283db2b3d Marc-Andre Lemburg: add new string token types u"..." and ur"..."
Guido van Rossum <guido@python.org>
parents: 9590
diff changeset
1159 case 'u':
574283db2b3d Marc-Andre Lemburg: add new string token types u"..." and ur"..."
Guido van Rossum <guido@python.org>
parents: 9590
diff changeset
1160 case 'U':
574283db2b3d Marc-Andre Lemburg: add new string token types u"..." and ur"..."
Guido van Rossum <guido@python.org>
parents: 9590
diff changeset
1161 c = tok_nextc(tok);
574283db2b3d Marc-Andre Lemburg: add new string token types u"..." and ur"..."
Guido van Rossum <guido@python.org>
parents: 9590
diff changeset
1162 if (c == 'r' || c == 'R')
574283db2b3d Marc-Andre Lemburg: add new string token types u"..." and ur"..."
Guido van Rossum <guido@python.org>
parents: 9590
diff changeset
1163 c = tok_nextc(tok);
574283db2b3d Marc-Andre Lemburg: add new string token types u"..." and ur"..."
Guido van Rossum <guido@python.org>
parents: 9590
diff changeset
1164 if (c == '"' || c == '\'')
574283db2b3d Marc-Andre Lemburg: add new string token types u"..." and ur"..."
Guido van Rossum <guido@python.org>
parents: 9590
diff changeset
1165 goto letter_quote;
574283db2b3d Marc-Andre Lemburg: add new string token types u"..." and ur"..."
Guido van Rossum <guido@python.org>
parents: 9590
diff changeset
1166 break;
5341
67e5237ce141 Now that the string-sig has settled on r"obin" strings, restrict the
Guido van Rossum <guido@python.org>
parents: 5311
diff changeset
1167 }
5265
3c1788ab9761 Support for alternative string quotes (a"xx", b"xx", c"xx", ...).
Guido van Rossum <guido@python.org>
parents: 4839
diff changeset
1168 while (isalnum(c) || c == '_') {
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1169 c = tok_nextc(tok);
5265
3c1788ab9761 Support for alternative string quotes (a"xx", b"xx", c"xx", ...).
Guido van Rossum <guido@python.org>
parents: 4839
diff changeset
1170 }
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1171 tok_backup(tok, c);
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1172 *p_start = tok->start;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1173 *p_end = tok->cur;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1174 return NAME;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1175 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1176
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1177 /* Newline */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1178 if (c == '\n') {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1179 tok->atbol = 1;
1284
dbce72afe261 * selectmodule.c: fix (another!) two memory leaks -- this time in list2set
Guido van Rossum <guido@python.org>
parents: 1282
diff changeset
1180 if (blankline || tok->level > 0)
326
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1181 goto nextline;
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1182 *p_start = tok->start;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1183 *p_end = tok->cur - 1; /* Leave '\n' out of the string */
25098
c02a668488d0 Ignore encoding declarations inside strings. Fixes #603509.
Martin v. Löwis <martin@v.loewis.de>
parents: 24867
diff changeset
1184 tok->cont_line = 0;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1185 return NEWLINE;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1186 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1187
5311
782597cf6151 (Jack:) On the Mac, give syntax error on \r.
Guido van Rossum <guido@python.org>
parents: 5265
diff changeset
1188 #ifdef macintosh
782597cf6151 (Jack:) On the Mac, give syntax error on \r.
Guido van Rossum <guido@python.org>
parents: 5265
diff changeset
1189 if (c == '\r') {
9047
6d2eb58f0418 Replace all calls to fprintf(stderr, ...) with PySys_WriteStderr(...).
Guido van Rossum <guido@python.org>
parents: 7945
diff changeset
1190 PySys_WriteStderr(
5379
1c9a84e0981d Another directory quickly renamed.
Guido van Rossum <guido@python.org>
parents: 5341
diff changeset
1191 "File contains \\r characters (incorrect line endings?)\n");
5311
782597cf6151 (Jack:) On the Mac, give syntax error on \r.
Guido van Rossum <guido@python.org>
parents: 5265
diff changeset
1192 tok->done = E_TOKEN;
782597cf6151 (Jack:) On the Mac, give syntax error on \r.
Guido van Rossum <guido@python.org>
parents: 5265
diff changeset
1193 tok->cur = tok->inp;
782597cf6151 (Jack:) On the Mac, give syntax error on \r.
Guido van Rossum <guido@python.org>
parents: 5265
diff changeset
1194 return ERRORTOKEN;
782597cf6151 (Jack:) On the Mac, give syntax error on \r.
Guido van Rossum <guido@python.org>
parents: 5265
diff changeset
1195 }
782597cf6151 (Jack:) On the Mac, give syntax error on \r.
Guido van Rossum <guido@python.org>
parents: 5265
diff changeset
1196 #endif
410
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
1197 /* Period or number starting with period? */
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
1198 if (c == '.') {
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
1199 c = tok_nextc(tok);
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
1200 if (isdigit(c)) {
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
1201 goto fraction;
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
1202 }
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
1203 else {
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
1204 tok_backup(tok, c);
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1205 *p_start = tok->start;
410
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
1206 *p_end = tok->cur;
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
1207 return DOT;
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
1208 }
fd061dae9e57 Added shift and mask ops.
Guido van Rossum <guido@python.org>
parents: 400
diff changeset
1209 }
3333
4b5bff95804e changes for pow(**) and complex
Guido van Rossum <guido@python.org>
parents: 3116
diff changeset
1210
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1211 /* Number */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1212 if (isdigit(c)) {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1213 if (c == '0') {
19621
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1214 /* Hex or octal -- maybe. */
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1215 c = tok_nextc(tok);
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1216 if (c == '.')
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1217 goto fraction;
3333
4b5bff95804e changes for pow(**) and complex
Guido van Rossum <guido@python.org>
parents: 3116
diff changeset
1218 #ifndef WITHOUT_COMPLEX
3364
bc17e8313ea0 use only j for imaginary constants
Guido van Rossum <guido@python.org>
parents: 3333
diff changeset
1219 if (c == 'j' || c == 'J')
3333
4b5bff95804e changes for pow(**) and complex
Guido van Rossum <guido@python.org>
parents: 3116
diff changeset
1220 goto imaginary;
4b5bff95804e changes for pow(**) and complex
Guido van Rossum <guido@python.org>
parents: 3116
diff changeset
1221 #endif
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1222 if (c == 'x' || c == 'X') {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1223 /* Hex */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1224 do {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1225 c = tok_nextc(tok);
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1226 } while (isxdigit(c));
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1227 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1228 else {
19621
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1229 int found_decimal = 0;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1230 /* Octal; c is first char of it */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1231 /* There's no 'isoctdigit' macro, sigh */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1232 while ('0' <= c && c < '8') {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1233 c = tok_nextc(tok);
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1234 }
19621
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1235 if (isdigit(c)) {
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1236 found_decimal = 1;
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1237 do {
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1238 c = tok_nextc(tok);
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1239 } while (isdigit(c));
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1240 }
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1241 if (c == '.')
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1242 goto fraction;
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1243 else if (c == 'e' || c == 'E')
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1244 goto exponent;
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1245 #ifndef WITHOUT_COMPLEX
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1246 else if (c == 'j' || c == 'J')
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1247 goto imaginary;
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1248 #endif
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1249 else if (found_decimal) {
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1250 tok->done = E_TOKEN;
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1251 tok_backup(tok, c);
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1252 return ERRORTOKEN;
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1253 }
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1254 }
236
0a92b6a1b9c0 Added recognition of 'l' or 'L' as long integer suffix
Guido van Rossum <guido@python.org>
parents: 145
diff changeset
1255 if (c == 'l' || c == 'L')
0a92b6a1b9c0 Added recognition of 'l' or 'L' as long integer suffix
Guido van Rossum <guido@python.org>
parents: 145
diff changeset
1256 c = tok_nextc(tok);
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1257 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1258 else {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1259 /* Decimal */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1260 do {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1261 c = tok_nextc(tok);
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1262 } while (isdigit(c));
236
0a92b6a1b9c0 Added recognition of 'l' or 'L' as long integer suffix
Guido van Rossum <guido@python.org>
parents: 145
diff changeset
1263 if (c == 'l' || c == 'L')
0a92b6a1b9c0 Added recognition of 'l' or 'L' as long integer suffix
Guido van Rossum <guido@python.org>
parents: 145
diff changeset
1264 c = tok_nextc(tok);
0a92b6a1b9c0 Added recognition of 'l' or 'L' as long integer suffix
Guido van Rossum <guido@python.org>
parents: 145
diff changeset
1265 else {
19501
ce4f1e98ba7e SF bug [#455775] float parsing discrepancy.
Tim Peters <tim.peters@gmail.com>
parents: 19051
diff changeset
1266 /* Accept floating point numbers. */
236
0a92b6a1b9c0 Added recognition of 'l' or 'L' as long integer suffix
Guido van Rossum <guido@python.org>
parents: 145
diff changeset
1267 if (c == '.') {
0a92b6a1b9c0 Added recognition of 'l' or 'L' as long integer suffix
Guido van Rossum <guido@python.org>
parents: 145
diff changeset
1268 fraction:
0a92b6a1b9c0 Added recognition of 'l' or 'L' as long integer suffix
Guido van Rossum <guido@python.org>
parents: 145
diff changeset
1269 /* Fraction */
0a92b6a1b9c0 Added recognition of 'l' or 'L' as long integer suffix
Guido van Rossum <guido@python.org>
parents: 145
diff changeset
1270 do {
0a92b6a1b9c0 Added recognition of 'l' or 'L' as long integer suffix
Guido van Rossum <guido@python.org>
parents: 145
diff changeset
1271 c = tok_nextc(tok);
0a92b6a1b9c0 Added recognition of 'l' or 'L' as long integer suffix
Guido van Rossum <guido@python.org>
parents: 145
diff changeset
1272 } while (isdigit(c));
0a92b6a1b9c0 Added recognition of 'l' or 'L' as long integer suffix
Guido van Rossum <guido@python.org>
parents: 145
diff changeset
1273 }
0a92b6a1b9c0 Added recognition of 'l' or 'L' as long integer suffix
Guido van Rossum <guido@python.org>
parents: 145
diff changeset
1274 if (c == 'e' || c == 'E') {
19621
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1275 exponent:
236
0a92b6a1b9c0 Added recognition of 'l' or 'L' as long integer suffix
Guido van Rossum <guido@python.org>
parents: 145
diff changeset
1276 /* Exponent part */
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1277 c = tok_nextc(tok);
236
0a92b6a1b9c0 Added recognition of 'l' or 'L' as long integer suffix
Guido van Rossum <guido@python.org>
parents: 145
diff changeset
1278 if (c == '+' || c == '-')
0a92b6a1b9c0 Added recognition of 'l' or 'L' as long integer suffix
Guido van Rossum <guido@python.org>
parents: 145
diff changeset
1279 c = tok_nextc(tok);
19501
ce4f1e98ba7e SF bug [#455775] float parsing discrepancy.
Tim Peters <tim.peters@gmail.com>
parents: 19051
diff changeset
1280 if (!isdigit(c)) {
ce4f1e98ba7e SF bug [#455775] float parsing discrepancy.
Tim Peters <tim.peters@gmail.com>
parents: 19051
diff changeset
1281 tok->done = E_TOKEN;
ce4f1e98ba7e SF bug [#455775] float parsing discrepancy.
Tim Peters <tim.peters@gmail.com>
parents: 19051
diff changeset
1282 tok_backup(tok, c);
ce4f1e98ba7e SF bug [#455775] float parsing discrepancy.
Tim Peters <tim.peters@gmail.com>
parents: 19051
diff changeset
1283 return ERRORTOKEN;
ce4f1e98ba7e SF bug [#455775] float parsing discrepancy.
Tim Peters <tim.peters@gmail.com>
parents: 19051
diff changeset
1284 }
ce4f1e98ba7e SF bug [#455775] float parsing discrepancy.
Tim Peters <tim.peters@gmail.com>
parents: 19051
diff changeset
1285 do {
236
0a92b6a1b9c0 Added recognition of 'l' or 'L' as long integer suffix
Guido van Rossum <guido@python.org>
parents: 145
diff changeset
1286 c = tok_nextc(tok);
19501
ce4f1e98ba7e SF bug [#455775] float parsing discrepancy.
Tim Peters <tim.peters@gmail.com>
parents: 19051
diff changeset
1287 } while (isdigit(c));
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1288 }
3333
4b5bff95804e changes for pow(**) and complex
Guido van Rossum <guido@python.org>
parents: 3116
diff changeset
1289 #ifndef WITHOUT_COMPLEX
3364
bc17e8313ea0 use only j for imaginary constants
Guido van Rossum <guido@python.org>
parents: 3333
diff changeset
1290 if (c == 'j' || c == 'J')
3333
4b5bff95804e changes for pow(**) and complex
Guido van Rossum <guido@python.org>
parents: 3116
diff changeset
1291 /* Imaginary part */
4b5bff95804e changes for pow(**) and complex
Guido van Rossum <guido@python.org>
parents: 3116
diff changeset
1292 imaginary:
4b5bff95804e changes for pow(**) and complex
Guido van Rossum <guido@python.org>
parents: 3116
diff changeset
1293 c = tok_nextc(tok);
4b5bff95804e changes for pow(**) and complex
Guido van Rossum <guido@python.org>
parents: 3116
diff changeset
1294 #endif
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1295 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1296 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1297 tok_backup(tok, c);
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1298 *p_start = tok->start;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1299 *p_end = tok->cur;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1300 return NUMBER;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1301 }
5265
3c1788ab9761 Support for alternative string quotes (a"xx", b"xx", c"xx", ...).
Guido van Rossum <guido@python.org>
parents: 4839
diff changeset
1302
3c1788ab9761 Support for alternative string quotes (a"xx", b"xx", c"xx", ...).
Guido van Rossum <guido@python.org>
parents: 4839
diff changeset
1303 letter_quote:
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1304 /* String */
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1305 if (c == '\'' || c == '"') {
7315
3a55fbb99f20 Fixed the bug in searching for triple quotes -- change the 'quote2'
Guido van Rossum <guido@python.org>
parents: 7118
diff changeset
1306 int quote2 = tok->cur - tok->start + 1;
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1307 int quote = c;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1308 int triple = 0;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1309 int tripcount = 0;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1310 for (;;) {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1311 c = tok_nextc(tok);
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1312 if (c == '\n') {
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1313 if (!triple) {
24839
17e95cc06f69 provide less mysterious error messages when seeing end-of-line in
Skip Montanaro <skip@pobox.com>
parents: 24630
diff changeset
1314 tok->done = E_EOLS;
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1315 tok_backup(tok, c);
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1316 return ERRORTOKEN;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1317 }
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1318 tripcount = 0;
25098
c02a668488d0 Ignore encoding declarations inside strings. Fixes #603509.
Martin v. Löwis <martin@v.loewis.de>
parents: 24867
diff changeset
1319 tok->cont_line = 1; /* multiline string. */
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1320 }
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1321 else if (c == EOF) {
24839
17e95cc06f69 provide less mysterious error messages when seeing end-of-line in
Skip Montanaro <skip@pobox.com>
parents: 24630
diff changeset
1322 if (triple)
17e95cc06f69 provide less mysterious error messages when seeing end-of-line in
Skip Montanaro <skip@pobox.com>
parents: 24630
diff changeset
1323 tok->done = E_EOFS;
17e95cc06f69 provide less mysterious error messages when seeing end-of-line in
Skip Montanaro <skip@pobox.com>
parents: 24630
diff changeset
1324 else
17e95cc06f69 provide less mysterious error messages when seeing end-of-line in
Skip Montanaro <skip@pobox.com>
parents: 24630
diff changeset
1325 tok->done = E_EOLS;
1282
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
1326 tok->cur = tok->inp;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1327 return ERRORTOKEN;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1328 }
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1329 else if (c == quote) {
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1330 tripcount++;
7315
3a55fbb99f20 Fixed the bug in searching for triple quotes -- change the 'quote2'
Guido van Rossum <guido@python.org>
parents: 7118
diff changeset
1331 if (tok->cur - tok->start == quote2) {
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1332 c = tok_nextc(tok);
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1333 if (c == quote) {
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1334 triple = 1;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1335 tripcount = 0;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1336 continue;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1337 }
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1338 tok_backup(tok, c);
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1339 }
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1340 if (!triple || tripcount == 3)
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1341 break;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1342 }
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1343 else if (c == '\\') {
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1344 tripcount = 0;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1345 c = tok_nextc(tok);
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1346 if (c == EOF) {
24839
17e95cc06f69 provide less mysterious error messages when seeing end-of-line in
Skip Montanaro <skip@pobox.com>
parents: 24630
diff changeset
1347 tok->done = E_EOLS;
1282
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
1348 tok->cur = tok->inp;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1349 return ERRORTOKEN;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1350 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1351 }
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1352 else
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1353 tripcount = 0;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1354 }
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1355 *p_start = tok->start;
1364
132602d854d8 Changes to accept double-quoted strings on input.
Guido van Rossum <guido@python.org>
parents: 1284
diff changeset
1356 *p_end = tok->cur;
132602d854d8 Changes to accept double-quoted strings on input.
Guido van Rossum <guido@python.org>
parents: 1284
diff changeset
1357 return STRING;
132602d854d8 Changes to accept double-quoted strings on input.
Guido van Rossum <guido@python.org>
parents: 1284
diff changeset
1358 }
132602d854d8 Changes to accept double-quoted strings on input.
Guido van Rossum <guido@python.org>
parents: 1284
diff changeset
1359
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1360 /* Line continuation */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1361 if (c == '\\') {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1362 c = tok_nextc(tok);
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1363 if (c != '\n') {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1364 tok->done = E_TOKEN;
1282
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
1365 tok->cur = tok->inp;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1366 return ERRORTOKEN;
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1367 }
25098
c02a668488d0 Ignore encoding declarations inside strings. Fixes #603509.
Martin v. Löwis <martin@v.loewis.de>
parents: 24867
diff changeset
1368 tok->cont_line = 1;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1369 goto again; /* Read next line */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1370 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1371
400
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1372 /* Check for two-character token */
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1373 {
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1374 int c2 = tok_nextc(tok);
5379
1c9a84e0981d Another directory quickly renamed.
Guido van Rossum <guido@python.org>
parents: 5341
diff changeset
1375 int token = PyToken_TwoChars(c, c2);
400
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1376 if (token != OP) {
14163
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1377 int c3 = tok_nextc(tok);
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1378 int token3 = PyToken_ThreeChars(c, c2, c3);
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1379 if (token3 != OP) {
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1380 token = token3;
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1381 } else {
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1382 tok_backup(tok, c3);
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1383 }
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1384 *p_start = tok->start;
400
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1385 *p_end = tok->cur;
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1386 return token;
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1387 }
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1388 tok_backup(tok, c2);
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1389 }
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1390
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1391 /* Keep track of parentheses nesting level */
1284
dbce72afe261 * selectmodule.c: fix (another!) two memory leaks -- this time in list2set
Guido van Rossum <guido@python.org>
parents: 1282
diff changeset
1392 switch (c) {
dbce72afe261 * selectmodule.c: fix (another!) two memory leaks -- this time in list2set
Guido van Rossum <guido@python.org>
parents: 1282
diff changeset
1393 case '(':
dbce72afe261 * selectmodule.c: fix (another!) two memory leaks -- this time in list2set
Guido van Rossum <guido@python.org>
parents: 1282
diff changeset
1394 case '[':
dbce72afe261 * selectmodule.c: fix (another!) two memory leaks -- this time in list2set
Guido van Rossum <guido@python.org>
parents: 1282
diff changeset
1395 case '{':
dbce72afe261 * selectmodule.c: fix (another!) two memory leaks -- this time in list2set
Guido van Rossum <guido@python.org>
parents: 1282
diff changeset
1396 tok->level++;
dbce72afe261 * selectmodule.c: fix (another!) two memory leaks -- this time in list2set
Guido van Rossum <guido@python.org>
parents: 1282
diff changeset
1397 break;
dbce72afe261 * selectmodule.c: fix (another!) two memory leaks -- this time in list2set
Guido van Rossum <guido@python.org>
parents: 1282
diff changeset
1398 case ')':
dbce72afe261 * selectmodule.c: fix (another!) two memory leaks -- this time in list2set
Guido van Rossum <guido@python.org>
parents: 1282
diff changeset
1399 case ']':
dbce72afe261 * selectmodule.c: fix (another!) two memory leaks -- this time in list2set
Guido van Rossum <guido@python.org>
parents: 1282
diff changeset
1400 case '}':
dbce72afe261 * selectmodule.c: fix (another!) two memory leaks -- this time in list2set
Guido van Rossum <guido@python.org>
parents: 1282
diff changeset
1401 tok->level--;
dbce72afe261 * selectmodule.c: fix (another!) two memory leaks -- this time in list2set
Guido van Rossum <guido@python.org>
parents: 1282
diff changeset
1402 break;
dbce72afe261 * selectmodule.c: fix (another!) two memory leaks -- this time in list2set
Guido van Rossum <guido@python.org>
parents: 1282
diff changeset
1403 }
dbce72afe261 * selectmodule.c: fix (another!) two memory leaks -- this time in list2set
Guido van Rossum <guido@python.org>
parents: 1282
diff changeset
1404
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1405 /* Punctuation character */
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
1406 *p_start = tok->start;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1407 *p_end = tok->cur;
5379
1c9a84e0981d Another directory quickly renamed.
Guido van Rossum <guido@python.org>
parents: 5341
diff changeset
1408 return PyToken_OneChar(c);
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1409 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1410
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
1411 int
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
1412 PyTokenizer_Get(struct tok_state *tok, char **p_start, char **p_end)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
1413 {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
1414 int result = tok_get(tok, p_start, p_end);
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
1415 if (tok->decoding_erred) {
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
1416 result = ERRORTOKEN;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
1417 tok->done = E_DECODE;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
1418 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
1419 return result;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
1420 }
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1421
4839
4567e158d20c Rename DEBUG macro to Py_DEBUG
Guido van Rossum <guido@python.org>
parents: 4577
diff changeset
1422 #ifdef Py_DEBUG
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1423
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1424 void
13659
e78153e179fb Mass ANSIfication.
Thomas Wouters <thomas@python.org>
parents: 13539
diff changeset
1425 tok_dump(int type, char *start, char *end)
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1426 {
5379
1c9a84e0981d Another directory quickly renamed.
Guido van Rossum <guido@python.org>
parents: 5341
diff changeset
1427 printf("%s", _PyParser_TokenNames[type]);
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1428 if (type == NAME || type == NUMBER || type == STRING || type == OP)
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1429 printf("(%.*s)", (int)(end - start), start);
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1430 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1431
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1432 #endif