Mercurial > cpython-lru_cache-weakref

annotate Parser/tokenizer.c @ 80661:94de77bd0b4b 2.7

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Issue #16194: document imp.load_dynamic problems when read several extension modules from the same binary object file.
author Andrew Svetlov <andrew.svetlov@gmail.com>
date Thu, 29 Nov 2012 19:51:28 +0200
parents b36cb4602e21
children 570b5b4040b1
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
145
853b736a6a9e Added copyright notice.
Guido van Rossum <guido@python.org>
parents: 96
diff changeset
1
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
2 /* Tokenizer implementation */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
3
23029
8291473fde3e Mass checkin of universal newline support.
Jack Jansen <jack.jansen@cwi.nl>
parents: 19621
diff changeset
4 #include "Python.h"
86
a6934380c6e7 "Compiling" version
Guido van Rossum <guido@python.org>
parents: 67
diff changeset
5 #include "pgenheaders.h"
a6934380c6e7 "Compiling" version
Guido van Rossum <guido@python.org>
parents: 67
diff changeset
6
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
7 #include <ctype.h>
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
8 #include <assert.h>
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
9
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
10 #include "tokenizer.h"
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
11 #include "errcode.h"
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
12
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
13 #ifndef PGEN
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
14 #include "unicodeobject.h"
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
15 #include "stringobject.h"
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
16 #include "fileobject.h"
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
17 #include "codecs.h"
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
18 #include "abstract.h"
43982
5f6a8e2b9e9c Applied patch #1754273 and #1754271 from Thomas Glee
Christian Heimes <christian@cheimes.de>
parents: 43572
diff changeset
19 #include "pydebug.h"
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
20 #endif /* PGEN */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
21
25924
dc4a0336a2a3 Patch #512981: Update readline input stream on sys.stdin/out change.
Martin v. Löwis <martin@v.loewis.de>
parents: 25101
diff changeset
22 extern char *PyOS_Readline(FILE *, FILE *, char *);
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
23 /* Return malloc'ed string including trailing \n;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
24 empty malloc'ed string for EOF;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
25 NULL if interrupted */
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
26
678
205b947b1d3c Make tabs always 8 spaces wide -- it's more portable.
Guido van Rossum <guido@python.org>
parents: 477
diff changeset
27 /* Don't ever change this -- it would break the portability of Python code */
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
28 #define TABSIZE 8
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
29
86
a6934380c6e7 "Compiling" version
Guido van Rossum <guido@python.org>
parents: 67
diff changeset
30 /* Forward */
13447
a89f7f268de4 Nuke all remaining occurrences of Py_PROTO and Py_FPROTO.
Tim Peters <tim.peters@gmail.com>
parents: 13256
diff changeset
31 static struct tok_state *tok_new(void);
a89f7f268de4 Nuke all remaining occurrences of Py_PROTO and Py_FPROTO.
Tim Peters <tim.peters@gmail.com>
parents: 13256
diff changeset
32 static int tok_nextc(struct tok_state *tok);
a89f7f268de4 Nuke all remaining occurrences of Py_PROTO and Py_FPROTO.
Tim Peters <tim.peters@gmail.com>
parents: 13256
diff changeset
33 static void tok_backup(struct tok_state *tok, int c);
86
a6934380c6e7 "Compiling" version
Guido van Rossum <guido@python.org>
parents: 67
diff changeset
34
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
35 /* Token names */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
36
5379
1c9a84e0981d Another directory quickly renamed.
Guido van Rossum <guido@python.org>
parents: 5341
diff changeset
37 char *_PyParser_TokenNames[] = {
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
38 "ENDMARKER",
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
39 "NAME",
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
40 "NUMBER",
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
41 "STRING",
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
42 "NEWLINE",
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
43 "INDENT",
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
44 "DEDENT",
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
45 "LPAR",
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
46 "RPAR",
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
47 "LSQB",
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
48 "RSQB",
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
49 "COLON",
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
50 "COMMA",
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
51 "SEMI",
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
52 "PLUS",
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
53 "MINUS",
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
54 "STAR",
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
55 "SLASH",
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
56 "VBAR",
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
57 "AMPER",
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
58 "LESS",
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
59 "GREATER",
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
60 "EQUAL",
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
61 "DOT",
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
62 "PERCENT",
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
63 "BACKQUOTE",
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
64 "LBRACE",
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
65 "RBRACE",
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
66 "EQEQUAL",
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
67 "NOTEQUAL",
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
68 "LESSEQUAL",
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
69 "GREATEREQUAL",
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
70 "TILDE",
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
71 "CIRCUMFLEX",
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
72 "LEFTSHIFT",
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
73 "RIGHTSHIFT",
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
74 "DOUBLESTAR",
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
75 "PLUSEQUAL",
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
76 "MINEQUAL",
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
77 "STAREQUAL",
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
78 "SLASHEQUAL",
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
79 "PERCENTEQUAL",
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
80 "AMPEREQUAL",
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
81 "VBAREQUAL",
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
82 "CIRCUMFLEXEQUAL",
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
83 "LEFTSHIFTEQUAL",
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
84 "RIGHTSHIFTEQUAL",
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
85 "DOUBLESTAREQUAL",
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
86 "DOUBLESLASH",
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
87 "DOUBLESLASHEQUAL",
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
88 "AT",
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
89 /* This table must match the #defines in token.h! */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
90 "OP",
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
91 "<ERRORTOKEN>",
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
92 "<N_TOKENS>"
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
93 };
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
94
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
95 /* Create and initialize a new tok_state structure */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
96
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
97 static struct tok_state *
13659
e78153e179fb Mass ANSIfication.
Thomas Wouters <thomas@python.org>
parents: 13539
diff changeset
98 tok_new(void)
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
99 {
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
100 struct tok_state *tok = (struct tok_state *)PyMem_MALLOC(
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
101 sizeof(struct tok_state));
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
102 if (tok == NULL)
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
103 return NULL;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
104 tok->buf = tok->cur = tok->end = tok->inp = tok->start = NULL;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
105 tok->done = E_OK;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
106 tok->fp = NULL;
61463
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 60289
diff changeset
107 tok->input = NULL;
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
108 tok->tabsize = TABSIZE;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
109 tok->indent = 0;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
110 tok->indstack[0] = 0;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
111 tok->atbol = 1;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
112 tok->pendin = 0;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
113 tok->prompt = tok->nextprompt = NULL;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
114 tok->lineno = 0;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
115 tok->level = 0;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
116 tok->filename = NULL;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
117 tok->altwarning = 0;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
118 tok->alterror = 0;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
119 tok->alttabsize = 1;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
120 tok->altindstack[0] = 0;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
121 tok->decoding_state = 0;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
122 tok->decoding_erred = 0;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
123 tok->read_coding_spec = 0;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
124 tok->encoding = NULL;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
125 tok->cont_line = 0;
24544
0bd521eb714e Make pgen compile with pydebug. Duplicate normalized names, as it may
Martin v. Löwis <martin@v.loewis.de>
parents: 24543
diff changeset
126 #ifndef PGEN
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
127 tok->decoding_readline = NULL;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
128 tok->decoding_buffer = NULL;
24544
0bd521eb714e Make pgen compile with pydebug. Duplicate normalized names, as it may
Martin v. Löwis <martin@v.loewis.de>
parents: 24543
diff changeset
129 #endif
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
130 return tok;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
131 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
132
57204
89f42938d5cc fix several compile() issues by translating newlines in the tokenizer
Benjamin Peterson <benjamin@python.org>
parents: 57131
diff changeset
133 static char *
89f42938d5cc fix several compile() issues by translating newlines in the tokenizer
Benjamin Peterson <benjamin@python.org>
parents: 57131
diff changeset
134 new_string(const char *s, Py_ssize_t len)
89f42938d5cc fix several compile() issues by translating newlines in the tokenizer
Benjamin Peterson <benjamin@python.org>
parents: 57131
diff changeset
135 {
61463
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 60289
diff changeset
136 char* result = (char *)PyMem_MALLOC(len + 1);
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 60289
diff changeset
137 if (result != NULL) {
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 60289
diff changeset
138 memcpy(result, s, len);
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 60289
diff changeset
139 result[len] = '\0';
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 60289
diff changeset
140 }
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 60289
diff changeset
141 return result;
57204
89f42938d5cc fix several compile() issues by translating newlines in the tokenizer
Benjamin Peterson <benjamin@python.org>
parents: 57131
diff changeset
142 }
89f42938d5cc fix several compile() issues by translating newlines in the tokenizer
Benjamin Peterson <benjamin@python.org>
parents: 57131
diff changeset
143
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
144 #ifdef PGEN
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
145
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
146 static char *
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
147 decoding_fgets(char *s, int size, struct tok_state *tok)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
148 {
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
149 return fgets(s, size, tok->fp);
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
150 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
151
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
152 static int
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
153 decoding_feof(struct tok_state *tok)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
154 {
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
155 return feof(tok->fp);
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
156 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
157
57204
89f42938d5cc fix several compile() issues by translating newlines in the tokenizer
Benjamin Peterson <benjamin@python.org>
parents: 57131
diff changeset
158 static char *
89f42938d5cc fix several compile() issues by translating newlines in the tokenizer
Benjamin Peterson <benjamin@python.org>
parents: 57131
diff changeset
159 decode_str(const char *str, int exec_input, struct tok_state *tok)
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
160 {
61463
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 60289
diff changeset
161 return new_string(str, strlen(str));
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
162 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
163
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
164 #else /* PGEN */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
165
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
166 static char *
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
167 error_ret(struct tok_state *tok) /* XXX */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
168 {
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
169 tok->decoding_erred = 1;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
170 if (tok->fp != NULL && tok->buf != NULL) /* see PyTokenizer_Free */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
171 PyMem_FREE(tok->buf);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
172 tok->buf = NULL;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
173 return NULL; /* as if it were EOF */
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
174 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
175
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
176
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
177 static char *
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
178 get_normal_name(char *s) /* for utf-8 and latin-1 */
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
179 {
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
180 char buf[13];
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
181 int i;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
182 for (i = 0; i < 12; i++) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
183 int c = s[i];
61463
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 60289
diff changeset
184 if (c == '\0')
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 60289
diff changeset
185 break;
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 60289
diff changeset
186 else if (c == '_')
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 60289
diff changeset
187 buf[i] = '-';
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 60289
diff changeset
188 else
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 60289
diff changeset
189 buf[i] = tolower(c);
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
190 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
191 buf[i] = '\0';
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
192 if (strcmp(buf, "utf-8") == 0 ||
61463
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 60289
diff changeset
193 strncmp(buf, "utf-8-", 6) == 0)
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 60289
diff changeset
194 return "utf-8";
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
195 else if (strcmp(buf, "latin-1") == 0 ||
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
196 strcmp(buf, "iso-8859-1") == 0 ||
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
197 strcmp(buf, "iso-latin-1") == 0 ||
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
198 strncmp(buf, "latin-1-", 8) == 0 ||
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
199 strncmp(buf, "iso-8859-1-", 11) == 0 ||
61463
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 60289
diff changeset
200 strncmp(buf, "iso-latin-1-", 12) == 0)
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 60289
diff changeset
201 return "iso-8859-1";
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 60289
diff changeset
202 else
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 60289
diff changeset
203 return s;
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
204 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
205
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
206 /* Return the coding spec in S, or NULL if none is found. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
207
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
208 static char *
36501
7fdc639bc5b4 Merge ssize_t branch.
Martin v. Löwis <martin@v.loewis.de>
parents: 36090
diff changeset
209 get_coding_spec(const char *s, Py_ssize_t size)
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
210 {
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
211 Py_ssize_t i;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
212 /* Coding spec must be in a comment, and that comment must be
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
213 * the only statement on the source code line. */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
214 for (i = 0; i < size - 6; i++) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
215 if (s[i] == '#')
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
216 break;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
217 if (s[i] != ' ' && s[i] != '\t' && s[i] != '\014')
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
218 return NULL;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
219 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
220 for (; i < size - 6; i++) { /* XXX inefficient search */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
221 const char* t = s + i;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
222 if (strncmp(t, "coding", 6) == 0) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
223 const char* begin = NULL;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
224 t += 6;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
225 if (t[0] != ':' && t[0] != '=')
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
226 continue;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
227 do {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
228 t++;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
229 } while (t[0] == '\x20' || t[0] == '\t');
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
230
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
231 begin = t;
61463
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 60289
diff changeset
232 while (Py_ISALNUM(t[0]) ||
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
233 t[0] == '-' || t[0] == '_' || t[0] == '.')
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
234 t++;
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
235
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
236 if (begin < t) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
237 char* r = new_string(begin, t - begin);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
238 char* q = get_normal_name(r);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
239 if (r != q) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
240 PyMem_FREE(r);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
241 r = new_string(q, strlen(q));
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
242 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
243 return r;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
244 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
245 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
246 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
247 return NULL;
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
248 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
249
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
250 /* Check whether the line contains a coding spec. If it does,
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
251 invoke the set_readline function for the new encoding.
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
252 This function receives the tok_state and the new encoding.
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
253 Return 1 on success, 0 on failure. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
254
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
255 static int
36501
7fdc639bc5b4 Merge ssize_t branch.
Martin v. Löwis <martin@v.loewis.de>
parents: 36090
diff changeset
256 check_coding_spec(const char* line, Py_ssize_t size, struct tok_state *tok,
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
257 int set_readline(struct tok_state *, const char *))
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
258 {
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
259 char * cs;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
260 int r = 1;
25101
32331d26d7cf Removed reliance on gcc/C99 extension.
Tim Peters <tim.peters@gmail.com>
parents: 25098
diff changeset
261
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
262 if (tok->cont_line)
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
263 /* It's a continuation line, so it can't be a coding spec. */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
264 return 1;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
265 cs = get_coding_spec(line, size);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
266 if (cs != NULL) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
267 tok->read_coding_spec = 1;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
268 if (tok->encoding == NULL) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
269 assert(tok->decoding_state == 1); /* raw */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
270 if (strcmp(cs, "utf-8") == 0 ||
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
271 strcmp(cs, "iso-8859-1") == 0) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
272 tok->encoding = cs;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
273 } else {
24624
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
274 #ifdef Py_USING_UNICODE
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
275 r = set_readline(tok, cs);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
276 if (r) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
277 tok->encoding = cs;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
278 tok->decoding_state = -1;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
279 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
280 else
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
281 PyMem_FREE(cs);
24624
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
282 #else
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
283 /* Without Unicode support, we cannot
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
284 process the coding spec. Since there
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
285 won't be any Unicode literals, that
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
286 won't matter. */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
287 PyMem_FREE(cs);
24624
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
288 #endif
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
289 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
290 } else { /* then, compare cs with BOM */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
291 r = (strcmp(tok->encoding, cs) == 0);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
292 PyMem_FREE(cs);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
293 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
294 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
295 if (!r) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
296 cs = tok->encoding;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
297 if (!cs)
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
298 cs = "with BOM";
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
299 PyErr_Format(PyExc_SyntaxError, "encoding problem: %s", cs);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
300 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
301 return r;
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
302 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
303
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
304 /* See whether the file starts with a BOM. If it does,
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
305 invoke the set_readline function with the new encoding.
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
306 Return 1 on success, 0 on failure. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
307
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
308 static int
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
309 check_bom(int get_char(struct tok_state *),
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
310 void unget_char(int, struct tok_state *),
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
311 int set_readline(struct tok_state *, const char *),
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
312 struct tok_state *tok)
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
313 {
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
314 int ch1, ch2, ch3;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
315 ch1 = get_char(tok);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
316 tok->decoding_state = 1;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
317 if (ch1 == EOF) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
318 return 1;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
319 } else if (ch1 == 0xEF) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
320 ch2 = get_char(tok);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
321 if (ch2 != 0xBB) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
322 unget_char(ch2, tok);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
323 unget_char(ch1, tok);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
324 return 1;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
325 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
326 ch3 = get_char(tok);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
327 if (ch3 != 0xBF) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
328 unget_char(ch3, tok);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
329 unget_char(ch2, tok);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
330 unget_char(ch1, tok);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
331 return 1;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
332 }
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
333 #if 0
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
334 /* Disable support for UTF-16 BOMs until a decision
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
335 is made whether this needs to be supported. */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
336 } else if (ch1 == 0xFE) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
337 ch2 = get_char(tok);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
338 if (ch2 != 0xFF) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
339 unget_char(ch2, tok);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
340 unget_char(ch1, tok);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
341 return 1;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
342 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
343 if (!set_readline(tok, "utf-16-be"))
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
344 return 0;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
345 tok->decoding_state = -1;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
346 } else if (ch1 == 0xFF) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
347 ch2 = get_char(tok);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
348 if (ch2 != 0xFE) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
349 unget_char(ch2, tok);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
350 unget_char(ch1, tok);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
351 return 1;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
352 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
353 if (!set_readline(tok, "utf-16-le"))
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
354 return 0;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
355 tok->decoding_state = -1;
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
356 #endif
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
357 } else {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
358 unget_char(ch1, tok);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
359 return 1;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
360 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
361 if (tok->encoding != NULL)
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
362 PyMem_FREE(tok->encoding);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
363 tok->encoding = new_string("utf-8", 5); /* resulting is in utf-8 */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
364 return 1;
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
365 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
366
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
367 /* Read a line of text from TOK into S, using the stream in TOK.
35231
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
368 Return NULL on failure, else S.
37229
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
369
35231
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
370 On entry, tok->decoding_buffer will be one of:
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
371 1) NULL: need to call tok->decoding_readline to get a new line
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
372 2) PyUnicodeObject *: decoding_feof has called tok->decoding_readline and
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
373 stored the result in tok->decoding_buffer
35231
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
374 3) PyStringObject *: previous call to fp_readl did not have enough room
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
375 (in the s buffer) to copy entire contents of the line read
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
376 by tok->decoding_readline. tok->decoding_buffer has the overflow.
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
377 In this case, fp_readl is called in a loop (with an expanded buffer)
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
378 until the buffer ends with a '\n' (or until the end of the file is
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
379 reached): see tok_nextc and its calls to decoding_fgets.
35231
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
380 */
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
381
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
382 static char *
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
383 fp_readl(char *s, int size, struct tok_state *tok)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
384 {
24624
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
385 #ifndef Py_USING_UNICODE
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
386 /* In a non-Unicode built, this should never be called. */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
387 Py_FatalError("fp_readl should not be called in this build.");
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
388 return NULL; /* Keep compiler happy (not reachable) */
24624
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
389 #else
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
390 PyObject* utf8 = NULL;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
391 PyObject* buf = tok->decoding_buffer;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
392 char *str;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
393 Py_ssize_t utf8len;
35231
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
394
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
395 /* Ask for one less byte so we can terminate it */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
396 assert(size > 0);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
397 size--;
35231
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
398
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
399 if (buf == NULL) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
400 buf = PyObject_CallObject(tok->decoding_readline, NULL);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
401 if (buf == NULL)
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
402 return error_ret(tok);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
403 } else {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
404 tok->decoding_buffer = NULL;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
405 if (PyString_CheckExact(buf))
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
406 utf8 = buf;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
407 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
408 if (utf8 == NULL) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
409 utf8 = PyUnicode_AsUTF8String(buf);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
410 Py_DECREF(buf);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
411 if (utf8 == NULL)
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
412 return error_ret(tok);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
413 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
414 str = PyString_AsString(utf8);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
415 utf8len = PyString_GET_SIZE(utf8);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
416 if (utf8len > size) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
417 tok->decoding_buffer = PyString_FromStringAndSize(str+size, utf8len-size);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
418 if (tok->decoding_buffer == NULL) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
419 Py_DECREF(utf8);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
420 return error_ret(tok);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
421 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
422 utf8len = size;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
423 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
424 memcpy(s, str, utf8len);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
425 s[utf8len] = '\0';
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
426 Py_DECREF(utf8);
61463
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 60289
diff changeset
427 if (utf8len == 0)
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 60289
diff changeset
428 return NULL; /* EOF */
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
429 return s;
24624
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
430 #endif
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
431 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
432
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
433 /* Set the readline function for TOK to a StreamReader's
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
434 readline function. The StreamReader is named ENC.
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
435
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
436 This function is called from check_bom and check_coding_spec.
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
437
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
438 ENC is usually identical to the future value of tok->encoding,
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
439 except for the (currently unsupported) case of UTF-16.
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
440
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
441 Return 1 on success, 0 on failure. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
442
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
443 static int
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
444 fp_setreadl(struct tok_state *tok, const char* enc)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
445 {
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
446 PyObject *reader, *stream, *readline;
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
447
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
448 /* XXX: constify filename argument. */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
449 stream = PyFile_FromFile(tok->fp, (char*)tok->filename, "rb", NULL);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
450 if (stream == NULL)
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
451 return 0;
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
452
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
453 reader = PyCodec_StreamReader(enc, stream, NULL);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
454 Py_DECREF(stream);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
455 if (reader == NULL)
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
456 return 0;
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
457
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
458 readline = PyObject_GetAttrString(reader, "readline");
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
459 Py_DECREF(reader);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
460 if (readline == NULL)
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
461 return 0;
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
462
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
463 tok->decoding_readline = readline;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
464 return 1;
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
465 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
466
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
467 /* Fetch the next byte from TOK. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
468
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
469 static int fp_getc(struct tok_state *tok) {
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
470 return getc(tok->fp);
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
471 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
472
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
473 /* Unfetch the last byte back into TOK. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
474
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
475 static void fp_ungetc(int c, struct tok_state *tok) {
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
476 ungetc(c, tok->fp);
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
477 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
478
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
479 /* Read a line of input from TOK. Determine encoding
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
480 if necessary. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
481
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
482 static char *
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
483 decoding_fgets(char *s, int size, struct tok_state *tok)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
484 {
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
485 char *line = NULL;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
486 int badchar = 0;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
487 for (;;) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
488 if (tok->decoding_state < 0) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
489 /* We already have a codec associated with
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
490 this input. */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
491 line = fp_readl(s, size, tok);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
492 break;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
493 } else if (tok->decoding_state > 0) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
494 /* We want a 'raw' read. */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
495 line = Py_UniversalNewlineFgets(s, size,
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
496 tok->fp, NULL);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
497 break;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
498 } else {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
499 /* We have not yet determined the encoding.
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
500 If an encoding is found, use the file-pointer
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
501 reader functions from now on. */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
502 if (!check_bom(fp_getc, fp_ungetc, fp_setreadl, tok))
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
503 return error_ret(tok);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
504 assert(tok->decoding_state != 0);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
505 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
506 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
507 if (line != NULL && tok->lineno < 2 && !tok->read_coding_spec) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
508 if (!check_coding_spec(line, strlen(line), tok, fp_setreadl)) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
509 return error_ret(tok);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
510 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
511 }
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
512 #ifndef PGEN
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
513 /* The default encoding is ASCII, so make sure we don't have any
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
514 non-ASCII bytes in it. */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
515 if (line && !tok->encoding) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
516 unsigned char *c;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
517 for (c = (unsigned char *)line; *c; c++)
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
518 if (*c > 127) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
519 badchar = *c;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
520 break;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
521 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
522 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
523 if (badchar) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
524 char buf[500];
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
525 /* Need to add 1 to the line number, since this line
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
526 has not been counted, yet. */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
527 sprintf(buf,
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
528 "Non-ASCII character '\\x%.2x' "
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
529 "in file %.200s on line %i, "
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
530 "but no encoding declared; "
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
531 "see http://www.python.org/peps/pep-0263.html for details",
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
532 badchar, tok->filename, tok->lineno + 1);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
533 PyErr_SetString(PyExc_SyntaxError, buf);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
534 return error_ret(tok);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
535 }
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
536 #endif
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
537 return line;
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
538 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
540 static int
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
541 decoding_feof(struct tok_state *tok)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
542 {
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
543 if (tok->decoding_state >= 0) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
544 return feof(tok->fp);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
545 } else {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
546 PyObject* buf = tok->decoding_buffer;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
547 if (buf == NULL) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
548 buf = PyObject_CallObject(tok->decoding_readline, NULL);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
549 if (buf == NULL) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
550 error_ret(tok);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
551 return 1;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
552 } else {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
553 tok->decoding_buffer = buf;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
554 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
555 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
556 return PyObject_Length(buf) == 0;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
557 }
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
558 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
559
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
560 /* Fetch a byte from TOK, using the string buffer. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
561
37229
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
562 static int
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
563 buf_getc(struct tok_state *tok) {
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
564 return Py_CHARMASK(*tok->str++);
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
565 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
566
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
567 /* Unfetch a byte from TOK, using the string buffer. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
568
37229
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
569 static void
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
570 buf_ungetc(int c, struct tok_state *tok) {
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
571 tok->str--;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
572 assert(Py_CHARMASK(*tok->str) == c); /* tok->cur may point to read-only segment */
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
573 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
574
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
575 /* Set the readline function for TOK to ENC. For the string-based
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
576 tokenizer, this means to just record the encoding. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
577
37229
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
578 static int
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
579 buf_setreadl(struct tok_state *tok, const char* enc) {
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
580 tok->enc = enc;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
581 return 1;
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
582 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
583
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
584 /* Return a UTF-8 encoding Python string object from the
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
585 C byte string STR, which is encoded with ENC. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
586
24624
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
587 #ifdef Py_USING_UNICODE
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
588 static PyObject *
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
589 translate_into_utf8(const char* str, const char* enc) {
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
590 PyObject *utf8;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
591 PyObject* buf = PyUnicode_Decode(str, strlen(str), enc, NULL);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
592 if (buf == NULL)
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
593 return NULL;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
594 utf8 = PyUnicode_AsUTF8String(buf);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
595 Py_DECREF(buf);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
596 return utf8;
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
597 }
24624
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
598 #endif
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
599
57204
89f42938d5cc fix several compile() issues by translating newlines in the tokenizer
Benjamin Peterson <benjamin@python.org>
parents: 57131
diff changeset
600
89f42938d5cc fix several compile() issues by translating newlines in the tokenizer
Benjamin Peterson <benjamin@python.org>
parents: 57131
diff changeset
601 static char *
89f42938d5cc fix several compile() issues by translating newlines in the tokenizer
Benjamin Peterson <benjamin@python.org>
parents: 57131
diff changeset
602 translate_newlines(const char *s, int exec_input, struct tok_state *tok) {
61463
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 60289
diff changeset
603 int skip_next_lf = 0, needed_length = strlen(s) + 2, final_length;
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 60289
diff changeset
604 char *buf, *current;
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 60289
diff changeset
605 char c = '\0';
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 60289
diff changeset
606 buf = PyMem_MALLOC(needed_length);
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 60289
diff changeset
607 if (buf == NULL) {
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 60289
diff changeset
608 tok->done = E_NOMEM;
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 60289
diff changeset
609 return NULL;
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 60289
diff changeset
610 }
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 60289
diff changeset
611 for (current = buf; *s; s++, current++) {
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 60289
diff changeset
612 c = *s;
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 60289
diff changeset
613 if (skip_next_lf) {
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 60289
diff changeset
614 skip_next_lf = 0;
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 60289
diff changeset
615 if (c == '\n') {
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 60289
diff changeset
616 c = *++s;
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 60289
diff changeset
617 if (!c)
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 60289
diff changeset
618 break;
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 60289
diff changeset
619 }
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 60289
diff changeset
620 }
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 60289
diff changeset
621 if (c == '\r') {
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 60289
diff changeset
622 skip_next_lf = 1;
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 60289
diff changeset
623 c = '\n';
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 60289
diff changeset
624 }
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 60289
diff changeset
625 *current = c;
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 60289
diff changeset
626 }
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 60289
diff changeset
627 /* If this is exec input, add a newline to the end of the string if
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 60289
diff changeset
628 there isn't one already. */
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 60289
diff changeset
629 if (exec_input && c != '\n') {
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 60289
diff changeset
630 *current = '\n';
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 60289
diff changeset
631 current++;
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 60289
diff changeset
632 }
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 60289
diff changeset
633 *current = '\0';
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 60289
diff changeset
634 final_length = current - buf + 1;
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 60289
diff changeset
635 if (final_length < needed_length && final_length)
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 60289
diff changeset
636 /* should never fail */
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 60289
diff changeset
637 buf = PyMem_REALLOC(buf, final_length);
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 60289
diff changeset
638 return buf;
57204
89f42938d5cc fix several compile() issues by translating newlines in the tokenizer
Benjamin Peterson <benjamin@python.org>
parents: 57131
diff changeset
639 }
89f42938d5cc fix several compile() issues by translating newlines in the tokenizer
Benjamin Peterson <benjamin@python.org>
parents: 57131
diff changeset
640
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
641 /* Decode a byte string STR for use as the buffer of TOK.
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
642 Look for encoding declarations inside STR, and record them
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
643 inside TOK. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
644
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
645 static const char *
57204
89f42938d5cc fix several compile() issues by translating newlines in the tokenizer
Benjamin Peterson <benjamin@python.org>
parents: 57131
diff changeset
646 decode_str(const char *input, int single, struct tok_state *tok)
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
647 {
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
648 PyObject* utf8 = NULL;
61463
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 60289
diff changeset
649 const char *str;
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
650 const char *s;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
651 const char *newl[2] = {NULL, NULL};
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
652 int lineno = 0;
61463
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 60289
diff changeset
653 tok->input = str = translate_newlines(input, single, tok);
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 60289
diff changeset
654 if (str == NULL)
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 60289
diff changeset
655 return NULL;
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
656 tok->enc = NULL;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
657 tok->str = str;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
658 if (!check_bom(buf_getc, buf_ungetc, buf_setreadl, tok))
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
659 return error_ret(tok);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
660 str = tok->str; /* string after BOM if any */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
661 assert(str);
24624
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
662 #ifdef Py_USING_UNICODE
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
663 if (tok->enc != NULL) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
664 utf8 = translate_into_utf8(str, tok->enc);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
665 if (utf8 == NULL)
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
666 return error_ret(tok);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
667 str = PyString_AsString(utf8);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
668 }
24624
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
669 #endif
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
670 for (s = str;; s++) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
671 if (*s == '\0') break;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
672 else if (*s == '\n') {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
673 assert(lineno < 2);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
674 newl[lineno] = s;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
675 lineno++;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
676 if (lineno == 2) break;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
677 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
678 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
679 tok->enc = NULL;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
680 /* need to check line 1 and 2 separately since check_coding_spec
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
681 assumes a single line as input */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
682 if (newl[0]) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
683 if (!check_coding_spec(str, newl[0] - str, tok, buf_setreadl))
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
684 return error_ret(tok);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
685 if (tok->enc == NULL && newl[1]) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
686 if (!check_coding_spec(newl[0]+1, newl[1] - newl[0],
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
687 tok, buf_setreadl))
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
688 return error_ret(tok);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
689 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
690 }
24624
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
691 #ifdef Py_USING_UNICODE
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
692 if (tok->enc != NULL) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
693 assert(utf8 == NULL);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
694 utf8 = translate_into_utf8(str, tok->enc);
61463
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 60289
diff changeset
695 if (utf8 == NULL)
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
696 return error_ret(tok);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
697 str = PyString_AsString(utf8);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
698 }
24624
6445e44fcb78 Fix PEP 263 code --without-unicode. Fixes #591943.
Martin v. Löwis <martin@v.loewis.de>
parents: 24565
diff changeset
699 #endif
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
700 assert(tok->decoding_buffer == NULL);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
701 tok->decoding_buffer = utf8; /* CAUTION */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
702 return str;
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
703 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
704
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
705 #endif /* PGEN */
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
706
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
707 /* Set up tokenizer for string */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
708
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
709 struct tok_state *
57204
89f42938d5cc fix several compile() issues by translating newlines in the tokenizer
Benjamin Peterson <benjamin@python.org>
parents: 57131
diff changeset
710 PyTokenizer_FromString(const char *str, int exec_input)
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
711 {
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
712 struct tok_state *tok = tok_new();
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
713 if (tok == NULL)
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
714 return NULL;
61463
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 60289
diff changeset
715 str = (char *)decode_str(str, exec_input, tok);
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
716 if (str == NULL) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
717 PyTokenizer_Free(tok);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
718 return NULL;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
719 }
35878
94a409da4bd4 Fix some more memory leaks.
Neal Norwitz <nnorwitz@gmail.com>
parents: 35751
diff changeset
720
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
721 /* XXX: constify members. */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
722 tok->buf = tok->cur = tok->end = tok->inp = (char*)str;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
723 return tok;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
724 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
725
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
726
326
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
727 /* Set up tokenizer for file */
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
728
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
729 struct tok_state *
13659
e78153e179fb Mass ANSIfication.
Thomas Wouters <thomas@python.org>
parents: 13539
diff changeset
730 PyTokenizer_FromFile(FILE *fp, char *ps1, char *ps2)
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
731 {
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
732 struct tok_state *tok = tok_new();
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
733 if (tok == NULL)
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
734 return NULL;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
735 if ((tok->buf = (char *)PyMem_MALLOC(BUFSIZ)) == NULL) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
736 PyTokenizer_Free(tok);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
737 return NULL;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
738 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
739 tok->cur = tok->inp = tok->buf;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
740 tok->end = tok->buf + BUFSIZ;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
741 tok->fp = fp;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
742 tok->prompt = ps1;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
743 tok->nextprompt = ps2;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
744 return tok;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
745 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
746
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
747
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
748 /* Free a tok_state structure */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
749
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
750 void
13659
e78153e179fb Mass ANSIfication.
Thomas Wouters <thomas@python.org>
parents: 13539
diff changeset
751 PyTokenizer_Free(struct tok_state *tok)
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
752 {
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
753 if (tok->encoding != NULL)
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
754 PyMem_FREE(tok->encoding);
24544
0bd521eb714e Make pgen compile with pydebug. Duplicate normalized names, as it may
Martin v. Löwis <martin@v.loewis.de>
parents: 24543
diff changeset
755 #ifndef PGEN
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
756 Py_XDECREF(tok->decoding_readline);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
757 Py_XDECREF(tok->decoding_buffer);
24544
0bd521eb714e Make pgen compile with pydebug. Duplicate normalized names, as it may
Martin v. Löwis <martin@v.loewis.de>
parents: 24543
diff changeset
758 #endif
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
759 if (tok->fp != NULL && tok->buf != NULL)
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
760 PyMem_FREE(tok->buf);
61463
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 60289
diff changeset
761 if (tok->input)
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 60289
diff changeset
762 PyMem_FREE((char *)tok->input);
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
763 PyMem_FREE(tok);
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
764 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
765
32961
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
766 #if !defined(PGEN) && defined(Py_USING_UNICODE)
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
767 static int
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
768 tok_stdin_decode(struct tok_state *tok, char **inp)
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
769 {
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
770 PyObject *enc, *sysstdin, *decoded, *utf8;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
771 const char *encoding;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
772 char *converted;
32961
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
773
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
774 if (PySys_GetFile((char *)"stdin", NULL) != stdin)
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
775 return 0;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
776 sysstdin = PySys_GetObject("stdin");
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
777 if (sysstdin == NULL || !PyFile_Check(sysstdin))
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
778 return 0;
32961
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
779
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
780 enc = ((PyFileObject *)sysstdin)->f_encoding;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
781 if (enc == NULL || !PyString_Check(enc))
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
782 return 0;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
783 Py_INCREF(enc);
32961
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
784
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
785 encoding = PyString_AsString(enc);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
786 decoded = PyUnicode_Decode(*inp, strlen(*inp), encoding, NULL);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
787 if (decoded == NULL)
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
788 goto error_clear;
32961
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
789
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
790 utf8 = PyUnicode_AsEncodedString(decoded, "utf-8", NULL);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
791 Py_DECREF(decoded);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
792 if (utf8 == NULL)
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
793 goto error_clear;
32961
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
794
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
795 assert(PyString_Check(utf8));
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
796 converted = new_string(PyString_AS_STRING(utf8),
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
797 PyString_GET_SIZE(utf8));
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
798 Py_DECREF(utf8);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
799 if (converted == NULL)
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
800 goto error_nomem;
32961
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
801
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
802 PyMem_FREE(*inp);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
803 *inp = converted;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
804 if (tok->encoding != NULL)
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
805 PyMem_FREE(tok->encoding);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
806 tok->encoding = new_string(encoding, strlen(encoding));
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
807 if (tok->encoding == NULL)
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
808 goto error_nomem;
32961
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
809
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
810 Py_DECREF(enc);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
811 return 0;
32961
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
812
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
813 error_nomem:
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
814 Py_DECREF(enc);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
815 tok->done = E_NOMEM;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
816 return -1;
32961
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
817
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
818 error_clear:
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
819 Py_DECREF(enc);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
820 if (!PyErr_ExceptionMatches(PyExc_UnicodeDecodeError)) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
821 tok->done = E_ERROR;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
822 return -1;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
823 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
824 /* Fallback to iso-8859-1: for backward compatibility */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
825 PyErr_Clear();
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
826 return 0;
32961
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
827 }
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
828 #endif
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
829
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
830 /* Get next char, updating state; error code goes into tok->done */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
831
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
832 static int
13659
e78153e179fb Mass ANSIfication.
Thomas Wouters <thomas@python.org>
parents: 13539
diff changeset
833 tok_nextc(register struct tok_state *tok)
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
834 {
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
835 for (;;) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
836 if (tok->cur != tok->inp) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
837 return Py_CHARMASK(*tok->cur++); /* Fast path */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
838 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
839 if (tok->done != E_OK)
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
840 return EOF;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
841 if (tok->fp == NULL) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
842 char *end = strchr(tok->inp, '\n');
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
843 if (end != NULL)
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
844 end++;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
845 else {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
846 end = strchr(tok->inp, '\0');
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
847 if (end == tok->inp) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
848 tok->done = E_EOF;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
849 return EOF;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
850 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
851 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
852 if (tok->start == NULL)
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
853 tok->buf = tok->cur;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
854 tok->line_start = tok->cur;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
855 tok->lineno++;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
856 tok->inp = end;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
857 return Py_CHARMASK(*tok->cur++);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
858 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
859 if (tok->prompt != NULL) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
860 char *newtok = PyOS_Readline(stdin, stdout, tok->prompt);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
861 if (tok->nextprompt != NULL)
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
862 tok->prompt = tok->nextprompt;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
863 if (newtok == NULL)
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
864 tok->done = E_INTR;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
865 else if (*newtok == '\0') {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
866 PyMem_FREE(newtok);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
867 tok->done = E_EOF;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
868 }
32961
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
869 #if !defined(PGEN) && defined(Py_USING_UNICODE)
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
870 else if (tok_stdin_decode(tok, &newtok) != 0)
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
871 PyMem_FREE(newtok);
32961
5e8e309c0bb0 SF #941229: Decode source code with sys.stdin.encoding in interactive
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32888
diff changeset
872 #endif
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
873 else if (tok->start != NULL) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
874 size_t start = tok->start - tok->buf;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
875 size_t oldlen = tok->cur - tok->buf;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
876 size_t newlen = oldlen + strlen(newtok);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
877 char *buf = tok->buf;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
878 buf = (char *)PyMem_REALLOC(buf, newlen+1);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
879 tok->lineno++;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
880 if (buf == NULL) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
881 PyMem_FREE(tok->buf);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
882 tok->buf = NULL;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
883 PyMem_FREE(newtok);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
884 tok->done = E_NOMEM;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
885 return EOF;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
886 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
887 tok->buf = buf;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
888 tok->cur = tok->buf + oldlen;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
889 tok->line_start = tok->cur;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
890 strcpy(tok->buf + oldlen, newtok);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
891 PyMem_FREE(newtok);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
892 tok->inp = tok->buf + newlen;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
893 tok->end = tok->inp + 1;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
894 tok->start = tok->buf + start;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
895 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
896 else {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
897 tok->lineno++;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
898 if (tok->buf != NULL)
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
899 PyMem_FREE(tok->buf);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
900 tok->buf = newtok;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
901 tok->line_start = tok->buf;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
902 tok->cur = tok->buf;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
903 tok->line_start = tok->buf;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
904 tok->inp = strchr(tok->buf, '\0');
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
905 tok->end = tok->inp + 1;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
906 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
907 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
908 else {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
909 int done = 0;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
910 Py_ssize_t cur = 0;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
911 char *pt;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
912 if (tok->start == NULL) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
913 if (tok->buf == NULL) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
914 tok->buf = (char *)
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
915 PyMem_MALLOC(BUFSIZ);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
916 if (tok->buf == NULL) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
917 tok->done = E_NOMEM;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
918 return EOF;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
919 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
920 tok->end = tok->buf + BUFSIZ;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
921 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
922 if (decoding_fgets(tok->buf, (int)(tok->end - tok->buf),
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
923 tok) == NULL) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
924 tok->done = E_EOF;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
925 done = 1;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
926 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
927 else {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
928 tok->done = E_OK;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
929 tok->inp = strchr(tok->buf, '\0');
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
930 done = tok->inp[-1] == '\n';
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
931 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
932 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
933 else {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
934 cur = tok->cur - tok->buf;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
935 if (decoding_feof(tok)) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
936 tok->done = E_EOF;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
937 done = 1;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
938 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
939 else
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
940 tok->done = E_OK;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
941 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
942 tok->lineno++;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
943 /* Read until '\n' or EOF */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
944 while (!done) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
945 Py_ssize_t curstart = tok->start == NULL ? -1 :
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
946 tok->start - tok->buf;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
947 Py_ssize_t curvalid = tok->inp - tok->buf;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
948 Py_ssize_t newsize = curvalid + BUFSIZ;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
949 char *newbuf = tok->buf;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
950 newbuf = (char *)PyMem_REALLOC(newbuf,
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
951 newsize);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
952 if (newbuf == NULL) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
953 tok->done = E_NOMEM;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
954 tok->cur = tok->inp;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
955 return EOF;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
956 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
957 tok->buf = newbuf;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
958 tok->inp = tok->buf + curvalid;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
959 tok->end = tok->buf + newsize;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
960 tok->start = curstart < 0 ? NULL :
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
961 tok->buf + curstart;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
962 if (decoding_fgets(tok->inp,
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
963 (int)(tok->end - tok->inp),
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
964 tok) == NULL) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
965 /* Break out early on decoding
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
966 errors, as tok->buf will be NULL
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
967 */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
968 if (tok->decoding_erred)
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
969 return EOF;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
970 /* Last line does not end in \n,
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
971 fake one */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
972 strcpy(tok->inp, "\n");
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
973 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
974 tok->inp = strchr(tok->inp, '\0');
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
975 done = tok->inp[-1] == '\n';
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
976 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
977 if (tok->buf != NULL) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
978 tok->cur = tok->buf + cur;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
979 tok->line_start = tok->cur;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
980 /* replace "\r\n" with "\n" */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
981 /* For Mac leave the \r, giving a syntax error */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
982 pt = tok->inp - 2;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
983 if (pt >= tok->buf && *pt == '\r') {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
984 *pt++ = '\n';
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
985 *pt = '\0';
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
986 tok->inp = pt;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
987 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
988 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
989 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
990 if (tok->done != E_OK) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
991 if (tok->prompt != NULL)
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
992 PySys_WriteStderr("\n");
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
993 tok->cur = tok->inp;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
994 return EOF;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
995 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
996 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
997 /*NOTREACHED*/
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
998 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
999
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1000
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1001 /* Back-up one character */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1002
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1003 static void
13659
e78153e179fb Mass ANSIfication.
Thomas Wouters <thomas@python.org>
parents: 13539
diff changeset
1004 tok_backup(register struct tok_state *tok, register int c)
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1005 {
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1006 if (c != EOF) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1007 if (--tok->cur < tok->buf)
61463
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 60289
diff changeset
1008 Py_FatalError("tok_backup: beginning of buffer");
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1009 if (*tok->cur != c)
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1010 *tok->cur = c;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1011 }
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1012 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1013
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1014
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1015 /* Return the token corresponding to a single character */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1016
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1017 int
13659
e78153e179fb Mass ANSIfication.
Thomas Wouters <thomas@python.org>
parents: 13539
diff changeset
1018 PyToken_OneChar(int c)
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1019 {
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1020 switch (c) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1021 case '(': return LPAR;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1022 case ')': return RPAR;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1023 case '[': return LSQB;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1024 case ']': return RSQB;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1025 case ':': return COLON;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1026 case ',': return COMMA;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1027 case ';': return SEMI;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1028 case '+': return PLUS;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1029 case '-': return MINUS;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1030 case '*': return STAR;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1031 case '/': return SLASH;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1032 case '|': return VBAR;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1033 case '&': return AMPER;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1034 case '<': return LESS;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1035 case '>': return GREATER;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1036 case '=': return EQUAL;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1037 case '.': return DOT;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1038 case '%': return PERCENT;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1039 case '`': return BACKQUOTE;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1040 case '{': return LBRACE;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1041 case '}': return RBRACE;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1042 case '^': return CIRCUMFLEX;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1043 case '~': return TILDE;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1044 case '@': return AT;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1045 default: return OP;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1046 }
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1047 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1048
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1049
400
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1050 int
13659
e78153e179fb Mass ANSIfication.
Thomas Wouters <thomas@python.org>
parents: 13539
diff changeset
1051 PyToken_TwoChars(int c1, int c2)
400
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1052 {
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1053 switch (c1) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1054 case '=':
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1055 switch (c2) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1056 case '=': return EQEQUAL;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1057 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1058 break;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1059 case '!':
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1060 switch (c2) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1061 case '=': return NOTEQUAL;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1062 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1063 break;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1064 case '<':
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1065 switch (c2) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1066 case '>': return NOTEQUAL;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1067 case '=': return LESSEQUAL;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1068 case '<': return LEFTSHIFT;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1069 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1070 break;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1071 case '>':
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1072 switch (c2) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1073 case '=': return GREATEREQUAL;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1074 case '>': return RIGHTSHIFT;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1075 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1076 break;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1077 case '+':
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1078 switch (c2) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1079 case '=': return PLUSEQUAL;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1080 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1081 break;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1082 case '-':
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1083 switch (c2) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1084 case '=': return MINEQUAL;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1085 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1086 break;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1087 case '*':
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1088 switch (c2) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1089 case '*': return DOUBLESTAR;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1090 case '=': return STAREQUAL;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1091 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1092 break;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1093 case '/':
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1094 switch (c2) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1095 case '/': return DOUBLESLASH;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1096 case '=': return SLASHEQUAL;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1097 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1098 break;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1099 case '|':
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1100 switch (c2) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1101 case '=': return VBAREQUAL;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1102 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1103 break;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1104 case '%':
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1105 switch (c2) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1106 case '=': return PERCENTEQUAL;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1107 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1108 break;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1109 case '&':
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1110 switch (c2) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1111 case '=': return AMPEREQUAL;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1112 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1113 break;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1114 case '^':
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1115 switch (c2) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1116 case '=': return CIRCUMFLEXEQUAL;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1117 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1118 break;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1119 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1120 return OP;
400
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1121 }
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1122
14163
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1123 int
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1124 PyToken_ThreeChars(int c1, int c2, int c3)
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1125 {
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1126 switch (c1) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1127 case '<':
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1128 switch (c2) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1129 case '<':
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1130 switch (c3) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1131 case '=':
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1132 return LEFTSHIFTEQUAL;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1133 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1134 break;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1135 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1136 break;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1137 case '>':
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1138 switch (c2) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1139 case '>':
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1140 switch (c3) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1141 case '=':
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1142 return RIGHTSHIFTEQUAL;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1143 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1144 break;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1145 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1146 break;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1147 case '*':
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1148 switch (c2) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1149 case '*':
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1150 switch (c3) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1151 case '=':
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1152 return DOUBLESTAREQUAL;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1153 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1154 break;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1155 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1156 break;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1157 case '/':
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1158 switch (c2) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1159 case '/':
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1160 switch (c3) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1161 case '=':
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1162 return DOUBLESLASHEQUAL;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1163 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1164 break;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1165 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1166 break;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1167 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1168 return OP;
14163
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1169 }
400
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1170
7945
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1171 static int
13659
e78153e179fb Mass ANSIfication.
Thomas Wouters <thomas@python.org>
parents: 13539
diff changeset
1172 indenterror(struct tok_state *tok)
7945
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1173 {
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1174 if (tok->alterror) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1175 tok->done = E_TABSPACE;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1176 tok->cur = tok->inp;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1177 return 1;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1178 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1179 if (tok->altwarning) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1180 PySys_WriteStderr("%s: inconsistent use of tabs and spaces "
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1181 "in indentation\n", tok->filename);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1182 tok->altwarning = 0;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1183 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1184 return 0;
7945
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1185 }
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1186
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1187 /* Get next token, after space stripping etc. */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1188
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
1189 static int
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
1190 tok_get(register struct tok_state *tok, char **p_start, char **p_end)
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1191 {
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1192 register int c;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1193 int blankline;
326
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1194
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1195 *p_start = *p_end = NULL;
326
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1196 nextline:
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1197 tok->start = NULL;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1198 blankline = 0;
326
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1199
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1200 /* Get indentation level */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1201 if (tok->atbol) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1202 register int col = 0;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1203 register int altcol = 0;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1204 tok->atbol = 0;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1205 for (;;) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1206 c = tok_nextc(tok);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1207 if (c == ' ')
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1208 col++, altcol++;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1209 else if (c == '\t') {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1210 col = (col/tok->tabsize + 1) * tok->tabsize;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1211 altcol = (altcol/tok->alttabsize + 1)
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1212 * tok->alttabsize;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1213 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1214 else if (c == '\014') /* Control-L (formfeed) */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1215 col = altcol = 0; /* For Emacs users */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1216 else
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1217 break;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1218 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1219 tok_backup(tok, c);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1220 if (c == '#' || c == '\n') {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1221 /* Lines with only whitespace and/or comments
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1222 shouldn't affect the indentation and are
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1223 not passed to the parser as NEWLINE tokens,
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1224 except *totally* empty lines in interactive
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1225 mode, which signal the end of a command group. */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1226 if (col == 0 && c == '\n' && tok->prompt != NULL)
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1227 blankline = 0; /* Let it through */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1228 else
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1229 blankline = 1; /* Ignore completely */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1230 /* We can't jump back right here since we still
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1231 may need to skip to the end of a comment */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1232 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1233 if (!blankline && tok->level == 0) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1234 if (col == tok->indstack[tok->indent]) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1235 /* No change */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1236 if (altcol != tok->altindstack[tok->indent]) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1237 if (indenterror(tok))
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1238 return ERRORTOKEN;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1239 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1240 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1241 else if (col > tok->indstack[tok->indent]) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1242 /* Indent -- always one */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1243 if (tok->indent+1 >= MAXINDENT) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1244 tok->done = E_TOODEEP;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1245 tok->cur = tok->inp;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1246 return ERRORTOKEN;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1247 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1248 if (altcol <= tok->altindstack[tok->indent]) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1249 if (indenterror(tok))
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1250 return ERRORTOKEN;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1251 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1252 tok->pendin++;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1253 tok->indstack[++tok->indent] = col;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1254 tok->altindstack[tok->indent] = altcol;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1255 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1256 else /* col < tok->indstack[tok->indent] */ {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1257 /* Dedent -- any number, must be consistent */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1258 while (tok->indent > 0 &&
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1259 col < tok->indstack[tok->indent]) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1260 tok->pendin--;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1261 tok->indent--;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1262 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1263 if (col != tok->indstack[tok->indent]) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1264 tok->done = E_DEDENT;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1265 tok->cur = tok->inp;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1266 return ERRORTOKEN;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1267 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1268 if (altcol != tok->altindstack[tok->indent]) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1269 if (indenterror(tok))
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1270 return ERRORTOKEN;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1271 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1272 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1273 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1274 }
37229
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
1275
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1276 tok->start = tok->cur;
37229
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
1277
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1278 /* Return pending indents/dedents */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1279 if (tok->pendin != 0) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1280 if (tok->pendin < 0) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1281 tok->pendin++;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1282 return DEDENT;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1283 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1284 else {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1285 tok->pendin--;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1286 return INDENT;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1287 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1288 }
37229
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
1289
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1290 again:
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1291 tok->start = NULL;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1292 /* Skip spaces */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1293 do {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1294 c = tok_nextc(tok);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1295 } while (c == ' ' || c == '\t' || c == '\014');
37229
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
1296
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1297 /* Set start of current token */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1298 tok->start = tok->cur - 1;
37229
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
1299
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1300 /* Skip comment, while looking for tab-setting magic */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1301 if (c == '#') {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1302 static char *tabforms[] = {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1303 "tab-width:", /* Emacs */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1304 ":tabstop=", /* vim, full form */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1305 ":ts=", /* vim, abbreviated form */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1306 "set tabsize=", /* will vi never die? */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1307 /* more templates can be added here to support other editors */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1308 };
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1309 char cbuf[80];
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1310 char *tp, **cp;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1311 tp = cbuf;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1312 do {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1313 *tp++ = c = tok_nextc(tok);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1314 } while (c != EOF && c != '\n' &&
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1315 (size_t)(tp - cbuf + 1) < sizeof(cbuf));
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1316 *tp = '\0';
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1317 for (cp = tabforms;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1318 cp < tabforms + sizeof(tabforms)/sizeof(tabforms[0]);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1319 cp++) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1320 if ((tp = strstr(cbuf, *cp))) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1321 int newsize = atoi(tp + strlen(*cp));
12142
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1322
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1323 if (newsize >= 1 && newsize <= 40) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1324 tok->tabsize = newsize;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1325 if (Py_VerboseFlag)
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1326 PySys_WriteStderr(
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1327 "Tab size set to %d\n",
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1328 newsize);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1329 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1330 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1331 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1332 while (c != EOF && c != '\n')
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1333 c = tok_nextc(tok);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1334 }
37229
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
1335
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1336 /* Check for EOF and errors now */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1337 if (c == EOF) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1338 return tok->done == E_EOF ? ENDMARKER : ERRORTOKEN;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1339 }
12142
43c36f63f867 Fix by Eric Raymond: make the code that looks for various bits of
Guido van Rossum <guido@python.org>
parents: 11971
diff changeset
1340
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1341 /* Identifier (most frequent token!) */
61463
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 60289
diff changeset
1342 if (Py_ISALPHA(c) || c == '_') {
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1343 /* Process r"", u"" and ur"" */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1344 switch (c) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1345 case 'b':
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1346 case 'B':
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1347 c = tok_nextc(tok);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1348 if (c == 'r' || c == 'R')
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1349 c = tok_nextc(tok);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1350 if (c == '"' || c == '\'')
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1351 goto letter_quote;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1352 break;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1353 case 'r':
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1354 case 'R':
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1355 c = tok_nextc(tok);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1356 if (c == '"' || c == '\'')
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1357 goto letter_quote;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1358 break;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1359 case 'u':
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1360 case 'U':
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1361 c = tok_nextc(tok);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1362 if (c == 'r' || c == 'R')
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1363 c = tok_nextc(tok);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1364 if (c == '"' || c == '\'')
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1365 goto letter_quote;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1366 break;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1367 }
62471
b17fcdd38a5b Issue #9020: The Py_IS* macros from pyctype.h should generally only be
Stefan Krah <stefan@bytereef.org>
parents: 61463
diff changeset
1368 while (c != EOF && (Py_ISALNUM(c) || c == '_')) {
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1369 c = tok_nextc(tok);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1370 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1371 tok_backup(tok, c);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1372 *p_start = tok->start;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1373 *p_end = tok->cur;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1374 return NAME;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1375 }
37229
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
1376
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1377 /* Newline */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1378 if (c == '\n') {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1379 tok->atbol = 1;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1380 if (blankline || tok->level > 0)
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1381 goto nextline;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1382 *p_start = tok->start;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1383 *p_end = tok->cur - 1; /* Leave '\n' out of the string */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1384 tok->cont_line = 0;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1385 return NEWLINE;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1386 }
37229
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
1387
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1388 /* Period or number starting with period? */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1389 if (c == '.') {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1390 c = tok_nextc(tok);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1391 if (isdigit(c)) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1392 goto fraction;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1393 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1394 else {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1395 tok_backup(tok, c);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1396 *p_start = tok->start;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1397 *p_end = tok->cur;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1398 return DOT;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1399 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1400 }
37229
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
1401
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1402 /* Number */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1403 if (isdigit(c)) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1404 if (c == '0') {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1405 /* Hex, octal or binary -- maybe. */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1406 c = tok_nextc(tok);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1407 if (c == '.')
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1408 goto fraction;
3333
4b5bff95804e changes for pow(**) and complex
Guido van Rossum <guido@python.org>
parents: 3116
diff changeset
1409 #ifndef WITHOUT_COMPLEX
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1410 if (c == 'j' || c == 'J')
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1411 goto imaginary;
19621
0bec699783d0 SF patch #455966: Allow leading 0 in float/imag literals.
Tim Peters <tim.peters@gmail.com>
parents: 19501
diff changeset
1412 #endif
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1413 if (c == 'x' || c == 'X') {
44780
61a11cc95632 Fix #1679: "0x" was taken as a valid integer literal.
Georg Brandl <georg@python.org>
parents: 44747
diff changeset
1414
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1415 /* Hex */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1416 c = tok_nextc(tok);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1417 if (!isxdigit(c)) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1418 tok->done = E_TOKEN;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1419 tok_backup(tok, c);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1420 return ERRORTOKEN;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1421 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1422 do {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1423 c = tok_nextc(tok);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1424 } while (isxdigit(c));
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1425 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1426 else if (c == 'o' || c == 'O') {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1427 /* Octal */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1428 c = tok_nextc(tok);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1429 if (c < '0' || c >= '8') {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1430 tok->done = E_TOKEN;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1431 tok_backup(tok, c);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1432 return ERRORTOKEN;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1433 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1434 do {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1435 c = tok_nextc(tok);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1436 } while ('0' <= c && c < '8');
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1437 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1438 else if (c == 'b' || c == 'B') {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1439 /* Binary */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1440 c = tok_nextc(tok);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1441 if (c != '0' && c != '1') {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1442 tok->done = E_TOKEN;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1443 tok_backup(tok, c);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1444 return ERRORTOKEN;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1445 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1446 do {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1447 c = tok_nextc(tok);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1448 } while (c == '0' || c == '1');
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1449 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1450 else {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1451 int found_decimal = 0;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1452 /* Octal; c is first char of it */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1453 /* There's no 'isoctdigit' macro, sigh */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1454 while ('0' <= c && c < '8') {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1455 c = tok_nextc(tok);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1456 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1457 if (isdigit(c)) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1458 found_decimal = 1;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1459 do {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1460 c = tok_nextc(tok);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1461 } while (isdigit(c));
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1462 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1463 if (c == '.')
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1464 goto fraction;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1465 else if (c == 'e' || c == 'E')
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1466 goto exponent;
3333
4b5bff95804e changes for pow(**) and complex
Guido van Rossum <guido@python.org>
parents: 3116
diff changeset
1467 #ifndef WITHOUT_COMPLEX
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1468 else if (c == 'j' || c == 'J')
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1469 goto imaginary;
3333
4b5bff95804e changes for pow(**) and complex
Guido van Rossum <guido@python.org>
parents: 3116
diff changeset
1470 #endif
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1471 else if (found_decimal) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1472 tok->done = E_TOKEN;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1473 tok_backup(tok, c);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1474 return ERRORTOKEN;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1475 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1476 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1477 if (c == 'l' || c == 'L')
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1478 c = tok_nextc(tok);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1479 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1480 else {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1481 /* Decimal */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1482 do {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1483 c = tok_nextc(tok);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1484 } while (isdigit(c));
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1485 if (c == 'l' || c == 'L')
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1486 c = tok_nextc(tok);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1487 else {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1488 /* Accept floating point numbers. */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1489 if (c == '.') {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1490 fraction:
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1491 /* Fraction */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1492 do {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1493 c = tok_nextc(tok);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1494 } while (isdigit(c));
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1495 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1496 if (c == 'e' || c == 'E') {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1497 exponent:
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1498 /* Exponent part */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1499 c = tok_nextc(tok);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1500 if (c == '+' || c == '-')
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1501 c = tok_nextc(tok);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1502 if (!isdigit(c)) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1503 tok->done = E_TOKEN;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1504 tok_backup(tok, c);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1505 return ERRORTOKEN;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1506 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1507 do {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1508 c = tok_nextc(tok);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1509 } while (isdigit(c));
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1510 }
3333
4b5bff95804e changes for pow(**) and complex
Guido van Rossum <guido@python.org>
parents: 3116
diff changeset
1511 #ifndef WITHOUT_COMPLEX
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1512 if (c == 'j' || c == 'J')
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1513 /* Imaginary part */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1514 imaginary:
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1515 c = tok_nextc(tok);
3333
4b5bff95804e changes for pow(**) and complex
Guido van Rossum <guido@python.org>
parents: 3116
diff changeset
1516 #endif
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1517 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1518 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1519 tok_backup(tok, c);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1520 *p_start = tok->start;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1521 *p_end = tok->cur;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1522 return NUMBER;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1523 }
5265
3c1788ab9761 Support for alternative string quotes (a"xx", b"xx", c"xx", ...).
Guido van Rossum <guido@python.org>
parents: 4839
diff changeset
1524
3c1788ab9761 Support for alternative string quotes (a"xx", b"xx", c"xx", ...).
Guido van Rossum <guido@python.org>
parents: 4839
diff changeset
1525 letter_quote:
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1526 /* String */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1527 if (c == '\'' || c == '"') {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1528 Py_ssize_t quote2 = tok->cur - tok->start + 1;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1529 int quote = c;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1530 int triple = 0;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1531 int tripcount = 0;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1532 for (;;) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1533 c = tok_nextc(tok);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1534 if (c == '\n') {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1535 if (!triple) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1536 tok->done = E_EOLS;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1537 tok_backup(tok, c);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1538 return ERRORTOKEN;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1539 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1540 tripcount = 0;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1541 tok->cont_line = 1; /* multiline string. */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1542 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1543 else if (c == EOF) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1544 if (triple)
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1545 tok->done = E_EOFS;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1546 else
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1547 tok->done = E_EOLS;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1548 tok->cur = tok->inp;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1549 return ERRORTOKEN;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1550 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1551 else if (c == quote) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1552 tripcount++;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1553 if (tok->cur - tok->start == quote2) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1554 c = tok_nextc(tok);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1555 if (c == quote) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1556 triple = 1;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1557 tripcount = 0;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1558 continue;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1559 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1560 tok_backup(tok, c);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1561 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1562 if (!triple || tripcount == 3)
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1563 break;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1564 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1565 else if (c == '\\') {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1566 tripcount = 0;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1567 c = tok_nextc(tok);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1568 if (c == EOF) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1569 tok->done = E_EOLS;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1570 tok->cur = tok->inp;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1571 return ERRORTOKEN;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1572 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1573 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1574 else
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1575 tripcount = 0;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1576 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1577 *p_start = tok->start;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1578 *p_end = tok->cur;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1579 return STRING;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1580 }
37229
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
1581
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1582 /* Line continuation */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1583 if (c == '\\') {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1584 c = tok_nextc(tok);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1585 if (c != '\n') {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1586 tok->done = E_LINECONT;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1587 tok->cur = tok->inp;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1588 return ERRORTOKEN;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1589 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1590 tok->cont_line = 1;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1591 goto again; /* Read next line */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1592 }
37229
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
1593
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1594 /* Check for two-character token */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1595 {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1596 int c2 = tok_nextc(tok);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1597 int token = PyToken_TwoChars(c, c2);
43983
e725166c8c81 Fixed problems in the last commit. Filenames and line numbers weren't reported correctly.
Christian Heimes <christian@cheimes.de>
parents: 43982
diff changeset
1598 #ifndef PGEN
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1599 if (Py_Py3kWarningFlag && token == NOTEQUAL && c == '<') {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1600 if (PyErr_WarnExplicit(PyExc_DeprecationWarning,
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1601 "<> not supported in 3.x; use !=",
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1602 tok->filename, tok->lineno,
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1603 NULL, NULL)) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1604 return ERRORTOKEN;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1605 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1606 }
43983
e725166c8c81 Fixed problems in the last commit. Filenames and line numbers weren't reported correctly.
Christian Heimes <christian@cheimes.de>
parents: 43982
diff changeset
1607 #endif
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1608 if (token != OP) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1609 int c3 = tok_nextc(tok);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1610 int token3 = PyToken_ThreeChars(c, c2, c3);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1611 if (token3 != OP) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1612 token = token3;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1613 } else {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1614 tok_backup(tok, c3);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1615 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1616 *p_start = tok->start;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1617 *p_end = tok->cur;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1618 return token;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1619 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1620 tok_backup(tok, c2);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1621 }
37229
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
1622
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1623 /* Keep track of parentheses nesting level */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1624 switch (c) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1625 case '(':
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1626 case '[':
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1627 case '{':
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1628 tok->level++;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1629 break;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1630 case ')':
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1631 case ']':
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1632 case '}':
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1633 tok->level--;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1634 break;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1635 }
37229
6e1a899cac8e Years in the making.
Tim Peters <tim.peters@gmail.com>
parents: 37086
diff changeset
1636
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1637 /* Punctuation character */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1638 *p_start = tok->start;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1639 *p_end = tok->cur;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1640 return PyToken_OneChar(c);
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1641 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1642
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
1643 int
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
1644 PyTokenizer_Get(struct tok_state *tok, char **p_start, char **p_end)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
1645 {
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1646 int result = tok_get(tok, p_start, p_end);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1647 if (tok->decoding_erred) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1648 result = ERRORTOKEN;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1649 tok->done = E_DECODE;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1650 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1651 return result;
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
1652 }
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1653
43230
425127c1d113 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 38762
diff changeset
1654 /* This function is only called from parsetok. However, it cannot live
425127c1d113 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 38762
diff changeset
1655 there, as it must be empty for PGEN, and we can check for PGEN only
425127c1d113 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 38762
diff changeset
1656 in this file. */
425127c1d113 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 38762
diff changeset
1657
44888
dfb6f367a406 Fixed bug #1915: Python compiles with --enable-unicode=no again. However several extension methods and modules do not work without unicode support.
Christian Heimes <christian@cheimes.de>
parents: 44853
diff changeset
1658 #if defined(PGEN) || !defined(Py_USING_UNICODE)
43230
425127c1d113 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 38762
diff changeset
1659 char*
425127c1d113 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 38762
diff changeset
1660 PyTokenizer_RestoreEncoding(struct tok_state* tok, int len, int* offset)
425127c1d113 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 38762
diff changeset
1661 {
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1662 return NULL;
43230
425127c1d113 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 38762
diff changeset
1663 }
425127c1d113 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 38762
diff changeset
1664 #else
44575
cf36e01e83f2 Fix #define ordering.
Georg Brandl <georg@python.org>
parents: 44572
diff changeset
1665 #ifdef Py_USING_UNICODE
43230
425127c1d113 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 38762
diff changeset
1666 static PyObject *
425127c1d113 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 38762
diff changeset
1667 dec_utf8(const char *enc, const char *text, size_t len) {
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1668 PyObject *ret = NULL;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1669 PyObject *unicode_text = PyUnicode_DecodeUTF8(text, len, "replace");
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1670 if (unicode_text) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1671 ret = PyUnicode_AsEncodedString(unicode_text, enc, "replace");
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1672 Py_DECREF(unicode_text);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1673 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1674 if (!ret) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1675 PyErr_Clear();
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1676 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1677 return ret;
43230
425127c1d113 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 38762
diff changeset
1678 }
425127c1d113 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 38762
diff changeset
1679 char *
425127c1d113 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 38762
diff changeset
1680 PyTokenizer_RestoreEncoding(struct tok_state* tok, int len, int *offset)
425127c1d113 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 38762
diff changeset
1681 {
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1682 char *text = NULL;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1683 if (tok->encoding) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1684 /* convert source to original encondig */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1685 PyObject *lineobj = dec_utf8(tok->encoding, tok->buf, len);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1686 if (lineobj != NULL) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1687 int linelen = PyString_Size(lineobj);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1688 const char *line = PyString_AsString(lineobj);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1689 text = PyObject_MALLOC(linelen + 1);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1690 if (text != NULL && line != NULL) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1691 if (linelen)
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1692 strncpy(text, line, linelen);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1693 text[linelen] = '\0';
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1694 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1695 Py_DECREF(lineobj);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1696
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1697 /* adjust error offset */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1698 if (*offset > 1) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1699 PyObject *offsetobj = dec_utf8(tok->encoding,
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1700 tok->buf, *offset-1);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1701 if (offsetobj) {
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1702 *offset = PyString_Size(offsetobj) + 1;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1703 Py_DECREF(offsetobj);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1704 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1705 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1706
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1707 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1708 }
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1709 return text;
43230
425127c1d113 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 38762
diff changeset
1710
425127c1d113 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 38762
diff changeset
1711 }
44575
cf36e01e83f2 Fix #define ordering.
Georg Brandl <georg@python.org>
parents: 44572
diff changeset
1712 #endif /* defined(Py_USING_UNICODE) */
43230
425127c1d113 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 38762
diff changeset
1713 #endif
425127c1d113 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 38762
diff changeset
1714
425127c1d113 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 38762
diff changeset
1715
4839
4567e158d20c Rename DEBUG macro to Py_DEBUG
Guido van Rossum <guido@python.org>
parents: 4577
diff changeset
1716 #ifdef Py_DEBUG
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1717
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1718 void
13659
e78153e179fb Mass ANSIfication.
Thomas Wouters <thomas@python.org>
parents: 13539
diff changeset
1719 tok_dump(int type, char *start, char *end)
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1720 {
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1721 printf("%s", _PyParser_TokenNames[type]);
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1722 if (type == NAME || type == NUMBER || type == STRING || type == OP)
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59822
diff changeset
1723 printf("(%.*s)", (int)(end - start), start);
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1724 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1725
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1726 #endif