Mercurial > cpython-lru_cache-weakref

annotate Parser/tokenizer.c @ 66807:d28126ebd60d

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Entries for datetime, callable, and collections.Counter.
author Raymond Hettinger <python@rcn.com>
date Thu, 09 Dec 2010 16:41:54 +0000
parents 3d9ef098daf2
children fc834b323f26
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
145
853b736a6a9e Added copyright notice.
Guido van Rossum <guido@python.org>
parents: 96
diff changeset
1
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
2 /* Tokenizer implementation */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
3
23029
8291473fde3e Mass checkin of universal newline support.
Jack Jansen <jack.jansen@cwi.nl>
parents: 19621
diff changeset
4 #include "Python.h"
86
a6934380c6e7 "Compiling" version
Guido van Rossum <guido@python.org>
parents: 67
diff changeset
5 #include "pgenheaders.h"
a6934380c6e7 "Compiling" version
Guido van Rossum <guido@python.org>
parents: 67
diff changeset
6
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
7 #include <ctype.h>
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
8 #include <assert.h>
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
9
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
10 #include "tokenizer.h"
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
11 #include "errcode.h"
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
12
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
13 #ifndef PGEN
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
14 #include "unicodeobject.h"
47302
811a80651232 Renamed files bytesobject.[ch] and stringobject.[ch]
Christian Heimes <christian@cheimes.de>
parents: 47301
diff changeset
15 #include "bytesobject.h"
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
16 #include "fileobject.h"
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
17 #include "codecs.h"
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
18 #include "abstract.h"
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
19 #endif /* PGEN */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
20
42013
cd125fe83051 Make identifiers str (not str8) objects throughout.
Martin v. Löwis <martin@v.loewis.de>
parents: 41970
diff changeset
21 #define is_potential_identifier_start(c) (\
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
22 (c >= 'a' && c <= 'z')\
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
23 || (c >= 'A' && c <= 'Z')\
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
24 || c == '_'\
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
25 || (c >= 128))
42013
cd125fe83051 Make identifiers str (not str8) objects throughout.
Martin v. Löwis <martin@v.loewis.de>
parents: 41970
diff changeset
26
cd125fe83051 Make identifiers str (not str8) objects throughout.
Martin v. Löwis <martin@v.loewis.de>
parents: 41970
diff changeset
27 #define is_potential_identifier_char(c) (\
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
28 (c >= 'a' && c <= 'z')\
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
29 || (c >= 'A' && c <= 'Z')\
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
30 || (c >= '0' && c <= '9')\
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
31 || c == '_'\
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
32 || (c >= 128))
42013
cd125fe83051 Make identifiers str (not str8) objects throughout.
Martin v. Löwis <martin@v.loewis.de>
parents: 41970
diff changeset
33
25924
dc4a0336a2a3 Patch #512981: Update readline input stream on sys.stdin/out change.
Martin v. Löwis <martin@v.loewis.de>
parents: 25101
diff changeset
34 extern char *PyOS_Readline(FILE *, FILE *, char *);
1835
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
35 /* Return malloc'ed string including trailing \n;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
36 empty malloc'ed string for EOF;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
37 NULL if interrupted */
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string
Guido van Rossum <guido@python.org>
parents: 1364
diff changeset
38
678
205b947b1d3c Make tabs always 8 spaces wide -- it's more portable.
Guido van Rossum <guido@python.org>
parents: 477
diff changeset
39 /* Don't ever change this -- it would break the portability of Python code */
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
40 #define TABSIZE 8
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
41
86
a6934380c6e7 "Compiling" version
Guido van Rossum <guido@python.org>
parents: 67
diff changeset
42 /* Forward */
13447
a89f7f268de4 Nuke all remaining occurrences of Py_PROTO and Py_FPROTO.
Tim Peters <tim.peters@gmail.com>
parents: 13256
diff changeset
43 static struct tok_state *tok_new(void);
a89f7f268de4 Nuke all remaining occurrences of Py_PROTO and Py_FPROTO.
Tim Peters <tim.peters@gmail.com>
parents: 13256
diff changeset
44 static int tok_nextc(struct tok_state *tok);
a89f7f268de4 Nuke all remaining occurrences of Py_PROTO and Py_FPROTO.
Tim Peters <tim.peters@gmail.com>
parents: 13256
diff changeset
45 static void tok_backup(struct tok_state *tok, int c);
86
a6934380c6e7 "Compiling" version
Guido van Rossum <guido@python.org>
parents: 67
diff changeset
46
43607
7593436058dd Plug a memory leak where a struct tok_state was not being freed.
Brett Cannon <bcannon@gmail.com>
parents: 43605
diff changeset
47
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
48 /* Token names */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
49
5379
1c9a84e0981d Another directory quickly renamed.
Guido van Rossum <guido@python.org>
parents: 5341
diff changeset
50 char *_PyParser_TokenNames[] = {
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
51 "ENDMARKER",
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
52 "NAME",
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
53 "NUMBER",
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
54 "STRING",
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
55 "NEWLINE",
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
56 "INDENT",
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
57 "DEDENT",
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
58 "LPAR",
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
59 "RPAR",
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
60 "LSQB",
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
61 "RSQB",
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
62 "COLON",
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
63 "COMMA",
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
64 "SEMI",
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
65 "PLUS",
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
66 "MINUS",
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
67 "STAR",
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
68 "SLASH",
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
69 "VBAR",
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
70 "AMPER",
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
71 "LESS",
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
72 "GREATER",
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
73 "EQUAL",
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
74 "DOT",
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
75 "PERCENT",
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
76 "LBRACE",
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
77 "RBRACE",
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
78 "EQEQUAL",
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
79 "NOTEQUAL",
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
80 "LESSEQUAL",
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
81 "GREATEREQUAL",
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
82 "TILDE",
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
83 "CIRCUMFLEX",
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
84 "LEFTSHIFT",
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
85 "RIGHTSHIFT",
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
86 "DOUBLESTAR",
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
87 "PLUSEQUAL",
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
88 "MINEQUAL",
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
89 "STAREQUAL",
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
90 "SLASHEQUAL",
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
91 "PERCENTEQUAL",
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
92 "AMPEREQUAL",
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
93 "VBAREQUAL",
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
94 "CIRCUMFLEXEQUAL",
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
95 "LEFTSHIFTEQUAL",
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
96 "RIGHTSHIFTEQUAL",
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
97 "DOUBLESTAREQUAL",
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
98 "DOUBLESLASH",
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
99 "DOUBLESLASHEQUAL",
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
100 "AT",
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
101 "RARROW",
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
102 "ELLIPSIS",
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
103 /* This table must match the #defines in token.h! */
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
104 "OP",
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
105 "<ERRORTOKEN>",
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
106 "<N_TOKENS>"
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
107 };
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
108
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
109
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
110 /* Create and initialize a new tok_state structure */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
111
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
112 static struct tok_state *
13659
e78153e179fb Mass ANSIfication.
Thomas Wouters <thomas@python.org>
parents: 13539
diff changeset
113 tok_new(void)
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
114 {
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
115 struct tok_state *tok = (struct tok_state *)PyMem_MALLOC(
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
116 sizeof(struct tok_state));
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
117 if (tok == NULL)
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
118 return NULL;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
119 tok->buf = tok->cur = tok->end = tok->inp = tok->start = NULL;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
120 tok->done = E_OK;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
121 tok->fp = NULL;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
122 tok->input = NULL;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
123 tok->tabsize = TABSIZE;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
124 tok->indent = 0;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
125 tok->indstack[0] = 0;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
126 tok->atbol = 1;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
127 tok->pendin = 0;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
128 tok->prompt = tok->nextprompt = NULL;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
129 tok->lineno = 0;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
130 tok->level = 0;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
131 tok->filename = NULL;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
132 tok->altwarning = 1;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
133 tok->alterror = 1;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
134 tok->alttabsize = 1;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
135 tok->altindstack[0] = 0;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
136 tok->decoding_state = STATE_INIT;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
137 tok->decoding_erred = 0;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
138 tok->read_coding_spec = 0;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
139 tok->enc = NULL;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
140 tok->encoding = NULL;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
141 tok->cont_line = 0;
24544
0bd521eb714e Make pgen compile with pydebug. Duplicate normalized names, as it may
Martin v. Löwis <martin@v.loewis.de>
parents: 24543
diff changeset
142 #ifndef PGEN
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
143 tok->decoding_readline = NULL;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
144 tok->decoding_buffer = NULL;
24544
0bd521eb714e Make pgen compile with pydebug. Duplicate normalized names, as it may
Martin v. Löwis <martin@v.loewis.de>
parents: 24543
diff changeset
145 #endif
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
146 return tok;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
147 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
148
57206
b2d90c1c637d Merged revisions 76230 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 56594
diff changeset
149 static char *
b2d90c1c637d Merged revisions 76230 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 56594
diff changeset
150 new_string(const char *s, Py_ssize_t len)
b2d90c1c637d Merged revisions 76230 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 56594
diff changeset
151 {
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
152 char* result = (char *)PyMem_MALLOC(len + 1);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
153 if (result != NULL) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
154 memcpy(result, s, len);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
155 result[len] = '\0';
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
156 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
157 return result;
57206
b2d90c1c637d Merged revisions 76230 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 56594
diff changeset
158 }
b2d90c1c637d Merged revisions 76230 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 56594
diff changeset
159
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
160 #ifdef PGEN
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
161
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
162 static char *
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
163 decoding_fgets(char *s, int size, struct tok_state *tok)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
164 {
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
165 return fgets(s, size, tok->fp);
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
166 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
167
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
168 static int
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
169 decoding_feof(struct tok_state *tok)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
170 {
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
171 return feof(tok->fp);
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
172 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
173
57206
b2d90c1c637d Merged revisions 76230 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 56594
diff changeset
174 static char *
b2d90c1c637d Merged revisions 76230 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 56594
diff changeset
175 decode_str(const char *str, int exec_input, struct tok_state *tok)
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
176 {
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
177 return new_string(str, strlen(str));
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
178 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
179
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
180 #else /* PGEN */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
181
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
182 static char *
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
183 error_ret(struct tok_state *tok) /* XXX */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
184 {
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
185 tok->decoding_erred = 1;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
186 if (tok->fp != NULL && tok->buf != NULL) /* see PyTokenizer_Free */
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
187 PyMem_FREE(tok->buf);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
188 tok->buf = NULL;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
189 return NULL; /* as if it were EOF */
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
190 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
191
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
192
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
193 static char *
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
194 get_normal_name(char *s) /* for utf-8 and latin-1 */
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
195 {
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
196 char buf[13];
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
197 int i;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
198 for (i = 0; i < 12; i++) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
199 int c = s[i];
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
200 if (c == '\0')
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
201 break;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
202 else if (c == '_')
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
203 buf[i] = '-';
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
204 else
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
205 buf[i] = tolower(c);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
206 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
207 buf[i] = '\0';
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
208 if (strcmp(buf, "utf-8") == 0 ||
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
209 strncmp(buf, "utf-8-", 6) == 0)
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
210 return "utf-8";
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
211 else if (strcmp(buf, "latin-1") == 0 ||
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
212 strcmp(buf, "iso-8859-1") == 0 ||
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
213 strcmp(buf, "iso-latin-1") == 0 ||
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
214 strncmp(buf, "latin-1-", 8) == 0 ||
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
215 strncmp(buf, "iso-8859-1-", 11) == 0 ||
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
216 strncmp(buf, "iso-latin-1-", 12) == 0)
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
217 return "iso-8859-1";
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
218 else
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
219 return s;
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
220 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
221
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
222 /* Return the coding spec in S, or NULL if none is found. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
223
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
224 static char *
36501
7fdc639bc5b4 Merge ssize_t branch.
Martin v. Löwis <martin@v.loewis.de>
parents: 36090
diff changeset
225 get_coding_spec(const char *s, Py_ssize_t size)
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
226 {
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
227 Py_ssize_t i;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
228 /* Coding spec must be in a comment, and that comment must be
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
229 * the only statement on the source code line. */
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
230 for (i = 0; i < size - 6; i++) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
231 if (s[i] == '#')
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
232 break;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
233 if (s[i] != ' ' && s[i] != '\t' && s[i] != '\014')
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
234 return NULL;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
235 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
236 for (; i < size - 6; i++) { /* XXX inefficient search */
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
237 const char* t = s + i;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
238 if (strncmp(t, "coding", 6) == 0) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
239 const char* begin = NULL;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
240 t += 6;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
241 if (t[0] != ':' && t[0] != '=')
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
242 continue;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
243 do {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
244 t++;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
245 } while (t[0] == '\x20' || t[0] == '\t');
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
246
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
247 begin = t;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
248 while (Py_ISALNUM(t[0]) ||
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
249 t[0] == '-' || t[0] == '_' || t[0] == '.')
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
250 t++;
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
251
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
252 if (begin < t) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
253 char* r = new_string(begin, t - begin);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
254 char* q = get_normal_name(r);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
255 if (r != q) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
256 PyMem_FREE(r);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
257 r = new_string(q, strlen(q));
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
258 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
259 return r;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
260 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
261 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
262 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
263 return NULL;
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
264 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
265
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
266 /* Check whether the line contains a coding spec. If it does,
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
267 invoke the set_readline function for the new encoding.
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
268 This function receives the tok_state and the new encoding.
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
269 Return 1 on success, 0 on failure. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
270
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
271 static int
36501
7fdc639bc5b4 Merge ssize_t branch.
Martin v. Löwis <martin@v.loewis.de>
parents: 36090
diff changeset
272 check_coding_spec(const char* line, Py_ssize_t size, struct tok_state *tok,
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
273 int set_readline(struct tok_state *, const char *))
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
274 {
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
275 char * cs;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
276 int r = 1;
25101
32331d26d7cf Removed reliance on gcc/C99 extension.
Tim Peters <tim.peters@gmail.com>
parents: 25098
diff changeset
277
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
278 if (tok->cont_line)
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
279 /* It's a continuation line, so it can't be a coding spec. */
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
280 return 1;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
281 cs = get_coding_spec(line, size);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
282 if (cs != NULL) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
283 tok->read_coding_spec = 1;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
284 if (tok->encoding == NULL) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
285 assert(tok->decoding_state == STATE_RAW);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
286 if (strcmp(cs, "utf-8") == 0) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
287 tok->encoding = cs;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
288 } else {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
289 r = set_readline(tok, cs);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
290 if (r) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
291 tok->encoding = cs;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
292 tok->decoding_state = STATE_NORMAL;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
293 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
294 else
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
295 PyMem_FREE(cs);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
296 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
297 } else { /* then, compare cs with BOM */
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
298 r = (strcmp(tok->encoding, cs) == 0);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
299 PyMem_FREE(cs);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
300 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
301 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
302 if (!r) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
303 cs = tok->encoding;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
304 if (!cs)
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
305 cs = "with BOM";
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
306 PyErr_Format(PyExc_SyntaxError, "encoding problem: %s", cs);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
307 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
308 return r;
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
309 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
310
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
311 /* See whether the file starts with a BOM. If it does,
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
312 invoke the set_readline function with the new encoding.
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
313 Return 1 on success, 0 on failure. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
314
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
315 static int
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
316 check_bom(int get_char(struct tok_state *),
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
317 void unget_char(int, struct tok_state *),
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
318 int set_readline(struct tok_state *, const char *),
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
319 struct tok_state *tok)
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
320 {
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
321 int ch1, ch2, ch3;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
322 ch1 = get_char(tok);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
323 tok->decoding_state = STATE_RAW;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
324 if (ch1 == EOF) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
325 return 1;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
326 } else if (ch1 == 0xEF) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
327 ch2 = get_char(tok);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
328 if (ch2 != 0xBB) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
329 unget_char(ch2, tok);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
330 unget_char(ch1, tok);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
331 return 1;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
332 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
333 ch3 = get_char(tok);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
334 if (ch3 != 0xBF) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
335 unget_char(ch3, tok);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
336 unget_char(ch2, tok);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
337 unget_char(ch1, tok);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
338 return 1;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
339 }
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
340 #if 0
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
341 /* Disable support for UTF-16 BOMs until a decision
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
342 is made whether this needs to be supported. */
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
343 } else if (ch1 == 0xFE) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
344 ch2 = get_char(tok);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
345 if (ch2 != 0xFF) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
346 unget_char(ch2, tok);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
347 unget_char(ch1, tok);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
348 return 1;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
349 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
350 if (!set_readline(tok, "utf-16-be"))
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
351 return 0;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
352 tok->decoding_state = STATE_NORMAL;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
353 } else if (ch1 == 0xFF) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
354 ch2 = get_char(tok);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
355 if (ch2 != 0xFE) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
356 unget_char(ch2, tok);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
357 unget_char(ch1, tok);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
358 return 1;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
359 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
360 if (!set_readline(tok, "utf-16-le"))
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
361 return 0;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
362 tok->decoding_state = STATE_NORMAL;
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
363 #endif
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
364 } else {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
365 unget_char(ch1, tok);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
366 return 1;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
367 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
368 if (tok->encoding != NULL)
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
369 PyMem_FREE(tok->encoding);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
370 tok->encoding = new_string("utf-8", 5); /* resulting is in utf-8 */
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
371 /* No need to set_readline: input is already utf-8 */
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
372 return 1;
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
373 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
374
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
375 /* Read a line of text from TOK into S, using the stream in TOK.
35231
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
376 Return NULL on failure, else S.
37879
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 37704
diff changeset
377
35231
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
378 On entry, tok->decoding_buffer will be one of:
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
379 1) NULL: need to call tok->decoding_readline to get a new line
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
380 2) PyUnicodeObject *: decoding_feof has called tok->decoding_readline and
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
381 stored the result in tok->decoding_buffer
47300
96b34263e448 Renamed PyBytes to PyByteArray
Christian Heimes <christian@cheimes.de>
parents: 46664
diff changeset
382 3) PyByteArrayObject *: previous call to fp_readl did not have enough room
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
383 (in the s buffer) to copy entire contents of the line read
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
384 by tok->decoding_readline. tok->decoding_buffer has the overflow.
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
385 In this case, fp_readl is called in a loop (with an expanded buffer)
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
386 until the buffer ends with a '\n' (or until the end of the file is
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
387 reached): see tok_nextc and its calls to decoding_fgets.
35231
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
388 */
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
389
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
390 static char *
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
391 fp_readl(char *s, int size, struct tok_state *tok)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
392 {
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
393 PyObject* bufobj;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
394 const char *buf;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
395 Py_ssize_t buflen;
35231
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
396
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
397 /* Ask for one less byte so we can terminate it */
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
398 assert(size > 0);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
399 size--;
35231
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file
Walter Dörwald <walter@livinglogic.de>
parents: 34664
diff changeset
400
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
401 if (tok->decoding_buffer) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
402 bufobj = tok->decoding_buffer;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
403 Py_INCREF(bufobj);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
404 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
405 else
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
406 {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
407 bufobj = PyObject_CallObject(tok->decoding_readline, NULL);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
408 if (bufobj == NULL)
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
409 goto error;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
410 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
411 if (PyUnicode_CheckExact(bufobj))
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
412 {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
413 buf = _PyUnicode_AsStringAndSize(bufobj, &buflen);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
414 if (buf == NULL) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
415 goto error;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
416 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
417 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
418 else
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
419 {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
420 buf = PyByteArray_AsString(bufobj);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
421 if (buf == NULL) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
422 goto error;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
423 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
424 buflen = PyByteArray_GET_SIZE(bufobj);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
425 }
43873
1739869cbc6e Correction for issue1134: all source files with a coding spec, except latin-1
Amaury Forgeot d'Arc <amauryfa@gmail.com>
parents: 43777
diff changeset
426
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
427 Py_XDECREF(tok->decoding_buffer);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
428 if (buflen > size) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
429 /* Too many chars, the rest goes into tok->decoding_buffer */
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
430 tok->decoding_buffer = PyByteArray_FromStringAndSize(buf+size,
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
431 buflen-size);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
432 if (tok->decoding_buffer == NULL)
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
433 goto error;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
434 buflen = size;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
435 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
436 else
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
437 tok->decoding_buffer = NULL;
43873
1739869cbc6e Correction for issue1134: all source files with a coding spec, except latin-1
Amaury Forgeot d'Arc <amauryfa@gmail.com>
parents: 43777
diff changeset
438
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
439 memcpy(s, buf, buflen);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
440 s[buflen] = '\0';
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
441 if (buflen == 0) /* EOF */
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
442 s = NULL;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
443 Py_DECREF(bufobj);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
444 return s;
42551
b6625d2b378a Fix refleaks from execfile('file that contains a # coding: line')
Neal Norwitz <nnorwitz@gmail.com>
parents: 42548
diff changeset
445
b6625d2b378a Fix refleaks from execfile('file that contains a # coding: line')
Neal Norwitz <nnorwitz@gmail.com>
parents: 42548
diff changeset
446 error:
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
447 Py_XDECREF(bufobj);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
448 return error_ret(tok);
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
449 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
450
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
451 /* Set the readline function for TOK to a StreamReader's
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
452 readline function. The StreamReader is named ENC.
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
453
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
454 This function is called from check_bom and check_coding_spec.
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
455
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
456 ENC is usually identical to the future value of tok->encoding,
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
457 except for the (currently unsupported) case of UTF-16.
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
458
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
459 Return 1 on success, 0 on failure. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
460
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
461 static int
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
462 fp_setreadl(struct tok_state *tok, const char* enc)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
463 {
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
464 PyObject *readline = NULL, *stream = NULL, *io = NULL;
65311
3d9ef098daf2 Issue #10095: fp_setreadl() doesn't reopen the file, reuse instead the file
Victor Stinner <victor.stinner@haypocalc.com>
parents: 61466
diff changeset
465 int fd;
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
466
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
467 io = PyImport_ImportModuleNoBlock("io");
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
468 if (io == NULL)
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
469 goto cleanup;
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
470
65311
3d9ef098daf2 Issue #10095: fp_setreadl() doesn't reopen the file, reuse instead the file
Victor Stinner <victor.stinner@haypocalc.com>
parents: 61466
diff changeset
471 fd = fileno(tok->fp);
3d9ef098daf2 Issue #10095: fp_setreadl() doesn't reopen the file, reuse instead the file
Victor Stinner <victor.stinner@haypocalc.com>
parents: 61466
diff changeset
472 if (lseek(fd, 0, SEEK_SET) == (off_t)-1) {
3d9ef098daf2 Issue #10095: fp_setreadl() doesn't reopen the file, reuse instead the file
Victor Stinner <victor.stinner@haypocalc.com>
parents: 61466
diff changeset
473 PyErr_SetFromErrnoWithFilename(PyExc_OSError, NULL);
3d9ef098daf2 Issue #10095: fp_setreadl() doesn't reopen the file, reuse instead the file
Victor Stinner <victor.stinner@haypocalc.com>
parents: 61466
diff changeset
474 goto cleanup;
3d9ef098daf2 Issue #10095: fp_setreadl() doesn't reopen the file, reuse instead the file
Victor Stinner <victor.stinner@haypocalc.com>
parents: 61466
diff changeset
475 }
3d9ef098daf2 Issue #10095: fp_setreadl() doesn't reopen the file, reuse instead the file
Victor Stinner <victor.stinner@haypocalc.com>
parents: 61466
diff changeset
476
3d9ef098daf2 Issue #10095: fp_setreadl() doesn't reopen the file, reuse instead the file
Victor Stinner <victor.stinner@haypocalc.com>
parents: 61466
diff changeset
477 stream = PyObject_CallMethod(io, "open", "isisOOO",
3d9ef098daf2 Issue #10095: fp_setreadl() doesn't reopen the file, reuse instead the file
Victor Stinner <victor.stinner@haypocalc.com>
parents: 61466
diff changeset
478 fd, "r", -1, enc, Py_None, Py_None, Py_False);
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
479 if (stream == NULL)
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
480 goto cleanup;
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
481
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
482 Py_XDECREF(tok->decoding_readline);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
483 readline = PyObject_GetAttrString(stream, "readline");
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
484 tok->decoding_readline = readline;
41970
b3900a51346b tokenizer.c: make coding markup work again.
Guido van Rossum <guido@python.org>
parents: 41670
diff changeset
485
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
486 /* The file has been reopened; parsing will restart from
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
487 * the beginning of the file, we have to reset the line number.
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
488 * But this function has been called from inside tok_nextc() which
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
489 * will increment lineno before it returns. So we set it -1 so that
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
490 * the next call to tok_nextc() will start with tok->lineno == 0.
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
491 */
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
492 tok->lineno = -1;
49484
0b7e023803fb Issues #2384 and #3975: Tracebacks were not correctly printed when the source file
Amaury Forgeot d'Arc <amauryfa@gmail.com>
parents: 49021
diff changeset
493
41970
b3900a51346b tokenizer.c: make coding markup work again.
Guido van Rossum <guido@python.org>
parents: 41670
diff changeset
494 cleanup:
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
495 Py_XDECREF(stream);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
496 Py_XDECREF(io);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
497 return readline != NULL;
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
498 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
499
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
500 /* Fetch the next byte from TOK. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
501
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
502 static int fp_getc(struct tok_state *tok) {
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
503 return getc(tok->fp);
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
504 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
505
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
506 /* Unfetch the last byte back into TOK. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
507
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
508 static void fp_ungetc(int c, struct tok_state *tok) {
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
509 ungetc(c, tok->fp);
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
510 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
511
42403
7e8469617b1c Implement PEP 3120.
Martin v. Löwis <martin@v.loewis.de>
parents: 42084
diff changeset
512 /* Check whether the characters at s start a valid
7e8469617b1c Implement PEP 3120.
Martin v. Löwis <martin@v.loewis.de>
parents: 42084
diff changeset
513 UTF-8 sequence. Return the number of characters forming
7e8469617b1c Implement PEP 3120.
Martin v. Löwis <martin@v.loewis.de>
parents: 42084
diff changeset
514 the sequence if yes, 0 if not. */
7e8469617b1c Implement PEP 3120.
Martin v. Löwis <martin@v.loewis.de>
parents: 42084
diff changeset
515 static int valid_utf8(const unsigned char* s)
7e8469617b1c Implement PEP 3120.
Martin v. Löwis <martin@v.loewis.de>
parents: 42084
diff changeset
516 {
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
517 int expected = 0;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
518 int length;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
519 if (*s < 0x80)
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
520 /* single-byte code */
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
521 return 1;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
522 if (*s < 0xc0)
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
523 /* following byte */
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
524 return 0;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
525 if (*s < 0xE0)
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
526 expected = 1;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
527 else if (*s < 0xF0)
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
528 expected = 2;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
529 else if (*s < 0xF8)
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
530 expected = 3;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
531 else
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
532 return 0;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
533 length = expected + 1;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
534 for (; expected; expected--)
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
535 if (s[expected] < 0x80 || s[expected] >= 0xC0)
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
536 return 0;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
537 return length;
42403
7e8469617b1c Implement PEP 3120.
Martin v. Löwis <martin@v.loewis.de>
parents: 42084
diff changeset
538 }
7e8469617b1c Implement PEP 3120.
Martin v. Löwis <martin@v.loewis.de>
parents: 42084
diff changeset
539
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
540 /* Read a line of input from TOK. Determine encoding
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
541 if necessary. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
542
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
543 static char *
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
544 decoding_fgets(char *s, int size, struct tok_state *tok)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
545 {
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
546 char *line = NULL;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
547 int badchar = 0;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
548 for (;;) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
549 if (tok->decoding_state == STATE_NORMAL) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
550 /* We already have a codec associated with
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
551 this input. */
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
552 line = fp_readl(s, size, tok);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
553 break;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
554 } else if (tok->decoding_state == STATE_RAW) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
555 /* We want a 'raw' read. */
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
556 line = Py_UniversalNewlineFgets(s, size,
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
557 tok->fp, NULL);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
558 break;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
559 } else {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
560 /* We have not yet determined the encoding.
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
561 If an encoding is found, use the file-pointer
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
562 reader functions from now on. */
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
563 if (!check_bom(fp_getc, fp_ungetc, fp_setreadl, tok))
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
564 return error_ret(tok);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
565 assert(tok->decoding_state != STATE_INIT);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
566 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
567 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
568 if (line != NULL && tok->lineno < 2 && !tok->read_coding_spec) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
569 if (!check_coding_spec(line, strlen(line), tok, fp_setreadl)) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
570 return error_ret(tok);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
571 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
572 }
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
573 #ifndef PGEN
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
574 /* The default encoding is UTF-8, so make sure we don't have any
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
575 non-UTF-8 sequences in it. */
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
576 if (line && !tok->encoding) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
577 unsigned char *c;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
578 int length;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
579 for (c = (unsigned char *)line; *c; c += length)
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
580 if (!(length = valid_utf8(c))) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
581 badchar = *c;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
582 break;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
583 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
584 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
585 if (badchar) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
586 /* Need to add 1 to the line number, since this line
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
587 has not been counted, yet. */
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
588 PyErr_Format(PyExc_SyntaxError,
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
589 "Non-UTF-8 code starting with '\\x%.2x' "
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
590 "in file %.200s on line %i, "
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
591 "but no encoding declared; "
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
592 "see http://python.org/dev/peps/pep-0263/ for details",
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
593 badchar, tok->filename, tok->lineno + 1);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
594 return error_ret(tok);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
595 }
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
596 #endif
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
597 return line;
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
598 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
599
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
600 static int
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
601 decoding_feof(struct tok_state *tok)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
602 {
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
603 if (tok->decoding_state != STATE_NORMAL) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
604 return feof(tok->fp);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
605 } else {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
606 PyObject* buf = tok->decoding_buffer;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
607 if (buf == NULL) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
608 buf = PyObject_CallObject(tok->decoding_readline, NULL);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
609 if (buf == NULL) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
610 error_ret(tok);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
611 return 1;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
612 } else {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
613 tok->decoding_buffer = buf;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
614 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
615 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
616 return PyObject_Length(buf) == 0;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
617 }
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
618 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
619
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
620 /* Fetch a byte from TOK, using the string buffer. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
621
37879
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 37704
diff changeset
622 static int
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 37704
diff changeset
623 buf_getc(struct tok_state *tok) {
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
624 return Py_CHARMASK(*tok->str++);
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
625 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
626
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
627 /* Unfetch a byte from TOK, using the string buffer. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
628
37879
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 37704
diff changeset
629 static void
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 37704
diff changeset
630 buf_ungetc(int c, struct tok_state *tok) {
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
631 tok->str--;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
632 assert(Py_CHARMASK(*tok->str) == c); /* tok->cur may point to read-only segment */
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
633 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
634
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
635 /* Set the readline function for TOK to ENC. For the string-based
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
636 tokenizer, this means to just record the encoding. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
637
37879
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 37704
diff changeset
638 static int
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 37704
diff changeset
639 buf_setreadl(struct tok_state *tok, const char* enc) {
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
640 tok->enc = enc;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
641 return 1;
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
642 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
643
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
644 /* Return a UTF-8 encoding Python string object from the
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
645 C byte string STR, which is encoded with ENC. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
646
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
647 static PyObject *
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
648 translate_into_utf8(const char* str, const char* enc) {
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
649 PyObject *utf8;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
650 PyObject* buf = PyUnicode_Decode(str, strlen(str), enc, NULL);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
651 if (buf == NULL)
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
652 return NULL;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
653 utf8 = PyUnicode_AsUTF8String(buf);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
654 Py_DECREF(buf);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
655 return utf8;
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
656 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
657
57206
b2d90c1c637d Merged revisions 76230 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 56594
diff changeset
658
b2d90c1c637d Merged revisions 76230 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 56594
diff changeset
659 static char *
b2d90c1c637d Merged revisions 76230 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 56594
diff changeset
660 translate_newlines(const char *s, int exec_input, struct tok_state *tok) {
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
661 int skip_next_lf = 0, needed_length = strlen(s) + 2, final_length;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
662 char *buf, *current;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
663 char c = '\0';
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
664 buf = PyMem_MALLOC(needed_length);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
665 if (buf == NULL) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
666 tok->done = E_NOMEM;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
667 return NULL;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
668 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
669 for (current = buf; *s; s++, current++) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
670 c = *s;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
671 if (skip_next_lf) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
672 skip_next_lf = 0;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
673 if (c == '\n') {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
674 c = *++s;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
675 if (!c)
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
676 break;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
677 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
678 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
679 if (c == '\r') {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
680 skip_next_lf = 1;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
681 c = '\n';
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
682 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
683 *current = c;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
684 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
685 /* If this is exec input, add a newline to the end of the string if
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
686 there isn't one already. */
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
687 if (exec_input && c != '\n') {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
688 *current = '\n';
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
689 current++;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
690 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
691 *current = '\0';
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
692 final_length = current - buf + 1;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
693 if (final_length < needed_length && final_length)
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
694 /* should never fail */
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
695 buf = PyMem_REALLOC(buf, final_length);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
696 return buf;
57206
b2d90c1c637d Merged revisions 76230 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 56594
diff changeset
697 }
b2d90c1c637d Merged revisions 76230 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 56594
diff changeset
698
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
699 /* Decode a byte string STR for use as the buffer of TOK.
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
700 Look for encoding declarations inside STR, and record them
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
701 inside TOK. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
702
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
703 static const char *
57206
b2d90c1c637d Merged revisions 76230 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 56594
diff changeset
704 decode_str(const char *input, int single, struct tok_state *tok)
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
705 {
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
706 PyObject* utf8 = NULL;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
707 const char *str;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
708 const char *s;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
709 const char *newl[2] = {NULL, NULL};
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
710 int lineno = 0;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
711 tok->input = str = translate_newlines(input, single, tok);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
712 if (str == NULL)
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
713 return NULL;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
714 tok->enc = NULL;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
715 tok->str = str;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
716 if (!check_bom(buf_getc, buf_ungetc, buf_setreadl, tok))
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
717 return error_ret(tok);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
718 str = tok->str; /* string after BOM if any */
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
719 assert(str);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
720 if (tok->enc != NULL) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
721 utf8 = translate_into_utf8(str, tok->enc);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
722 if (utf8 == NULL)
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
723 return error_ret(tok);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
724 str = PyBytes_AsString(utf8);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
725 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
726 for (s = str;; s++) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
727 if (*s == '\0') break;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
728 else if (*s == '\n') {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
729 assert(lineno < 2);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
730 newl[lineno] = s;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
731 lineno++;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
732 if (lineno == 2) break;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
733 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
734 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
735 tok->enc = NULL;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
736 /* need to check line 1 and 2 separately since check_coding_spec
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
737 assumes a single line as input */
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
738 if (newl[0]) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
739 if (!check_coding_spec(str, newl[0] - str, tok, buf_setreadl))
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
740 return error_ret(tok);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
741 if (tok->enc == NULL && newl[1]) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
742 if (!check_coding_spec(newl[0]+1, newl[1] - newl[0],
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
743 tok, buf_setreadl))
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
744 return error_ret(tok);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
745 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
746 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
747 if (tok->enc != NULL) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
748 assert(utf8 == NULL);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
749 utf8 = translate_into_utf8(str, tok->enc);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
750 if (utf8 == NULL)
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
751 return error_ret(tok);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
752 str = PyBytes_AS_STRING(utf8);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
753 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
754 assert(tok->decoding_buffer == NULL);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
755 tok->decoding_buffer = utf8; /* CAUTION */
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
756 return str;
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
757 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
758
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
759 #endif /* PGEN */
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
760
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
761 /* Set up tokenizer for string */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
762
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
763 struct tok_state *
57206
b2d90c1c637d Merged revisions 76230 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 56594
diff changeset
764 PyTokenizer_FromString(const char *str, int exec_input)
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
765 {
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
766 struct tok_state *tok = tok_new();
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
767 if (tok == NULL)
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
768 return NULL;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
769 str = (char *)decode_str(str, exec_input, tok);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
770 if (str == NULL) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
771 PyTokenizer_Free(tok);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
772 return NULL;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
773 }
35878
94a409da4bd4 Fix some more memory leaks.
Neal Norwitz <nnorwitz@gmail.com>
parents: 35751
diff changeset
774
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
775 /* XXX: constify members. */
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
776 tok->buf = tok->cur = tok->end = tok->inp = (char*)str;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
777 return tok;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
778 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
779
51956
8fe2988cf550 ignore the coding cookie in compile(), exec(), and eval() if the source is a string #4626
Benjamin Peterson <benjamin@python.org>
parents: 50540
diff changeset
780 struct tok_state *
57206
b2d90c1c637d Merged revisions 76230 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 56594
diff changeset
781 PyTokenizer_FromUTF8(const char *str, int exec_input)
51956
8fe2988cf550 ignore the coding cookie in compile(), exec(), and eval() if the source is a string #4626
Benjamin Peterson <benjamin@python.org>
parents: 50540
diff changeset
782 {
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
783 struct tok_state *tok = tok_new();
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
784 if (tok == NULL)
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
785 return NULL;
57206
b2d90c1c637d Merged revisions 76230 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 56594
diff changeset
786 #ifndef PGEN
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
787 tok->input = str = translate_newlines(str, exec_input, tok);
57206
b2d90c1c637d Merged revisions 76230 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 56594
diff changeset
788 #endif
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
789 if (str == NULL) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
790 PyTokenizer_Free(tok);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
791 return NULL;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
792 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
793 tok->decoding_state = STATE_RAW;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
794 tok->read_coding_spec = 1;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
795 tok->enc = NULL;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
796 tok->str = str;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
797 tok->encoding = (char *)PyMem_MALLOC(6);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
798 if (!tok->encoding) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
799 PyTokenizer_Free(tok);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
800 return NULL;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
801 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
802 strcpy(tok->encoding, "utf-8");
51956
8fe2988cf550 ignore the coding cookie in compile(), exec(), and eval() if the source is a string #4626
Benjamin Peterson <benjamin@python.org>
parents: 50540
diff changeset
803
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
804 /* XXX: constify members. */
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
805 tok->buf = tok->cur = tok->end = tok->inp = (char*)str;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
806 return tok;
51956
8fe2988cf550 ignore the coding cookie in compile(), exec(), and eval() if the source is a string #4626
Benjamin Peterson <benjamin@python.org>
parents: 50540
diff changeset
807 }
8fe2988cf550 ignore the coding cookie in compile(), exec(), and eval() if the source is a string #4626
Benjamin Peterson <benjamin@python.org>
parents: 50540
diff changeset
808
326
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
809 /* Set up tokenizer for file */
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
810
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
811 struct tok_state *
43225
38f3f9ef193d Convert code from sys.stdin.encoding to UTF-8 in
Martin v. Löwis <martin@v.loewis.de>
parents: 43018
diff changeset
812 PyTokenizer_FromFile(FILE *fp, char* enc, char *ps1, char *ps2)
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
813 {
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
814 struct tok_state *tok = tok_new();
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
815 if (tok == NULL)
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
816 return NULL;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
817 if ((tok->buf = (char *)PyMem_MALLOC(BUFSIZ)) == NULL) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
818 PyTokenizer_Free(tok);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
819 return NULL;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
820 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
821 tok->cur = tok->inp = tok->buf;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
822 tok->end = tok->buf + BUFSIZ;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
823 tok->fp = fp;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
824 tok->prompt = ps1;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
825 tok->nextprompt = ps2;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
826 if (enc != NULL) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
827 /* Must copy encoding declaration since it
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
828 gets copied into the parse tree. */
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
829 tok->encoding = PyMem_MALLOC(strlen(enc)+1);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
830 if (!tok->encoding) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
831 PyTokenizer_Free(tok);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
832 return NULL;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
833 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
834 strcpy(tok->encoding, enc);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
835 tok->decoding_state = STATE_NORMAL;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
836 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
837 return tok;
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
838 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
839
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
840
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
841 /* Free a tok_state structure */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
842
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
843 void
13659
e78153e179fb Mass ANSIfication.
Thomas Wouters <thomas@python.org>
parents: 13539
diff changeset
844 PyTokenizer_Free(struct tok_state *tok)
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
845 {
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
846 if (tok->encoding != NULL)
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
847 PyMem_FREE(tok->encoding);
24544
0bd521eb714e Make pgen compile with pydebug. Duplicate normalized names, as it may
Martin v. Löwis <martin@v.loewis.de>
parents: 24543
diff changeset
848 #ifndef PGEN
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
849 Py_XDECREF(tok->decoding_readline);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
850 Py_XDECREF(tok->decoding_buffer);
24544
0bd521eb714e Make pgen compile with pydebug. Duplicate normalized names, as it may
Martin v. Löwis <martin@v.loewis.de>
parents: 24543
diff changeset
851 #endif
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
852 if (tok->fp != NULL && tok->buf != NULL)
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
853 PyMem_FREE(tok->buf);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
854 if (tok->input)
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
855 PyMem_FREE((char *)tok->input);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
856 PyMem_FREE(tok);
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
857 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
858
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
859 /* Get next char, updating state; error code goes into tok->done */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
860
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
861 static int
13659
e78153e179fb Mass ANSIfication.
Thomas Wouters <thomas@python.org>
parents: 13539
diff changeset
862 tok_nextc(register struct tok_state *tok)
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
863 {
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
864 for (;;) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
865 if (tok->cur != tok->inp) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
866 return Py_CHARMASK(*tok->cur++); /* Fast path */
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
867 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
868 if (tok->done != E_OK)
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
869 return EOF;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
870 if (tok->fp == NULL) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
871 char *end = strchr(tok->inp, '\n');
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
872 if (end != NULL)
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
873 end++;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
874 else {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
875 end = strchr(tok->inp, '\0');
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
876 if (end == tok->inp) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
877 tok->done = E_EOF;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
878 return EOF;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
879 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
880 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
881 if (tok->start == NULL)
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
882 tok->buf = tok->cur;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
883 tok->line_start = tok->cur;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
884 tok->lineno++;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
885 tok->inp = end;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
886 return Py_CHARMASK(*tok->cur++);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
887 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
888 if (tok->prompt != NULL) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
889 char *newtok = PyOS_Readline(stdin, stdout, tok->prompt);
43225
38f3f9ef193d Convert code from sys.stdin.encoding to UTF-8 in
Martin v. Löwis <martin@v.loewis.de>
parents: 43018
diff changeset
890 #ifndef PGEN
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
891 if (tok->encoding && newtok && *newtok) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
892 /* Recode to UTF-8 */
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
893 Py_ssize_t buflen;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
894 const char* buf;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
895 PyObject *u = translate_into_utf8(newtok, tok->encoding);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
896 PyMem_FREE(newtok);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
897 if (!u) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
898 tok->done = E_DECODE;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
899 return EOF;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
900 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
901 buflen = PyBytes_GET_SIZE(u);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
902 buf = PyBytes_AS_STRING(u);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
903 if (!buf) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
904 Py_DECREF(u);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
905 tok->done = E_DECODE;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
906 return EOF;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
907 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
908 newtok = PyMem_MALLOC(buflen+1);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
909 strcpy(newtok, buf);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
910 Py_DECREF(u);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
911 }
43225
38f3f9ef193d Convert code from sys.stdin.encoding to UTF-8 in
Martin v. Löwis <martin@v.loewis.de>
parents: 43018
diff changeset
912 #endif
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
913 if (tok->nextprompt != NULL)
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
914 tok->prompt = tok->nextprompt;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
915 if (newtok == NULL)
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
916 tok->done = E_INTR;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
917 else if (*newtok == '\0') {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
918 PyMem_FREE(newtok);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
919 tok->done = E_EOF;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
920 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
921 else if (tok->start != NULL) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
922 size_t start = tok->start - tok->buf;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
923 size_t oldlen = tok->cur - tok->buf;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
924 size_t newlen = oldlen + strlen(newtok);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
925 char *buf = tok->buf;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
926 buf = (char *)PyMem_REALLOC(buf, newlen+1);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
927 tok->lineno++;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
928 if (buf == NULL) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
929 PyMem_FREE(tok->buf);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
930 tok->buf = NULL;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
931 PyMem_FREE(newtok);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
932 tok->done = E_NOMEM;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
933 return EOF;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
934 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
935 tok->buf = buf;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
936 tok->cur = tok->buf + oldlen;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
937 tok->line_start = tok->cur;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
938 strcpy(tok->buf + oldlen, newtok);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
939 PyMem_FREE(newtok);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
940 tok->inp = tok->buf + newlen;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
941 tok->end = tok->inp + 1;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
942 tok->start = tok->buf + start;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
943 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
944 else {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
945 tok->lineno++;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
946 if (tok->buf != NULL)
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
947 PyMem_FREE(tok->buf);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
948 tok->buf = newtok;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
949 tok->line_start = tok->buf;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
950 tok->cur = tok->buf;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
951 tok->line_start = tok->buf;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
952 tok->inp = strchr(tok->buf, '\0');
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
953 tok->end = tok->inp + 1;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
954 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
955 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
956 else {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
957 int done = 0;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
958 Py_ssize_t cur = 0;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
959 char *pt;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
960 if (tok->start == NULL) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
961 if (tok->buf == NULL) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
962 tok->buf = (char *)
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
963 PyMem_MALLOC(BUFSIZ);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
964 if (tok->buf == NULL) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
965 tok->done = E_NOMEM;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
966 return EOF;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
967 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
968 tok->end = tok->buf + BUFSIZ;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
969 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
970 if (decoding_fgets(tok->buf, (int)(tok->end - tok->buf),
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
971 tok) == NULL) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
972 tok->done = E_EOF;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
973 done = 1;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
974 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
975 else {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
976 tok->done = E_OK;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
977 tok->inp = strchr(tok->buf, '\0');
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
978 done = tok->inp[-1] == '\n';
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
979 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
980 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
981 else {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
982 cur = tok->cur - tok->buf;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
983 if (decoding_feof(tok)) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
984 tok->done = E_EOF;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
985 done = 1;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
986 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
987 else
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
988 tok->done = E_OK;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
989 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
990 tok->lineno++;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
991 /* Read until '\n' or EOF */
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
992 while (!done) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
993 Py_ssize_t curstart = tok->start == NULL ? -1 :
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
994 tok->start - tok->buf;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
995 Py_ssize_t curvalid = tok->inp - tok->buf;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
996 Py_ssize_t newsize = curvalid + BUFSIZ;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
997 char *newbuf = tok->buf;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
998 newbuf = (char *)PyMem_REALLOC(newbuf,
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
999 newsize);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1000 if (newbuf == NULL) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1001 tok->done = E_NOMEM;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1002 tok->cur = tok->inp;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1003 return EOF;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1004 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1005 tok->buf = newbuf;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1006 tok->inp = tok->buf + curvalid;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1007 tok->end = tok->buf + newsize;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1008 tok->start = curstart < 0 ? NULL :
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1009 tok->buf + curstart;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1010 if (decoding_fgets(tok->inp,
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1011 (int)(tok->end - tok->inp),
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1012 tok) == NULL) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1013 /* Break out early on decoding
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1014 errors, as tok->buf will be NULL
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1015 */
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1016 if (tok->decoding_erred)
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1017 return EOF;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1018 /* Last line does not end in \n,
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1019 fake one */
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1020 strcpy(tok->inp, "\n");
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1021 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1022 tok->inp = strchr(tok->inp, '\0');
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1023 done = tok->inp[-1] == '\n';
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1024 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1025 if (tok->buf != NULL) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1026 tok->cur = tok->buf + cur;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1027 tok->line_start = tok->cur;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1028 /* replace "\r\n" with "\n" */
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1029 /* For Mac leave the \r, giving a syntax error */
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1030 pt = tok->inp - 2;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1031 if (pt >= tok->buf && *pt == '\r') {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1032 *pt++ = '\n';
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1033 *pt = '\0';
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1034 tok->inp = pt;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1035 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1036 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1037 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1038 if (tok->done != E_OK) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1039 if (tok->prompt != NULL)
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1040 PySys_WriteStderr("\n");
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1041 tok->cur = tok->inp;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1042 return EOF;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1043 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1044 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1045 /*NOTREACHED*/
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1046 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1047
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1048
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1049 /* Back-up one character */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1050
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1051 static void
13659
e78153e179fb Mass ANSIfication.
Thomas Wouters <thomas@python.org>
parents: 13539
diff changeset
1052 tok_backup(register struct tok_state *tok, register int c)
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1053 {
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1054 if (c != EOF) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1055 if (--tok->cur < tok->buf)
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1056 Py_FatalError("tok_backup: beginning of buffer");
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1057 if (*tok->cur != c)
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1058 *tok->cur = c;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1059 }
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1060 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1061
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1062
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1063 /* Return the token corresponding to a single character */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1064
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1065 int
13659
e78153e179fb Mass ANSIfication.
Thomas Wouters <thomas@python.org>
parents: 13539
diff changeset
1066 PyToken_OneChar(int c)
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1067 {
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1068 switch (c) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1069 case '(': return LPAR;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1070 case ')': return RPAR;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1071 case '[': return LSQB;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1072 case ']': return RSQB;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1073 case ':': return COLON;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1074 case ',': return COMMA;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1075 case ';': return SEMI;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1076 case '+': return PLUS;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1077 case '-': return MINUS;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1078 case '*': return STAR;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1079 case '/': return SLASH;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1080 case '|': return VBAR;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1081 case '&': return AMPER;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1082 case '<': return LESS;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1083 case '>': return GREATER;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1084 case '=': return EQUAL;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1085 case '.': return DOT;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1086 case '%': return PERCENT;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1087 case '{': return LBRACE;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1088 case '}': return RBRACE;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1089 case '^': return CIRCUMFLEX;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1090 case '~': return TILDE;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1091 case '@': return AT;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1092 default: return OP;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1093 }
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1094 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1095
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1096
400
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1097 int
13659
e78153e179fb Mass ANSIfication.
Thomas Wouters <thomas@python.org>
parents: 13539
diff changeset
1098 PyToken_TwoChars(int c1, int c2)
400
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1099 {
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1100 switch (c1) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1101 case '=':
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1102 switch (c2) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1103 case '=': return EQEQUAL;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1104 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1105 break;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1106 case '!':
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1107 switch (c2) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1108 case '=': return NOTEQUAL;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1109 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1110 break;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1111 case '<':
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1112 switch (c2) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1113 case '>': return NOTEQUAL;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1114 case '=': return LESSEQUAL;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1115 case '<': return LEFTSHIFT;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1116 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1117 break;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1118 case '>':
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1119 switch (c2) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1120 case '=': return GREATEREQUAL;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1121 case '>': return RIGHTSHIFT;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1122 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1123 break;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1124 case '+':
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1125 switch (c2) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1126 case '=': return PLUSEQUAL;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1127 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1128 break;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1129 case '-':
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1130 switch (c2) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1131 case '=': return MINEQUAL;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1132 case '>': return RARROW;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1133 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1134 break;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1135 case '*':
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1136 switch (c2) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1137 case '*': return DOUBLESTAR;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1138 case '=': return STAREQUAL;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1139 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1140 break;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1141 case '/':
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1142 switch (c2) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1143 case '/': return DOUBLESLASH;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1144 case '=': return SLASHEQUAL;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1145 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1146 break;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1147 case '|':
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1148 switch (c2) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1149 case '=': return VBAREQUAL;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1150 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1151 break;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1152 case '%':
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1153 switch (c2) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1154 case '=': return PERCENTEQUAL;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1155 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1156 break;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1157 case '&':
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1158 switch (c2) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1159 case '=': return AMPEREQUAL;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1160 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1161 break;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1162 case '^':
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1163 switch (c2) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1164 case '=': return CIRCUMFLEXEQUAL;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1165 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1166 break;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1167 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1168 return OP;
400
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1169 }
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1170
14163
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1171 int
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1172 PyToken_ThreeChars(int c1, int c2, int c3)
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1173 {
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1174 switch (c1) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1175 case '<':
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1176 switch (c2) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1177 case '<':
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1178 switch (c3) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1179 case '=':
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1180 return LEFTSHIFTEQUAL;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1181 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1182 break;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1183 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1184 break;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1185 case '>':
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1186 switch (c2) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1187 case '>':
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1188 switch (c3) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1189 case '=':
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1190 return RIGHTSHIFTEQUAL;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1191 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1192 break;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1193 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1194 break;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1195 case '*':
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1196 switch (c2) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1197 case '*':
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1198 switch (c3) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1199 case '=':
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1200 return DOUBLESTAREQUAL;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1201 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1202 break;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1203 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1204 break;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1205 case '/':
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1206 switch (c2) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1207 case '/':
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1208 switch (c3) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1209 case '=':
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1210 return DOUBLESLASHEQUAL;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1211 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1212 break;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1213 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1214 break;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1215 case '.':
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1216 switch (c2) {
41230
b1a18c7ed5ca Make ELLIPSIS a separate token. This makes it a syntax error to write ". . ." for Ellipsis.
Georg Brandl <georg@python.org>
parents: 40894
diff changeset
1217 case '.':
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1218 switch (c3) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1219 case '.':
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1220 return ELLIPSIS;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1221 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1222 break;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1223 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1224 break;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1225 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1226 return OP;
14163
1b9fd0cbd914 Support for three-token characters (**=, >>=, <<=) which was written by
Thomas Wouters <thomas@python.org>
parents: 13659
diff changeset
1227 }
400
d189a2f681a6 Added 2-char tokens and new versions of comparisons
Guido van Rossum <guido@python.org>
parents: 326
diff changeset
1228
7945
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1229 static int
13659
e78153e179fb Mass ANSIfication.
Thomas Wouters <thomas@python.org>
parents: 13539
diff changeset
1230 indenterror(struct tok_state *tok)
7945
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1231 {
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1232 if (tok->alterror) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1233 tok->done = E_TABSPACE;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1234 tok->cur = tok->inp;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1235 return 1;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1236 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1237 if (tok->altwarning) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1238 PySys_WriteStderr("%s: inconsistent use of tabs and spaces "
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1239 "in indentation\n", tok->filename);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1240 tok->altwarning = 0;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1241 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1242 return 0;
7945
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1243 }
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1244
42597
20477cbfd996 Implement PEP 3131. Add isidentifier to str.
Martin v. Löwis <martin@v.loewis.de>
parents: 42551
diff changeset
1245 #ifdef PGEN
59536
42261c6ca564 Merged revisions 78826 via svnmerge from
Victor Stinner <victor.stinner@haypocalc.com>
parents: 59288
diff changeset
1246 #define verify_identifier(tok) 1
42597
20477cbfd996 Implement PEP 3131. Add isidentifier to str.
Martin v. Löwis <martin@v.loewis.de>
parents: 42551
diff changeset
1247 #else
20477cbfd996 Implement PEP 3131. Add isidentifier to str.
Martin v. Löwis <martin@v.loewis.de>
parents: 42551
diff changeset
1248 /* Verify that the identifier follows PEP 3131. */
20477cbfd996 Implement PEP 3131. Add isidentifier to str.
Martin v. Löwis <martin@v.loewis.de>
parents: 42551
diff changeset
1249 static int
59536
42261c6ca564 Merged revisions 78826 via svnmerge from
Victor Stinner <victor.stinner@haypocalc.com>
parents: 59288
diff changeset
1250 verify_identifier(struct tok_state *tok)
42597
20477cbfd996 Implement PEP 3131. Add isidentifier to str.
Martin v. Löwis <martin@v.loewis.de>
parents: 42551
diff changeset
1251 {
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1252 PyObject *s;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1253 int result;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1254 s = PyUnicode_DecodeUTF8(tok->start, tok->cur - tok->start, NULL);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1255 if (s == NULL) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1256 if (PyErr_ExceptionMatches(PyExc_UnicodeDecodeError)) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1257 PyErr_Clear();
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1258 tok->done = E_IDENTIFIER;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1259 } else {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1260 tok->done = E_ERROR;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1261 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1262 return 0;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1263 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1264 result = PyUnicode_IsIdentifier(s);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1265 Py_DECREF(s);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1266 if (result == 0)
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1267 tok->done = E_IDENTIFIER;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1268 return result;
42597
20477cbfd996 Implement PEP 3131. Add isidentifier to str.
Martin v. Löwis <martin@v.loewis.de>
parents: 42551
diff changeset
1269 }
20477cbfd996 Implement PEP 3131. Add isidentifier to str.
Martin v. Löwis <martin@v.loewis.de>
parents: 42551
diff changeset
1270 #endif
7945
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 7324
diff changeset
1271
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1272 /* Get next token, after space stripping etc. */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1273
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
1274 static int
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
1275 tok_get(register struct tok_state *tok, char **p_start, char **p_end)
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1276 {
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1277 register int c;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1278 int blankline, nonascii;
326
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1279
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1280 *p_start = *p_end = NULL;
326
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1281 nextline:
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1282 tok->start = NULL;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1283 blankline = 0;
326
ff0862e3d63a Completely ignore lines with only a newline token on them, except
Guido van Rossum <guido@python.org>
parents: 293
diff changeset
1284
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1285 /* Get indentation level */
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1286 if (tok->atbol) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1287 register int col = 0;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1288 register int altcol = 0;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1289 tok->atbol = 0;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1290 for (;;) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1291 c = tok_nextc(tok);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1292 if (c == ' ')
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1293 col++, altcol++;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1294 else if (c == '\t') {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1295 col = (col/tok->tabsize + 1) * tok->tabsize;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1296 altcol = (altcol/tok->alttabsize + 1)
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1297 * tok->alttabsize;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1298 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1299 else if (c == '\014') /* Control-L (formfeed) */
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1300 col = altcol = 0; /* For Emacs users */
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1301 else
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1302 break;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1303 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1304 tok_backup(tok, c);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1305 if (c == '#' || c == '\n') {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1306 /* Lines with only whitespace and/or comments
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1307 shouldn't affect the indentation and are
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1308 not passed to the parser as NEWLINE tokens,
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1309 except *totally* empty lines in interactive
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1310 mode, which signal the end of a command group. */
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1311 if (col == 0 && c == '\n' && tok->prompt != NULL)
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1312 blankline = 0; /* Let it through */
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1313 else
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1314 blankline = 1; /* Ignore completely */
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1315 /* We can't jump back right here since we still
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1316 may need to skip to the end of a comment */
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1317 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1318 if (!blankline && tok->level == 0) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1319 if (col == tok->indstack[tok->indent]) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1320 /* No change */
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1321 if (altcol != tok->altindstack[tok->indent]) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1322 if (indenterror(tok))
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1323 return ERRORTOKEN;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1324 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1325 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1326 else if (col > tok->indstack[tok->indent]) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1327 /* Indent -- always one */
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1328 if (tok->indent+1 >= MAXINDENT) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1329 tok->done = E_TOODEEP;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1330 tok->cur = tok->inp;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1331 return ERRORTOKEN;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1332 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1333 if (altcol <= tok->altindstack[tok->indent]) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1334 if (indenterror(tok))
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1335 return ERRORTOKEN;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1336 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1337 tok->pendin++;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1338 tok->indstack[++tok->indent] = col;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1339 tok->altindstack[tok->indent] = altcol;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1340 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1341 else /* col < tok->indstack[tok->indent] */ {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1342 /* Dedent -- any number, must be consistent */
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1343 while (tok->indent > 0 &&
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1344 col < tok->indstack[tok->indent]) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1345 tok->pendin--;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1346 tok->indent--;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1347 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1348 if (col != tok->indstack[tok->indent]) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1349 tok->done = E_DEDENT;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1350 tok->cur = tok->inp;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1351 return ERRORTOKEN;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1352 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1353 if (altcol != tok->altindstack[tok->indent]) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1354 if (indenterror(tok))
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1355 return ERRORTOKEN;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1356 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1357 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1358 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1359 }
37879
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 37704
diff changeset
1360
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1361 tok->start = tok->cur;
37879
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 37704
diff changeset
1362
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1363 /* Return pending indents/dedents */
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1364 if (tok->pendin != 0) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1365 if (tok->pendin < 0) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1366 tok->pendin++;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1367 return DEDENT;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1368 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1369 else {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1370 tok->pendin--;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1371 return INDENT;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1372 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1373 }
37879
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 37704
diff changeset
1374
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1375 again:
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1376 tok->start = NULL;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1377 /* Skip spaces */
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1378 do {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1379 c = tok_nextc(tok);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1380 } while (c == ' ' || c == '\t' || c == '\014');
37879
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 37704
diff changeset
1381
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1382 /* Set start of current token */
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1383 tok->start = tok->cur - 1;
37879
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 37704
diff changeset
1384
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1385 /* Skip comment */
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1386 if (c == '#')
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1387 while (c != EOF && c != '\n')
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1388 c = tok_nextc(tok);
37879
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 37704
diff changeset
1389
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1390 /* Check for EOF and errors now */
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1391 if (c == EOF) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1392 return tok->done == E_EOF ? ENDMARKER : ERRORTOKEN;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1393 }
37879
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 37704
diff changeset
1394
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1395 /* Identifier (most frequent token!) */
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1396 nonascii = 0;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1397 if (is_potential_identifier_start(c)) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1398 /* Process b"", r"" and br"" */
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1399 if (c == 'b' || c == 'B') {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1400 c = tok_nextc(tok);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1401 if (c == '"' || c == '\'')
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1402 goto letter_quote;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1403 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1404 if (c == 'r' || c == 'R') {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1405 c = tok_nextc(tok);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1406 if (c == '"' || c == '\'')
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1407 goto letter_quote;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1408 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1409 while (is_potential_identifier_char(c)) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1410 if (c >= 128)
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1411 nonascii = 1;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1412 c = tok_nextc(tok);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1413 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1414 tok_backup(tok, c);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1415 if (nonascii &&
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1416 !verify_identifier(tok)) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1417 tok->done = E_IDENTIFIER;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1418 return ERRORTOKEN;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1419 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1420 *p_start = tok->start;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1421 *p_end = tok->cur;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1422 return NAME;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1423 }
37879
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 37704
diff changeset
1424
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1425 /* Newline */
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1426 if (c == '\n') {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1427 tok->atbol = 1;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1428 if (blankline || tok->level > 0)
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1429 goto nextline;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1430 *p_start = tok->start;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1431 *p_end = tok->cur - 1; /* Leave '\n' out of the string */
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1432 tok->cont_line = 0;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1433 return NEWLINE;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1434 }
37879
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 37704
diff changeset
1435
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1436 /* Period or number starting with period? */
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1437 if (c == '.') {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1438 c = tok_nextc(tok);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1439 if (isdigit(c)) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1440 goto fraction;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1441 } else if (c == '.') {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1442 c = tok_nextc(tok);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1443 if (c == '.') {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1444 *p_start = tok->start;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1445 *p_end = tok->cur;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1446 return ELLIPSIS;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1447 } else {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1448 tok_backup(tok, c);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1449 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1450 tok_backup(tok, '.');
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1451 } else {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1452 tok_backup(tok, c);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1453 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1454 *p_start = tok->start;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1455 *p_end = tok->cur;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1456 return DOT;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1457 }
3333
4b5bff95804e changes for pow(**) and complex
Guido van Rossum <guido@python.org>
parents: 3116
diff changeset
1458
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1459 /* Number */
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1460 if (isdigit(c)) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1461 if (c == '0') {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1462 /* Hex, octal or binary -- maybe. */
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1463 c = tok_nextc(tok);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1464 if (c == '.')
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1465 goto fraction;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1466 if (c == 'j' || c == 'J')
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1467 goto imaginary;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1468 if (c == 'x' || c == 'X') {
44782
ca2683a67943 Merged revisions 60080-60089,60091-60093 via svnmerge from
Georg Brandl <georg@python.org>
parents: 44442
diff changeset
1469
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1470 /* Hex */
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1471 c = tok_nextc(tok);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1472 if (!isxdigit(c)) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1473 tok->done = E_TOKEN;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1474 tok_backup(tok, c);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1475 return ERRORTOKEN;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1476 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1477 do {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1478 c = tok_nextc(tok);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1479 } while (isxdigit(c));
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1480 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1481 else if (c == 'o' || c == 'O') {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1482 /* Octal */
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1483 c = tok_nextc(tok);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1484 if (c < '0' || c >= '8') {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1485 tok->done = E_TOKEN;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1486 tok_backup(tok, c);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1487 return ERRORTOKEN;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1488 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1489 do {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1490 c = tok_nextc(tok);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1491 } while ('0' <= c && c < '8');
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1492 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1493 else if (c == 'b' || c == 'B') {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1494 /* Binary */
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1495 c = tok_nextc(tok);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1496 if (c != '0' && c != '1') {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1497 tok->done = E_TOKEN;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1498 tok_backup(tok, c);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1499 return ERRORTOKEN;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1500 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1501 do {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1502 c = tok_nextc(tok);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1503 } while (c == '0' || c == '1');
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1504 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1505 else {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1506 int nonzero = 0;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1507 /* maybe old-style octal; c is first char of it */
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1508 /* in any case, allow '0' as a literal */
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1509 while (c == '0')
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1510 c = tok_nextc(tok);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1511 while (isdigit(c)) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1512 nonzero = 1;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1513 c = tok_nextc(tok);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1514 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1515 if (c == '.')
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1516 goto fraction;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1517 else if (c == 'e' || c == 'E')
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1518 goto exponent;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1519 else if (c == 'j' || c == 'J')
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1520 goto imaginary;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1521 else if (nonzero) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1522 tok->done = E_TOKEN;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1523 tok_backup(tok, c);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1524 return ERRORTOKEN;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1525 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1526 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1527 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1528 else {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1529 /* Decimal */
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1530 do {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1531 c = tok_nextc(tok);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1532 } while (isdigit(c));
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1533 {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1534 /* Accept floating point numbers. */
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1535 if (c == '.') {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1536 fraction:
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1537 /* Fraction */
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1538 do {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1539 c = tok_nextc(tok);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1540 } while (isdigit(c));
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1541 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1542 if (c == 'e' || c == 'E') {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1543 exponent:
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1544 /* Exponent part */
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1545 c = tok_nextc(tok);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1546 if (c == '+' || c == '-')
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1547 c = tok_nextc(tok);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1548 if (!isdigit(c)) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1549 tok->done = E_TOKEN;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1550 tok_backup(tok, c);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1551 return ERRORTOKEN;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1552 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1553 do {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1554 c = tok_nextc(tok);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1555 } while (isdigit(c));
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1556 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1557 if (c == 'j' || c == 'J')
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1558 /* Imaginary part */
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1559 imaginary:
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1560 c = tok_nextc(tok);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1561 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1562 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1563 tok_backup(tok, c);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1564 *p_start = tok->start;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1565 *p_end = tok->cur;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1566 return NUMBER;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1567 }
5265
3c1788ab9761 Support for alternative string quotes (a"xx", b"xx", c"xx", ...).
Guido van Rossum <guido@python.org>
parents: 4839
diff changeset
1568
3c1788ab9761 Support for alternative string quotes (a"xx", b"xx", c"xx", ...).
Guido van Rossum <guido@python.org>
parents: 4839
diff changeset
1569 letter_quote:
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1570 /* String */
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1571 if (c == '\'' || c == '"') {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1572 int quote = c;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1573 int quote_size = 1; /* 1 or 3 */
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1574 int end_quote_size = 0;
43879
5590157f4fc6 Cleanup of tokenizer.c.
Guido van Rossum <guido@python.org>
parents: 43873
diff changeset
1575
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1576 /* Find the quote size and start of string */
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1577 c = tok_nextc(tok);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1578 if (c == quote) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1579 c = tok_nextc(tok);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1580 if (c == quote)
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1581 quote_size = 3;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1582 else
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1583 end_quote_size = 1; /* empty string found */
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1584 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1585 if (c != quote)
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1586 tok_backup(tok, c);
43879
5590157f4fc6 Cleanup of tokenizer.c.
Guido van Rossum <guido@python.org>
parents: 43873
diff changeset
1587
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1588 /* Get rest of string */
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1589 while (end_quote_size != quote_size) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1590 c = tok_nextc(tok);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1591 if (c == EOF) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1592 if (quote_size == 3)
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1593 tok->done = E_EOFS;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1594 else
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1595 tok->done = E_EOLS;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1596 tok->cur = tok->inp;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1597 return ERRORTOKEN;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1598 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1599 if (quote_size == 1 && c == '\n') {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1600 tok->done = E_EOLS;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1601 tok->cur = tok->inp;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1602 return ERRORTOKEN;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1603 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1604 if (c == quote)
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1605 end_quote_size += 1;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1606 else {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1607 end_quote_size = 0;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1608 if (c == '\\')
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1609 c = tok_nextc(tok); /* skip escaped char */
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1610 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1611 }
43879
5590157f4fc6 Cleanup of tokenizer.c.
Guido van Rossum <guido@python.org>
parents: 43873
diff changeset
1612
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1613 *p_start = tok->start;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1614 *p_end = tok->cur;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1615 return STRING;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1616 }
37879
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 37704
diff changeset
1617
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1618 /* Line continuation */
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1619 if (c == '\\') {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1620 c = tok_nextc(tok);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1621 if (c != '\n') {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1622 tok->done = E_LINECONT;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1623 tok->cur = tok->inp;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1624 return ERRORTOKEN;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1625 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1626 tok->cont_line = 1;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1627 goto again; /* Read next line */
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1628 }
37879
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 37704
diff changeset
1629
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1630 /* Check for two-character token */
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1631 {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1632 int c2 = tok_nextc(tok);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1633 int token = PyToken_TwoChars(c, c2);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1634 if (token != OP) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1635 int c3 = tok_nextc(tok);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1636 int token3 = PyToken_ThreeChars(c, c2, c3);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1637 if (token3 != OP) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1638 token = token3;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1639 } else {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1640 tok_backup(tok, c3);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1641 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1642 *p_start = tok->start;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1643 *p_end = tok->cur;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1644 return token;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1645 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1646 tok_backup(tok, c2);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1647 }
37879
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 37704
diff changeset
1648
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1649 /* Keep track of parentheses nesting level */
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1650 switch (c) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1651 case '(':
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1652 case '[':
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1653 case '{':
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1654 tok->level++;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1655 break;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1656 case ')':
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1657 case ']':
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1658 case '}':
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1659 tok->level--;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1660 break;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1661 }
37879
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 37704
diff changeset
1662
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1663 /* Punctuation character */
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1664 *p_start = tok->start;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1665 *p_end = tok->cur;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1666 return PyToken_OneChar(c);
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1667 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1668
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
1669 int
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
1670 PyTokenizer_Get(struct tok_state *tok, char **p_start, char **p_end)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
1671 {
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1672 int result = tok_get(tok, p_start, p_end);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1673 if (tok->decoding_erred) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1674 result = ERRORTOKEN;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1675 tok->done = E_DECODE;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1676 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1677 return result;
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 23029
diff changeset
1678 }
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1679
43632
0067a6ab4854 Issue 1267, continued.
Guido van Rossum <guido@python.org>
parents: 43622
diff changeset
1680 /* Get -*- encoding -*- from a Python file.
43605
a2bb76d11d66 Patch 1267 by Christian Heimes.
Guido van Rossum <guido@python.org>
parents: 43523
diff changeset
1681
a2bb76d11d66 Patch 1267 by Christian Heimes.
Guido van Rossum <guido@python.org>
parents: 43523
diff changeset
1682 PyTokenizer_FindEncoding returns NULL when it can't find the encoding in
43879
5590157f4fc6 Cleanup of tokenizer.c.
Guido van Rossum <guido@python.org>
parents: 43873
diff changeset
1683 the first or second line of the file (in which case the encoding
43609
d378acdc30ee Fix PyTokenizer_FindEncoding() for OS X 10.4. Turns out that seeking to the
Brett Cannon <bcannon@gmail.com>
parents: 43607
diff changeset
1684 should be assumed to be PyUnicode_GetDefaultEncoding()).
d378acdc30ee Fix PyTokenizer_FindEncoding() for OS X 10.4. Turns out that seeking to the
Brett Cannon <bcannon@gmail.com>
parents: 43607
diff changeset
1685
43632
0067a6ab4854 Issue 1267, continued.
Guido van Rossum <guido@python.org>
parents: 43622
diff changeset
1686 The char * returned is malloc'ed via PyMem_MALLOC() and thus must be freed
0067a6ab4854 Issue 1267, continued.
Guido van Rossum <guido@python.org>
parents: 43622
diff changeset
1687 by the caller.
43605
a2bb76d11d66 Patch 1267 by Christian Heimes.
Guido van Rossum <guido@python.org>
parents: 43523
diff changeset
1688 */
a2bb76d11d66 Patch 1267 by Christian Heimes.
Guido van Rossum <guido@python.org>
parents: 43523
diff changeset
1689 char *
43632
0067a6ab4854 Issue 1267, continued.
Guido van Rossum <guido@python.org>
parents: 43622
diff changeset
1690 PyTokenizer_FindEncoding(int fd)
0067a6ab4854 Issue 1267, continued.
Guido van Rossum <guido@python.org>
parents: 43622
diff changeset
1691 {
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1692 struct tok_state *tok;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1693 FILE *fp;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1694 char *p_start =NULL , *p_end =NULL , *encoding = NULL;
43605
a2bb76d11d66 Patch 1267 by Christian Heimes.
Guido van Rossum <guido@python.org>
parents: 43523
diff changeset
1695
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1696 fd = dup(fd);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1697 if (fd < 0) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1698 return NULL;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1699 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1700 fp = fdopen(fd, "r");
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1701 if (fp == NULL) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1702 return NULL;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1703 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1704 tok = PyTokenizer_FromFile(fp, NULL, NULL, NULL);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1705 if (tok == NULL) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1706 fclose(fp);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1707 return NULL;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1708 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1709 while (tok->lineno < 2 && tok->done == E_OK) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1710 PyTokenizer_Get(tok, &p_start, &p_end);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1711 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1712 fclose(fp);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1713 if (tok->encoding) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1714 encoding = (char *)PyMem_MALLOC(strlen(tok->encoding) + 1);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1715 if (encoding)
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1716 strcpy(encoding, tok->encoding);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1717 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1718 PyTokenizer_Free(tok);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1719 return encoding;
43605
a2bb76d11d66 Patch 1267 by Christian Heimes.
Guido van Rossum <guido@python.org>
parents: 43523
diff changeset
1720 }
43292
876b7ef4775d Merged revisions 57778-58052 via svnmerge from
Thomas Wouters <thomas@python.org>
parents: 43225
diff changeset
1721
4839
4567e158d20c Rename DEBUG macro to Py_DEBUG
Guido van Rossum <guido@python.org>
parents: 4577
diff changeset
1722 #ifdef Py_DEBUG
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1723
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1724 void
13659
e78153e179fb Mass ANSIfication.
Thomas Wouters <thomas@python.org>
parents: 13539
diff changeset
1725 tok_dump(int type, char *start, char *end)
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1726 {
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1727 printf("%s", _PyParser_TokenNames[type]);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1728 if (type == NAME || type == NUMBER || type == STRING || type == OP)
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 61055
diff changeset
1729 printf("(%.*s)", (int)(end - start), start);
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1730 }
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1731
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
1732 #endif