cpython-lru_cache-weakref: Parser/tokenizer.c annotate

annotate Parser/tokenizer.c @ 78069:a8aa77458e12 3.2

Add c_size_t and c_ssize_t to the ctypes summary table.

author	Antoine Pitrou <solipsis@pitrou.net>
date	Thu, 12 Jul 2012 20:31:50 +0200
parents	fa5e348889c2
children	bb62908896fe

rev	line source
145 853b736a6a9e Added copyright notice. Guido van Rossum <guido@python.org> parents: 96 diff changeset	1
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	2 /* Tokenizer implementation */
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	3
23029 8291473fde3e Mass checkin of universal newline support. Jack Jansen <jack.jansen@cwi.nl> parents: 19621 diff changeset	4 #include "Python.h"
86 a6934380c6e7 "Compiling" version Guido van Rossum <guido@python.org> parents: 67 diff changeset	5 #include "pgenheaders.h"
a6934380c6e7 "Compiling" version Guido van Rossum <guido@python.org> parents: 67 diff changeset	6
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	7 #include <ctype.h>
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	8 #include <assert.h>
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	9
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	10 #include "tokenizer.h"
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	11 #include "errcode.h"
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	12
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	13 #ifndef PGEN
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	14 #include "unicodeobject.h"
47302 811a80651232 Renamed files bytesobject.[ch] and stringobject.[ch] Christian Heimes <christian@cheimes.de> parents: 47301 diff changeset	15 #include "bytesobject.h"
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	16 #include "fileobject.h"
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	17 #include "codecs.h"
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	18 #include "abstract.h"
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	19 #endif /* PGEN */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	20
42013 cd125fe83051 Make identifiers str (not str8) objects throughout. Martin v. Löwis <martin@v.loewis.de> parents: 41970 diff changeset	21 #define is_potential_identifier_start(c) (\
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	22 (c >= 'a' && c <= 'z')\
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	23 \|\| (c >= 'A' && c <= 'Z')\
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	24 \|\| c == '_'\
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	25 \|\| (c >= 128))
42013 cd125fe83051 Make identifiers str (not str8) objects throughout. Martin v. Löwis <martin@v.loewis.de> parents: 41970 diff changeset	26
cd125fe83051 Make identifiers str (not str8) objects throughout. Martin v. Löwis <martin@v.loewis.de> parents: 41970 diff changeset	27 #define is_potential_identifier_char(c) (\
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	28 (c >= 'a' && c <= 'z')\
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	29 \|\| (c >= 'A' && c <= 'Z')\
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	30 \|\| (c >= '0' && c <= '9')\
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	31 \|\| c == '_'\
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	32 \|\| (c >= 128))
42013 cd125fe83051 Make identifiers str (not str8) objects throughout. Martin v. Löwis <martin@v.loewis.de> parents: 41970 diff changeset	33
25924 dc4a0336a2a3 Patch #512981: Update readline input stream on sys.stdin/out change. Martin v. Löwis <martin@v.loewis.de> parents: 25101 diff changeset	34 extern char PyOS_Readline(FILE , FILE , char );
1835 d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string Guido van Rossum <guido@python.org> parents: 1364 diff changeset	35 /* Return malloc'ed string including trailing \n;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string Guido van Rossum <guido@python.org> parents: 1364 diff changeset	36 empty malloc'ed string for EOF;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string Guido van Rossum <guido@python.org> parents: 1364 diff changeset	37 NULL if interrupted */
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string Guido van Rossum <guido@python.org> parents: 1364 diff changeset	38
678 205b947b1d3c Make tabs always 8 spaces wide -- it's more portable. Guido van Rossum <guido@python.org> parents: 477 diff changeset	39 /* Don't ever change this -- it would break the portability of Python code */
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	40 #define TABSIZE 8
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	41
86 a6934380c6e7 "Compiling" version Guido van Rossum <guido@python.org> parents: 67 diff changeset	42 /* Forward */
13447 a89f7f268de4 Nuke all remaining occurrences of Py_PROTO and Py_FPROTO. Tim Peters <tim.peters@gmail.com> parents: 13256 diff changeset	43 static struct tok_state *tok_new(void);
a89f7f268de4 Nuke all remaining occurrences of Py_PROTO and Py_FPROTO. Tim Peters <tim.peters@gmail.com> parents: 13256 diff changeset	44 static int tok_nextc(struct tok_state *tok);
a89f7f268de4 Nuke all remaining occurrences of Py_PROTO and Py_FPROTO. Tim Peters <tim.peters@gmail.com> parents: 13256 diff changeset	45 static void tok_backup(struct tok_state *tok, int c);
86 a6934380c6e7 "Compiling" version Guido van Rossum <guido@python.org> parents: 67 diff changeset	46
43607 7593436058dd Plug a memory leak where a struct tok_state was not being freed. Brett Cannon <bcannon@gmail.com> parents: 43605 diff changeset	47
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	48 /* Token names */
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	49
5379 1c9a84e0981d Another directory quickly renamed. Guido van Rossum <guido@python.org> parents: 5341 diff changeset	50 char *_PyParser_TokenNames[] = {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	51 "ENDMARKER",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	52 "NAME",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	53 "NUMBER",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	54 "STRING",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	55 "NEWLINE",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	56 "INDENT",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	57 "DEDENT",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	58 "LPAR",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	59 "RPAR",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	60 "LSQB",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	61 "RSQB",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	62 "COLON",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	63 "COMMA",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	64 "SEMI",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	65 "PLUS",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	66 "MINUS",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	67 "STAR",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	68 "SLASH",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	69 "VBAR",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	70 "AMPER",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	71 "LESS",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	72 "GREATER",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	73 "EQUAL",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	74 "DOT",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	75 "PERCENT",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	76 "LBRACE",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	77 "RBRACE",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	78 "EQEQUAL",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	79 "NOTEQUAL",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	80 "LESSEQUAL",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	81 "GREATEREQUAL",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	82 "TILDE",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	83 "CIRCUMFLEX",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	84 "LEFTSHIFT",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	85 "RIGHTSHIFT",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	86 "DOUBLESTAR",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	87 "PLUSEQUAL",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	88 "MINEQUAL",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	89 "STAREQUAL",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	90 "SLASHEQUAL",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	91 "PERCENTEQUAL",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	92 "AMPEREQUAL",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	93 "VBAREQUAL",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	94 "CIRCUMFLEXEQUAL",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	95 "LEFTSHIFTEQUAL",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	96 "RIGHTSHIFTEQUAL",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	97 "DOUBLESTAREQUAL",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	98 "DOUBLESLASH",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	99 "DOUBLESLASHEQUAL",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	100 "AT",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	101 "RARROW",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	102 "ELLIPSIS",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	103 /* This table must match the #defines in token.h! */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	104 "OP",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	105 "<ERRORTOKEN>",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	106 "<N_TOKENS>"
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	107 };
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	108
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	109
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	110 /* Create and initialize a new tok_state structure */
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	111
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	112 static struct tok_state *
13659 e78153e179fb Mass ANSIfication. Thomas Wouters <thomas@python.org> parents: 13539 diff changeset	113 tok_new(void)
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	114 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	115 struct tok_state tok = (struct tok_state )PyMem_MALLOC(
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	116 sizeof(struct tok_state));
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	117 if (tok == NULL)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	118 return NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	119 tok->buf = tok->cur = tok->end = tok->inp = tok->start = NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	120 tok->done = E_OK;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	121 tok->fp = NULL;
61466 cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	122 tok->input = NULL;
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	123 tok->tabsize = TABSIZE;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	124 tok->indent = 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	125 tok->indstack[0] = 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	126 tok->atbol = 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	127 tok->pendin = 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	128 tok->prompt = tok->nextprompt = NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	129 tok->lineno = 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	130 tok->level = 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	131 tok->filename = NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	132 tok->altwarning = 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	133 tok->alterror = 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	134 tok->alttabsize = 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	135 tok->altindstack[0] = 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	136 tok->decoding_state = STATE_INIT;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	137 tok->decoding_erred = 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	138 tok->read_coding_spec = 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	139 tok->enc = NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	140 tok->encoding = NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	141 tok->cont_line = 0;
24544 0bd521eb714e Make pgen compile with pydebug. Duplicate normalized names, as it may Martin v. Löwis <martin@v.loewis.de> parents: 24543 diff changeset	142 #ifndef PGEN
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	143 tok->decoding_readline = NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	144 tok->decoding_buffer = NULL;
24544 0bd521eb714e Make pgen compile with pydebug. Duplicate normalized names, as it may Martin v. Löwis <martin@v.loewis.de> parents: 24543 diff changeset	145 #endif
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	146 return tok;
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	147 }
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	148
57206 b2d90c1c637d Merged revisions 76230 via svnmerge from Benjamin Peterson <benjamin@python.org> parents: 56594 diff changeset	149 static char *
b2d90c1c637d Merged revisions 76230 via svnmerge from Benjamin Peterson <benjamin@python.org> parents: 56594 diff changeset	150 new_string(const char *s, Py_ssize_t len)
b2d90c1c637d Merged revisions 76230 via svnmerge from Benjamin Peterson <benjamin@python.org> parents: 56594 diff changeset	151 {
61466 cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	152 char* result = (char *)PyMem_MALLOC(len + 1);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	153 if (result != NULL) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	154 memcpy(result, s, len);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	155 result[len] = '\0';
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	156 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	157 return result;
57206 b2d90c1c637d Merged revisions 76230 via svnmerge from Benjamin Peterson <benjamin@python.org> parents: 56594 diff changeset	158 }
b2d90c1c637d Merged revisions 76230 via svnmerge from Benjamin Peterson <benjamin@python.org> parents: 56594 diff changeset	159
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	160 #ifdef PGEN
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	161
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	162 static char *
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	163 decoding_fgets(char s, int size, struct tok_state tok)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	164 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	165 return fgets(s, size, tok->fp);
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	166 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	167
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	168 static int
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	169 decoding_feof(struct tok_state *tok)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	170 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	171 return feof(tok->fp);
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	172 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	173
57206 b2d90c1c637d Merged revisions 76230 via svnmerge from Benjamin Peterson <benjamin@python.org> parents: 56594 diff changeset	174 static char *
b2d90c1c637d Merged revisions 76230 via svnmerge from Benjamin Peterson <benjamin@python.org> parents: 56594 diff changeset	175 decode_str(const char str, int exec_input, struct tok_state tok)
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	176 {
61466 cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	177 return new_string(str, strlen(str));
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	178 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	179
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	180 #else /* PGEN */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	181
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	182 static char *
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	183 error_ret(struct tok_state tok) / XXX */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	184 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	185 tok->decoding_erred = 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	186 if (tok->fp != NULL && tok->buf != NULL) /* see PyTokenizer_Free */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	187 PyMem_FREE(tok->buf);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	188 tok->buf = NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	189 return NULL; /* as if it were EOF */
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	190 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	191
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	192
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	193 static char *
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	194 get_normal_name(char s) / for utf-8 and latin-1 */
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	195 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	196 char buf[13];
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	197 int i;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	198 for (i = 0; i < 12; i++) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	199 int c = s[i];
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	200 if (c == '\0')
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	201 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	202 else if (c == '_')
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	203 buf[i] = '-';
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	204 else
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	205 buf[i] = tolower(c);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	206 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	207 buf[i] = '\0';
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	208 if (strcmp(buf, "utf-8") == 0 \|\|
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	209 strncmp(buf, "utf-8-", 6) == 0)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	210 return "utf-8";
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	211 else if (strcmp(buf, "latin-1") == 0 \|\|
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	212 strcmp(buf, "iso-8859-1") == 0 \|\|
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	213 strcmp(buf, "iso-latin-1") == 0 \|\|
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	214 strncmp(buf, "latin-1-", 8) == 0 \|\|
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	215 strncmp(buf, "iso-8859-1-", 11) == 0 \|\|
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	216 strncmp(buf, "iso-latin-1-", 12) == 0)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	217 return "iso-8859-1";
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	218 else
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	219 return s;
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	220 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	221
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	222 /* Return the coding spec in S, or NULL if none is found. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	223
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	224 static char *
36501 7fdc639bc5b4 Merge ssize_t branch. Martin v. Löwis <martin@v.loewis.de> parents: 36090 diff changeset	225 get_coding_spec(const char *s, Py_ssize_t size)
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	226 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	227 Py_ssize_t i;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	228 /* Coding spec must be in a comment, and that comment must be
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	229 * the only statement on the source code line. */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	230 for (i = 0; i < size - 6; i++) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	231 if (s[i] == '#')
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	232 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	233 if (s[i] != ' ' && s[i] != '\t' && s[i] != '\014')
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	234 return NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	235 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	236 for (; i < size - 6; i++) { /* XXX inefficient search */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	237 const char* t = s + i;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	238 if (strncmp(t, "coding", 6) == 0) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	239 const char* begin = NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	240 t += 6;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	241 if (t[0] != ':' && t[0] != '=')
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	242 continue;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	243 do {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	244 t++;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	245 } while (t[0] == '\x20' \|\| t[0] == '\t');
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	246
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	247 begin = t;
61466 cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	248 while (Py_ISALNUM(t[0]) \|\|
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	249 t[0] == '-' \|\| t[0] == '_' \|\| t[0] == '.')
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	250 t++;
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	251
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	252 if (begin < t) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	253 char* r = new_string(begin, t - begin);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	254 char* q = get_normal_name(r);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	255 if (r != q) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	256 PyMem_FREE(r);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	257 r = new_string(q, strlen(q));
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	258 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	259 return r;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	260 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	261 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	262 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	263 return NULL;
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	264 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	265
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	266 /* Check whether the line contains a coding spec. If it does,
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	267 invoke the set_readline function for the new encoding.
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	268 This function receives the tok_state and the new encoding.
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	269 Return 1 on success, 0 on failure. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	270
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	271 static int
36501 7fdc639bc5b4 Merge ssize_t branch. Martin v. Löwis <martin@v.loewis.de> parents: 36090 diff changeset	272 check_coding_spec(const char* line, Py_ssize_t size, struct tok_state *tok,
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	273 int set_readline(struct tok_state , const char ))
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	274 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	275 char * cs;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	276 int r = 1;
25101 32331d26d7cf Removed reliance on gcc/C99 extension. Tim Peters <tim.peters@gmail.com> parents: 25098 diff changeset	277
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	278 if (tok->cont_line)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	279 /* It's a continuation line, so it can't be a coding spec. */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	280 return 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	281 cs = get_coding_spec(line, size);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	282 if (cs != NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	283 tok->read_coding_spec = 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	284 if (tok->encoding == NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	285 assert(tok->decoding_state == STATE_RAW);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	286 if (strcmp(cs, "utf-8") == 0) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	287 tok->encoding = cs;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	288 } else {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	289 r = set_readline(tok, cs);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	290 if (r) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	291 tok->encoding = cs;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	292 tok->decoding_state = STATE_NORMAL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	293 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	294 else
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	295 PyMem_FREE(cs);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	296 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	297 } else { /* then, compare cs with BOM */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	298 r = (strcmp(tok->encoding, cs) == 0);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	299 PyMem_FREE(cs);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	300 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	301 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	302 if (!r) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	303 cs = tok->encoding;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	304 if (!cs)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	305 cs = "with BOM";
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	306 PyErr_Format(PyExc_SyntaxError, "encoding problem: %s", cs);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	307 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	308 return r;
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	309 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	310
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	311 /* See whether the file starts with a BOM. If it does,
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	312 invoke the set_readline function with the new encoding.
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	313 Return 1 on success, 0 on failure. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	314
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	315 static int
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	316 check_bom(int get_char(struct tok_state *),
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	317 void unget_char(int, struct tok_state *),
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	318 int set_readline(struct tok_state , const char ),
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	319 struct tok_state *tok)
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	320 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	321 int ch1, ch2, ch3;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	322 ch1 = get_char(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	323 tok->decoding_state = STATE_RAW;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	324 if (ch1 == EOF) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	325 return 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	326 } else if (ch1 == 0xEF) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	327 ch2 = get_char(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	328 if (ch2 != 0xBB) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	329 unget_char(ch2, tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	330 unget_char(ch1, tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	331 return 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	332 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	333 ch3 = get_char(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	334 if (ch3 != 0xBF) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	335 unget_char(ch3, tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	336 unget_char(ch2, tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	337 unget_char(ch1, tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	338 return 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	339 }
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	340 #if 0
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	341 /* Disable support for UTF-16 BOMs until a decision
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	342 is made whether this needs to be supported. */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	343 } else if (ch1 == 0xFE) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	344 ch2 = get_char(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	345 if (ch2 != 0xFF) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	346 unget_char(ch2, tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	347 unget_char(ch1, tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	348 return 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	349 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	350 if (!set_readline(tok, "utf-16-be"))
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	351 return 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	352 tok->decoding_state = STATE_NORMAL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	353 } else if (ch1 == 0xFF) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	354 ch2 = get_char(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	355 if (ch2 != 0xFE) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	356 unget_char(ch2, tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	357 unget_char(ch1, tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	358 return 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	359 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	360 if (!set_readline(tok, "utf-16-le"))
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	361 return 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	362 tok->decoding_state = STATE_NORMAL;
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	363 #endif
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	364 } else {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	365 unget_char(ch1, tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	366 return 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	367 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	368 if (tok->encoding != NULL)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	369 PyMem_FREE(tok->encoding);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	370 tok->encoding = new_string("utf-8", 5); /* resulting is in utf-8 */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	371 /* No need to set_readline: input is already utf-8 */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	372 return 1;
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	373 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	374
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	375 /* Read a line of text from TOK into S, using the stream in TOK.
35231 763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file Walter Dörwald <walter@livinglogic.de> parents: 34664 diff changeset	376 Return NULL on failure, else S.
37879 f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair Thomas Wouters <thomas@python.org> parents: 37704 diff changeset	377
35231 763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file Walter Dörwald <walter@livinglogic.de> parents: 34664 diff changeset	378 On entry, tok->decoding_buffer will be one of:
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file Walter Dörwald <walter@livinglogic.de> parents: 34664 diff changeset	379 1) NULL: need to call tok->decoding_readline to get a new line
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file Walter Dörwald <walter@livinglogic.de> parents: 34664 diff changeset	380 2) PyUnicodeObject *: decoding_feof has called tok->decoding_readline and
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	381 stored the result in tok->decoding_buffer
47300 96b34263e448 Renamed PyBytes to PyByteArray Christian Heimes <christian@cheimes.de> parents: 46664 diff changeset	382 3) PyByteArrayObject *: previous call to fp_readl did not have enough room
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	383 (in the s buffer) to copy entire contents of the line read
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	384 by tok->decoding_readline. tok->decoding_buffer has the overflow.
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	385 In this case, fp_readl is called in a loop (with an expanded buffer)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	386 until the buffer ends with a '\n' (or until the end of the file is
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	387 reached): see tok_nextc and its calls to decoding_fgets.
35231 763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file Walter Dörwald <walter@livinglogic.de> parents: 34664 diff changeset	388 */
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	389
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	390 static char *
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	391 fp_readl(char s, int size, struct tok_state tok)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	392 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	393 PyObject* bufobj;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	394 const char *buf;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	395 Py_ssize_t buflen;
35231 763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file Walter Dörwald <walter@livinglogic.de> parents: 34664 diff changeset	396
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	397 /* Ask for one less byte so we can terminate it */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	398 assert(size > 0);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	399 size--;
35231 763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file Walter Dörwald <walter@livinglogic.de> parents: 34664 diff changeset	400
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	401 if (tok->decoding_buffer) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	402 bufobj = tok->decoding_buffer;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	403 Py_INCREF(bufobj);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	404 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	405 else
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	406 {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	407 bufobj = PyObject_CallObject(tok->decoding_readline, NULL);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	408 if (bufobj == NULL)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	409 goto error;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	410 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	411 if (PyUnicode_CheckExact(bufobj))
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	412 {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	413 buf = _PyUnicode_AsStringAndSize(bufobj, &buflen);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	414 if (buf == NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	415 goto error;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	416 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	417 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	418 else
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	419 {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	420 buf = PyByteArray_AsString(bufobj);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	421 if (buf == NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	422 goto error;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	423 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	424 buflen = PyByteArray_GET_SIZE(bufobj);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	425 }
43873 1739869cbc6e Correction for issue1134: all source files with a coding spec, except latin-1 Amaury Forgeot d'Arc <amauryfa@gmail.com> parents: 43777 diff changeset	426
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	427 Py_XDECREF(tok->decoding_buffer);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	428 if (buflen > size) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	429 /* Too many chars, the rest goes into tok->decoding_buffer */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	430 tok->decoding_buffer = PyByteArray_FromStringAndSize(buf+size,
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	431 buflen-size);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	432 if (tok->decoding_buffer == NULL)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	433 goto error;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	434 buflen = size;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	435 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	436 else
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	437 tok->decoding_buffer = NULL;
43873 1739869cbc6e Correction for issue1134: all source files with a coding spec, except latin-1 Amaury Forgeot d'Arc <amauryfa@gmail.com> parents: 43777 diff changeset	438
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	439 memcpy(s, buf, buflen);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	440 s[buflen] = '\0';
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	441 if (buflen == 0) /* EOF */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	442 s = NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	443 Py_DECREF(bufobj);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	444 return s;
42551 b6625d2b378a Fix refleaks from execfile('file that contains a # coding: line') Neal Norwitz <nnorwitz@gmail.com> parents: 42548 diff changeset	445
b6625d2b378a Fix refleaks from execfile('file that contains a # coding: line') Neal Norwitz <nnorwitz@gmail.com> parents: 42548 diff changeset	446 error:
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	447 Py_XDECREF(bufobj);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	448 return error_ret(tok);
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	449 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	450
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	451 /* Set the readline function for TOK to a StreamReader's
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	452 readline function. The StreamReader is named ENC.
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	453
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	454 This function is called from check_bom and check_coding_spec.
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	455
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	456 ENC is usually identical to the future value of tok->encoding,
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	457 except for the (currently unsupported) case of UTF-16.
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	458
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	459 Return 1 on success, 0 on failure. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	460
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	461 static int
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	462 fp_setreadl(struct tok_state tok, const char enc)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	463 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	464 PyObject readline = NULL, stream = NULL, *io = NULL;
65311 3d9ef098daf2 Issue #10095: fp_setreadl() doesn't reopen the file, reuse instead the file Victor Stinner <victor.stinner@haypocalc.com> parents: 61466 diff changeset	465 int fd;
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	466
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	467 io = PyImport_ImportModuleNoBlock("io");
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	468 if (io == NULL)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	469 goto cleanup;
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	470
65311 3d9ef098daf2 Issue #10095: fp_setreadl() doesn't reopen the file, reuse instead the file Victor Stinner <victor.stinner@haypocalc.com> parents: 61466 diff changeset	471 fd = fileno(tok->fp);
3d9ef098daf2 Issue #10095: fp_setreadl() doesn't reopen the file, reuse instead the file Victor Stinner <victor.stinner@haypocalc.com> parents: 61466 diff changeset	472 if (lseek(fd, 0, SEEK_SET) == (off_t)-1) {
3d9ef098daf2 Issue #10095: fp_setreadl() doesn't reopen the file, reuse instead the file Victor Stinner <victor.stinner@haypocalc.com> parents: 61466 diff changeset	473 PyErr_SetFromErrnoWithFilename(PyExc_OSError, NULL);
3d9ef098daf2 Issue #10095: fp_setreadl() doesn't reopen the file, reuse instead the file Victor Stinner <victor.stinner@haypocalc.com> parents: 61466 diff changeset	474 goto cleanup;
3d9ef098daf2 Issue #10095: fp_setreadl() doesn't reopen the file, reuse instead the file Victor Stinner <victor.stinner@haypocalc.com> parents: 61466 diff changeset	475 }
3d9ef098daf2 Issue #10095: fp_setreadl() doesn't reopen the file, reuse instead the file Victor Stinner <victor.stinner@haypocalc.com> parents: 61466 diff changeset	476
3d9ef098daf2 Issue #10095: fp_setreadl() doesn't reopen the file, reuse instead the file Victor Stinner <victor.stinner@haypocalc.com> parents: 61466 diff changeset	477 stream = PyObject_CallMethod(io, "open", "isisOOO",
3d9ef098daf2 Issue #10095: fp_setreadl() doesn't reopen the file, reuse instead the file Victor Stinner <victor.stinner@haypocalc.com> parents: 61466 diff changeset	478 fd, "r", -1, enc, Py_None, Py_None, Py_False);
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	479 if (stream == NULL)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	480 goto cleanup;
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	481
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	482 Py_XDECREF(tok->decoding_readline);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	483 readline = PyObject_GetAttrString(stream, "readline");
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	484 tok->decoding_readline = readline;
41970 b3900a51346b tokenizer.c: make coding markup work again. Guido van Rossum <guido@python.org> parents: 41670 diff changeset	485
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	486 /* The file has been reopened; parsing will restart from
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	487 * the beginning of the file, we have to reset the line number.
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	488 * But this function has been called from inside tok_nextc() which
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	489 * will increment lineno before it returns. So we set it -1 so that
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	490 * the next call to tok_nextc() will start with tok->lineno == 0.
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	491 */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	492 tok->lineno = -1;
49484 0b7e023803fb Issues #2384 and #3975: Tracebacks were not correctly printed when the source file Amaury Forgeot d'Arc <amauryfa@gmail.com> parents: 49021 diff changeset	493
41970 b3900a51346b tokenizer.c: make coding markup work again. Guido van Rossum <guido@python.org> parents: 41670 diff changeset	494 cleanup:
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	495 Py_XDECREF(stream);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	496 Py_XDECREF(io);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	497 return readline != NULL;
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	498 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	499
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	500 /* Fetch the next byte from TOK. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	501
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	502 static int fp_getc(struct tok_state *tok) {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	503 return getc(tok->fp);
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	504 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	505
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	506 /* Unfetch the last byte back into TOK. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	507
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	508 static void fp_ungetc(int c, struct tok_state *tok) {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	509 ungetc(c, tok->fp);
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	510 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	511
42403 7e8469617b1c Implement PEP 3120. Martin v. Löwis <martin@v.loewis.de> parents: 42084 diff changeset	512 /* Check whether the characters at s start a valid
7e8469617b1c Implement PEP 3120. Martin v. Löwis <martin@v.loewis.de> parents: 42084 diff changeset	513 UTF-8 sequence. Return the number of characters forming
7e8469617b1c Implement PEP 3120. Martin v. Löwis <martin@v.loewis.de> parents: 42084 diff changeset	514 the sequence if yes, 0 if not. */
7e8469617b1c Implement PEP 3120. Martin v. Löwis <martin@v.loewis.de> parents: 42084 diff changeset	515 static int valid_utf8(const unsigned char* s)
7e8469617b1c Implement PEP 3120. Martin v. Löwis <martin@v.loewis.de> parents: 42084 diff changeset	516 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	517 int expected = 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	518 int length;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	519 if (*s < 0x80)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	520 /* single-byte code */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	521 return 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	522 if (*s < 0xc0)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	523 /* following byte */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	524 return 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	525 if (*s < 0xE0)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	526 expected = 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	527 else if (*s < 0xF0)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	528 expected = 2;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	529 else if (*s < 0xF8)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	530 expected = 3;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	531 else
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	532 return 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	533 length = expected + 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	534 for (; expected; expected--)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	535 if (s[expected] < 0x80 \|\| s[expected] >= 0xC0)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	536 return 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	537 return length;
42403 7e8469617b1c Implement PEP 3120. Martin v. Löwis <martin@v.loewis.de> parents: 42084 diff changeset	538 }
7e8469617b1c Implement PEP 3120. Martin v. Löwis <martin@v.loewis.de> parents: 42084 diff changeset	539
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	540 /* Read a line of input from TOK. Determine encoding
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	541 if necessary. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	542
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	543 static char *
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	544 decoding_fgets(char s, int size, struct tok_state tok)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	545 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	546 char *line = NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	547 int badchar = 0;
67162 fc834b323f26 Issue #10778: decoding_fgets() decodes the filename from the filesystem Victor Stinner <victor.stinner@haypocalc.com> parents: 65311 diff changeset	548 PyObject *filename;
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	549 for (;;) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	550 if (tok->decoding_state == STATE_NORMAL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	551 /* We already have a codec associated with
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	552 this input. */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	553 line = fp_readl(s, size, tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	554 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	555 } else if (tok->decoding_state == STATE_RAW) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	556 /* We want a 'raw' read. */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	557 line = Py_UniversalNewlineFgets(s, size,
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	558 tok->fp, NULL);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	559 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	560 } else {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	561 /* We have not yet determined the encoding.
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	562 If an encoding is found, use the file-pointer
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	563 reader functions from now on. */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	564 if (!check_bom(fp_getc, fp_ungetc, fp_setreadl, tok))
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	565 return error_ret(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	566 assert(tok->decoding_state != STATE_INIT);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	567 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	568 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	569 if (line != NULL && tok->lineno < 2 && !tok->read_coding_spec) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	570 if (!check_coding_spec(line, strlen(line), tok, fp_setreadl)) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	571 return error_ret(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	572 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	573 }
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	574 #ifndef PGEN
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	575 /* The default encoding is UTF-8, so make sure we don't have any
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	576 non-UTF-8 sequences in it. */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	577 if (line && !tok->encoding) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	578 unsigned char *c;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	579 int length;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	580 for (c = (unsigned char )line; c; c += length)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	581 if (!(length = valid_utf8(c))) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	582 badchar = *c;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	583 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	584 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	585 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	586 if (badchar) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	587 /* Need to add 1 to the line number, since this line
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	588 has not been counted, yet. */
69529 fa5e348889c2 Issue #9319: Fix a crash on parsing a Python source code without encoding Victor Stinner <victor.stinner@haypocalc.com> parents: 68257 diff changeset	589 if (tok->filename != NULL)
fa5e348889c2 Issue #9319: Fix a crash on parsing a Python source code without encoding Victor Stinner <victor.stinner@haypocalc.com> parents: 68257 diff changeset	590 filename = PyUnicode_DecodeFSDefault(tok->filename);
fa5e348889c2 Issue #9319: Fix a crash on parsing a Python source code without encoding Victor Stinner <victor.stinner@haypocalc.com> parents: 68257 diff changeset	591 else
fa5e348889c2 Issue #9319: Fix a crash on parsing a Python source code without encoding Victor Stinner <victor.stinner@haypocalc.com> parents: 68257 diff changeset	592 filename = PyUnicode_FromString("<file>");
67162 fc834b323f26 Issue #10778: decoding_fgets() decodes the filename from the filesystem Victor Stinner <victor.stinner@haypocalc.com> parents: 65311 diff changeset	593 if (filename != NULL) {
fc834b323f26 Issue #10778: decoding_fgets() decodes the filename from the filesystem Victor Stinner <victor.stinner@haypocalc.com> parents: 65311 diff changeset	594 PyErr_Format(PyExc_SyntaxError,
fc834b323f26 Issue #10778: decoding_fgets() decodes the filename from the filesystem Victor Stinner <victor.stinner@haypocalc.com> parents: 65311 diff changeset	595 "Non-UTF-8 code starting with '\\x%.2x' "
67367 a0d4eda8da0a Remove arbitrary string length limits Victor Stinner <victor.stinner@haypocalc.com> parents: 67162 diff changeset	596 "in file %U on line %i, "
67162 fc834b323f26 Issue #10778: decoding_fgets() decodes the filename from the filesystem Victor Stinner <victor.stinner@haypocalc.com> parents: 65311 diff changeset	597 "but no encoding declared; "
fc834b323f26 Issue #10778: decoding_fgets() decodes the filename from the filesystem Victor Stinner <victor.stinner@haypocalc.com> parents: 65311 diff changeset	598 "see http://python.org/dev/peps/pep-0263/ for details",
fc834b323f26 Issue #10778: decoding_fgets() decodes the filename from the filesystem Victor Stinner <victor.stinner@haypocalc.com> parents: 65311 diff changeset	599 badchar, filename, tok->lineno + 1);
fc834b323f26 Issue #10778: decoding_fgets() decodes the filename from the filesystem Victor Stinner <victor.stinner@haypocalc.com> parents: 65311 diff changeset	600 Py_DECREF(filename);
fc834b323f26 Issue #10778: decoding_fgets() decodes the filename from the filesystem Victor Stinner <victor.stinner@haypocalc.com> parents: 65311 diff changeset	601 }
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	602 return error_ret(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	603 }
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	604 #endif
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	605 return line;
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	606 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	607
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	608 static int
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	609 decoding_feof(struct tok_state *tok)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	610 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	611 if (tok->decoding_state != STATE_NORMAL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	612 return feof(tok->fp);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	613 } else {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	614 PyObject* buf = tok->decoding_buffer;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	615 if (buf == NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	616 buf = PyObject_CallObject(tok->decoding_readline, NULL);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	617 if (buf == NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	618 error_ret(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	619 return 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	620 } else {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	621 tok->decoding_buffer = buf;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	622 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	623 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	624 return PyObject_Length(buf) == 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	625 }
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	626 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	627
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	628 /* Fetch a byte from TOK, using the string buffer. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	629
37879 f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair Thomas Wouters <thomas@python.org> parents: 37704 diff changeset	630 static int
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair Thomas Wouters <thomas@python.org> parents: 37704 diff changeset	631 buf_getc(struct tok_state *tok) {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	632 return Py_CHARMASK(*tok->str++);
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	633 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	634
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	635 /* Unfetch a byte from TOK, using the string buffer. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	636
37879 f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair Thomas Wouters <thomas@python.org> parents: 37704 diff changeset	637 static void
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair Thomas Wouters <thomas@python.org> parents: 37704 diff changeset	638 buf_ungetc(int c, struct tok_state *tok) {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	639 tok->str--;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	640 assert(Py_CHARMASK(tok->str) == c); / tok->cur may point to read-only segment */
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	641 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	642
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	643 /* Set the readline function for TOK to ENC. For the string-based
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	644 tokenizer, this means to just record the encoding. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	645
37879 f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair Thomas Wouters <thomas@python.org> parents: 37704 diff changeset	646 static int
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair Thomas Wouters <thomas@python.org> parents: 37704 diff changeset	647 buf_setreadl(struct tok_state tok, const char enc) {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	648 tok->enc = enc;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	649 return 1;
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	650 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	651
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	652 /* Return a UTF-8 encoding Python string object from the
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	653 C byte string STR, which is encoded with ENC. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	654
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	655 static PyObject *
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	656 translate_into_utf8(const char* str, const char* enc) {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	657 PyObject *utf8;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	658 PyObject* buf = PyUnicode_Decode(str, strlen(str), enc, NULL);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	659 if (buf == NULL)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	660 return NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	661 utf8 = PyUnicode_AsUTF8String(buf);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	662 Py_DECREF(buf);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	663 return utf8;
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	664 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	665
57206 b2d90c1c637d Merged revisions 76230 via svnmerge from Benjamin Peterson <benjamin@python.org> parents: 56594 diff changeset	666
b2d90c1c637d Merged revisions 76230 via svnmerge from Benjamin Peterson <benjamin@python.org> parents: 56594 diff changeset	667 static char *
b2d90c1c637d Merged revisions 76230 via svnmerge from Benjamin Peterson <benjamin@python.org> parents: 56594 diff changeset	668 translate_newlines(const char s, int exec_input, struct tok_state tok) {
61466 cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	669 int skip_next_lf = 0, needed_length = strlen(s) + 2, final_length;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	670 char buf, current;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	671 char c = '\0';
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	672 buf = PyMem_MALLOC(needed_length);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	673 if (buf == NULL) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	674 tok->done = E_NOMEM;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	675 return NULL;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	676 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	677 for (current = buf; *s; s++, current++) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	678 c = *s;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	679 if (skip_next_lf) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	680 skip_next_lf = 0;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	681 if (c == '\n') {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	682 c = *++s;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	683 if (!c)
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	684 break;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	685 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	686 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	687 if (c == '\r') {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	688 skip_next_lf = 1;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	689 c = '\n';
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	690 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	691 *current = c;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	692 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	693 /* If this is exec input, add a newline to the end of the string if
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	694 there isn't one already. */
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	695 if (exec_input && c != '\n') {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	696 *current = '\n';
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	697 current++;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	698 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	699 *current = '\0';
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	700 final_length = current - buf + 1;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	701 if (final_length < needed_length && final_length)
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	702 /* should never fail */
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	703 buf = PyMem_REALLOC(buf, final_length);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	704 return buf;
57206 b2d90c1c637d Merged revisions 76230 via svnmerge from Benjamin Peterson <benjamin@python.org> parents: 56594 diff changeset	705 }
b2d90c1c637d Merged revisions 76230 via svnmerge from Benjamin Peterson <benjamin@python.org> parents: 56594 diff changeset	706
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	707 /* Decode a byte string STR for use as the buffer of TOK.
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	708 Look for encoding declarations inside STR, and record them
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	709 inside TOK. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	710
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	711 static const char *
57206 b2d90c1c637d Merged revisions 76230 via svnmerge from Benjamin Peterson <benjamin@python.org> parents: 56594 diff changeset	712 decode_str(const char input, int single, struct tok_state tok)
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	713 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	714 PyObject* utf8 = NULL;
61466 cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	715 const char *str;
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	716 const char *s;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	717 const char *newl[2] = {NULL, NULL};
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	718 int lineno = 0;
61466 cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	719 tok->input = str = translate_newlines(input, single, tok);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	720 if (str == NULL)
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	721 return NULL;
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	722 tok->enc = NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	723 tok->str = str;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	724 if (!check_bom(buf_getc, buf_ungetc, buf_setreadl, tok))
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	725 return error_ret(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	726 str = tok->str; /* string after BOM if any */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	727 assert(str);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	728 if (tok->enc != NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	729 utf8 = translate_into_utf8(str, tok->enc);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	730 if (utf8 == NULL)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	731 return error_ret(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	732 str = PyBytes_AsString(utf8);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	733 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	734 for (s = str;; s++) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	735 if (*s == '\0') break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	736 else if (*s == '\n') {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	737 assert(lineno < 2);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	738 newl[lineno] = s;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	739 lineno++;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	740 if (lineno == 2) break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	741 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	742 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	743 tok->enc = NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	744 /* need to check line 1 and 2 separately since check_coding_spec
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	745 assumes a single line as input */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	746 if (newl[0]) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	747 if (!check_coding_spec(str, newl[0] - str, tok, buf_setreadl))
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	748 return error_ret(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	749 if (tok->enc == NULL && newl[1]) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	750 if (!check_coding_spec(newl[0]+1, newl[1] - newl[0],
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	751 tok, buf_setreadl))
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	752 return error_ret(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	753 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	754 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	755 if (tok->enc != NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	756 assert(utf8 == NULL);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	757 utf8 = translate_into_utf8(str, tok->enc);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	758 if (utf8 == NULL)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	759 return error_ret(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	760 str = PyBytes_AS_STRING(utf8);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	761 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	762 assert(tok->decoding_buffer == NULL);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	763 tok->decoding_buffer = utf8; /* CAUTION */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	764 return str;
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	765 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	766
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	767 #endif /* PGEN */
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	768
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	769 /* Set up tokenizer for string */
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	770
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	771 struct tok_state *
57206 b2d90c1c637d Merged revisions 76230 via svnmerge from Benjamin Peterson <benjamin@python.org> parents: 56594 diff changeset	772 PyTokenizer_FromString(const char *str, int exec_input)
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	773 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	774 struct tok_state *tok = tok_new();
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	775 if (tok == NULL)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	776 return NULL;
61466 cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	777 str = (char *)decode_str(str, exec_input, tok);
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	778 if (str == NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	779 PyTokenizer_Free(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	780 return NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	781 }
35878 94a409da4bd4 Fix some more memory leaks. Neal Norwitz <nnorwitz@gmail.com> parents: 35751 diff changeset	782
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	783 /* XXX: constify members. */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	784 tok->buf = tok->cur = tok->end = tok->inp = (char*)str;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	785 return tok;
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	786 }
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	787
51956 8fe2988cf550 ignore the coding cookie in compile(), exec(), and eval() if the source is a string #4626 Benjamin Peterson <benjamin@python.org> parents: 50540 diff changeset	788 struct tok_state *
57206 b2d90c1c637d Merged revisions 76230 via svnmerge from Benjamin Peterson <benjamin@python.org> parents: 56594 diff changeset	789 PyTokenizer_FromUTF8(const char *str, int exec_input)
51956 8fe2988cf550 ignore the coding cookie in compile(), exec(), and eval() if the source is a string #4626 Benjamin Peterson <benjamin@python.org> parents: 50540 diff changeset	790 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	791 struct tok_state *tok = tok_new();
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	792 if (tok == NULL)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	793 return NULL;
57206 b2d90c1c637d Merged revisions 76230 via svnmerge from Benjamin Peterson <benjamin@python.org> parents: 56594 diff changeset	794 #ifndef PGEN
61466 cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	795 tok->input = str = translate_newlines(str, exec_input, tok);
57206 b2d90c1c637d Merged revisions 76230 via svnmerge from Benjamin Peterson <benjamin@python.org> parents: 56594 diff changeset	796 #endif
61466 cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	797 if (str == NULL) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	798 PyTokenizer_Free(tok);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	799 return NULL;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	800 }
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	801 tok->decoding_state = STATE_RAW;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	802 tok->read_coding_spec = 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	803 tok->enc = NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	804 tok->str = str;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	805 tok->encoding = (char *)PyMem_MALLOC(6);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	806 if (!tok->encoding) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	807 PyTokenizer_Free(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	808 return NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	809 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	810 strcpy(tok->encoding, "utf-8");
51956 8fe2988cf550 ignore the coding cookie in compile(), exec(), and eval() if the source is a string #4626 Benjamin Peterson <benjamin@python.org> parents: 50540 diff changeset	811
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	812 /* XXX: constify members. */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	813 tok->buf = tok->cur = tok->end = tok->inp = (char*)str;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	814 return tok;
51956 8fe2988cf550 ignore the coding cookie in compile(), exec(), and eval() if the source is a string #4626 Benjamin Peterson <benjamin@python.org> parents: 50540 diff changeset	815 }
8fe2988cf550 ignore the coding cookie in compile(), exec(), and eval() if the source is a string #4626 Benjamin Peterson <benjamin@python.org> parents: 50540 diff changeset	816
326 ff0862e3d63a Completely ignore lines with only a newline token on them, except Guido van Rossum <guido@python.org> parents: 293 diff changeset	817 /* Set up tokenizer for file */
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	818
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	819 struct tok_state *
43225 38f3f9ef193d Convert code from sys.stdin.encoding to UTF-8 in Martin v. Löwis <martin@v.loewis.de> parents: 43018 diff changeset	820 PyTokenizer_FromFile(FILE fp, char enc, char ps1, char ps2)
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	821 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	822 struct tok_state *tok = tok_new();
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	823 if (tok == NULL)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	824 return NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	825 if ((tok->buf = (char *)PyMem_MALLOC(BUFSIZ)) == NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	826 PyTokenizer_Free(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	827 return NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	828 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	829 tok->cur = tok->inp = tok->buf;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	830 tok->end = tok->buf + BUFSIZ;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	831 tok->fp = fp;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	832 tok->prompt = ps1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	833 tok->nextprompt = ps2;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	834 if (enc != NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	835 /* Must copy encoding declaration since it
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	836 gets copied into the parse tree. */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	837 tok->encoding = PyMem_MALLOC(strlen(enc)+1);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	838 if (!tok->encoding) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	839 PyTokenizer_Free(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	840 return NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	841 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	842 strcpy(tok->encoding, enc);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	843 tok->decoding_state = STATE_NORMAL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	844 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	845 return tok;
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	846 }
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	847
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	848
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	849 /* Free a tok_state structure */
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	850
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	851 void
13659 e78153e179fb Mass ANSIfication. Thomas Wouters <thomas@python.org> parents: 13539 diff changeset	852 PyTokenizer_Free(struct tok_state *tok)
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	853 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	854 if (tok->encoding != NULL)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	855 PyMem_FREE(tok->encoding);
24544 0bd521eb714e Make pgen compile with pydebug. Duplicate normalized names, as it may Martin v. Löwis <martin@v.loewis.de> parents: 24543 diff changeset	856 #ifndef PGEN
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	857 Py_XDECREF(tok->decoding_readline);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	858 Py_XDECREF(tok->decoding_buffer);
24544 0bd521eb714e Make pgen compile with pydebug. Duplicate normalized names, as it may Martin v. Löwis <martin@v.loewis.de> parents: 24543 diff changeset	859 #endif
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	860 if (tok->fp != NULL && tok->buf != NULL)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	861 PyMem_FREE(tok->buf);
61466 cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	862 if (tok->input)
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	863 PyMem_FREE((char *)tok->input);
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	864 PyMem_FREE(tok);
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	865 }
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	866
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	867 /* Get next char, updating state; error code goes into tok->done */
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	868
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	869 static int
13659 e78153e179fb Mass ANSIfication. Thomas Wouters <thomas@python.org> parents: 13539 diff changeset	870 tok_nextc(register struct tok_state *tok)
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	871 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	872 for (;;) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	873 if (tok->cur != tok->inp) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	874 return Py_CHARMASK(tok->cur++); / Fast path */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	875 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	876 if (tok->done != E_OK)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	877 return EOF;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	878 if (tok->fp == NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	879 char *end = strchr(tok->inp, '\n');
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	880 if (end != NULL)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	881 end++;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	882 else {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	883 end = strchr(tok->inp, '\0');
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	884 if (end == tok->inp) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	885 tok->done = E_EOF;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	886 return EOF;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	887 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	888 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	889 if (tok->start == NULL)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	890 tok->buf = tok->cur;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	891 tok->line_start = tok->cur;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	892 tok->lineno++;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	893 tok->inp = end;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	894 return Py_CHARMASK(*tok->cur++);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	895 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	896 if (tok->prompt != NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	897 char *newtok = PyOS_Readline(stdin, stdout, tok->prompt);
43225 38f3f9ef193d Convert code from sys.stdin.encoding to UTF-8 in Martin v. Löwis <martin@v.loewis.de> parents: 43018 diff changeset	898 #ifndef PGEN
67431 0933c3753a71 Issue #10841: set binary mode on files; the parser translates newlines Victor Stinner <victor.stinner@haypocalc.com> parents: 67367 diff changeset	899 if (newtok != NULL) {
0933c3753a71 Issue #10841: set binary mode on files; the parser translates newlines Victor Stinner <victor.stinner@haypocalc.com> parents: 67367 diff changeset	900 char *translated = translate_newlines(newtok, 0, tok);
0933c3753a71 Issue #10841: set binary mode on files; the parser translates newlines Victor Stinner <victor.stinner@haypocalc.com> parents: 67367 diff changeset	901 PyMem_FREE(newtok);
0933c3753a71 Issue #10841: set binary mode on files; the parser translates newlines Victor Stinner <victor.stinner@haypocalc.com> parents: 67367 diff changeset	902 if (translated == NULL)
0933c3753a71 Issue #10841: set binary mode on files; the parser translates newlines Victor Stinner <victor.stinner@haypocalc.com> parents: 67367 diff changeset	903 return EOF;
0933c3753a71 Issue #10841: set binary mode on files; the parser translates newlines Victor Stinner <victor.stinner@haypocalc.com> parents: 67367 diff changeset	904 newtok = translated;
0933c3753a71 Issue #10841: set binary mode on files; the parser translates newlines Victor Stinner <victor.stinner@haypocalc.com> parents: 67367 diff changeset	905 }
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	906 if (tok->encoding && newtok && *newtok) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	907 /* Recode to UTF-8 */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	908 Py_ssize_t buflen;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	909 const char* buf;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	910 PyObject *u = translate_into_utf8(newtok, tok->encoding);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	911 PyMem_FREE(newtok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	912 if (!u) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	913 tok->done = E_DECODE;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	914 return EOF;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	915 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	916 buflen = PyBytes_GET_SIZE(u);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	917 buf = PyBytes_AS_STRING(u);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	918 if (!buf) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	919 Py_DECREF(u);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	920 tok->done = E_DECODE;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	921 return EOF;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	922 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	923 newtok = PyMem_MALLOC(buflen+1);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	924 strcpy(newtok, buf);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	925 Py_DECREF(u);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	926 }
43225 38f3f9ef193d Convert code from sys.stdin.encoding to UTF-8 in Martin v. Löwis <martin@v.loewis.de> parents: 43018 diff changeset	927 #endif
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	928 if (tok->nextprompt != NULL)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	929 tok->prompt = tok->nextprompt;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	930 if (newtok == NULL)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	931 tok->done = E_INTR;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	932 else if (*newtok == '\0') {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	933 PyMem_FREE(newtok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	934 tok->done = E_EOF;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	935 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	936 else if (tok->start != NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	937 size_t start = tok->start - tok->buf;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	938 size_t oldlen = tok->cur - tok->buf;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	939 size_t newlen = oldlen + strlen(newtok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	940 char *buf = tok->buf;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	941 buf = (char *)PyMem_REALLOC(buf, newlen+1);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	942 tok->lineno++;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	943 if (buf == NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	944 PyMem_FREE(tok->buf);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	945 tok->buf = NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	946 PyMem_FREE(newtok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	947 tok->done = E_NOMEM;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	948 return EOF;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	949 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	950 tok->buf = buf;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	951 tok->cur = tok->buf + oldlen;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	952 tok->line_start = tok->cur;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	953 strcpy(tok->buf + oldlen, newtok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	954 PyMem_FREE(newtok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	955 tok->inp = tok->buf + newlen;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	956 tok->end = tok->inp + 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	957 tok->start = tok->buf + start;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	958 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	959 else {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	960 tok->lineno++;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	961 if (tok->buf != NULL)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	962 PyMem_FREE(tok->buf);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	963 tok->buf = newtok;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	964 tok->line_start = tok->buf;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	965 tok->cur = tok->buf;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	966 tok->line_start = tok->buf;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	967 tok->inp = strchr(tok->buf, '\0');
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	968 tok->end = tok->inp + 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	969 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	970 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	971 else {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	972 int done = 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	973 Py_ssize_t cur = 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	974 char *pt;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	975 if (tok->start == NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	976 if (tok->buf == NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	977 tok->buf = (char *)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	978 PyMem_MALLOC(BUFSIZ);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	979 if (tok->buf == NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	980 tok->done = E_NOMEM;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	981 return EOF;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	982 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	983 tok->end = tok->buf + BUFSIZ;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	984 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	985 if (decoding_fgets(tok->buf, (int)(tok->end - tok->buf),
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	986 tok) == NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	987 tok->done = E_EOF;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	988 done = 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	989 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	990 else {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	991 tok->done = E_OK;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	992 tok->inp = strchr(tok->buf, '\0');
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	993 done = tok->inp[-1] == '\n';
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	994 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	995 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	996 else {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	997 cur = tok->cur - tok->buf;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	998 if (decoding_feof(tok)) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	999 tok->done = E_EOF;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1000 done = 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1001 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1002 else
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1003 tok->done = E_OK;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1004 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1005 tok->lineno++;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1006 /* Read until '\n' or EOF */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1007 while (!done) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1008 Py_ssize_t curstart = tok->start == NULL ? -1 :
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1009 tok->start - tok->buf;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1010 Py_ssize_t curvalid = tok->inp - tok->buf;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1011 Py_ssize_t newsize = curvalid + BUFSIZ;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1012 char *newbuf = tok->buf;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1013 newbuf = (char *)PyMem_REALLOC(newbuf,
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1014 newsize);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1015 if (newbuf == NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1016 tok->done = E_NOMEM;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1017 tok->cur = tok->inp;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1018 return EOF;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1019 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1020 tok->buf = newbuf;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1021 tok->inp = tok->buf + curvalid;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1022 tok->end = tok->buf + newsize;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1023 tok->start = curstart < 0 ? NULL :
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1024 tok->buf + curstart;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1025 if (decoding_fgets(tok->inp,
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1026 (int)(tok->end - tok->inp),
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1027 tok) == NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1028 /* Break out early on decoding
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1029 errors, as tok->buf will be NULL
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1030 */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1031 if (tok->decoding_erred)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1032 return EOF;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1033 /* Last line does not end in \n,
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1034 fake one */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1035 strcpy(tok->inp, "\n");
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1036 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1037 tok->inp = strchr(tok->inp, '\0');
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1038 done = tok->inp[-1] == '\n';
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1039 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1040 if (tok->buf != NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1041 tok->cur = tok->buf + cur;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1042 tok->line_start = tok->cur;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1043 /* replace "\r\n" with "\n" */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1044 /* For Mac leave the \r, giving a syntax error */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1045 pt = tok->inp - 2;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1046 if (pt >= tok->buf && *pt == '\r') {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1047 *pt++ = '\n';
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1048 *pt = '\0';
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1049 tok->inp = pt;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1050 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1051 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1052 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1053 if (tok->done != E_OK) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1054 if (tok->prompt != NULL)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1055 PySys_WriteStderr("\n");
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1056 tok->cur = tok->inp;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1057 return EOF;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1058 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1059 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1060 /NOTREACHED/
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1061 }
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1062
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1063
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1064 /* Back-up one character */
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1065
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1066 static void
13659 e78153e179fb Mass ANSIfication. Thomas Wouters <thomas@python.org> parents: 13539 diff changeset	1067 tok_backup(register struct tok_state *tok, register int c)
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1068 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1069 if (c != EOF) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1070 if (--tok->cur < tok->buf)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1071 Py_FatalError("tok_backup: beginning of buffer");
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1072 if (*tok->cur != c)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1073 *tok->cur = c;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1074 }
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1075 }
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1076
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1077
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1078 /* Return the token corresponding to a single character */
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1079
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1080 int
13659 e78153e179fb Mass ANSIfication. Thomas Wouters <thomas@python.org> parents: 13539 diff changeset	1081 PyToken_OneChar(int c)
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1082 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1083 switch (c) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1084 case '(': return LPAR;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1085 case ')': return RPAR;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1086 case '[': return LSQB;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1087 case ']': return RSQB;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1088 case ':': return COLON;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1089 case ',': return COMMA;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1090 case ';': return SEMI;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1091 case '+': return PLUS;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1092 case '-': return MINUS;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1093 case '*': return STAR;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1094 case '/': return SLASH;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1095 case '\|': return VBAR;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1096 case '&': return AMPER;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1097 case '<': return LESS;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1098 case '>': return GREATER;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1099 case '=': return EQUAL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1100 case '.': return DOT;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1101 case '%': return PERCENT;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1102 case '{': return LBRACE;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1103 case '}': return RBRACE;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1104 case '^': return CIRCUMFLEX;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1105 case '~': return TILDE;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1106 case '@': return AT;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1107 default: return OP;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1108 }
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1109 }
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1110
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1111
400 d189a2f681a6 Added 2-char tokens and new versions of comparisons Guido van Rossum <guido@python.org> parents: 326 diff changeset	1112 int
13659 e78153e179fb Mass ANSIfication. Thomas Wouters <thomas@python.org> parents: 13539 diff changeset	1113 PyToken_TwoChars(int c1, int c2)
400 d189a2f681a6 Added 2-char tokens and new versions of comparisons Guido van Rossum <guido@python.org> parents: 326 diff changeset	1114 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1115 switch (c1) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1116 case '=':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1117 switch (c2) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1118 case '=': return EQEQUAL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1119 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1120 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1121 case '!':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1122 switch (c2) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1123 case '=': return NOTEQUAL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1124 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1125 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1126 case '<':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1127 switch (c2) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1128 case '>': return NOTEQUAL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1129 case '=': return LESSEQUAL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1130 case '<': return LEFTSHIFT;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1131 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1132 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1133 case '>':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1134 switch (c2) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1135 case '=': return GREATEREQUAL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1136 case '>': return RIGHTSHIFT;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1137 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1138 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1139 case '+':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1140 switch (c2) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1141 case '=': return PLUSEQUAL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1142 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1143 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1144 case '-':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1145 switch (c2) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1146 case '=': return MINEQUAL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1147 case '>': return RARROW;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1148 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1149 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1150 case '*':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1151 switch (c2) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1152 case '*': return DOUBLESTAR;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1153 case '=': return STAREQUAL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1154 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1155 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1156 case '/':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1157 switch (c2) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1158 case '/': return DOUBLESLASH;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1159 case '=': return SLASHEQUAL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1160 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1161 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1162 case '\|':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1163 switch (c2) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1164 case '=': return VBAREQUAL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1165 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1166 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1167 case '%':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1168 switch (c2) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1169 case '=': return PERCENTEQUAL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1170 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1171 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1172 case '&':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1173 switch (c2) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1174 case '=': return AMPEREQUAL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1175 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1176 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1177 case '^':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1178 switch (c2) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1179 case '=': return CIRCUMFLEXEQUAL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1180 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1181 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1182 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1183 return OP;
400 d189a2f681a6 Added 2-char tokens and new versions of comparisons Guido van Rossum <guido@python.org> parents: 326 diff changeset	1184 }
d189a2f681a6 Added 2-char tokens and new versions of comparisons Guido van Rossum <guido@python.org> parents: 326 diff changeset	1185
14163 1b9fd0cbd914 Support for three-token characters (*=, >>=, <<=) which was written by Thomas Wouters <thomas@python.org>* parents: 13659 diff changeset	1186 int
1b9fd0cbd914 Support for three-token characters (*=, >>=, <<=) which was written by Thomas Wouters <thomas@python.org>* parents: 13659 diff changeset	1187 PyToken_ThreeChars(int c1, int c2, int c3)
1b9fd0cbd914 Support for three-token characters (*=, >>=, <<=) which was written by Thomas Wouters <thomas@python.org>* parents: 13659 diff changeset	1188 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1189 switch (c1) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1190 case '<':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1191 switch (c2) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1192 case '<':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1193 switch (c3) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1194 case '=':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1195 return LEFTSHIFTEQUAL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1196 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1197 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1198 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1199 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1200 case '>':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1201 switch (c2) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1202 case '>':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1203 switch (c3) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1204 case '=':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1205 return RIGHTSHIFTEQUAL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1206 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1207 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1208 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1209 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1210 case '*':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1211 switch (c2) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1212 case '*':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1213 switch (c3) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1214 case '=':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1215 return DOUBLESTAREQUAL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1216 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1217 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1218 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1219 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1220 case '/':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1221 switch (c2) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1222 case '/':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1223 switch (c3) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1224 case '=':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1225 return DOUBLESLASHEQUAL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1226 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1227 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1228 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1229 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1230 case '.':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1231 switch (c2) {
41230 b1a18c7ed5ca Make ELLIPSIS a separate token. This makes it a syntax error to write ". . ." for Ellipsis. Georg Brandl <georg@python.org> parents: 40894 diff changeset	1232 case '.':
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1233 switch (c3) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1234 case '.':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1235 return ELLIPSIS;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1236 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1237 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1238 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1239 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1240 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1241 return OP;
14163 1b9fd0cbd914 Support for three-token characters (*=, >>=, <<=) which was written by Thomas Wouters <thomas@python.org>* parents: 13659 diff changeset	1242 }
400 d189a2f681a6 Added 2-char tokens and new versions of comparisons Guido van Rossum <guido@python.org> parents: 326 diff changeset	1243
7945 dc132cc39573 Add checking for inconsistent tab usage Guido van Rossum <guido@python.org> parents: 7324 diff changeset	1244 static int
13659 e78153e179fb Mass ANSIfication. Thomas Wouters <thomas@python.org> parents: 13539 diff changeset	1245 indenterror(struct tok_state *tok)
7945 dc132cc39573 Add checking for inconsistent tab usage Guido van Rossum <guido@python.org> parents: 7324 diff changeset	1246 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1247 if (tok->alterror) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1248 tok->done = E_TABSPACE;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1249 tok->cur = tok->inp;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1250 return 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1251 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1252 if (tok->altwarning) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1253 PySys_WriteStderr("%s: inconsistent use of tabs and spaces "
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1254 "in indentation\n", tok->filename);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1255 tok->altwarning = 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1256 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1257 return 0;
7945 dc132cc39573 Add checking for inconsistent tab usage Guido van Rossum <guido@python.org> parents: 7324 diff changeset	1258 }
dc132cc39573 Add checking for inconsistent tab usage Guido van Rossum <guido@python.org> parents: 7324 diff changeset	1259
42597 20477cbfd996 Implement PEP 3131. Add isidentifier to str. Martin v. Löwis <martin@v.loewis.de> parents: 42551 diff changeset	1260 #ifdef PGEN
59865 c2d2e26bcf7f Merged revisions 78872 via svnmerge from Victor Stinner <victor.stinner@haypocalc.com> parents: 59289 diff changeset	1261 #define verify_identifier(tok) 1
42597 20477cbfd996 Implement PEP 3131. Add isidentifier to str. Martin v. Löwis <martin@v.loewis.de> parents: 42551 diff changeset	1262 #else
20477cbfd996 Implement PEP 3131. Add isidentifier to str. Martin v. Löwis <martin@v.loewis.de> parents: 42551 diff changeset	1263 /* Verify that the identifier follows PEP 3131. */
20477cbfd996 Implement PEP 3131. Add isidentifier to str. Martin v. Löwis <martin@v.loewis.de> parents: 42551 diff changeset	1264 static int
59865 c2d2e26bcf7f Merged revisions 78872 via svnmerge from Victor Stinner <victor.stinner@haypocalc.com> parents: 59289 diff changeset	1265 verify_identifier(struct tok_state *tok)
42597 20477cbfd996 Implement PEP 3131. Add isidentifier to str. Martin v. Löwis <martin@v.loewis.de> parents: 42551 diff changeset	1266 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1267 PyObject *s;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1268 int result;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1269 s = PyUnicode_DecodeUTF8(tok->start, tok->cur - tok->start, NULL);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1270 if (s == NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1271 if (PyErr_ExceptionMatches(PyExc_UnicodeDecodeError)) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1272 PyErr_Clear();
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1273 tok->done = E_IDENTIFIER;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1274 } else {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1275 tok->done = E_ERROR;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1276 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1277 return 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1278 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1279 result = PyUnicode_IsIdentifier(s);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1280 Py_DECREF(s);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1281 if (result == 0)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1282 tok->done = E_IDENTIFIER;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1283 return result;
42597 20477cbfd996 Implement PEP 3131. Add isidentifier to str. Martin v. Löwis <martin@v.loewis.de> parents: 42551 diff changeset	1284 }
20477cbfd996 Implement PEP 3131. Add isidentifier to str. Martin v. Löwis <martin@v.loewis.de> parents: 42551 diff changeset	1285 #endif
7945 dc132cc39573 Add checking for inconsistent tab usage Guido van Rossum <guido@python.org> parents: 7324 diff changeset	1286
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1287 /* Get next token, after space stripping etc. */
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1288
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	1289 static int
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	1290 tok_get(register struct tok_state tok, char p_start, char *p_end)
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1291 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1292 register int c;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1293 int blankline, nonascii;
326 ff0862e3d63a Completely ignore lines with only a newline token on them, except Guido van Rossum <guido@python.org> parents: 293 diff changeset	1294
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1295 p_start = p_end = NULL;
326 ff0862e3d63a Completely ignore lines with only a newline token on them, except Guido van Rossum <guido@python.org> parents: 293 diff changeset	1296 nextline:
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1297 tok->start = NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1298 blankline = 0;
326 ff0862e3d63a Completely ignore lines with only a newline token on them, except Guido van Rossum <guido@python.org> parents: 293 diff changeset	1299
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1300 /* Get indentation level */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1301 if (tok->atbol) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1302 register int col = 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1303 register int altcol = 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1304 tok->atbol = 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1305 for (;;) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1306 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1307 if (c == ' ')
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1308 col++, altcol++;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1309 else if (c == '\t') {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1310 col = (col/tok->tabsize + 1) * tok->tabsize;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1311 altcol = (altcol/tok->alttabsize + 1)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1312 * tok->alttabsize;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1313 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1314 else if (c == '\014') /* Control-L (formfeed) */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1315 col = altcol = 0; /* For Emacs users */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1316 else
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1317 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1318 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1319 tok_backup(tok, c);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1320 if (c == '#' \|\| c == '\n') {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1321 /* Lines with only whitespace and/or comments
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1322 shouldn't affect the indentation and are
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1323 not passed to the parser as NEWLINE tokens,
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1324 except totally empty lines in interactive
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1325 mode, which signal the end of a command group. */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1326 if (col == 0 && c == '\n' && tok->prompt != NULL)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1327 blankline = 0; /* Let it through */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1328 else
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1329 blankline = 1; /* Ignore completely */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1330 /* We can't jump back right here since we still
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1331 may need to skip to the end of a comment */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1332 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1333 if (!blankline && tok->level == 0) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1334 if (col == tok->indstack[tok->indent]) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1335 /* No change */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1336 if (altcol != tok->altindstack[tok->indent]) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1337 if (indenterror(tok))
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1338 return ERRORTOKEN;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1339 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1340 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1341 else if (col > tok->indstack[tok->indent]) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1342 /* Indent -- always one */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1343 if (tok->indent+1 >= MAXINDENT) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1344 tok->done = E_TOODEEP;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1345 tok->cur = tok->inp;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1346 return ERRORTOKEN;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1347 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1348 if (altcol <= tok->altindstack[tok->indent]) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1349 if (indenterror(tok))
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1350 return ERRORTOKEN;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1351 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1352 tok->pendin++;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1353 tok->indstack[++tok->indent] = col;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1354 tok->altindstack[tok->indent] = altcol;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1355 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1356 else /* col < tok->indstack[tok->indent] */ {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1357 /* Dedent -- any number, must be consistent */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1358 while (tok->indent > 0 &&
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1359 col < tok->indstack[tok->indent]) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1360 tok->pendin--;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1361 tok->indent--;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1362 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1363 if (col != tok->indstack[tok->indent]) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1364 tok->done = E_DEDENT;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1365 tok->cur = tok->inp;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1366 return ERRORTOKEN;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1367 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1368 if (altcol != tok->altindstack[tok->indent]) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1369 if (indenterror(tok))
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1370 return ERRORTOKEN;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1371 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1372 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1373 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1374 }
37879 f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair Thomas Wouters <thomas@python.org> parents: 37704 diff changeset	1375
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1376 tok->start = tok->cur;
37879 f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair Thomas Wouters <thomas@python.org> parents: 37704 diff changeset	1377
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1378 /* Return pending indents/dedents */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1379 if (tok->pendin != 0) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1380 if (tok->pendin < 0) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1381 tok->pendin++;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1382 return DEDENT;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1383 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1384 else {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1385 tok->pendin--;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1386 return INDENT;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1387 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1388 }
37879 f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair Thomas Wouters <thomas@python.org> parents: 37704 diff changeset	1389
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1390 again:
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1391 tok->start = NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1392 /* Skip spaces */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1393 do {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1394 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1395 } while (c == ' ' \|\| c == '\t' \|\| c == '\014');
37879 f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair Thomas Wouters <thomas@python.org> parents: 37704 diff changeset	1396
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1397 /* Set start of current token */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1398 tok->start = tok->cur - 1;
37879 f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair Thomas Wouters <thomas@python.org> parents: 37704 diff changeset	1399
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1400 /* Skip comment */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1401 if (c == '#')
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1402 while (c != EOF && c != '\n')
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1403 c = tok_nextc(tok);
37879 f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair Thomas Wouters <thomas@python.org> parents: 37704 diff changeset	1404
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1405 /* Check for EOF and errors now */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1406 if (c == EOF) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1407 return tok->done == E_EOF ? ENDMARKER : ERRORTOKEN;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1408 }
37879 f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair Thomas Wouters <thomas@python.org> parents: 37704 diff changeset	1409
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1410 /* Identifier (most frequent token!) */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1411 nonascii = 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1412 if (is_potential_identifier_start(c)) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1413 /* Process b"", r"" and br"" */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1414 if (c == 'b' \|\| c == 'B') {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1415 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1416 if (c == '"' \|\| c == '\'')
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1417 goto letter_quote;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1418 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1419 if (c == 'r' \|\| c == 'R') {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1420 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1421 if (c == '"' \|\| c == '\'')
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1422 goto letter_quote;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1423 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1424 while (is_potential_identifier_char(c)) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1425 if (c >= 128)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1426 nonascii = 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1427 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1428 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1429 tok_backup(tok, c);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1430 if (nonascii &&
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1431 !verify_identifier(tok)) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1432 tok->done = E_IDENTIFIER;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1433 return ERRORTOKEN;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1434 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1435 *p_start = tok->start;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1436 *p_end = tok->cur;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1437 return NAME;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1438 }
37879 f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair Thomas Wouters <thomas@python.org> parents: 37704 diff changeset	1439
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1440 /* Newline */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1441 if (c == '\n') {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1442 tok->atbol = 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1443 if (blankline \|\| tok->level > 0)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1444 goto nextline;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1445 *p_start = tok->start;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1446 p_end = tok->cur - 1; / Leave '\n' out of the string */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1447 tok->cont_line = 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1448 return NEWLINE;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1449 }
37879 f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair Thomas Wouters <thomas@python.org> parents: 37704 diff changeset	1450
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1451 /* Period or number starting with period? */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1452 if (c == '.') {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1453 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1454 if (isdigit(c)) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1455 goto fraction;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1456 } else if (c == '.') {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1457 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1458 if (c == '.') {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1459 *p_start = tok->start;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1460 *p_end = tok->cur;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1461 return ELLIPSIS;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1462 } else {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1463 tok_backup(tok, c);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1464 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1465 tok_backup(tok, '.');
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1466 } else {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1467 tok_backup(tok, c);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1468 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1469 *p_start = tok->start;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1470 *p_end = tok->cur;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1471 return DOT;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1472 }
3333 4b5bff95804e changes for pow(*) and complex Guido van Rossum <guido@python.org>* parents: 3116 diff changeset	1473
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1474 /* Number */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1475 if (isdigit(c)) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1476 if (c == '0') {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1477 /* Hex, octal or binary -- maybe. */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1478 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1479 if (c == '.')
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1480 goto fraction;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1481 if (c == 'j' \|\| c == 'J')
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1482 goto imaginary;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1483 if (c == 'x' \|\| c == 'X') {
44782 ca2683a67943 Merged revisions 60080-60089,60091-60093 via svnmerge from Georg Brandl <georg@python.org> parents: 44442 diff changeset	1484
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1485 /* Hex */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1486 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1487 if (!isxdigit(c)) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1488 tok->done = E_TOKEN;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1489 tok_backup(tok, c);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1490 return ERRORTOKEN;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1491 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1492 do {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1493 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1494 } while (isxdigit(c));
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1495 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1496 else if (c == 'o' \|\| c == 'O') {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1497 /* Octal */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1498 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1499 if (c < '0' \|\| c >= '8') {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1500 tok->done = E_TOKEN;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1501 tok_backup(tok, c);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1502 return ERRORTOKEN;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1503 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1504 do {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1505 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1506 } while ('0' <= c && c < '8');
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1507 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1508 else if (c == 'b' \|\| c == 'B') {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1509 /* Binary */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1510 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1511 if (c != '0' && c != '1') {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1512 tok->done = E_TOKEN;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1513 tok_backup(tok, c);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1514 return ERRORTOKEN;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1515 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1516 do {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1517 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1518 } while (c == '0' \|\| c == '1');
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1519 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1520 else {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1521 int nonzero = 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1522 /* maybe old-style octal; c is first char of it */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1523 /* in any case, allow '0' as a literal */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1524 while (c == '0')
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1525 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1526 while (isdigit(c)) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1527 nonzero = 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1528 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1529 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1530 if (c == '.')
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1531 goto fraction;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1532 else if (c == 'e' \|\| c == 'E')
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1533 goto exponent;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1534 else if (c == 'j' \|\| c == 'J')
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1535 goto imaginary;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1536 else if (nonzero) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1537 tok->done = E_TOKEN;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1538 tok_backup(tok, c);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1539 return ERRORTOKEN;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1540 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1541 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1542 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1543 else {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1544 /* Decimal */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1545 do {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1546 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1547 } while (isdigit(c));
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1548 {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1549 /* Accept floating point numbers. */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1550 if (c == '.') {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1551 fraction:
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1552 /* Fraction */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1553 do {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1554 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1555 } while (isdigit(c));
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1556 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1557 if (c == 'e' \|\| c == 'E') {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1558 exponent:
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1559 /* Exponent part */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1560 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1561 if (c == '+' \|\| c == '-')
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1562 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1563 if (!isdigit(c)) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1564 tok->done = E_TOKEN;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1565 tok_backup(tok, c);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1566 return ERRORTOKEN;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1567 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1568 do {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1569 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1570 } while (isdigit(c));
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1571 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1572 if (c == 'j' \|\| c == 'J')
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1573 /* Imaginary part */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1574 imaginary:
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1575 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1576 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1577 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1578 tok_backup(tok, c);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1579 *p_start = tok->start;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1580 *p_end = tok->cur;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1581 return NUMBER;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1582 }
5265 3c1788ab9761 Support for alternative string quotes (a"xx", b"xx", c"xx", ...). Guido van Rossum <guido@python.org> parents: 4839 diff changeset	1583
3c1788ab9761 Support for alternative string quotes (a"xx", b"xx", c"xx", ...). Guido van Rossum <guido@python.org> parents: 4839 diff changeset	1584 letter_quote:
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1585 /* String */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1586 if (c == '\'' \|\| c == '"') {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1587 int quote = c;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1588 int quote_size = 1; /* 1 or 3 */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1589 int end_quote_size = 0;
43879 5590157f4fc6 Cleanup of tokenizer.c. Guido van Rossum <guido@python.org> parents: 43873 diff changeset	1590
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1591 /* Find the quote size and start of string */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1592 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1593 if (c == quote) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1594 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1595 if (c == quote)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1596 quote_size = 3;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1597 else
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1598 end_quote_size = 1; /* empty string found */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1599 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1600 if (c != quote)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1601 tok_backup(tok, c);
43879 5590157f4fc6 Cleanup of tokenizer.c. Guido van Rossum <guido@python.org> parents: 43873 diff changeset	1602
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1603 /* Get rest of string */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1604 while (end_quote_size != quote_size) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1605 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1606 if (c == EOF) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1607 if (quote_size == 3)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1608 tok->done = E_EOFS;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1609 else
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1610 tok->done = E_EOLS;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1611 tok->cur = tok->inp;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1612 return ERRORTOKEN;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1613 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1614 if (quote_size == 1 && c == '\n') {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1615 tok->done = E_EOLS;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1616 tok->cur = tok->inp;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1617 return ERRORTOKEN;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1618 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1619 if (c == quote)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1620 end_quote_size += 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1621 else {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1622 end_quote_size = 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1623 if (c == '\\')
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1624 c = tok_nextc(tok); /* skip escaped char */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1625 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1626 }
43879 5590157f4fc6 Cleanup of tokenizer.c. Guido van Rossum <guido@python.org> parents: 43873 diff changeset	1627
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1628 *p_start = tok->start;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1629 *p_end = tok->cur;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1630 return STRING;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1631 }
37879 f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair Thomas Wouters <thomas@python.org> parents: 37704 diff changeset	1632
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1633 /* Line continuation */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1634 if (c == '\\') {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1635 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1636 if (c != '\n') {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1637 tok->done = E_LINECONT;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1638 tok->cur = tok->inp;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1639 return ERRORTOKEN;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1640 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1641 tok->cont_line = 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1642 goto again; /* Read next line */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1643 }
37879 f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair Thomas Wouters <thomas@python.org> parents: 37704 diff changeset	1644
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1645 /* Check for two-character token */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1646 {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1647 int c2 = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1648 int token = PyToken_TwoChars(c, c2);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1649 if (token != OP) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1650 int c3 = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1651 int token3 = PyToken_ThreeChars(c, c2, c3);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1652 if (token3 != OP) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1653 token = token3;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1654 } else {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1655 tok_backup(tok, c3);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1656 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1657 *p_start = tok->start;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1658 *p_end = tok->cur;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1659 return token;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1660 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1661 tok_backup(tok, c2);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1662 }
37879 f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair Thomas Wouters <thomas@python.org> parents: 37704 diff changeset	1663
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1664 /* Keep track of parentheses nesting level */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1665 switch (c) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1666 case '(':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1667 case '[':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1668 case '{':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1669 tok->level++;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1670 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1671 case ')':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1672 case ']':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1673 case '}':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1674 tok->level--;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1675 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1676 }
37879 f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair Thomas Wouters <thomas@python.org> parents: 37704 diff changeset	1677
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1678 /* Punctuation character */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1679 *p_start = tok->start;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1680 *p_end = tok->cur;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1681 return PyToken_OneChar(c);
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1682 }
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1683
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	1684 int
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	1685 PyTokenizer_Get(struct tok_state tok, char p_start, char *p_end)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	1686 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1687 int result = tok_get(tok, p_start, p_end);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1688 if (tok->decoding_erred) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1689 result = ERRORTOKEN;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1690 tok->done = E_DECODE;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1691 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1692 return result;
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	1693 }
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1694
43632 0067a6ab4854 Issue 1267, continued. Guido van Rossum <guido@python.org> parents: 43622 diff changeset	1695 /* Get -- encoding -- from a Python file.
43605 a2bb76d11d66 Patch 1267 by Christian Heimes. Guido van Rossum <guido@python.org> parents: 43523 diff changeset	1696
a2bb76d11d66 Patch 1267 by Christian Heimes. Guido van Rossum <guido@python.org> parents: 43523 diff changeset	1697 PyTokenizer_FindEncoding returns NULL when it can't find the encoding in
43879 5590157f4fc6 Cleanup of tokenizer.c. Guido van Rossum <guido@python.org> parents: 43873 diff changeset	1698 the first or second line of the file (in which case the encoding
43609 d378acdc30ee Fix PyTokenizer_FindEncoding() for OS X 10.4. Turns out that seeking to the Brett Cannon <bcannon@gmail.com> parents: 43607 diff changeset	1699 should be assumed to be PyUnicode_GetDefaultEncoding()).
d378acdc30ee Fix PyTokenizer_FindEncoding() for OS X 10.4. Turns out that seeking to the Brett Cannon <bcannon@gmail.com> parents: 43607 diff changeset	1700
43632 0067a6ab4854 Issue 1267, continued. Guido van Rossum <guido@python.org> parents: 43622 diff changeset	1701 The char * returned is malloc'ed via PyMem_MALLOC() and thus must be freed
0067a6ab4854 Issue 1267, continued. Guido van Rossum <guido@python.org> parents: 43622 diff changeset	1702 by the caller.
43605 a2bb76d11d66 Patch 1267 by Christian Heimes. Guido van Rossum <guido@python.org> parents: 43523 diff changeset	1703 */
a2bb76d11d66 Patch 1267 by Christian Heimes. Guido van Rossum <guido@python.org> parents: 43523 diff changeset	1704 char *
43632 0067a6ab4854 Issue 1267, continued. Guido van Rossum <guido@python.org> parents: 43622 diff changeset	1705 PyTokenizer_FindEncoding(int fd)
0067a6ab4854 Issue 1267, continued. Guido van Rossum <guido@python.org> parents: 43622 diff changeset	1706 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1707 struct tok_state *tok;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1708 FILE *fp;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1709 char p_start =NULL , p_end =NULL , *encoding = NULL;
43605 a2bb76d11d66 Patch 1267 by Christian Heimes. Guido van Rossum <guido@python.org> parents: 43523 diff changeset	1710
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1711 fd = dup(fd);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1712 if (fd < 0) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1713 return NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1714 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1715 fp = fdopen(fd, "r");
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1716 if (fp == NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1717 return NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1718 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1719 tok = PyTokenizer_FromFile(fp, NULL, NULL, NULL);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1720 if (tok == NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1721 fclose(fp);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1722 return NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1723 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1724 while (tok->lineno < 2 && tok->done == E_OK) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1725 PyTokenizer_Get(tok, &p_start, &p_end);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1726 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1727 fclose(fp);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1728 if (tok->encoding) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1729 encoding = (char *)PyMem_MALLOC(strlen(tok->encoding) + 1);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1730 if (encoding)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1731 strcpy(encoding, tok->encoding);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1732 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1733 PyTokenizer_Free(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1734 return encoding;
43605 a2bb76d11d66 Patch 1267 by Christian Heimes. Guido van Rossum <guido@python.org> parents: 43523 diff changeset	1735 }
43292 876b7ef4775d Merged revisions 57778-58052 via svnmerge from Thomas Wouters <thomas@python.org> parents: 43225 diff changeset	1736
4839 4567e158d20c Rename DEBUG macro to Py_DEBUG Guido van Rossum <guido@python.org> parents: 4577 diff changeset	1737 #ifdef Py_DEBUG
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1738
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1739 void
13659 e78153e179fb Mass ANSIfication. Thomas Wouters <thomas@python.org> parents: 13539 diff changeset	1740 tok_dump(int type, char start, char end)
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1741 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1742 printf("%s", _PyParser_TokenNames[type]);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1743 if (type == NAME \|\| type == NUMBER \|\| type == STRING \|\| type == OP)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1744 printf("(%.*s)", (int)(end - start), start);
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1745 }
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1746
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1747 #endif

Mercurial > cpython-lru_cache-weakref

annotate Parser/tokenizer.c @ 78069:a8aa77458e12 3.2