cpython-lru_cache-weakref: Parser/tokenizer.c annotate

annotate Parser/tokenizer.c @ 86127:a3d09c19bb93

merge with 3.3

author	Georg Brandl <georg@python.org>
date	Sun, 06 Oct 2013 19:24:00 +0200
parents	ef889c3d5dc6
children	3055e61f1e66

rev	line source
145 853b736a6a9e Added copyright notice. Guido van Rossum <guido@python.org> parents: 96 diff changeset	1
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	2 /* Tokenizer implementation */
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	3
23029 8291473fde3e Mass checkin of universal newline support. Jack Jansen <jack.jansen@cwi.nl> parents: 19621 diff changeset	4 #include "Python.h"
86 a6934380c6e7 "Compiling" version Guido van Rossum <guido@python.org> parents: 67 diff changeset	5 #include "pgenheaders.h"
a6934380c6e7 "Compiling" version Guido van Rossum <guido@python.org> parents: 67 diff changeset	6
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	7 #include <ctype.h>
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	8 #include <assert.h>
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	9
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	10 #include "tokenizer.h"
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	11 #include "errcode.h"
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	12
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	13 #ifndef PGEN
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	14 #include "unicodeobject.h"
47302 811a80651232 Renamed files bytesobject.[ch] and stringobject.[ch] Christian Heimes <christian@cheimes.de> parents: 47301 diff changeset	15 #include "bytesobject.h"
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	16 #include "fileobject.h"
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	17 #include "codecs.h"
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	18 #include "abstract.h"
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	19 #endif /* PGEN */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	20
42013 cd125fe83051 Make identifiers str (not str8) objects throughout. Martin v. Löwis <martin@v.loewis.de> parents: 41970 diff changeset	21 #define is_potential_identifier_start(c) (\
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	22 (c >= 'a' && c <= 'z')\
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	23 \|\| (c >= 'A' && c <= 'Z')\
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	24 \|\| c == '_'\
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	25 \|\| (c >= 128))
42013 cd125fe83051 Make identifiers str (not str8) objects throughout. Martin v. Löwis <martin@v.loewis.de> parents: 41970 diff changeset	26
cd125fe83051 Make identifiers str (not str8) objects throughout. Martin v. Löwis <martin@v.loewis.de> parents: 41970 diff changeset	27 #define is_potential_identifier_char(c) (\
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	28 (c >= 'a' && c <= 'z')\
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	29 \|\| (c >= 'A' && c <= 'Z')\
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	30 \|\| (c >= '0' && c <= '9')\
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	31 \|\| c == '_'\
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	32 \|\| (c >= 128))
42013 cd125fe83051 Make identifiers str (not str8) objects throughout. Martin v. Löwis <martin@v.loewis.de> parents: 41970 diff changeset	33
25924 dc4a0336a2a3 Patch #512981: Update readline input stream on sys.stdin/out change. Martin v. Löwis <martin@v.loewis.de> parents: 25101 diff changeset	34 extern char PyOS_Readline(FILE , FILE , char );
1835 d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string Guido van Rossum <guido@python.org> parents: 1364 diff changeset	35 /* Return malloc'ed string including trailing \n;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string Guido van Rossum <guido@python.org> parents: 1364 diff changeset	36 empty malloc'ed string for EOF;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string Guido van Rossum <guido@python.org> parents: 1364 diff changeset	37 NULL if interrupted */
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string Guido van Rossum <guido@python.org> parents: 1364 diff changeset	38
678 205b947b1d3c Make tabs always 8 spaces wide -- it's more portable. Guido van Rossum <guido@python.org> parents: 477 diff changeset	39 /* Don't ever change this -- it would break the portability of Python code */
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	40 #define TABSIZE 8
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	41
86 a6934380c6e7 "Compiling" version Guido van Rossum <guido@python.org> parents: 67 diff changeset	42 /* Forward */
13447 a89f7f268de4 Nuke all remaining occurrences of Py_PROTO and Py_FPROTO. Tim Peters <tim.peters@gmail.com> parents: 13256 diff changeset	43 static struct tok_state *tok_new(void);
a89f7f268de4 Nuke all remaining occurrences of Py_PROTO and Py_FPROTO. Tim Peters <tim.peters@gmail.com> parents: 13256 diff changeset	44 static int tok_nextc(struct tok_state *tok);
a89f7f268de4 Nuke all remaining occurrences of Py_PROTO and Py_FPROTO. Tim Peters <tim.peters@gmail.com> parents: 13256 diff changeset	45 static void tok_backup(struct tok_state *tok, int c);
86 a6934380c6e7 "Compiling" version Guido van Rossum <guido@python.org> parents: 67 diff changeset	46
43607 7593436058dd Plug a memory leak where a struct tok_state was not being freed. Brett Cannon <bcannon@gmail.com> parents: 43605 diff changeset	47
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	48 /* Token names */
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	49
79925 7e0e15d9957f make _PyParser_TokenNames const Benjamin Peterson <benjamin@python.org> parents: 77522 diff changeset	50 const char *_PyParser_TokenNames[] = {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	51 "ENDMARKER",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	52 "NAME",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	53 "NUMBER",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	54 "STRING",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	55 "NEWLINE",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	56 "INDENT",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	57 "DEDENT",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	58 "LPAR",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	59 "RPAR",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	60 "LSQB",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	61 "RSQB",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	62 "COLON",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	63 "COMMA",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	64 "SEMI",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	65 "PLUS",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	66 "MINUS",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	67 "STAR",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	68 "SLASH",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	69 "VBAR",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	70 "AMPER",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	71 "LESS",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	72 "GREATER",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	73 "EQUAL",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	74 "DOT",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	75 "PERCENT",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	76 "LBRACE",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	77 "RBRACE",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	78 "EQEQUAL",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	79 "NOTEQUAL",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	80 "LESSEQUAL",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	81 "GREATEREQUAL",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	82 "TILDE",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	83 "CIRCUMFLEX",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	84 "LEFTSHIFT",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	85 "RIGHTSHIFT",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	86 "DOUBLESTAR",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	87 "PLUSEQUAL",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	88 "MINEQUAL",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	89 "STAREQUAL",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	90 "SLASHEQUAL",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	91 "PERCENTEQUAL",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	92 "AMPEREQUAL",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	93 "VBAREQUAL",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	94 "CIRCUMFLEXEQUAL",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	95 "LEFTSHIFTEQUAL",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	96 "RIGHTSHIFTEQUAL",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	97 "DOUBLESTAREQUAL",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	98 "DOUBLESLASH",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	99 "DOUBLESLASHEQUAL",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	100 "AT",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	101 "RARROW",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	102 "ELLIPSIS",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	103 /* This table must match the #defines in token.h! */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	104 "OP",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	105 "<ERRORTOKEN>",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	106 "<N_TOKENS>"
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	107 };
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	108
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	109
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	110 /* Create and initialize a new tok_state structure */
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	111
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	112 static struct tok_state *
13659 e78153e179fb Mass ANSIfication. Thomas Wouters <thomas@python.org> parents: 13539 diff changeset	113 tok_new(void)
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	114 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	115 struct tok_state tok = (struct tok_state )PyMem_MALLOC(
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	116 sizeof(struct tok_state));
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	117 if (tok == NULL)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	118 return NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	119 tok->buf = tok->cur = tok->end = tok->inp = tok->start = NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	120 tok->done = E_OK;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	121 tok->fp = NULL;
61466 cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	122 tok->input = NULL;
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	123 tok->tabsize = TABSIZE;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	124 tok->indent = 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	125 tok->indstack[0] = 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	126 tok->atbol = 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	127 tok->pendin = 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	128 tok->prompt = tok->nextprompt = NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	129 tok->lineno = 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	130 tok->level = 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	131 tok->altwarning = 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	132 tok->alterror = 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	133 tok->alttabsize = 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	134 tok->altindstack[0] = 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	135 tok->decoding_state = STATE_INIT;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	136 tok->decoding_erred = 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	137 tok->read_coding_spec = 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	138 tok->enc = NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	139 tok->encoding = NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	140 tok->cont_line = 0;
24544 0bd521eb714e Make pgen compile with pydebug. Duplicate normalized names, as it may Martin v. Löwis <martin@v.loewis.de> parents: 24543 diff changeset	141 #ifndef PGEN
69147 6e9dc970ac0e Issue #10785: Store the filename as Unicode in the Python parser. Victor Stinner <victor.stinner@haypocalc.com> parents: 68257 diff changeset	142 tok->filename = NULL;
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	143 tok->decoding_readline = NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	144 tok->decoding_buffer = NULL;
24544 0bd521eb714e Make pgen compile with pydebug. Duplicate normalized names, as it may Martin v. Löwis <martin@v.loewis.de> parents: 24543 diff changeset	145 #endif
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	146 return tok;
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	147 }
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	148
57206 b2d90c1c637d Merged revisions 76230 via svnmerge from Benjamin Peterson <benjamin@python.org> parents: 56594 diff changeset	149 static char *
84653 c3a510b22218 check the return value of new_string() (closes #18470) Benjamin Peterson <benjamin@python.org> parents: 84063 diff changeset	150 new_string(const char s, Py_ssize_t len, struct tok_state tok)
57206 b2d90c1c637d Merged revisions 76230 via svnmerge from Benjamin Peterson <benjamin@python.org> parents: 56594 diff changeset	151 {
61466 cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	152 char* result = (char *)PyMem_MALLOC(len + 1);
84653 c3a510b22218 check the return value of new_string() (closes #18470) Benjamin Peterson <benjamin@python.org> parents: 84063 diff changeset	153 if (!result) {
c3a510b22218 check the return value of new_string() (closes #18470) Benjamin Peterson <benjamin@python.org> parents: 84063 diff changeset	154 tok->done = E_NOMEM;
c3a510b22218 check the return value of new_string() (closes #18470) Benjamin Peterson <benjamin@python.org> parents: 84063 diff changeset	155 return NULL;
61466 cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	156 }
84653 c3a510b22218 check the return value of new_string() (closes #18470) Benjamin Peterson <benjamin@python.org> parents: 84063 diff changeset	157 memcpy(result, s, len);
c3a510b22218 check the return value of new_string() (closes #18470) Benjamin Peterson <benjamin@python.org> parents: 84063 diff changeset	158 result[len] = '\0';
61466 cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	159 return result;
57206 b2d90c1c637d Merged revisions 76230 via svnmerge from Benjamin Peterson <benjamin@python.org> parents: 56594 diff changeset	160 }
b2d90c1c637d Merged revisions 76230 via svnmerge from Benjamin Peterson <benjamin@python.org> parents: 56594 diff changeset	161
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	162 #ifdef PGEN
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	163
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	164 static char *
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	165 decoding_fgets(char s, int size, struct tok_state tok)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	166 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	167 return fgets(s, size, tok->fp);
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	168 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	169
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	170 static int
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	171 decoding_feof(struct tok_state *tok)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	172 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	173 return feof(tok->fp);
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	174 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	175
57206 b2d90c1c637d Merged revisions 76230 via svnmerge from Benjamin Peterson <benjamin@python.org> parents: 56594 diff changeset	176 static char *
b2d90c1c637d Merged revisions 76230 via svnmerge from Benjamin Peterson <benjamin@python.org> parents: 56594 diff changeset	177 decode_str(const char str, int exec_input, struct tok_state tok)
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	178 {
84653 c3a510b22218 check the return value of new_string() (closes #18470) Benjamin Peterson <benjamin@python.org> parents: 84063 diff changeset	179 return new_string(str, strlen(str), tok);
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	180 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	181
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	182 #else /* PGEN */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	183
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	184 static char *
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	185 error_ret(struct tok_state tok) / XXX */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	186 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	187 tok->decoding_erred = 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	188 if (tok->fp != NULL && tok->buf != NULL) /* see PyTokenizer_Free */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	189 PyMem_FREE(tok->buf);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	190 tok->buf = NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	191 return NULL; /* as if it were EOF */
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	192 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	193
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	194
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	195 static char *
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	196 get_normal_name(char s) / for utf-8 and latin-1 */
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	197 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	198 char buf[13];
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	199 int i;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	200 for (i = 0; i < 12; i++) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	201 int c = s[i];
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	202 if (c == '\0')
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	203 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	204 else if (c == '_')
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	205 buf[i] = '-';
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	206 else
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	207 buf[i] = tolower(c);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	208 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	209 buf[i] = '\0';
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	210 if (strcmp(buf, "utf-8") == 0 \|\|
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	211 strncmp(buf, "utf-8-", 6) == 0)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	212 return "utf-8";
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	213 else if (strcmp(buf, "latin-1") == 0 \|\|
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	214 strcmp(buf, "iso-8859-1") == 0 \|\|
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	215 strcmp(buf, "iso-latin-1") == 0 \|\|
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	216 strncmp(buf, "latin-1-", 8) == 0 \|\|
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	217 strncmp(buf, "iso-8859-1-", 11) == 0 \|\|
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	218 strncmp(buf, "iso-latin-1-", 12) == 0)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	219 return "iso-8859-1";
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	220 else
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	221 return s;
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	222 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	223
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	224 /* Return the coding spec in S, or NULL if none is found. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	225
84653 c3a510b22218 check the return value of new_string() (closes #18470) Benjamin Peterson <benjamin@python.org> parents: 84063 diff changeset	226 static int
c3a510b22218 check the return value of new_string() (closes #18470) Benjamin Peterson <benjamin@python.org> parents: 84063 diff changeset	227 get_coding_spec(const char s, char spec, Py_ssize_t size, struct tok_state tok)
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	228 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	229 Py_ssize_t i;
84653 c3a510b22218 check the return value of new_string() (closes #18470) Benjamin Peterson <benjamin@python.org> parents: 84063 diff changeset	230 *spec = NULL;
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	231 /* Coding spec must be in a comment, and that comment must be
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	232 * the only statement on the source code line. */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	233 for (i = 0; i < size - 6; i++) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	234 if (s[i] == '#')
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	235 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	236 if (s[i] != ' ' && s[i] != '\t' && s[i] != '\014')
84653 c3a510b22218 check the return value of new_string() (closes #18470) Benjamin Peterson <benjamin@python.org> parents: 84063 diff changeset	237 return 1;
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	238 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	239 for (; i < size - 6; i++) { /* XXX inefficient search */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	240 const char* t = s + i;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	241 if (strncmp(t, "coding", 6) == 0) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	242 const char* begin = NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	243 t += 6;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	244 if (t[0] != ':' && t[0] != '=')
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	245 continue;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	246 do {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	247 t++;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	248 } while (t[0] == '\x20' \|\| t[0] == '\t');
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	249
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	250 begin = t;
61466 cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	251 while (Py_ISALNUM(t[0]) \|\|
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	252 t[0] == '-' \|\| t[0] == '_' \|\| t[0] == '.')
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	253 t++;
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	254
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	255 if (begin < t) {
84653 c3a510b22218 check the return value of new_string() (closes #18470) Benjamin Peterson <benjamin@python.org> parents: 84063 diff changeset	256 char* r = new_string(begin, t - begin, tok);
84655 72312ff5f712 move declaration to top of block Benjamin Peterson <benjamin@python.org> parents: 84653 diff changeset	257 char* q;
84653 c3a510b22218 check the return value of new_string() (closes #18470) Benjamin Peterson <benjamin@python.org> parents: 84063 diff changeset	258 if (!r)
c3a510b22218 check the return value of new_string() (closes #18470) Benjamin Peterson <benjamin@python.org> parents: 84063 diff changeset	259 return 0;
84655 72312ff5f712 move declaration to top of block Benjamin Peterson <benjamin@python.org> parents: 84653 diff changeset	260 q = get_normal_name(r);
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	261 if (r != q) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	262 PyMem_FREE(r);
84653 c3a510b22218 check the return value of new_string() (closes #18470) Benjamin Peterson <benjamin@python.org> parents: 84063 diff changeset	263 r = new_string(q, strlen(q), tok);
c3a510b22218 check the return value of new_string() (closes #18470) Benjamin Peterson <benjamin@python.org> parents: 84063 diff changeset	264 if (!r)
c3a510b22218 check the return value of new_string() (closes #18470) Benjamin Peterson <benjamin@python.org> parents: 84063 diff changeset	265 return 0;
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	266 }
84653 c3a510b22218 check the return value of new_string() (closes #18470) Benjamin Peterson <benjamin@python.org> parents: 84063 diff changeset	267 *spec = r;
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	268 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	269 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	270 }
84653 c3a510b22218 check the return value of new_string() (closes #18470) Benjamin Peterson <benjamin@python.org> parents: 84063 diff changeset	271 return 1;
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	272 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	273
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	274 /* Check whether the line contains a coding spec. If it does,
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	275 invoke the set_readline function for the new encoding.
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	276 This function receives the tok_state and the new encoding.
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	277 Return 1 on success, 0 on failure. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	278
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	279 static int
36501 7fdc639bc5b4 Merge ssize_t branch. Martin v. Löwis <martin@v.loewis.de> parents: 36090 diff changeset	280 check_coding_spec(const char* line, Py_ssize_t size, struct tok_state *tok,
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	281 int set_readline(struct tok_state , const char ))
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	282 {
84653 c3a510b22218 check the return value of new_string() (closes #18470) Benjamin Peterson <benjamin@python.org> parents: 84063 diff changeset	283 char *cs;
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	284 int r = 1;
25101 32331d26d7cf Removed reliance on gcc/C99 extension. Tim Peters <tim.peters@gmail.com> parents: 25098 diff changeset	285
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	286 if (tok->cont_line)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	287 /* It's a continuation line, so it can't be a coding spec. */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	288 return 1;
84653 c3a510b22218 check the return value of new_string() (closes #18470) Benjamin Peterson <benjamin@python.org> parents: 84063 diff changeset	289 if (!get_coding_spec(line, &cs, size, tok))
c3a510b22218 check the return value of new_string() (closes #18470) Benjamin Peterson <benjamin@python.org> parents: 84063 diff changeset	290 return 0;
c3a510b22218 check the return value of new_string() (closes #18470) Benjamin Peterson <benjamin@python.org> parents: 84063 diff changeset	291 if (!cs)
c3a510b22218 check the return value of new_string() (closes #18470) Benjamin Peterson <benjamin@python.org> parents: 84063 diff changeset	292 return 1;
c3a510b22218 check the return value of new_string() (closes #18470) Benjamin Peterson <benjamin@python.org> parents: 84063 diff changeset	293 tok->read_coding_spec = 1;
c3a510b22218 check the return value of new_string() (closes #18470) Benjamin Peterson <benjamin@python.org> parents: 84063 diff changeset	294 if (tok->encoding == NULL) {
c3a510b22218 check the return value of new_string() (closes #18470) Benjamin Peterson <benjamin@python.org> parents: 84063 diff changeset	295 assert(tok->decoding_state == STATE_RAW);
c3a510b22218 check the return value of new_string() (closes #18470) Benjamin Peterson <benjamin@python.org> parents: 84063 diff changeset	296 if (strcmp(cs, "utf-8") == 0) {
c3a510b22218 check the return value of new_string() (closes #18470) Benjamin Peterson <benjamin@python.org> parents: 84063 diff changeset	297 tok->encoding = cs;
c3a510b22218 check the return value of new_string() (closes #18470) Benjamin Peterson <benjamin@python.org> parents: 84063 diff changeset	298 } else {
c3a510b22218 check the return value of new_string() (closes #18470) Benjamin Peterson <benjamin@python.org> parents: 84063 diff changeset	299 r = set_readline(tok, cs);
c3a510b22218 check the return value of new_string() (closes #18470) Benjamin Peterson <benjamin@python.org> parents: 84063 diff changeset	300 if (r) {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	301 tok->encoding = cs;
84653 c3a510b22218 check the return value of new_string() (closes #18470) Benjamin Peterson <benjamin@python.org> parents: 84063 diff changeset	302 tok->decoding_state = STATE_NORMAL;
c3a510b22218 check the return value of new_string() (closes #18470) Benjamin Peterson <benjamin@python.org> parents: 84063 diff changeset	303 }
c3a510b22218 check the return value of new_string() (closes #18470) Benjamin Peterson <benjamin@python.org> parents: 84063 diff changeset	304 else {
c3a510b22218 check the return value of new_string() (closes #18470) Benjamin Peterson <benjamin@python.org> parents: 84063 diff changeset	305 PyErr_Format(PyExc_SyntaxError,
c3a510b22218 check the return value of new_string() (closes #18470) Benjamin Peterson <benjamin@python.org> parents: 84063 diff changeset	306 "encoding problem: %s", cs);
c3a510b22218 check the return value of new_string() (closes #18470) Benjamin Peterson <benjamin@python.org> parents: 84063 diff changeset	307 PyMem_FREE(cs);
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	308 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	309 }
84653 c3a510b22218 check the return value of new_string() (closes #18470) Benjamin Peterson <benjamin@python.org> parents: 84063 diff changeset	310 } else { /* then, compare cs with BOM */
c3a510b22218 check the return value of new_string() (closes #18470) Benjamin Peterson <benjamin@python.org> parents: 84063 diff changeset	311 r = (strcmp(tok->encoding, cs) == 0);
c3a510b22218 check the return value of new_string() (closes #18470) Benjamin Peterson <benjamin@python.org> parents: 84063 diff changeset	312 if (!r)
c3a510b22218 check the return value of new_string() (closes #18470) Benjamin Peterson <benjamin@python.org> parents: 84063 diff changeset	313 PyErr_Format(PyExc_SyntaxError,
c3a510b22218 check the return value of new_string() (closes #18470) Benjamin Peterson <benjamin@python.org> parents: 84063 diff changeset	314 "encoding problem: %s with BOM", cs);
c3a510b22218 check the return value of new_string() (closes #18470) Benjamin Peterson <benjamin@python.org> parents: 84063 diff changeset	315 PyMem_FREE(cs);
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	316 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	317 return r;
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	318 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	319
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	320 /* See whether the file starts with a BOM. If it does,
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	321 invoke the set_readline function with the new encoding.
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	322 Return 1 on success, 0 on failure. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	323
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	324 static int
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	325 check_bom(int get_char(struct tok_state *),
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	326 void unget_char(int, struct tok_state *),
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	327 int set_readline(struct tok_state , const char ),
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	328 struct tok_state *tok)
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	329 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	330 int ch1, ch2, ch3;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	331 ch1 = get_char(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	332 tok->decoding_state = STATE_RAW;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	333 if (ch1 == EOF) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	334 return 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	335 } else if (ch1 == 0xEF) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	336 ch2 = get_char(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	337 if (ch2 != 0xBB) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	338 unget_char(ch2, tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	339 unget_char(ch1, tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	340 return 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	341 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	342 ch3 = get_char(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	343 if (ch3 != 0xBF) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	344 unget_char(ch3, tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	345 unget_char(ch2, tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	346 unget_char(ch1, tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	347 return 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	348 }
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	349 #if 0
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	350 /* Disable support for UTF-16 BOMs until a decision
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	351 is made whether this needs to be supported. */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	352 } else if (ch1 == 0xFE) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	353 ch2 = get_char(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	354 if (ch2 != 0xFF) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	355 unget_char(ch2, tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	356 unget_char(ch1, tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	357 return 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	358 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	359 if (!set_readline(tok, "utf-16-be"))
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	360 return 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	361 tok->decoding_state = STATE_NORMAL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	362 } else if (ch1 == 0xFF) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	363 ch2 = get_char(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	364 if (ch2 != 0xFE) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	365 unget_char(ch2, tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	366 unget_char(ch1, tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	367 return 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	368 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	369 if (!set_readline(tok, "utf-16-le"))
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	370 return 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	371 tok->decoding_state = STATE_NORMAL;
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	372 #endif
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	373 } else {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	374 unget_char(ch1, tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	375 return 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	376 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	377 if (tok->encoding != NULL)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	378 PyMem_FREE(tok->encoding);
84653 c3a510b22218 check the return value of new_string() (closes #18470) Benjamin Peterson <benjamin@python.org> parents: 84063 diff changeset	379 tok->encoding = new_string("utf-8", 5, tok);
c3a510b22218 check the return value of new_string() (closes #18470) Benjamin Peterson <benjamin@python.org> parents: 84063 diff changeset	380 if (!tok->encoding)
c3a510b22218 check the return value of new_string() (closes #18470) Benjamin Peterson <benjamin@python.org> parents: 84063 diff changeset	381 return 0;
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	382 /* No need to set_readline: input is already utf-8 */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	383 return 1;
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	384 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	385
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	386 /* Read a line of text from TOK into S, using the stream in TOK.
35231 763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file Walter Dörwald <walter@livinglogic.de> parents: 34664 diff changeset	387 Return NULL on failure, else S.
37879 f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair Thomas Wouters <thomas@python.org> parents: 37704 diff changeset	388
35231 763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file Walter Dörwald <walter@livinglogic.de> parents: 34664 diff changeset	389 On entry, tok->decoding_buffer will be one of:
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file Walter Dörwald <walter@livinglogic.de> parents: 34664 diff changeset	390 1) NULL: need to call tok->decoding_readline to get a new line
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file Walter Dörwald <walter@livinglogic.de> parents: 34664 diff changeset	391 2) PyUnicodeObject *: decoding_feof has called tok->decoding_readline and
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	392 stored the result in tok->decoding_buffer
47300 96b34263e448 Renamed PyBytes to PyByteArray Christian Heimes <christian@cheimes.de> parents: 46664 diff changeset	393 3) PyByteArrayObject *: previous call to fp_readl did not have enough room
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	394 (in the s buffer) to copy entire contents of the line read
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	395 by tok->decoding_readline. tok->decoding_buffer has the overflow.
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	396 In this case, fp_readl is called in a loop (with an expanded buffer)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	397 until the buffer ends with a '\n' (or until the end of the file is
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	398 reached): see tok_nextc and its calls to decoding_fgets.
35231 763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file Walter Dörwald <walter@livinglogic.de> parents: 34664 diff changeset	399 */
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	400
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	401 static char *
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	402 fp_readl(char s, int size, struct tok_state tok)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	403 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	404 PyObject* bufobj;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	405 const char *buf;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	406 Py_ssize_t buflen;
35231 763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file Walter Dörwald <walter@livinglogic.de> parents: 34664 diff changeset	407
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	408 /* Ask for one less byte so we can terminate it */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	409 assert(size > 0);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	410 size--;
35231 763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file Walter Dörwald <walter@livinglogic.de> parents: 34664 diff changeset	411
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	412 if (tok->decoding_buffer) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	413 bufobj = tok->decoding_buffer;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	414 Py_INCREF(bufobj);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	415 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	416 else
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	417 {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	418 bufobj = PyObject_CallObject(tok->decoding_readline, NULL);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	419 if (bufobj == NULL)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	420 goto error;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	421 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	422 if (PyUnicode_CheckExact(bufobj))
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	423 {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	424 buf = _PyUnicode_AsStringAndSize(bufobj, &buflen);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	425 if (buf == NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	426 goto error;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	427 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	428 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	429 else
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	430 {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	431 buf = PyByteArray_AsString(bufobj);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	432 if (buf == NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	433 goto error;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	434 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	435 buflen = PyByteArray_GET_SIZE(bufobj);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	436 }
43873 1739869cbc6e Correction for issue1134: all source files with a coding spec, except latin-1 Amaury Forgeot d'Arc <amauryfa@gmail.com> parents: 43777 diff changeset	437
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	438 Py_XDECREF(tok->decoding_buffer);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	439 if (buflen > size) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	440 /* Too many chars, the rest goes into tok->decoding_buffer */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	441 tok->decoding_buffer = PyByteArray_FromStringAndSize(buf+size,
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	442 buflen-size);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	443 if (tok->decoding_buffer == NULL)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	444 goto error;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	445 buflen = size;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	446 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	447 else
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	448 tok->decoding_buffer = NULL;
43873 1739869cbc6e Correction for issue1134: all source files with a coding spec, except latin-1 Amaury Forgeot d'Arc <amauryfa@gmail.com> parents: 43777 diff changeset	449
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	450 memcpy(s, buf, buflen);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	451 s[buflen] = '\0';
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	452 if (buflen == 0) /* EOF */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	453 s = NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	454 Py_DECREF(bufobj);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	455 return s;
42551 b6625d2b378a Fix refleaks from execfile('file that contains a # coding: line') Neal Norwitz <nnorwitz@gmail.com> parents: 42548 diff changeset	456
b6625d2b378a Fix refleaks from execfile('file that contains a # coding: line') Neal Norwitz <nnorwitz@gmail.com> parents: 42548 diff changeset	457 error:
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	458 Py_XDECREF(bufobj);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	459 return error_ret(tok);
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	460 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	461
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	462 /* Set the readline function for TOK to a StreamReader's
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	463 readline function. The StreamReader is named ENC.
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	464
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	465 This function is called from check_bom and check_coding_spec.
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	466
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	467 ENC is usually identical to the future value of tok->encoding,
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	468 except for the (currently unsupported) case of UTF-16.
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	469
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	470 Return 1 on success, 0 on failure. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	471
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	472 static int
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	473 fp_setreadl(struct tok_state tok, const char enc)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	474 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	475 PyObject readline = NULL, stream = NULL, *io = NULL;
72925 7109f31300fb Rename _Py_identifier to _Py_IDENTIFIER. Martin v. Löwis <martin@v.loewis.de> parents: 72848 diff changeset	476 _Py_IDENTIFIER(open);
7109f31300fb Rename _Py_identifier to _Py_IDENTIFIER. Martin v. Löwis <martin@v.loewis.de> parents: 72848 diff changeset	477 _Py_IDENTIFIER(readline);
65311 3d9ef098daf2 Issue #10095: fp_setreadl() doesn't reopen the file, reuse instead the file Victor Stinner <victor.stinner@haypocalc.com> parents: 61466 diff changeset	478 int fd;
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	479
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	480 io = PyImport_ImportModuleNoBlock("io");
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	481 if (io == NULL)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	482 goto cleanup;
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	483
65311 3d9ef098daf2 Issue #10095: fp_setreadl() doesn't reopen the file, reuse instead the file Victor Stinner <victor.stinner@haypocalc.com> parents: 61466 diff changeset	484 fd = fileno(tok->fp);
3d9ef098daf2 Issue #10095: fp_setreadl() doesn't reopen the file, reuse instead the file Victor Stinner <victor.stinner@haypocalc.com> parents: 61466 diff changeset	485 if (lseek(fd, 0, SEEK_SET) == (off_t)-1) {
3d9ef098daf2 Issue #10095: fp_setreadl() doesn't reopen the file, reuse instead the file Victor Stinner <victor.stinner@haypocalc.com> parents: 61466 diff changeset	486 PyErr_SetFromErrnoWithFilename(PyExc_OSError, NULL);
3d9ef098daf2 Issue #10095: fp_setreadl() doesn't reopen the file, reuse instead the file Victor Stinner <victor.stinner@haypocalc.com> parents: 61466 diff changeset	487 goto cleanup;
3d9ef098daf2 Issue #10095: fp_setreadl() doesn't reopen the file, reuse instead the file Victor Stinner <victor.stinner@haypocalc.com> parents: 61466 diff changeset	488 }
3d9ef098daf2 Issue #10095: fp_setreadl() doesn't reopen the file, reuse instead the file Victor Stinner <victor.stinner@haypocalc.com> parents: 61466 diff changeset	489
72835 5e33219492bc Add API for static strings, primarily good for identifiers. Martin v. Löwis <martin@v.loewis.de> parents: 72475 diff changeset	490 stream = _PyObject_CallMethodId(io, &PyId_open, "isisOOO",
65311 3d9ef098daf2 Issue #10095: fp_setreadl() doesn't reopen the file, reuse instead the file Victor Stinner <victor.stinner@haypocalc.com> parents: 61466 diff changeset	491 fd, "r", -1, enc, Py_None, Py_None, Py_False);
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	492 if (stream == NULL)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	493 goto cleanup;
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	494
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	495 Py_XDECREF(tok->decoding_readline);
72848 81380082d216 Use identifier API for PyObject_GetAttrString. Martin v. Löwis <martin@v.loewis.de> parents: 72835 diff changeset	496 readline = _PyObject_GetAttrId(stream, &PyId_readline);
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	497 tok->decoding_readline = readline;
41970 b3900a51346b tokenizer.c: make coding markup work again. Guido van Rossum <guido@python.org> parents: 41670 diff changeset	498
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	499 /* The file has been reopened; parsing will restart from
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	500 * the beginning of the file, we have to reset the line number.
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	501 * But this function has been called from inside tok_nextc() which
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	502 * will increment lineno before it returns. So we set it -1 so that
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	503 * the next call to tok_nextc() will start with tok->lineno == 0.
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	504 */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	505 tok->lineno = -1;
49484 0b7e023803fb Issues #2384 and #3975: Tracebacks were not correctly printed when the source file Amaury Forgeot d'Arc <amauryfa@gmail.com> parents: 49021 diff changeset	506
41970 b3900a51346b tokenizer.c: make coding markup work again. Guido van Rossum <guido@python.org> parents: 41670 diff changeset	507 cleanup:
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	508 Py_XDECREF(stream);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	509 Py_XDECREF(io);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	510 return readline != NULL;
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	511 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	512
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	513 /* Fetch the next byte from TOK. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	514
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	515 static int fp_getc(struct tok_state *tok) {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	516 return getc(tok->fp);
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	517 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	518
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	519 /* Unfetch the last byte back into TOK. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	520
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	521 static void fp_ungetc(int c, struct tok_state *tok) {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	522 ungetc(c, tok->fp);
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	523 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	524
42403 7e8469617b1c Implement PEP 3120. Martin v. Löwis <martin@v.loewis.de> parents: 42084 diff changeset	525 /* Check whether the characters at s start a valid
7e8469617b1c Implement PEP 3120. Martin v. Löwis <martin@v.loewis.de> parents: 42084 diff changeset	526 UTF-8 sequence. Return the number of characters forming
7e8469617b1c Implement PEP 3120. Martin v. Löwis <martin@v.loewis.de> parents: 42084 diff changeset	527 the sequence if yes, 0 if not. */
7e8469617b1c Implement PEP 3120. Martin v. Löwis <martin@v.loewis.de> parents: 42084 diff changeset	528 static int valid_utf8(const unsigned char* s)
7e8469617b1c Implement PEP 3120. Martin v. Löwis <martin@v.loewis.de> parents: 42084 diff changeset	529 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	530 int expected = 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	531 int length;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	532 if (*s < 0x80)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	533 /* single-byte code */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	534 return 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	535 if (*s < 0xc0)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	536 /* following byte */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	537 return 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	538 if (*s < 0xE0)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	539 expected = 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	540 else if (*s < 0xF0)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	541 expected = 2;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	542 else if (*s < 0xF8)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	543 expected = 3;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	544 else
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	545 return 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	546 length = expected + 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	547 for (; expected; expected--)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	548 if (s[expected] < 0x80 \|\| s[expected] >= 0xC0)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	549 return 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	550 return length;
42403 7e8469617b1c Implement PEP 3120. Martin v. Löwis <martin@v.loewis.de> parents: 42084 diff changeset	551 }
7e8469617b1c Implement PEP 3120. Martin v. Löwis <martin@v.loewis.de> parents: 42084 diff changeset	552
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	553 /* Read a line of input from TOK. Determine encoding
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	554 if necessary. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	555
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	556 static char *
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	557 decoding_fgets(char s, int size, struct tok_state tok)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	558 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	559 char *line = NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	560 int badchar = 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	561 for (;;) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	562 if (tok->decoding_state == STATE_NORMAL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	563 /* We already have a codec associated with
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	564 this input. */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	565 line = fp_readl(s, size, tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	566 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	567 } else if (tok->decoding_state == STATE_RAW) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	568 /* We want a 'raw' read. */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	569 line = Py_UniversalNewlineFgets(s, size,
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	570 tok->fp, NULL);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	571 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	572 } else {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	573 /* We have not yet determined the encoding.
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	574 If an encoding is found, use the file-pointer
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	575 reader functions from now on. */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	576 if (!check_bom(fp_getc, fp_ungetc, fp_setreadl, tok))
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	577 return error_ret(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	578 assert(tok->decoding_state != STATE_INIT);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	579 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	580 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	581 if (line != NULL && tok->lineno < 2 && !tok->read_coding_spec) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	582 if (!check_coding_spec(line, strlen(line), tok, fp_setreadl)) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	583 return error_ret(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	584 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	585 }
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	586 #ifndef PGEN
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	587 /* The default encoding is UTF-8, so make sure we don't have any
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	588 non-UTF-8 sequences in it. */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	589 if (line && !tok->encoding) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	590 unsigned char *c;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	591 int length;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	592 for (c = (unsigned char )line; c; c += length)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	593 if (!(length = valid_utf8(c))) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	594 badchar = *c;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	595 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	596 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	597 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	598 if (badchar) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	599 /* Need to add 1 to the line number, since this line
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	600 has not been counted, yet. */
69550 4658762a77cf Revert bb62908896fe, but keep the test Jesus Cea <jcea@jcea.es> parents: 69548 diff changeset	601 PyErr_Format(PyExc_SyntaxError,
4658762a77cf Revert bb62908896fe, but keep the test Jesus Cea <jcea@jcea.es> parents: 69548 diff changeset	602 "Non-UTF-8 code starting with '\\x%.2x' "
4658762a77cf Revert bb62908896fe, but keep the test Jesus Cea <jcea@jcea.es> parents: 69548 diff changeset	603 "in file %U on line %i, "
4658762a77cf Revert bb62908896fe, but keep the test Jesus Cea <jcea@jcea.es> parents: 69548 diff changeset	604 "but no encoding declared; "
4658762a77cf Revert bb62908896fe, but keep the test Jesus Cea <jcea@jcea.es> parents: 69548 diff changeset	605 "see http://python.org/dev/peps/pep-0263/ for details",
4658762a77cf Revert bb62908896fe, but keep the test Jesus Cea <jcea@jcea.es> parents: 69548 diff changeset	606 badchar, tok->filename, tok->lineno + 1);
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	607 return error_ret(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	608 }
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	609 #endif
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	610 return line;
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	611 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	612
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	613 static int
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	614 decoding_feof(struct tok_state *tok)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	615 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	616 if (tok->decoding_state != STATE_NORMAL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	617 return feof(tok->fp);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	618 } else {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	619 PyObject* buf = tok->decoding_buffer;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	620 if (buf == NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	621 buf = PyObject_CallObject(tok->decoding_readline, NULL);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	622 if (buf == NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	623 error_ret(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	624 return 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	625 } else {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	626 tok->decoding_buffer = buf;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	627 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	628 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	629 return PyObject_Length(buf) == 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	630 }
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	631 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	632
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	633 /* Fetch a byte from TOK, using the string buffer. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	634
37879 f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair Thomas Wouters <thomas@python.org> parents: 37704 diff changeset	635 static int
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair Thomas Wouters <thomas@python.org> parents: 37704 diff changeset	636 buf_getc(struct tok_state *tok) {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	637 return Py_CHARMASK(*tok->str++);
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	638 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	639
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	640 /* Unfetch a byte from TOK, using the string buffer. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	641
37879 f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair Thomas Wouters <thomas@python.org> parents: 37704 diff changeset	642 static void
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair Thomas Wouters <thomas@python.org> parents: 37704 diff changeset	643 buf_ungetc(int c, struct tok_state *tok) {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	644 tok->str--;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	645 assert(Py_CHARMASK(tok->str) == c); / tok->cur may point to read-only segment */
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	646 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	647
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	648 /* Set the readline function for TOK to ENC. For the string-based
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	649 tokenizer, this means to just record the encoding. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	650
37879 f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair Thomas Wouters <thomas@python.org> parents: 37704 diff changeset	651 static int
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair Thomas Wouters <thomas@python.org> parents: 37704 diff changeset	652 buf_setreadl(struct tok_state tok, const char enc) {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	653 tok->enc = enc;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	654 return 1;
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	655 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	656
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	657 /* Return a UTF-8 encoding Python string object from the
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	658 C byte string STR, which is encoded with ENC. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	659
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	660 static PyObject *
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	661 translate_into_utf8(const char* str, const char* enc) {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	662 PyObject *utf8;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	663 PyObject* buf = PyUnicode_Decode(str, strlen(str), enc, NULL);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	664 if (buf == NULL)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	665 return NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	666 utf8 = PyUnicode_AsUTF8String(buf);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	667 Py_DECREF(buf);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	668 return utf8;
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	669 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	670
57206 b2d90c1c637d Merged revisions 76230 via svnmerge from Benjamin Peterson <benjamin@python.org> parents: 56594 diff changeset	671
b2d90c1c637d Merged revisions 76230 via svnmerge from Benjamin Peterson <benjamin@python.org> parents: 56594 diff changeset	672 static char *
b2d90c1c637d Merged revisions 76230 via svnmerge from Benjamin Peterson <benjamin@python.org> parents: 56594 diff changeset	673 translate_newlines(const char s, int exec_input, struct tok_state tok) {
84033 36c35a1893fe Issue #9566: Fix compiler warning on Windows 64-bit Victor Stinner <victor.stinner@gmail.com> parents: 79925 diff changeset	674 int skip_next_lf = 0;
36c35a1893fe Issue #9566: Fix compiler warning on Windows 64-bit Victor Stinner <victor.stinner@gmail.com> parents: 79925 diff changeset	675 size_t needed_length = strlen(s) + 2, final_length;
61466 cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	676 char buf, current;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	677 char c = '\0';
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	678 buf = PyMem_MALLOC(needed_length);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	679 if (buf == NULL) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	680 tok->done = E_NOMEM;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	681 return NULL;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	682 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	683 for (current = buf; *s; s++, current++) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	684 c = *s;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	685 if (skip_next_lf) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	686 skip_next_lf = 0;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	687 if (c == '\n') {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	688 c = *++s;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	689 if (!c)
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	690 break;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	691 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	692 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	693 if (c == '\r') {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	694 skip_next_lf = 1;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	695 c = '\n';
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	696 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	697 *current = c;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	698 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	699 /* If this is exec input, add a newline to the end of the string if
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	700 there isn't one already. */
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	701 if (exec_input && c != '\n') {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	702 *current = '\n';
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	703 current++;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	704 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	705 *current = '\0';
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	706 final_length = current - buf + 1;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	707 if (final_length < needed_length && final_length)
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	708 /* should never fail */
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	709 buf = PyMem_REALLOC(buf, final_length);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	710 return buf;
57206 b2d90c1c637d Merged revisions 76230 via svnmerge from Benjamin Peterson <benjamin@python.org> parents: 56594 diff changeset	711 }
b2d90c1c637d Merged revisions 76230 via svnmerge from Benjamin Peterson <benjamin@python.org> parents: 56594 diff changeset	712
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	713 /* Decode a byte string STR for use as the buffer of TOK.
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	714 Look for encoding declarations inside STR, and record them
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	715 inside TOK. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	716
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	717 static const char *
57206 b2d90c1c637d Merged revisions 76230 via svnmerge from Benjamin Peterson <benjamin@python.org> parents: 56594 diff changeset	718 decode_str(const char input, int single, struct tok_state tok)
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	719 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	720 PyObject* utf8 = NULL;
61466 cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	721 const char *str;
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	722 const char *s;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	723 const char *newl[2] = {NULL, NULL};
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	724 int lineno = 0;
61466 cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	725 tok->input = str = translate_newlines(input, single, tok);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	726 if (str == NULL)
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	727 return NULL;
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	728 tok->enc = NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	729 tok->str = str;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	730 if (!check_bom(buf_getc, buf_ungetc, buf_setreadl, tok))
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	731 return error_ret(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	732 str = tok->str; /* string after BOM if any */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	733 assert(str);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	734 if (tok->enc != NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	735 utf8 = translate_into_utf8(str, tok->enc);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	736 if (utf8 == NULL)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	737 return error_ret(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	738 str = PyBytes_AsString(utf8);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	739 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	740 for (s = str;; s++) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	741 if (*s == '\0') break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	742 else if (*s == '\n') {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	743 assert(lineno < 2);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	744 newl[lineno] = s;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	745 lineno++;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	746 if (lineno == 2) break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	747 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	748 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	749 tok->enc = NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	750 /* need to check line 1 and 2 separately since check_coding_spec
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	751 assumes a single line as input */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	752 if (newl[0]) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	753 if (!check_coding_spec(str, newl[0] - str, tok, buf_setreadl))
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	754 return error_ret(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	755 if (tok->enc == NULL && newl[1]) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	756 if (!check_coding_spec(newl[0]+1, newl[1] - newl[0],
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	757 tok, buf_setreadl))
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	758 return error_ret(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	759 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	760 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	761 if (tok->enc != NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	762 assert(utf8 == NULL);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	763 utf8 = translate_into_utf8(str, tok->enc);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	764 if (utf8 == NULL)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	765 return error_ret(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	766 str = PyBytes_AS_STRING(utf8);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	767 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	768 assert(tok->decoding_buffer == NULL);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	769 tok->decoding_buffer = utf8; /* CAUTION */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	770 return str;
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	771 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	772
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	773 #endif /* PGEN */
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	774
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	775 /* Set up tokenizer for string */
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	776
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	777 struct tok_state *
57206 b2d90c1c637d Merged revisions 76230 via svnmerge from Benjamin Peterson <benjamin@python.org> parents: 56594 diff changeset	778 PyTokenizer_FromString(const char *str, int exec_input)
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	779 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	780 struct tok_state *tok = tok_new();
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	781 if (tok == NULL)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	782 return NULL;
61466 cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	783 str = (char *)decode_str(str, exec_input, tok);
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	784 if (str == NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	785 PyTokenizer_Free(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	786 return NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	787 }
35878 94a409da4bd4 Fix some more memory leaks. Neal Norwitz <nnorwitz@gmail.com> parents: 35751 diff changeset	788
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	789 /* XXX: constify members. */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	790 tok->buf = tok->cur = tok->end = tok->inp = (char*)str;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	791 return tok;
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	792 }
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	793
51956 8fe2988cf550 ignore the coding cookie in compile(), exec(), and eval() if the source is a string #4626 Benjamin Peterson <benjamin@python.org> parents: 50540 diff changeset	794 struct tok_state *
57206 b2d90c1c637d Merged revisions 76230 via svnmerge from Benjamin Peterson <benjamin@python.org> parents: 56594 diff changeset	795 PyTokenizer_FromUTF8(const char *str, int exec_input)
51956 8fe2988cf550 ignore the coding cookie in compile(), exec(), and eval() if the source is a string #4626 Benjamin Peterson <benjamin@python.org> parents: 50540 diff changeset	796 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	797 struct tok_state *tok = tok_new();
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	798 if (tok == NULL)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	799 return NULL;
57206 b2d90c1c637d Merged revisions 76230 via svnmerge from Benjamin Peterson <benjamin@python.org> parents: 56594 diff changeset	800 #ifndef PGEN
61466 cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	801 tok->input = str = translate_newlines(str, exec_input, tok);
57206 b2d90c1c637d Merged revisions 76230 via svnmerge from Benjamin Peterson <benjamin@python.org> parents: 56594 diff changeset	802 #endif
61466 cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	803 if (str == NULL) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	804 PyTokenizer_Free(tok);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	805 return NULL;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	806 }
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	807 tok->decoding_state = STATE_RAW;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	808 tok->read_coding_spec = 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	809 tok->enc = NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	810 tok->str = str;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	811 tok->encoding = (char *)PyMem_MALLOC(6);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	812 if (!tok->encoding) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	813 PyTokenizer_Free(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	814 return NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	815 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	816 strcpy(tok->encoding, "utf-8");
51956 8fe2988cf550 ignore the coding cookie in compile(), exec(), and eval() if the source is a string #4626 Benjamin Peterson <benjamin@python.org> parents: 50540 diff changeset	817
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	818 /* XXX: constify members. */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	819 tok->buf = tok->cur = tok->end = tok->inp = (char*)str;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	820 return tok;
51956 8fe2988cf550 ignore the coding cookie in compile(), exec(), and eval() if the source is a string #4626 Benjamin Peterson <benjamin@python.org> parents: 50540 diff changeset	821 }
8fe2988cf550 ignore the coding cookie in compile(), exec(), and eval() if the source is a string #4626 Benjamin Peterson <benjamin@python.org> parents: 50540 diff changeset	822
326 ff0862e3d63a Completely ignore lines with only a newline token on them, except Guido van Rossum <guido@python.org> parents: 293 diff changeset	823 /* Set up tokenizer for file */
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	824
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	825 struct tok_state *
43225 38f3f9ef193d Convert code from sys.stdin.encoding to UTF-8 in Martin v. Löwis <martin@v.loewis.de> parents: 43018 diff changeset	826 PyTokenizer_FromFile(FILE fp, char enc, char ps1, char ps2)
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	827 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	828 struct tok_state *tok = tok_new();
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	829 if (tok == NULL)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	830 return NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	831 if ((tok->buf = (char *)PyMem_MALLOC(BUFSIZ)) == NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	832 PyTokenizer_Free(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	833 return NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	834 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	835 tok->cur = tok->inp = tok->buf;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	836 tok->end = tok->buf + BUFSIZ;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	837 tok->fp = fp;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	838 tok->prompt = ps1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	839 tok->nextprompt = ps2;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	840 if (enc != NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	841 /* Must copy encoding declaration since it
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	842 gets copied into the parse tree. */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	843 tok->encoding = PyMem_MALLOC(strlen(enc)+1);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	844 if (!tok->encoding) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	845 PyTokenizer_Free(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	846 return NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	847 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	848 strcpy(tok->encoding, enc);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	849 tok->decoding_state = STATE_NORMAL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	850 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	851 return tok;
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	852 }
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	853
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	854
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	855 /* Free a tok_state structure */
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	856
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	857 void
13659 e78153e179fb Mass ANSIfication. Thomas Wouters <thomas@python.org> parents: 13539 diff changeset	858 PyTokenizer_Free(struct tok_state *tok)
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	859 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	860 if (tok->encoding != NULL)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	861 PyMem_FREE(tok->encoding);
24544 0bd521eb714e Make pgen compile with pydebug. Duplicate normalized names, as it may Martin v. Löwis <martin@v.loewis.de> parents: 24543 diff changeset	862 #ifndef PGEN
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	863 Py_XDECREF(tok->decoding_readline);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	864 Py_XDECREF(tok->decoding_buffer);
69147 6e9dc970ac0e Issue #10785: Store the filename as Unicode in the Python parser. Victor Stinner <victor.stinner@haypocalc.com> parents: 68257 diff changeset	865 Py_XDECREF(tok->filename);
24544 0bd521eb714e Make pgen compile with pydebug. Duplicate normalized names, as it may Martin v. Löwis <martin@v.loewis.de> parents: 24543 diff changeset	866 #endif
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	867 if (tok->fp != NULL && tok->buf != NULL)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	868 PyMem_FREE(tok->buf);
61466 cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	869 if (tok->input)
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	870 PyMem_FREE((char *)tok->input);
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	871 PyMem_FREE(tok);
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	872 }
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	873
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	874 /* Get next char, updating state; error code goes into tok->done */
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	875
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	876 static int
85157 e7f6cef7a4cc Issue #18722: Remove uses of the "register" keyword in C code. Antoine Pitrou <solipsis@pitrou.net> parents: 84656 diff changeset	877 tok_nextc(struct tok_state *tok)
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	878 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	879 for (;;) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	880 if (tok->cur != tok->inp) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	881 return Py_CHARMASK(tok->cur++); / Fast path */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	882 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	883 if (tok->done != E_OK)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	884 return EOF;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	885 if (tok->fp == NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	886 char *end = strchr(tok->inp, '\n');
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	887 if (end != NULL)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	888 end++;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	889 else {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	890 end = strchr(tok->inp, '\0');
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	891 if (end == tok->inp) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	892 tok->done = E_EOF;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	893 return EOF;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	894 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	895 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	896 if (tok->start == NULL)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	897 tok->buf = tok->cur;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	898 tok->line_start = tok->cur;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	899 tok->lineno++;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	900 tok->inp = end;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	901 return Py_CHARMASK(*tok->cur++);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	902 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	903 if (tok->prompt != NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	904 char *newtok = PyOS_Readline(stdin, stdout, tok->prompt);
43225 38f3f9ef193d Convert code from sys.stdin.encoding to UTF-8 in Martin v. Löwis <martin@v.loewis.de> parents: 43018 diff changeset	905 #ifndef PGEN
67431 0933c3753a71 Issue #10841: set binary mode on files; the parser translates newlines Victor Stinner <victor.stinner@haypocalc.com> parents: 67367 diff changeset	906 if (newtok != NULL) {
0933c3753a71 Issue #10841: set binary mode on files; the parser translates newlines Victor Stinner <victor.stinner@haypocalc.com> parents: 67367 diff changeset	907 char *translated = translate_newlines(newtok, 0, tok);
0933c3753a71 Issue #10841: set binary mode on files; the parser translates newlines Victor Stinner <victor.stinner@haypocalc.com> parents: 67367 diff changeset	908 PyMem_FREE(newtok);
0933c3753a71 Issue #10841: set binary mode on files; the parser translates newlines Victor Stinner <victor.stinner@haypocalc.com> parents: 67367 diff changeset	909 if (translated == NULL)
0933c3753a71 Issue #10841: set binary mode on files; the parser translates newlines Victor Stinner <victor.stinner@haypocalc.com> parents: 67367 diff changeset	910 return EOF;
0933c3753a71 Issue #10841: set binary mode on files; the parser translates newlines Victor Stinner <victor.stinner@haypocalc.com> parents: 67367 diff changeset	911 newtok = translated;
0933c3753a71 Issue #10841: set binary mode on files; the parser translates newlines Victor Stinner <victor.stinner@haypocalc.com> parents: 67367 diff changeset	912 }
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	913 if (tok->encoding && newtok && *newtok) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	914 /* Recode to UTF-8 */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	915 Py_ssize_t buflen;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	916 const char* buf;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	917 PyObject *u = translate_into_utf8(newtok, tok->encoding);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	918 PyMem_FREE(newtok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	919 if (!u) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	920 tok->done = E_DECODE;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	921 return EOF;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	922 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	923 buflen = PyBytes_GET_SIZE(u);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	924 buf = PyBytes_AS_STRING(u);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	925 if (!buf) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	926 Py_DECREF(u);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	927 tok->done = E_DECODE;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	928 return EOF;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	929 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	930 newtok = PyMem_MALLOC(buflen+1);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	931 strcpy(newtok, buf);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	932 Py_DECREF(u);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	933 }
43225 38f3f9ef193d Convert code from sys.stdin.encoding to UTF-8 in Martin v. Löwis <martin@v.loewis.de> parents: 43018 diff changeset	934 #endif
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	935 if (tok->nextprompt != NULL)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	936 tok->prompt = tok->nextprompt;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	937 if (newtok == NULL)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	938 tok->done = E_INTR;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	939 else if (*newtok == '\0') {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	940 PyMem_FREE(newtok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	941 tok->done = E_EOF;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	942 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	943 else if (tok->start != NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	944 size_t start = tok->start - tok->buf;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	945 size_t oldlen = tok->cur - tok->buf;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	946 size_t newlen = oldlen + strlen(newtok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	947 char *buf = tok->buf;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	948 buf = (char *)PyMem_REALLOC(buf, newlen+1);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	949 tok->lineno++;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	950 if (buf == NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	951 PyMem_FREE(tok->buf);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	952 tok->buf = NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	953 PyMem_FREE(newtok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	954 tok->done = E_NOMEM;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	955 return EOF;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	956 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	957 tok->buf = buf;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	958 tok->cur = tok->buf + oldlen;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	959 tok->line_start = tok->cur;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	960 strcpy(tok->buf + oldlen, newtok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	961 PyMem_FREE(newtok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	962 tok->inp = tok->buf + newlen;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	963 tok->end = tok->inp + 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	964 tok->start = tok->buf + start;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	965 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	966 else {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	967 tok->lineno++;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	968 if (tok->buf != NULL)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	969 PyMem_FREE(tok->buf);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	970 tok->buf = newtok;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	971 tok->line_start = tok->buf;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	972 tok->cur = tok->buf;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	973 tok->line_start = tok->buf;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	974 tok->inp = strchr(tok->buf, '\0');
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	975 tok->end = tok->inp + 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	976 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	977 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	978 else {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	979 int done = 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	980 Py_ssize_t cur = 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	981 char *pt;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	982 if (tok->start == NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	983 if (tok->buf == NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	984 tok->buf = (char *)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	985 PyMem_MALLOC(BUFSIZ);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	986 if (tok->buf == NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	987 tok->done = E_NOMEM;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	988 return EOF;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	989 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	990 tok->end = tok->buf + BUFSIZ;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	991 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	992 if (decoding_fgets(tok->buf, (int)(tok->end - tok->buf),
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	993 tok) == NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	994 tok->done = E_EOF;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	995 done = 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	996 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	997 else {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	998 tok->done = E_OK;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	999 tok->inp = strchr(tok->buf, '\0');
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1000 done = tok->inp[-1] == '\n';
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1001 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1002 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1003 else {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1004 cur = tok->cur - tok->buf;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1005 if (decoding_feof(tok)) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1006 tok->done = E_EOF;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1007 done = 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1008 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1009 else
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1010 tok->done = E_OK;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1011 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1012 tok->lineno++;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1013 /* Read until '\n' or EOF */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1014 while (!done) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1015 Py_ssize_t curstart = tok->start == NULL ? -1 :
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1016 tok->start - tok->buf;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1017 Py_ssize_t curvalid = tok->inp - tok->buf;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1018 Py_ssize_t newsize = curvalid + BUFSIZ;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1019 char *newbuf = tok->buf;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1020 newbuf = (char *)PyMem_REALLOC(newbuf,
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1021 newsize);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1022 if (newbuf == NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1023 tok->done = E_NOMEM;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1024 tok->cur = tok->inp;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1025 return EOF;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1026 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1027 tok->buf = newbuf;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1028 tok->inp = tok->buf + curvalid;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1029 tok->end = tok->buf + newsize;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1030 tok->start = curstart < 0 ? NULL :
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1031 tok->buf + curstart;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1032 if (decoding_fgets(tok->inp,
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1033 (int)(tok->end - tok->inp),
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1034 tok) == NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1035 /* Break out early on decoding
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1036 errors, as tok->buf will be NULL
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1037 */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1038 if (tok->decoding_erred)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1039 return EOF;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1040 /* Last line does not end in \n,
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1041 fake one */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1042 strcpy(tok->inp, "\n");
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1043 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1044 tok->inp = strchr(tok->inp, '\0');
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1045 done = tok->inp[-1] == '\n';
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1046 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1047 if (tok->buf != NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1048 tok->cur = tok->buf + cur;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1049 tok->line_start = tok->cur;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1050 /* replace "\r\n" with "\n" */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1051 /* For Mac leave the \r, giving a syntax error */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1052 pt = tok->inp - 2;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1053 if (pt >= tok->buf && *pt == '\r') {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1054 *pt++ = '\n';
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1055 *pt = '\0';
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1056 tok->inp = pt;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1057 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1058 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1059 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1060 if (tok->done != E_OK) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1061 if (tok->prompt != NULL)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1062 PySys_WriteStderr("\n");
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1063 tok->cur = tok->inp;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1064 return EOF;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1065 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1066 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1067 /NOTREACHED/
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1068 }
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1069
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1070
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1071 /* Back-up one character */
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1072
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1073 static void
85157 e7f6cef7a4cc Issue #18722: Remove uses of the "register" keyword in C code. Antoine Pitrou <solipsis@pitrou.net> parents: 84656 diff changeset	1074 tok_backup(struct tok_state *tok, int c)
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1075 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1076 if (c != EOF) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1077 if (--tok->cur < tok->buf)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1078 Py_FatalError("tok_backup: beginning of buffer");
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1079 if (*tok->cur != c)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1080 *tok->cur = c;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1081 }
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1082 }
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1083
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1084
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1085 /* Return the token corresponding to a single character */
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1086
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1087 int
13659 e78153e179fb Mass ANSIfication. Thomas Wouters <thomas@python.org> parents: 13539 diff changeset	1088 PyToken_OneChar(int c)
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1089 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1090 switch (c) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1091 case '(': return LPAR;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1092 case ')': return RPAR;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1093 case '[': return LSQB;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1094 case ']': return RSQB;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1095 case ':': return COLON;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1096 case ',': return COMMA;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1097 case ';': return SEMI;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1098 case '+': return PLUS;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1099 case '-': return MINUS;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1100 case '*': return STAR;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1101 case '/': return SLASH;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1102 case '\|': return VBAR;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1103 case '&': return AMPER;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1104 case '<': return LESS;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1105 case '>': return GREATER;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1106 case '=': return EQUAL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1107 case '.': return DOT;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1108 case '%': return PERCENT;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1109 case '{': return LBRACE;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1110 case '}': return RBRACE;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1111 case '^': return CIRCUMFLEX;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1112 case '~': return TILDE;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1113 case '@': return AT;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1114 default: return OP;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1115 }
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1116 }
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1117
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1118
400 d189a2f681a6 Added 2-char tokens and new versions of comparisons Guido van Rossum <guido@python.org> parents: 326 diff changeset	1119 int
13659 e78153e179fb Mass ANSIfication. Thomas Wouters <thomas@python.org> parents: 13539 diff changeset	1120 PyToken_TwoChars(int c1, int c2)
400 d189a2f681a6 Added 2-char tokens and new versions of comparisons Guido van Rossum <guido@python.org> parents: 326 diff changeset	1121 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1122 switch (c1) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1123 case '=':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1124 switch (c2) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1125 case '=': return EQEQUAL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1126 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1127 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1128 case '!':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1129 switch (c2) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1130 case '=': return NOTEQUAL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1131 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1132 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1133 case '<':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1134 switch (c2) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1135 case '>': return NOTEQUAL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1136 case '=': return LESSEQUAL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1137 case '<': return LEFTSHIFT;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1138 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1139 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1140 case '>':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1141 switch (c2) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1142 case '=': return GREATEREQUAL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1143 case '>': return RIGHTSHIFT;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1144 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1145 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1146 case '+':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1147 switch (c2) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1148 case '=': return PLUSEQUAL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1149 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1150 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1151 case '-':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1152 switch (c2) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1153 case '=': return MINEQUAL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1154 case '>': return RARROW;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1155 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1156 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1157 case '*':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1158 switch (c2) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1159 case '*': return DOUBLESTAR;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1160 case '=': return STAREQUAL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1161 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1162 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1163 case '/':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1164 switch (c2) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1165 case '/': return DOUBLESLASH;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1166 case '=': return SLASHEQUAL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1167 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1168 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1169 case '\|':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1170 switch (c2) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1171 case '=': return VBAREQUAL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1172 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1173 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1174 case '%':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1175 switch (c2) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1176 case '=': return PERCENTEQUAL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1177 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1178 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1179 case '&':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1180 switch (c2) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1181 case '=': return AMPEREQUAL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1182 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1183 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1184 case '^':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1185 switch (c2) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1186 case '=': return CIRCUMFLEXEQUAL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1187 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1188 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1189 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1190 return OP;
400 d189a2f681a6 Added 2-char tokens and new versions of comparisons Guido van Rossum <guido@python.org> parents: 326 diff changeset	1191 }
d189a2f681a6 Added 2-char tokens and new versions of comparisons Guido van Rossum <guido@python.org> parents: 326 diff changeset	1192
14163 1b9fd0cbd914 Support for three-token characters (*=, >>=, <<=) which was written by Thomas Wouters <thomas@python.org>* parents: 13659 diff changeset	1193 int
1b9fd0cbd914 Support for three-token characters (*=, >>=, <<=) which was written by Thomas Wouters <thomas@python.org>* parents: 13659 diff changeset	1194 PyToken_ThreeChars(int c1, int c2, int c3)
1b9fd0cbd914 Support for three-token characters (*=, >>=, <<=) which was written by Thomas Wouters <thomas@python.org>* parents: 13659 diff changeset	1195 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1196 switch (c1) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1197 case '<':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1198 switch (c2) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1199 case '<':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1200 switch (c3) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1201 case '=':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1202 return LEFTSHIFTEQUAL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1203 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1204 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1205 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1206 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1207 case '>':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1208 switch (c2) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1209 case '>':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1210 switch (c3) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1211 case '=':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1212 return RIGHTSHIFTEQUAL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1213 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1214 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1215 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1216 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1217 case '*':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1218 switch (c2) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1219 case '*':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1220 switch (c3) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1221 case '=':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1222 return DOUBLESTAREQUAL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1223 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1224 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1225 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1226 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1227 case '/':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1228 switch (c2) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1229 case '/':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1230 switch (c3) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1231 case '=':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1232 return DOUBLESLASHEQUAL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1233 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1234 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1235 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1236 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1237 case '.':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1238 switch (c2) {
41230 b1a18c7ed5ca Make ELLIPSIS a separate token. This makes it a syntax error to write ". . ." for Ellipsis. Georg Brandl <georg@python.org> parents: 40894 diff changeset	1239 case '.':
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1240 switch (c3) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1241 case '.':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1242 return ELLIPSIS;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1243 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1244 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1245 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1246 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1247 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1248 return OP;
14163 1b9fd0cbd914 Support for three-token characters (*=, >>=, <<=) which was written by Thomas Wouters <thomas@python.org>* parents: 13659 diff changeset	1249 }
400 d189a2f681a6 Added 2-char tokens and new versions of comparisons Guido van Rossum <guido@python.org> parents: 326 diff changeset	1250
7945 dc132cc39573 Add checking for inconsistent tab usage Guido van Rossum <guido@python.org> parents: 7324 diff changeset	1251 static int
13659 e78153e179fb Mass ANSIfication. Thomas Wouters <thomas@python.org> parents: 13539 diff changeset	1252 indenterror(struct tok_state *tok)
7945 dc132cc39573 Add checking for inconsistent tab usage Guido van Rossum <guido@python.org> parents: 7324 diff changeset	1253 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1254 if (tok->alterror) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1255 tok->done = E_TABSPACE;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1256 tok->cur = tok->inp;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1257 return 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1258 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1259 if (tok->altwarning) {
69147 6e9dc970ac0e Issue #10785: Store the filename as Unicode in the Python parser. Victor Stinner <victor.stinner@haypocalc.com> parents: 68257 diff changeset	1260 #ifdef PGEN
6e9dc970ac0e Issue #10785: Store the filename as Unicode in the Python parser. Victor Stinner <victor.stinner@haypocalc.com> parents: 68257 diff changeset	1261 PySys_WriteStderr("inconsistent use of tabs and spaces "
6e9dc970ac0e Issue #10785: Store the filename as Unicode in the Python parser. Victor Stinner <victor.stinner@haypocalc.com> parents: 68257 diff changeset	1262 "in indentation\n");
6e9dc970ac0e Issue #10785: Store the filename as Unicode in the Python parser. Victor Stinner <victor.stinner@haypocalc.com> parents: 68257 diff changeset	1263 #else
6e9dc970ac0e Issue #10785: Store the filename as Unicode in the Python parser. Victor Stinner <victor.stinner@haypocalc.com> parents: 68257 diff changeset	1264 PySys_FormatStderr("%U: inconsistent use of tabs and spaces "
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1265 "in indentation\n", tok->filename);
69147 6e9dc970ac0e Issue #10785: Store the filename as Unicode in the Python parser. Victor Stinner <victor.stinner@haypocalc.com> parents: 68257 diff changeset	1266 #endif
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1267 tok->altwarning = 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1268 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1269 return 0;
7945 dc132cc39573 Add checking for inconsistent tab usage Guido van Rossum <guido@python.org> parents: 7324 diff changeset	1270 }
dc132cc39573 Add checking for inconsistent tab usage Guido van Rossum <guido@python.org> parents: 7324 diff changeset	1271
42597 20477cbfd996 Implement PEP 3131. Add isidentifier to str. Martin v. Löwis <martin@v.loewis.de> parents: 42551 diff changeset	1272 #ifdef PGEN
59865 c2d2e26bcf7f Merged revisions 78872 via svnmerge from Victor Stinner <victor.stinner@haypocalc.com> parents: 59289 diff changeset	1273 #define verify_identifier(tok) 1
42597 20477cbfd996 Implement PEP 3131. Add isidentifier to str. Martin v. Löwis <martin@v.loewis.de> parents: 42551 diff changeset	1274 #else
72475 8beaa9a37387 Implement PEP 393. Martin v. Löwis <martin@v.loewis.de> parents: 69550 diff changeset	1275 /* Verify that the identifier follows PEP 3131.
8beaa9a37387 Implement PEP 393. Martin v. Löwis <martin@v.loewis.de> parents: 69550 diff changeset	1276 All identifier strings are guaranteed to be "ready" unicode objects.
8beaa9a37387 Implement PEP 393. Martin v. Löwis <martin@v.loewis.de> parents: 69550 diff changeset	1277 */
42597 20477cbfd996 Implement PEP 3131. Add isidentifier to str. Martin v. Löwis <martin@v.loewis.de> parents: 42551 diff changeset	1278 static int
59865 c2d2e26bcf7f Merged revisions 78872 via svnmerge from Victor Stinner <victor.stinner@haypocalc.com> parents: 59289 diff changeset	1279 verify_identifier(struct tok_state *tok)
42597 20477cbfd996 Implement PEP 3131. Add isidentifier to str. Martin v. Löwis <martin@v.loewis.de> parents: 42551 diff changeset	1280 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1281 PyObject *s;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1282 int result;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1283 s = PyUnicode_DecodeUTF8(tok->start, tok->cur - tok->start, NULL);
72475 8beaa9a37387 Implement PEP 393. Martin v. Löwis <martin@v.loewis.de> parents: 69550 diff changeset	1284 if (s == NULL \|\| PyUnicode_READY(s) == -1) {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1285 if (PyErr_ExceptionMatches(PyExc_UnicodeDecodeError)) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1286 PyErr_Clear();
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1287 tok->done = E_IDENTIFIER;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1288 } else {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1289 tok->done = E_ERROR;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1290 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1291 return 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1292 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1293 result = PyUnicode_IsIdentifier(s);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1294 Py_DECREF(s);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1295 if (result == 0)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1296 tok->done = E_IDENTIFIER;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1297 return result;
42597 20477cbfd996 Implement PEP 3131. Add isidentifier to str. Martin v. Löwis <martin@v.loewis.de> parents: 42551 diff changeset	1298 }
20477cbfd996 Implement PEP 3131. Add isidentifier to str. Martin v. Löwis <martin@v.loewis.de> parents: 42551 diff changeset	1299 #endif
7945 dc132cc39573 Add checking for inconsistent tab usage Guido van Rossum <guido@python.org> parents: 7324 diff changeset	1300
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1301 /* Get next token, after space stripping etc. */
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1302
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	1303 static int
85157 e7f6cef7a4cc Issue #18722: Remove uses of the "register" keyword in C code. Antoine Pitrou <solipsis@pitrou.net> parents: 84656 diff changeset	1304 tok_get(struct tok_state tok, char p_start, char *p_end)
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1305 {
85157 e7f6cef7a4cc Issue #18722: Remove uses of the "register" keyword in C code. Antoine Pitrou <solipsis@pitrou.net> parents: 84656 diff changeset	1306 int c;
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1307 int blankline, nonascii;
326 ff0862e3d63a Completely ignore lines with only a newline token on them, except Guido van Rossum <guido@python.org> parents: 293 diff changeset	1308
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1309 p_start = p_end = NULL;
326 ff0862e3d63a Completely ignore lines with only a newline token on them, except Guido van Rossum <guido@python.org> parents: 293 diff changeset	1310 nextline:
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1311 tok->start = NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1312 blankline = 0;
326 ff0862e3d63a Completely ignore lines with only a newline token on them, except Guido van Rossum <guido@python.org> parents: 293 diff changeset	1313
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1314 /* Get indentation level */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1315 if (tok->atbol) {
85157 e7f6cef7a4cc Issue #18722: Remove uses of the "register" keyword in C code. Antoine Pitrou <solipsis@pitrou.net> parents: 84656 diff changeset	1316 int col = 0;
e7f6cef7a4cc Issue #18722: Remove uses of the "register" keyword in C code. Antoine Pitrou <solipsis@pitrou.net> parents: 84656 diff changeset	1317 int altcol = 0;
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1318 tok->atbol = 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1319 for (;;) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1320 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1321 if (c == ' ')
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1322 col++, altcol++;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1323 else if (c == '\t') {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1324 col = (col/tok->tabsize + 1) * tok->tabsize;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1325 altcol = (altcol/tok->alttabsize + 1)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1326 * tok->alttabsize;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1327 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1328 else if (c == '\014') /* Control-L (formfeed) */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1329 col = altcol = 0; /* For Emacs users */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1330 else
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1331 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1332 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1333 tok_backup(tok, c);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1334 if (c == '#' \|\| c == '\n') {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1335 /* Lines with only whitespace and/or comments
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1336 shouldn't affect the indentation and are
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1337 not passed to the parser as NEWLINE tokens,
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1338 except totally empty lines in interactive
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1339 mode, which signal the end of a command group. */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1340 if (col == 0 && c == '\n' && tok->prompt != NULL)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1341 blankline = 0; /* Let it through */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1342 else
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1343 blankline = 1; /* Ignore completely */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1344 /* We can't jump back right here since we still
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1345 may need to skip to the end of a comment */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1346 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1347 if (!blankline && tok->level == 0) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1348 if (col == tok->indstack[tok->indent]) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1349 /* No change */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1350 if (altcol != tok->altindstack[tok->indent]) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1351 if (indenterror(tok))
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1352 return ERRORTOKEN;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1353 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1354 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1355 else if (col > tok->indstack[tok->indent]) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1356 /* Indent -- always one */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1357 if (tok->indent+1 >= MAXINDENT) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1358 tok->done = E_TOODEEP;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1359 tok->cur = tok->inp;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1360 return ERRORTOKEN;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1361 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1362 if (altcol <= tok->altindstack[tok->indent]) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1363 if (indenterror(tok))
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1364 return ERRORTOKEN;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1365 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1366 tok->pendin++;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1367 tok->indstack[++tok->indent] = col;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1368 tok->altindstack[tok->indent] = altcol;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1369 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1370 else /* col < tok->indstack[tok->indent] */ {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1371 /* Dedent -- any number, must be consistent */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1372 while (tok->indent > 0 &&
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1373 col < tok->indstack[tok->indent]) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1374 tok->pendin--;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1375 tok->indent--;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1376 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1377 if (col != tok->indstack[tok->indent]) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1378 tok->done = E_DEDENT;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1379 tok->cur = tok->inp;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1380 return ERRORTOKEN;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1381 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1382 if (altcol != tok->altindstack[tok->indent]) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1383 if (indenterror(tok))
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1384 return ERRORTOKEN;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1385 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1386 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1387 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1388 }
37879 f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair Thomas Wouters <thomas@python.org> parents: 37704 diff changeset	1389
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1390 tok->start = tok->cur;
37879 f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair Thomas Wouters <thomas@python.org> parents: 37704 diff changeset	1391
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1392 /* Return pending indents/dedents */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1393 if (tok->pendin != 0) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1394 if (tok->pendin < 0) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1395 tok->pendin++;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1396 return DEDENT;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1397 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1398 else {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1399 tok->pendin--;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1400 return INDENT;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1401 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1402 }
37879 f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair Thomas Wouters <thomas@python.org> parents: 37704 diff changeset	1403
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1404 again:
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1405 tok->start = NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1406 /* Skip spaces */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1407 do {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1408 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1409 } while (c == ' ' \|\| c == '\t' \|\| c == '\014');
37879 f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair Thomas Wouters <thomas@python.org> parents: 37704 diff changeset	1410
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1411 /* Set start of current token */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1412 tok->start = tok->cur - 1;
37879 f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair Thomas Wouters <thomas@python.org> parents: 37704 diff changeset	1413
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1414 /* Skip comment */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1415 if (c == '#')
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1416 while (c != EOF && c != '\n')
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1417 c = tok_nextc(tok);
37879 f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair Thomas Wouters <thomas@python.org> parents: 37704 diff changeset	1418
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1419 /* Check for EOF and errors now */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1420 if (c == EOF) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1421 return tok->done == E_EOF ? ENDMARKER : ERRORTOKEN;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1422 }
37879 f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair Thomas Wouters <thomas@python.org> parents: 37704 diff changeset	1423
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1424 /* Identifier (most frequent token!) */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1425 nonascii = 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1426 if (is_potential_identifier_start(c)) {
77522 8e47e9af826e Issue #15096: Drop support for the ur string prefix Christian Heimes <c.heimes@cheimes.de> parents: 75381 diff changeset	1427 /* Process b"", r"", u"", br"" and rb"" */
75381 ed9497d6a216 Basic support for PEP 414 without docs or tests. Armin Ronacher <armin.ronacher@active-4.com> parents: 74354 diff changeset	1428 int saw_b = 0, saw_r = 0, saw_u = 0;
74354 bbed36370b08 Issue #13748: Raw bytes literals can now be written with the `rb` prefix as well as `br`. Antoine Pitrou <solipsis@pitrou.net> parents: 72925 diff changeset	1429 while (1) {
75381 ed9497d6a216 Basic support for PEP 414 without docs or tests. Armin Ronacher <armin.ronacher@active-4.com> parents: 74354 diff changeset	1430 if (!(saw_b \|\| saw_u) && (c == 'b' \|\| c == 'B'))
74354 bbed36370b08 Issue #13748: Raw bytes literals can now be written with the `rb` prefix as well as `br`. Antoine Pitrou <solipsis@pitrou.net> parents: 72925 diff changeset	1431 saw_b = 1;
75381 ed9497d6a216 Basic support for PEP 414 without docs or tests. Armin Ronacher <armin.ronacher@active-4.com> parents: 74354 diff changeset	1432 /* Since this is a backwards compatibility support literal we don't
ed9497d6a216 Basic support for PEP 414 without docs or tests. Armin Ronacher <armin.ronacher@active-4.com> parents: 74354 diff changeset	1433 want to support it in arbitrary order like byte literals. */
ed9497d6a216 Basic support for PEP 414 without docs or tests. Armin Ronacher <armin.ronacher@active-4.com> parents: 74354 diff changeset	1434 else if (!(saw_b \|\| saw_u \|\| saw_r) && (c == 'u' \|\| c == 'U'))
ed9497d6a216 Basic support for PEP 414 without docs or tests. Armin Ronacher <armin.ronacher@active-4.com> parents: 74354 diff changeset	1435 saw_u = 1;
77522 8e47e9af826e Issue #15096: Drop support for the ur string prefix Christian Heimes <c.heimes@cheimes.de> parents: 75381 diff changeset	1436 /* ur"" and ru"" are not supported */
8e47e9af826e Issue #15096: Drop support for the ur string prefix Christian Heimes <c.heimes@cheimes.de> parents: 75381 diff changeset	1437 else if (!(saw_r \|\| saw_u) && (c == 'r' \|\| c == 'R'))
74354 bbed36370b08 Issue #13748: Raw bytes literals can now be written with the `rb` prefix as well as `br`. Antoine Pitrou <solipsis@pitrou.net> parents: 72925 diff changeset	1438 saw_r = 1;
bbed36370b08 Issue #13748: Raw bytes literals can now be written with the `rb` prefix as well as `br`. Antoine Pitrou <solipsis@pitrou.net> parents: 72925 diff changeset	1439 else
bbed36370b08 Issue #13748: Raw bytes literals can now be written with the `rb` prefix as well as `br`. Antoine Pitrou <solipsis@pitrou.net> parents: 72925 diff changeset	1440 break;
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1441 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1442 if (c == '"' \|\| c == '\'')
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1443 goto letter_quote;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1444 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1445 while (is_potential_identifier_char(c)) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1446 if (c >= 128)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1447 nonascii = 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1448 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1449 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1450 tok_backup(tok, c);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1451 if (nonascii &&
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1452 !verify_identifier(tok)) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1453 tok->done = E_IDENTIFIER;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1454 return ERRORTOKEN;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1455 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1456 *p_start = tok->start;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1457 *p_end = tok->cur;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1458 return NAME;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1459 }
37879 f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair Thomas Wouters <thomas@python.org> parents: 37704 diff changeset	1460
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1461 /* Newline */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1462 if (c == '\n') {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1463 tok->atbol = 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1464 if (blankline \|\| tok->level > 0)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1465 goto nextline;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1466 *p_start = tok->start;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1467 p_end = tok->cur - 1; / Leave '\n' out of the string */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1468 tok->cont_line = 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1469 return NEWLINE;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1470 }
37879 f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair Thomas Wouters <thomas@python.org> parents: 37704 diff changeset	1471
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1472 /* Period or number starting with period? */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1473 if (c == '.') {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1474 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1475 if (isdigit(c)) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1476 goto fraction;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1477 } else if (c == '.') {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1478 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1479 if (c == '.') {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1480 *p_start = tok->start;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1481 *p_end = tok->cur;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1482 return ELLIPSIS;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1483 } else {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1484 tok_backup(tok, c);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1485 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1486 tok_backup(tok, '.');
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1487 } else {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1488 tok_backup(tok, c);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1489 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1490 *p_start = tok->start;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1491 *p_end = tok->cur;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1492 return DOT;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1493 }
3333 4b5bff95804e changes for pow(*) and complex Guido van Rossum <guido@python.org>* parents: 3116 diff changeset	1494
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1495 /* Number */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1496 if (isdigit(c)) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1497 if (c == '0') {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1498 /* Hex, octal or binary -- maybe. */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1499 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1500 if (c == '.')
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1501 goto fraction;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1502 if (c == 'j' \|\| c == 'J')
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1503 goto imaginary;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1504 if (c == 'x' \|\| c == 'X') {
44782 ca2683a67943 Merged revisions 60080-60089,60091-60093 via svnmerge from Georg Brandl <georg@python.org> parents: 44442 diff changeset	1505
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1506 /* Hex */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1507 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1508 if (!isxdigit(c)) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1509 tok->done = E_TOKEN;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1510 tok_backup(tok, c);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1511 return ERRORTOKEN;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1512 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1513 do {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1514 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1515 } while (isxdigit(c));
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1516 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1517 else if (c == 'o' \|\| c == 'O') {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1518 /* Octal */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1519 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1520 if (c < '0' \|\| c >= '8') {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1521 tok->done = E_TOKEN;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1522 tok_backup(tok, c);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1523 return ERRORTOKEN;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1524 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1525 do {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1526 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1527 } while ('0' <= c && c < '8');
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1528 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1529 else if (c == 'b' \|\| c == 'B') {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1530 /* Binary */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1531 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1532 if (c != '0' && c != '1') {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1533 tok->done = E_TOKEN;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1534 tok_backup(tok, c);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1535 return ERRORTOKEN;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1536 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1537 do {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1538 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1539 } while (c == '0' \|\| c == '1');
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1540 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1541 else {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1542 int nonzero = 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1543 /* maybe old-style octal; c is first char of it */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1544 /* in any case, allow '0' as a literal */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1545 while (c == '0')
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1546 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1547 while (isdigit(c)) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1548 nonzero = 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1549 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1550 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1551 if (c == '.')
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1552 goto fraction;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1553 else if (c == 'e' \|\| c == 'E')
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1554 goto exponent;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1555 else if (c == 'j' \|\| c == 'J')
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1556 goto imaginary;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1557 else if (nonzero) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1558 tok->done = E_TOKEN;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1559 tok_backup(tok, c);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1560 return ERRORTOKEN;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1561 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1562 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1563 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1564 else {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1565 /* Decimal */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1566 do {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1567 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1568 } while (isdigit(c));
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1569 {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1570 /* Accept floating point numbers. */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1571 if (c == '.') {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1572 fraction:
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1573 /* Fraction */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1574 do {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1575 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1576 } while (isdigit(c));
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1577 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1578 if (c == 'e' \|\| c == 'E') {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1579 exponent:
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1580 /* Exponent part */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1581 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1582 if (c == '+' \|\| c == '-')
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1583 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1584 if (!isdigit(c)) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1585 tok->done = E_TOKEN;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1586 tok_backup(tok, c);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1587 return ERRORTOKEN;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1588 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1589 do {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1590 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1591 } while (isdigit(c));
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1592 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1593 if (c == 'j' \|\| c == 'J')
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1594 /* Imaginary part */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1595 imaginary:
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1596 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1597 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1598 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1599 tok_backup(tok, c);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1600 *p_start = tok->start;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1601 *p_end = tok->cur;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1602 return NUMBER;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1603 }
5265 3c1788ab9761 Support for alternative string quotes (a"xx", b"xx", c"xx", ...). Guido van Rossum <guido@python.org> parents: 4839 diff changeset	1604
3c1788ab9761 Support for alternative string quotes (a"xx", b"xx", c"xx", ...). Guido van Rossum <guido@python.org> parents: 4839 diff changeset	1605 letter_quote:
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1606 /* String */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1607 if (c == '\'' \|\| c == '"') {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1608 int quote = c;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1609 int quote_size = 1; /* 1 or 3 */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1610 int end_quote_size = 0;
43879 5590157f4fc6 Cleanup of tokenizer.c. Guido van Rossum <guido@python.org> parents: 43873 diff changeset	1611
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1612 /* Find the quote size and start of string */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1613 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1614 if (c == quote) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1615 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1616 if (c == quote)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1617 quote_size = 3;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1618 else
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1619 end_quote_size = 1; /* empty string found */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1620 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1621 if (c != quote)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1622 tok_backup(tok, c);
43879 5590157f4fc6 Cleanup of tokenizer.c. Guido van Rossum <guido@python.org> parents: 43873 diff changeset	1623
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1624 /* Get rest of string */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1625 while (end_quote_size != quote_size) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1626 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1627 if (c == EOF) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1628 if (quote_size == 3)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1629 tok->done = E_EOFS;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1630 else
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1631 tok->done = E_EOLS;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1632 tok->cur = tok->inp;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1633 return ERRORTOKEN;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1634 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1635 if (quote_size == 1 && c == '\n') {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1636 tok->done = E_EOLS;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1637 tok->cur = tok->inp;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1638 return ERRORTOKEN;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1639 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1640 if (c == quote)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1641 end_quote_size += 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1642 else {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1643 end_quote_size = 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1644 if (c == '\\')
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1645 c = tok_nextc(tok); /* skip escaped char */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1646 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1647 }
43879 5590157f4fc6 Cleanup of tokenizer.c. Guido van Rossum <guido@python.org> parents: 43873 diff changeset	1648
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1649 *p_start = tok->start;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1650 *p_end = tok->cur;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1651 return STRING;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1652 }
37879 f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair Thomas Wouters <thomas@python.org> parents: 37704 diff changeset	1653
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1654 /* Line continuation */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1655 if (c == '\\') {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1656 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1657 if (c != '\n') {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1658 tok->done = E_LINECONT;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1659 tok->cur = tok->inp;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1660 return ERRORTOKEN;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1661 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1662 tok->cont_line = 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1663 goto again; /* Read next line */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1664 }
37879 f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair Thomas Wouters <thomas@python.org> parents: 37704 diff changeset	1665
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1666 /* Check for two-character token */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1667 {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1668 int c2 = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1669 int token = PyToken_TwoChars(c, c2);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1670 if (token != OP) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1671 int c3 = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1672 int token3 = PyToken_ThreeChars(c, c2, c3);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1673 if (token3 != OP) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1674 token = token3;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1675 } else {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1676 tok_backup(tok, c3);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1677 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1678 *p_start = tok->start;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1679 *p_end = tok->cur;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1680 return token;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1681 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1682 tok_backup(tok, c2);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1683 }
37879 f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair Thomas Wouters <thomas@python.org> parents: 37704 diff changeset	1684
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1685 /* Keep track of parentheses nesting level */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1686 switch (c) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1687 case '(':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1688 case '[':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1689 case '{':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1690 tok->level++;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1691 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1692 case ')':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1693 case ']':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1694 case '}':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1695 tok->level--;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1696 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1697 }
37879 f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair Thomas Wouters <thomas@python.org> parents: 37704 diff changeset	1698
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1699 /* Punctuation character */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1700 *p_start = tok->start;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1701 *p_end = tok->cur;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1702 return PyToken_OneChar(c);
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1703 }
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1704
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	1705 int
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	1706 PyTokenizer_Get(struct tok_state tok, char p_start, char *p_end)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	1707 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1708 int result = tok_get(tok, p_start, p_end);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1709 if (tok->decoding_erred) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1710 result = ERRORTOKEN;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1711 tok->done = E_DECODE;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1712 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1713 return result;
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	1714 }
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1715
69148 7b8d625eb6e4 Issue #9319: Include the filename in "Non-UTF8 code ..." syntax error. Victor Stinner <victor.stinner@haypocalc.com> parents: 69147 diff changeset	1716 /* Get the encoding of a Python file. Check for the coding cookie and check if
7b8d625eb6e4 Issue #9319: Include the filename in "Non-UTF8 code ..." syntax error. Victor Stinner <victor.stinner@haypocalc.com> parents: 69147 diff changeset	1717 the file starts with a BOM.
43609 d378acdc30ee Fix PyTokenizer_FindEncoding() for OS X 10.4. Turns out that seeking to the Brett Cannon <bcannon@gmail.com> parents: 43607 diff changeset	1718
69148 7b8d625eb6e4 Issue #9319: Include the filename in "Non-UTF8 code ..." syntax error. Victor Stinner <victor.stinner@haypocalc.com> parents: 69147 diff changeset	1719 PyTokenizer_FindEncodingFilename() returns NULL when it can't find the
7b8d625eb6e4 Issue #9319: Include the filename in "Non-UTF8 code ..." syntax error. Victor Stinner <victor.stinner@haypocalc.com> parents: 69147 diff changeset	1720 encoding in the first or second line of the file (in which case the encoding
7b8d625eb6e4 Issue #9319: Include the filename in "Non-UTF8 code ..." syntax error. Victor Stinner <victor.stinner@haypocalc.com> parents: 69147 diff changeset	1721 should be assumed to be UTF-8).
7b8d625eb6e4 Issue #9319: Include the filename in "Non-UTF8 code ..." syntax error. Victor Stinner <victor.stinner@haypocalc.com> parents: 69147 diff changeset	1722
7b8d625eb6e4 Issue #9319: Include the filename in "Non-UTF8 code ..." syntax error. Victor Stinner <victor.stinner@haypocalc.com> parents: 69147 diff changeset	1723 The char* returned is malloc'ed via PyMem_MALLOC() and thus must be freed
7b8d625eb6e4 Issue #9319: Include the filename in "Non-UTF8 code ..." syntax error. Victor Stinner <victor.stinner@haypocalc.com> parents: 69147 diff changeset	1724 by the caller. */
7b8d625eb6e4 Issue #9319: Include the filename in "Non-UTF8 code ..." syntax error. Victor Stinner <victor.stinner@haypocalc.com> parents: 69147 diff changeset	1725
43605 a2bb76d11d66 Patch 1267 by Christian Heimes. Guido van Rossum <guido@python.org> parents: 43523 diff changeset	1726 char *
69148 7b8d625eb6e4 Issue #9319: Include the filename in "Non-UTF8 code ..." syntax error. Victor Stinner <victor.stinner@haypocalc.com> parents: 69147 diff changeset	1727 PyTokenizer_FindEncodingFilename(int fd, PyObject *filename)
43632 0067a6ab4854 Issue 1267, continued. Guido van Rossum <guido@python.org> parents: 43622 diff changeset	1728 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1729 struct tok_state *tok;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1730 FILE *fp;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1731 char p_start =NULL , p_end =NULL , *encoding = NULL;
43605 a2bb76d11d66 Patch 1267 by Christian Heimes. Guido van Rossum <guido@python.org> parents: 43523 diff changeset	1732
85420 ef889c3d5dc6 Issue #18571: Implementation of the PEP 446: file descriptors and file handles Victor Stinner <victor.stinner@gmail.com> parents: 85157 diff changeset	1733 #ifndef PGEN
ef889c3d5dc6 Issue #18571: Implementation of the PEP 446: file descriptors and file handles Victor Stinner <victor.stinner@gmail.com> parents: 85157 diff changeset	1734 fd = _Py_dup(fd);
ef889c3d5dc6 Issue #18571: Implementation of the PEP 446: file descriptors and file handles Victor Stinner <victor.stinner@gmail.com> parents: 85157 diff changeset	1735 #else
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1736 fd = dup(fd);
85420 ef889c3d5dc6 Issue #18571: Implementation of the PEP 446: file descriptors and file handles Victor Stinner <victor.stinner@gmail.com> parents: 85157 diff changeset	1737 #endif
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1738 if (fd < 0) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1739 return NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1740 }
85420 ef889c3d5dc6 Issue #18571: Implementation of the PEP 446: file descriptors and file handles Victor Stinner <victor.stinner@gmail.com> parents: 85157 diff changeset	1741
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1742 fp = fdopen(fd, "r");
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1743 if (fp == NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1744 return NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1745 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1746 tok = PyTokenizer_FromFile(fp, NULL, NULL, NULL);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1747 if (tok == NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1748 fclose(fp);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1749 return NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1750 }
69147 6e9dc970ac0e Issue #10785: Store the filename as Unicode in the Python parser. Victor Stinner <victor.stinner@haypocalc.com> parents: 68257 diff changeset	1751 #ifndef PGEN
69148 7b8d625eb6e4 Issue #9319: Include the filename in "Non-UTF8 code ..." syntax error. Victor Stinner <victor.stinner@haypocalc.com> parents: 69147 diff changeset	1752 if (filename != NULL) {
7b8d625eb6e4 Issue #9319: Include the filename in "Non-UTF8 code ..." syntax error. Victor Stinner <victor.stinner@haypocalc.com> parents: 69147 diff changeset	1753 Py_INCREF(filename);
7b8d625eb6e4 Issue #9319: Include the filename in "Non-UTF8 code ..." syntax error. Victor Stinner <victor.stinner@haypocalc.com> parents: 69147 diff changeset	1754 tok->filename = filename;
7b8d625eb6e4 Issue #9319: Include the filename in "Non-UTF8 code ..." syntax error. Victor Stinner <victor.stinner@haypocalc.com> parents: 69147 diff changeset	1755 }
7b8d625eb6e4 Issue #9319: Include the filename in "Non-UTF8 code ..." syntax error. Victor Stinner <victor.stinner@haypocalc.com> parents: 69147 diff changeset	1756 else {
7b8d625eb6e4 Issue #9319: Include the filename in "Non-UTF8 code ..." syntax error. Victor Stinner <victor.stinner@haypocalc.com> parents: 69147 diff changeset	1757 tok->filename = PyUnicode_FromString("<string>");
7b8d625eb6e4 Issue #9319: Include the filename in "Non-UTF8 code ..." syntax error. Victor Stinner <victor.stinner@haypocalc.com> parents: 69147 diff changeset	1758 if (tok->filename == NULL) {
7b8d625eb6e4 Issue #9319: Include the filename in "Non-UTF8 code ..." syntax error. Victor Stinner <victor.stinner@haypocalc.com> parents: 69147 diff changeset	1759 fclose(fp);
7b8d625eb6e4 Issue #9319: Include the filename in "Non-UTF8 code ..." syntax error. Victor Stinner <victor.stinner@haypocalc.com> parents: 69147 diff changeset	1760 PyTokenizer_Free(tok);
7b8d625eb6e4 Issue #9319: Include the filename in "Non-UTF8 code ..." syntax error. Victor Stinner <victor.stinner@haypocalc.com> parents: 69147 diff changeset	1761 return encoding;
7b8d625eb6e4 Issue #9319: Include the filename in "Non-UTF8 code ..." syntax error. Victor Stinner <victor.stinner@haypocalc.com> parents: 69147 diff changeset	1762 }
7b8d625eb6e4 Issue #9319: Include the filename in "Non-UTF8 code ..." syntax error. Victor Stinner <victor.stinner@haypocalc.com> parents: 69147 diff changeset	1763 }
69147 6e9dc970ac0e Issue #10785: Store the filename as Unicode in the Python parser. Victor Stinner <victor.stinner@haypocalc.com> parents: 68257 diff changeset	1764 #endif
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1765 while (tok->lineno < 2 && tok->done == E_OK) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1766 PyTokenizer_Get(tok, &p_start, &p_end);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1767 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1768 fclose(fp);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1769 if (tok->encoding) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1770 encoding = (char *)PyMem_MALLOC(strlen(tok->encoding) + 1);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1771 if (encoding)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1772 strcpy(encoding, tok->encoding);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1773 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1774 PyTokenizer_Free(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1775 return encoding;
43605 a2bb76d11d66 Patch 1267 by Christian Heimes. Guido van Rossum <guido@python.org> parents: 43523 diff changeset	1776 }
43292 876b7ef4775d Merged revisions 57778-58052 via svnmerge from Thomas Wouters <thomas@python.org> parents: 43225 diff changeset	1777
69148 7b8d625eb6e4 Issue #9319: Include the filename in "Non-UTF8 code ..." syntax error. Victor Stinner <victor.stinner@haypocalc.com> parents: 69147 diff changeset	1778 char *
7b8d625eb6e4 Issue #9319: Include the filename in "Non-UTF8 code ..." syntax error. Victor Stinner <victor.stinner@haypocalc.com> parents: 69147 diff changeset	1779 PyTokenizer_FindEncoding(int fd)
7b8d625eb6e4 Issue #9319: Include the filename in "Non-UTF8 code ..." syntax error. Victor Stinner <victor.stinner@haypocalc.com> parents: 69147 diff changeset	1780 {
7b8d625eb6e4 Issue #9319: Include the filename in "Non-UTF8 code ..." syntax error. Victor Stinner <victor.stinner@haypocalc.com> parents: 69147 diff changeset	1781 return PyTokenizer_FindEncodingFilename(fd, NULL);
7b8d625eb6e4 Issue #9319: Include the filename in "Non-UTF8 code ..." syntax error. Victor Stinner <victor.stinner@haypocalc.com> parents: 69147 diff changeset	1782 }
7b8d625eb6e4 Issue #9319: Include the filename in "Non-UTF8 code ..." syntax error. Victor Stinner <victor.stinner@haypocalc.com> parents: 69147 diff changeset	1783
4839 4567e158d20c Rename DEBUG macro to Py_DEBUG Guido van Rossum <guido@python.org> parents: 4577 diff changeset	1784 #ifdef Py_DEBUG
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1785
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1786 void
13659 e78153e179fb Mass ANSIfication. Thomas Wouters <thomas@python.org> parents: 13539 diff changeset	1787 tok_dump(int type, char start, char end)
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1788 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1789 printf("%s", _PyParser_TokenNames[type]);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1790 if (type == NAME \|\| type == NUMBER \|\| type == STRING \|\| type == OP)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1791 printf("(%.*s)", (int)(end - start), start);
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1792 }
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1793
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1794 #endif

Mercurial > cpython-lru_cache-weakref

annotate Parser/tokenizer.c @ 86127:a3d09c19bb93