cpython-lru_cache-weakref: Parser/tokenizer.c annotate

annotate Parser/tokenizer.c @ 84007:dbfbdf2b5c19 3.3

Fix #17967: For ftp urls CWD to target instead of hopping to each directory towards target. This fixes a bug where target is accessible, but parent directories are restricted.

author	Senthil Kumaran <senthil@uthcode.com>
date	Sat, 01 Jun 2013 08:27:06 -0700
parents	8e47e9af826e
children	15aa786b723b 7e0e15d9957f

rev	line source
145 853b736a6a9e Added copyright notice. Guido van Rossum <guido@python.org> parents: 96 diff changeset	1
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	2 /* Tokenizer implementation */
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	3
23029 8291473fde3e Mass checkin of universal newline support. Jack Jansen <jack.jansen@cwi.nl> parents: 19621 diff changeset	4 #include "Python.h"
86 a6934380c6e7 "Compiling" version Guido van Rossum <guido@python.org> parents: 67 diff changeset	5 #include "pgenheaders.h"
a6934380c6e7 "Compiling" version Guido van Rossum <guido@python.org> parents: 67 diff changeset	6
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	7 #include <ctype.h>
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	8 #include <assert.h>
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	9
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	10 #include "tokenizer.h"
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	11 #include "errcode.h"
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	12
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	13 #ifndef PGEN
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	14 #include "unicodeobject.h"
47302 811a80651232 Renamed files bytesobject.[ch] and stringobject.[ch] Christian Heimes <christian@cheimes.de> parents: 47301 diff changeset	15 #include "bytesobject.h"
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	16 #include "fileobject.h"
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	17 #include "codecs.h"
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	18 #include "abstract.h"
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	19 #endif /* PGEN */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	20
42013 cd125fe83051 Make identifiers str (not str8) objects throughout. Martin v. Löwis <martin@v.loewis.de> parents: 41970 diff changeset	21 #define is_potential_identifier_start(c) (\
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	22 (c >= 'a' && c <= 'z')\
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	23 \|\| (c >= 'A' && c <= 'Z')\
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	24 \|\| c == '_'\
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	25 \|\| (c >= 128))
42013 cd125fe83051 Make identifiers str (not str8) objects throughout. Martin v. Löwis <martin@v.loewis.de> parents: 41970 diff changeset	26
cd125fe83051 Make identifiers str (not str8) objects throughout. Martin v. Löwis <martin@v.loewis.de> parents: 41970 diff changeset	27 #define is_potential_identifier_char(c) (\
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	28 (c >= 'a' && c <= 'z')\
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	29 \|\| (c >= 'A' && c <= 'Z')\
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	30 \|\| (c >= '0' && c <= '9')\
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	31 \|\| c == '_'\
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	32 \|\| (c >= 128))
42013 cd125fe83051 Make identifiers str (not str8) objects throughout. Martin v. Löwis <martin@v.loewis.de> parents: 41970 diff changeset	33
25924 dc4a0336a2a3 Patch #512981: Update readline input stream on sys.stdin/out change. Martin v. Löwis <martin@v.loewis.de> parents: 25101 diff changeset	34 extern char PyOS_Readline(FILE , FILE , char );
1835 d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string Guido van Rossum <guido@python.org> parents: 1364 diff changeset	35 /* Return malloc'ed string including trailing \n;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string Guido van Rossum <guido@python.org> parents: 1364 diff changeset	36 empty malloc'ed string for EOF;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string Guido van Rossum <guido@python.org> parents: 1364 diff changeset	37 NULL if interrupted */
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string Guido van Rossum <guido@python.org> parents: 1364 diff changeset	38
678 205b947b1d3c Make tabs always 8 spaces wide -- it's more portable. Guido van Rossum <guido@python.org> parents: 477 diff changeset	39 /* Don't ever change this -- it would break the portability of Python code */
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	40 #define TABSIZE 8
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	41
86 a6934380c6e7 "Compiling" version Guido van Rossum <guido@python.org> parents: 67 diff changeset	42 /* Forward */
13447 a89f7f268de4 Nuke all remaining occurrences of Py_PROTO and Py_FPROTO. Tim Peters <tim.peters@gmail.com> parents: 13256 diff changeset	43 static struct tok_state *tok_new(void);
a89f7f268de4 Nuke all remaining occurrences of Py_PROTO and Py_FPROTO. Tim Peters <tim.peters@gmail.com> parents: 13256 diff changeset	44 static int tok_nextc(struct tok_state *tok);
a89f7f268de4 Nuke all remaining occurrences of Py_PROTO and Py_FPROTO. Tim Peters <tim.peters@gmail.com> parents: 13256 diff changeset	45 static void tok_backup(struct tok_state *tok, int c);
86 a6934380c6e7 "Compiling" version Guido van Rossum <guido@python.org> parents: 67 diff changeset	46
43607 7593436058dd Plug a memory leak where a struct tok_state was not being freed. Brett Cannon <bcannon@gmail.com> parents: 43605 diff changeset	47
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	48 /* Token names */
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	49
5379 1c9a84e0981d Another directory quickly renamed. Guido van Rossum <guido@python.org> parents: 5341 diff changeset	50 char *_PyParser_TokenNames[] = {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	51 "ENDMARKER",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	52 "NAME",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	53 "NUMBER",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	54 "STRING",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	55 "NEWLINE",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	56 "INDENT",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	57 "DEDENT",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	58 "LPAR",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	59 "RPAR",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	60 "LSQB",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	61 "RSQB",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	62 "COLON",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	63 "COMMA",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	64 "SEMI",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	65 "PLUS",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	66 "MINUS",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	67 "STAR",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	68 "SLASH",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	69 "VBAR",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	70 "AMPER",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	71 "LESS",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	72 "GREATER",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	73 "EQUAL",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	74 "DOT",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	75 "PERCENT",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	76 "LBRACE",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	77 "RBRACE",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	78 "EQEQUAL",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	79 "NOTEQUAL",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	80 "LESSEQUAL",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	81 "GREATEREQUAL",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	82 "TILDE",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	83 "CIRCUMFLEX",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	84 "LEFTSHIFT",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	85 "RIGHTSHIFT",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	86 "DOUBLESTAR",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	87 "PLUSEQUAL",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	88 "MINEQUAL",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	89 "STAREQUAL",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	90 "SLASHEQUAL",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	91 "PERCENTEQUAL",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	92 "AMPEREQUAL",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	93 "VBAREQUAL",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	94 "CIRCUMFLEXEQUAL",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	95 "LEFTSHIFTEQUAL",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	96 "RIGHTSHIFTEQUAL",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	97 "DOUBLESTAREQUAL",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	98 "DOUBLESLASH",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	99 "DOUBLESLASHEQUAL",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	100 "AT",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	101 "RARROW",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	102 "ELLIPSIS",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	103 /* This table must match the #defines in token.h! */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	104 "OP",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	105 "<ERRORTOKEN>",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	106 "<N_TOKENS>"
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	107 };
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	108
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	109
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	110 /* Create and initialize a new tok_state structure */
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	111
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	112 static struct tok_state *
13659 e78153e179fb Mass ANSIfication. Thomas Wouters <thomas@python.org> parents: 13539 diff changeset	113 tok_new(void)
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	114 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	115 struct tok_state tok = (struct tok_state )PyMem_MALLOC(
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	116 sizeof(struct tok_state));
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	117 if (tok == NULL)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	118 return NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	119 tok->buf = tok->cur = tok->end = tok->inp = tok->start = NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	120 tok->done = E_OK;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	121 tok->fp = NULL;
61466 cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	122 tok->input = NULL;
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	123 tok->tabsize = TABSIZE;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	124 tok->indent = 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	125 tok->indstack[0] = 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	126 tok->atbol = 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	127 tok->pendin = 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	128 tok->prompt = tok->nextprompt = NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	129 tok->lineno = 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	130 tok->level = 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	131 tok->altwarning = 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	132 tok->alterror = 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	133 tok->alttabsize = 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	134 tok->altindstack[0] = 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	135 tok->decoding_state = STATE_INIT;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	136 tok->decoding_erred = 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	137 tok->read_coding_spec = 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	138 tok->enc = NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	139 tok->encoding = NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	140 tok->cont_line = 0;
24544 0bd521eb714e Make pgen compile with pydebug. Duplicate normalized names, as it may Martin v. Löwis <martin@v.loewis.de> parents: 24543 diff changeset	141 #ifndef PGEN
69147 6e9dc970ac0e Issue #10785: Store the filename as Unicode in the Python parser. Victor Stinner <victor.stinner@haypocalc.com> parents: 68257 diff changeset	142 tok->filename = NULL;
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	143 tok->decoding_readline = NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	144 tok->decoding_buffer = NULL;
24544 0bd521eb714e Make pgen compile with pydebug. Duplicate normalized names, as it may Martin v. Löwis <martin@v.loewis.de> parents: 24543 diff changeset	145 #endif
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	146 return tok;
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	147 }
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	148
57206 b2d90c1c637d Merged revisions 76230 via svnmerge from Benjamin Peterson <benjamin@python.org> parents: 56594 diff changeset	149 static char *
b2d90c1c637d Merged revisions 76230 via svnmerge from Benjamin Peterson <benjamin@python.org> parents: 56594 diff changeset	150 new_string(const char *s, Py_ssize_t len)
b2d90c1c637d Merged revisions 76230 via svnmerge from Benjamin Peterson <benjamin@python.org> parents: 56594 diff changeset	151 {
61466 cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	152 char* result = (char *)PyMem_MALLOC(len + 1);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	153 if (result != NULL) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	154 memcpy(result, s, len);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	155 result[len] = '\0';
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	156 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	157 return result;
57206 b2d90c1c637d Merged revisions 76230 via svnmerge from Benjamin Peterson <benjamin@python.org> parents: 56594 diff changeset	158 }
b2d90c1c637d Merged revisions 76230 via svnmerge from Benjamin Peterson <benjamin@python.org> parents: 56594 diff changeset	159
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	160 #ifdef PGEN
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	161
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	162 static char *
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	163 decoding_fgets(char s, int size, struct tok_state tok)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	164 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	165 return fgets(s, size, tok->fp);
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	166 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	167
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	168 static int
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	169 decoding_feof(struct tok_state *tok)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	170 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	171 return feof(tok->fp);
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	172 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	173
57206 b2d90c1c637d Merged revisions 76230 via svnmerge from Benjamin Peterson <benjamin@python.org> parents: 56594 diff changeset	174 static char *
b2d90c1c637d Merged revisions 76230 via svnmerge from Benjamin Peterson <benjamin@python.org> parents: 56594 diff changeset	175 decode_str(const char str, int exec_input, struct tok_state tok)
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	176 {
61466 cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	177 return new_string(str, strlen(str));
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	178 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	179
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	180 #else /* PGEN */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	181
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	182 static char *
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	183 error_ret(struct tok_state tok) / XXX */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	184 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	185 tok->decoding_erred = 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	186 if (tok->fp != NULL && tok->buf != NULL) /* see PyTokenizer_Free */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	187 PyMem_FREE(tok->buf);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	188 tok->buf = NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	189 return NULL; /* as if it were EOF */
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	190 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	191
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	192
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	193 static char *
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	194 get_normal_name(char s) / for utf-8 and latin-1 */
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	195 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	196 char buf[13];
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	197 int i;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	198 for (i = 0; i < 12; i++) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	199 int c = s[i];
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	200 if (c == '\0')
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	201 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	202 else if (c == '_')
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	203 buf[i] = '-';
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	204 else
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	205 buf[i] = tolower(c);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	206 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	207 buf[i] = '\0';
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	208 if (strcmp(buf, "utf-8") == 0 \|\|
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	209 strncmp(buf, "utf-8-", 6) == 0)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	210 return "utf-8";
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	211 else if (strcmp(buf, "latin-1") == 0 \|\|
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	212 strcmp(buf, "iso-8859-1") == 0 \|\|
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	213 strcmp(buf, "iso-latin-1") == 0 \|\|
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	214 strncmp(buf, "latin-1-", 8) == 0 \|\|
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	215 strncmp(buf, "iso-8859-1-", 11) == 0 \|\|
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	216 strncmp(buf, "iso-latin-1-", 12) == 0)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	217 return "iso-8859-1";
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	218 else
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	219 return s;
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	220 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	221
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	222 /* Return the coding spec in S, or NULL if none is found. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	223
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	224 static char *
36501 7fdc639bc5b4 Merge ssize_t branch. Martin v. Löwis <martin@v.loewis.de> parents: 36090 diff changeset	225 get_coding_spec(const char *s, Py_ssize_t size)
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	226 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	227 Py_ssize_t i;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	228 /* Coding spec must be in a comment, and that comment must be
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	229 * the only statement on the source code line. */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	230 for (i = 0; i < size - 6; i++) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	231 if (s[i] == '#')
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	232 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	233 if (s[i] != ' ' && s[i] != '\t' && s[i] != '\014')
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	234 return NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	235 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	236 for (; i < size - 6; i++) { /* XXX inefficient search */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	237 const char* t = s + i;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	238 if (strncmp(t, "coding", 6) == 0) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	239 const char* begin = NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	240 t += 6;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	241 if (t[0] != ':' && t[0] != '=')
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	242 continue;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	243 do {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	244 t++;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	245 } while (t[0] == '\x20' \|\| t[0] == '\t');
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	246
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	247 begin = t;
61466 cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	248 while (Py_ISALNUM(t[0]) \|\|
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	249 t[0] == '-' \|\| t[0] == '_' \|\| t[0] == '.')
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	250 t++;
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	251
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	252 if (begin < t) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	253 char* r = new_string(begin, t - begin);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	254 char* q = get_normal_name(r);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	255 if (r != q) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	256 PyMem_FREE(r);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	257 r = new_string(q, strlen(q));
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	258 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	259 return r;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	260 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	261 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	262 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	263 return NULL;
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	264 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	265
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	266 /* Check whether the line contains a coding spec. If it does,
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	267 invoke the set_readline function for the new encoding.
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	268 This function receives the tok_state and the new encoding.
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	269 Return 1 on success, 0 on failure. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	270
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	271 static int
36501 7fdc639bc5b4 Merge ssize_t branch. Martin v. Löwis <martin@v.loewis.de> parents: 36090 diff changeset	272 check_coding_spec(const char* line, Py_ssize_t size, struct tok_state *tok,
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	273 int set_readline(struct tok_state , const char ))
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	274 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	275 char * cs;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	276 int r = 1;
25101 32331d26d7cf Removed reliance on gcc/C99 extension. Tim Peters <tim.peters@gmail.com> parents: 25098 diff changeset	277
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	278 if (tok->cont_line)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	279 /* It's a continuation line, so it can't be a coding spec. */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	280 return 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	281 cs = get_coding_spec(line, size);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	282 if (cs != NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	283 tok->read_coding_spec = 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	284 if (tok->encoding == NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	285 assert(tok->decoding_state == STATE_RAW);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	286 if (strcmp(cs, "utf-8") == 0) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	287 tok->encoding = cs;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	288 } else {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	289 r = set_readline(tok, cs);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	290 if (r) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	291 tok->encoding = cs;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	292 tok->decoding_state = STATE_NORMAL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	293 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	294 else
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	295 PyMem_FREE(cs);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	296 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	297 } else { /* then, compare cs with BOM */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	298 r = (strcmp(tok->encoding, cs) == 0);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	299 PyMem_FREE(cs);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	300 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	301 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	302 if (!r) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	303 cs = tok->encoding;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	304 if (!cs)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	305 cs = "with BOM";
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	306 PyErr_Format(PyExc_SyntaxError, "encoding problem: %s", cs);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	307 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	308 return r;
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	309 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	310
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	311 /* See whether the file starts with a BOM. If it does,
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	312 invoke the set_readline function with the new encoding.
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	313 Return 1 on success, 0 on failure. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	314
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	315 static int
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	316 check_bom(int get_char(struct tok_state *),
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	317 void unget_char(int, struct tok_state *),
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	318 int set_readline(struct tok_state , const char ),
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	319 struct tok_state *tok)
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	320 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	321 int ch1, ch2, ch3;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	322 ch1 = get_char(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	323 tok->decoding_state = STATE_RAW;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	324 if (ch1 == EOF) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	325 return 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	326 } else if (ch1 == 0xEF) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	327 ch2 = get_char(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	328 if (ch2 != 0xBB) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	329 unget_char(ch2, tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	330 unget_char(ch1, tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	331 return 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	332 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	333 ch3 = get_char(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	334 if (ch3 != 0xBF) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	335 unget_char(ch3, tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	336 unget_char(ch2, tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	337 unget_char(ch1, tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	338 return 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	339 }
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	340 #if 0
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	341 /* Disable support for UTF-16 BOMs until a decision
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	342 is made whether this needs to be supported. */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	343 } else if (ch1 == 0xFE) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	344 ch2 = get_char(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	345 if (ch2 != 0xFF) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	346 unget_char(ch2, tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	347 unget_char(ch1, tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	348 return 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	349 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	350 if (!set_readline(tok, "utf-16-be"))
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	351 return 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	352 tok->decoding_state = STATE_NORMAL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	353 } else if (ch1 == 0xFF) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	354 ch2 = get_char(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	355 if (ch2 != 0xFE) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	356 unget_char(ch2, tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	357 unget_char(ch1, tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	358 return 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	359 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	360 if (!set_readline(tok, "utf-16-le"))
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	361 return 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	362 tok->decoding_state = STATE_NORMAL;
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	363 #endif
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	364 } else {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	365 unget_char(ch1, tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	366 return 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	367 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	368 if (tok->encoding != NULL)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	369 PyMem_FREE(tok->encoding);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	370 tok->encoding = new_string("utf-8", 5); /* resulting is in utf-8 */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	371 /* No need to set_readline: input is already utf-8 */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	372 return 1;
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	373 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	374
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	375 /* Read a line of text from TOK into S, using the stream in TOK.
35231 763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file Walter Dörwald <walter@livinglogic.de> parents: 34664 diff changeset	376 Return NULL on failure, else S.
37879 f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair Thomas Wouters <thomas@python.org> parents: 37704 diff changeset	377
35231 763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file Walter Dörwald <walter@livinglogic.de> parents: 34664 diff changeset	378 On entry, tok->decoding_buffer will be one of:
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file Walter Dörwald <walter@livinglogic.de> parents: 34664 diff changeset	379 1) NULL: need to call tok->decoding_readline to get a new line
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file Walter Dörwald <walter@livinglogic.de> parents: 34664 diff changeset	380 2) PyUnicodeObject *: decoding_feof has called tok->decoding_readline and
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	381 stored the result in tok->decoding_buffer
47300 96b34263e448 Renamed PyBytes to PyByteArray Christian Heimes <christian@cheimes.de> parents: 46664 diff changeset	382 3) PyByteArrayObject *: previous call to fp_readl did not have enough room
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	383 (in the s buffer) to copy entire contents of the line read
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	384 by tok->decoding_readline. tok->decoding_buffer has the overflow.
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	385 In this case, fp_readl is called in a loop (with an expanded buffer)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	386 until the buffer ends with a '\n' (or until the end of the file is
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	387 reached): see tok_nextc and its calls to decoding_fgets.
35231 763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file Walter Dörwald <walter@livinglogic.de> parents: 34664 diff changeset	388 */
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	389
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	390 static char *
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	391 fp_readl(char s, int size, struct tok_state tok)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	392 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	393 PyObject* bufobj;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	394 const char *buf;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	395 Py_ssize_t buflen;
35231 763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file Walter Dörwald <walter@livinglogic.de> parents: 34664 diff changeset	396
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	397 /* Ask for one less byte so we can terminate it */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	398 assert(size > 0);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	399 size--;
35231 763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file Walter Dörwald <walter@livinglogic.de> parents: 34664 diff changeset	400
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	401 if (tok->decoding_buffer) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	402 bufobj = tok->decoding_buffer;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	403 Py_INCREF(bufobj);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	404 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	405 else
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	406 {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	407 bufobj = PyObject_CallObject(tok->decoding_readline, NULL);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	408 if (bufobj == NULL)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	409 goto error;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	410 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	411 if (PyUnicode_CheckExact(bufobj))
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	412 {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	413 buf = _PyUnicode_AsStringAndSize(bufobj, &buflen);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	414 if (buf == NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	415 goto error;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	416 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	417 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	418 else
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	419 {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	420 buf = PyByteArray_AsString(bufobj);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	421 if (buf == NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	422 goto error;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	423 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	424 buflen = PyByteArray_GET_SIZE(bufobj);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	425 }
43873 1739869cbc6e Correction for issue1134: all source files with a coding spec, except latin-1 Amaury Forgeot d'Arc <amauryfa@gmail.com> parents: 43777 diff changeset	426
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	427 Py_XDECREF(tok->decoding_buffer);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	428 if (buflen > size) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	429 /* Too many chars, the rest goes into tok->decoding_buffer */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	430 tok->decoding_buffer = PyByteArray_FromStringAndSize(buf+size,
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	431 buflen-size);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	432 if (tok->decoding_buffer == NULL)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	433 goto error;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	434 buflen = size;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	435 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	436 else
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	437 tok->decoding_buffer = NULL;
43873 1739869cbc6e Correction for issue1134: all source files with a coding spec, except latin-1 Amaury Forgeot d'Arc <amauryfa@gmail.com> parents: 43777 diff changeset	438
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	439 memcpy(s, buf, buflen);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	440 s[buflen] = '\0';
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	441 if (buflen == 0) /* EOF */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	442 s = NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	443 Py_DECREF(bufobj);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	444 return s;
42551 b6625d2b378a Fix refleaks from execfile('file that contains a # coding: line') Neal Norwitz <nnorwitz@gmail.com> parents: 42548 diff changeset	445
b6625d2b378a Fix refleaks from execfile('file that contains a # coding: line') Neal Norwitz <nnorwitz@gmail.com> parents: 42548 diff changeset	446 error:
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	447 Py_XDECREF(bufobj);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	448 return error_ret(tok);
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	449 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	450
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	451 /* Set the readline function for TOK to a StreamReader's
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	452 readline function. The StreamReader is named ENC.
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	453
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	454 This function is called from check_bom and check_coding_spec.
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	455
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	456 ENC is usually identical to the future value of tok->encoding,
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	457 except for the (currently unsupported) case of UTF-16.
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	458
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	459 Return 1 on success, 0 on failure. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	460
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	461 static int
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	462 fp_setreadl(struct tok_state tok, const char enc)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	463 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	464 PyObject readline = NULL, stream = NULL, *io = NULL;
72925 7109f31300fb Rename _Py_identifier to _Py_IDENTIFIER. Martin v. Löwis <martin@v.loewis.de> parents: 72848 diff changeset	465 _Py_IDENTIFIER(open);
7109f31300fb Rename _Py_identifier to _Py_IDENTIFIER. Martin v. Löwis <martin@v.loewis.de> parents: 72848 diff changeset	466 _Py_IDENTIFIER(readline);
65311 3d9ef098daf2 Issue #10095: fp_setreadl() doesn't reopen the file, reuse instead the file Victor Stinner <victor.stinner@haypocalc.com> parents: 61466 diff changeset	467 int fd;
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	468
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	469 io = PyImport_ImportModuleNoBlock("io");
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	470 if (io == NULL)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	471 goto cleanup;
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	472
65311 3d9ef098daf2 Issue #10095: fp_setreadl() doesn't reopen the file, reuse instead the file Victor Stinner <victor.stinner@haypocalc.com> parents: 61466 diff changeset	473 fd = fileno(tok->fp);
3d9ef098daf2 Issue #10095: fp_setreadl() doesn't reopen the file, reuse instead the file Victor Stinner <victor.stinner@haypocalc.com> parents: 61466 diff changeset	474 if (lseek(fd, 0, SEEK_SET) == (off_t)-1) {
3d9ef098daf2 Issue #10095: fp_setreadl() doesn't reopen the file, reuse instead the file Victor Stinner <victor.stinner@haypocalc.com> parents: 61466 diff changeset	475 PyErr_SetFromErrnoWithFilename(PyExc_OSError, NULL);
3d9ef098daf2 Issue #10095: fp_setreadl() doesn't reopen the file, reuse instead the file Victor Stinner <victor.stinner@haypocalc.com> parents: 61466 diff changeset	476 goto cleanup;
3d9ef098daf2 Issue #10095: fp_setreadl() doesn't reopen the file, reuse instead the file Victor Stinner <victor.stinner@haypocalc.com> parents: 61466 diff changeset	477 }
3d9ef098daf2 Issue #10095: fp_setreadl() doesn't reopen the file, reuse instead the file Victor Stinner <victor.stinner@haypocalc.com> parents: 61466 diff changeset	478
72835 5e33219492bc Add API for static strings, primarily good for identifiers. Martin v. Löwis <martin@v.loewis.de> parents: 72475 diff changeset	479 stream = _PyObject_CallMethodId(io, &PyId_open, "isisOOO",
65311 3d9ef098daf2 Issue #10095: fp_setreadl() doesn't reopen the file, reuse instead the file Victor Stinner <victor.stinner@haypocalc.com> parents: 61466 diff changeset	480 fd, "r", -1, enc, Py_None, Py_None, Py_False);
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	481 if (stream == NULL)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	482 goto cleanup;
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	483
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	484 Py_XDECREF(tok->decoding_readline);
72848 81380082d216 Use identifier API for PyObject_GetAttrString. Martin v. Löwis <martin@v.loewis.de> parents: 72835 diff changeset	485 readline = _PyObject_GetAttrId(stream, &PyId_readline);
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	486 tok->decoding_readline = readline;
41970 b3900a51346b tokenizer.c: make coding markup work again. Guido van Rossum <guido@python.org> parents: 41670 diff changeset	487
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	488 /* The file has been reopened; parsing will restart from
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	489 * the beginning of the file, we have to reset the line number.
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	490 * But this function has been called from inside tok_nextc() which
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	491 * will increment lineno before it returns. So we set it -1 so that
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	492 * the next call to tok_nextc() will start with tok->lineno == 0.
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	493 */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	494 tok->lineno = -1;
49484 0b7e023803fb Issues #2384 and #3975: Tracebacks were not correctly printed when the source file Amaury Forgeot d'Arc <amauryfa@gmail.com> parents: 49021 diff changeset	495
41970 b3900a51346b tokenizer.c: make coding markup work again. Guido van Rossum <guido@python.org> parents: 41670 diff changeset	496 cleanup:
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	497 Py_XDECREF(stream);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	498 Py_XDECREF(io);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	499 return readline != NULL;
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	500 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	501
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	502 /* Fetch the next byte from TOK. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	503
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	504 static int fp_getc(struct tok_state *tok) {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	505 return getc(tok->fp);
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	506 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	507
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	508 /* Unfetch the last byte back into TOK. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	509
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	510 static void fp_ungetc(int c, struct tok_state *tok) {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	511 ungetc(c, tok->fp);
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	512 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	513
42403 7e8469617b1c Implement PEP 3120. Martin v. Löwis <martin@v.loewis.de> parents: 42084 diff changeset	514 /* Check whether the characters at s start a valid
7e8469617b1c Implement PEP 3120. Martin v. Löwis <martin@v.loewis.de> parents: 42084 diff changeset	515 UTF-8 sequence. Return the number of characters forming
7e8469617b1c Implement PEP 3120. Martin v. Löwis <martin@v.loewis.de> parents: 42084 diff changeset	516 the sequence if yes, 0 if not. */
7e8469617b1c Implement PEP 3120. Martin v. Löwis <martin@v.loewis.de> parents: 42084 diff changeset	517 static int valid_utf8(const unsigned char* s)
7e8469617b1c Implement PEP 3120. Martin v. Löwis <martin@v.loewis.de> parents: 42084 diff changeset	518 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	519 int expected = 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	520 int length;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	521 if (*s < 0x80)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	522 /* single-byte code */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	523 return 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	524 if (*s < 0xc0)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	525 /* following byte */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	526 return 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	527 if (*s < 0xE0)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	528 expected = 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	529 else if (*s < 0xF0)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	530 expected = 2;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	531 else if (*s < 0xF8)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	532 expected = 3;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	533 else
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	534 return 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	535 length = expected + 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	536 for (; expected; expected--)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	537 if (s[expected] < 0x80 \|\| s[expected] >= 0xC0)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	538 return 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	539 return length;
42403 7e8469617b1c Implement PEP 3120. Martin v. Löwis <martin@v.loewis.de> parents: 42084 diff changeset	540 }
7e8469617b1c Implement PEP 3120. Martin v. Löwis <martin@v.loewis.de> parents: 42084 diff changeset	541
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	542 /* Read a line of input from TOK. Determine encoding
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	543 if necessary. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	544
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	545 static char *
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	546 decoding_fgets(char s, int size, struct tok_state tok)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	547 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	548 char *line = NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	549 int badchar = 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	550 for (;;) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	551 if (tok->decoding_state == STATE_NORMAL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	552 /* We already have a codec associated with
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	553 this input. */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	554 line = fp_readl(s, size, tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	555 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	556 } else if (tok->decoding_state == STATE_RAW) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	557 /* We want a 'raw' read. */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	558 line = Py_UniversalNewlineFgets(s, size,
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	559 tok->fp, NULL);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	560 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	561 } else {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	562 /* We have not yet determined the encoding.
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	563 If an encoding is found, use the file-pointer
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	564 reader functions from now on. */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	565 if (!check_bom(fp_getc, fp_ungetc, fp_setreadl, tok))
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	566 return error_ret(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	567 assert(tok->decoding_state != STATE_INIT);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	568 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	569 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	570 if (line != NULL && tok->lineno < 2 && !tok->read_coding_spec) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	571 if (!check_coding_spec(line, strlen(line), tok, fp_setreadl)) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	572 return error_ret(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	573 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	574 }
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	575 #ifndef PGEN
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	576 /* The default encoding is UTF-8, so make sure we don't have any
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	577 non-UTF-8 sequences in it. */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	578 if (line && !tok->encoding) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	579 unsigned char *c;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	580 int length;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	581 for (c = (unsigned char )line; c; c += length)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	582 if (!(length = valid_utf8(c))) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	583 badchar = *c;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	584 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	585 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	586 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	587 if (badchar) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	588 /* Need to add 1 to the line number, since this line
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	589 has not been counted, yet. */
69550 4658762a77cf Revert bb62908896fe, but keep the test Jesus Cea <jcea@jcea.es> parents: 69548 diff changeset	590 PyErr_Format(PyExc_SyntaxError,
4658762a77cf Revert bb62908896fe, but keep the test Jesus Cea <jcea@jcea.es> parents: 69548 diff changeset	591 "Non-UTF-8 code starting with '\\x%.2x' "
4658762a77cf Revert bb62908896fe, but keep the test Jesus Cea <jcea@jcea.es> parents: 69548 diff changeset	592 "in file %U on line %i, "
4658762a77cf Revert bb62908896fe, but keep the test Jesus Cea <jcea@jcea.es> parents: 69548 diff changeset	593 "but no encoding declared; "
4658762a77cf Revert bb62908896fe, but keep the test Jesus Cea <jcea@jcea.es> parents: 69548 diff changeset	594 "see http://python.org/dev/peps/pep-0263/ for details",
4658762a77cf Revert bb62908896fe, but keep the test Jesus Cea <jcea@jcea.es> parents: 69548 diff changeset	595 badchar, tok->filename, tok->lineno + 1);
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	596 return error_ret(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	597 }
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	598 #endif
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	599 return line;
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	600 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	601
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	602 static int
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	603 decoding_feof(struct tok_state *tok)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	604 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	605 if (tok->decoding_state != STATE_NORMAL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	606 return feof(tok->fp);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	607 } else {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	608 PyObject* buf = tok->decoding_buffer;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	609 if (buf == NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	610 buf = PyObject_CallObject(tok->decoding_readline, NULL);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	611 if (buf == NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	612 error_ret(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	613 return 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	614 } else {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	615 tok->decoding_buffer = buf;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	616 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	617 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	618 return PyObject_Length(buf) == 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	619 }
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	620 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	621
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	622 /* Fetch a byte from TOK, using the string buffer. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	623
37879 f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair Thomas Wouters <thomas@python.org> parents: 37704 diff changeset	624 static int
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair Thomas Wouters <thomas@python.org> parents: 37704 diff changeset	625 buf_getc(struct tok_state *tok) {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	626 return Py_CHARMASK(*tok->str++);
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	627 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	628
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	629 /* Unfetch a byte from TOK, using the string buffer. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	630
37879 f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair Thomas Wouters <thomas@python.org> parents: 37704 diff changeset	631 static void
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair Thomas Wouters <thomas@python.org> parents: 37704 diff changeset	632 buf_ungetc(int c, struct tok_state *tok) {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	633 tok->str--;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	634 assert(Py_CHARMASK(tok->str) == c); / tok->cur may point to read-only segment */
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	635 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	636
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	637 /* Set the readline function for TOK to ENC. For the string-based
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	638 tokenizer, this means to just record the encoding. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	639
37879 f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair Thomas Wouters <thomas@python.org> parents: 37704 diff changeset	640 static int
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair Thomas Wouters <thomas@python.org> parents: 37704 diff changeset	641 buf_setreadl(struct tok_state tok, const char enc) {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	642 tok->enc = enc;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	643 return 1;
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	644 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	645
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	646 /* Return a UTF-8 encoding Python string object from the
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	647 C byte string STR, which is encoded with ENC. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	648
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	649 static PyObject *
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	650 translate_into_utf8(const char* str, const char* enc) {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	651 PyObject *utf8;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	652 PyObject* buf = PyUnicode_Decode(str, strlen(str), enc, NULL);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	653 if (buf == NULL)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	654 return NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	655 utf8 = PyUnicode_AsUTF8String(buf);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	656 Py_DECREF(buf);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	657 return utf8;
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	658 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	659
57206 b2d90c1c637d Merged revisions 76230 via svnmerge from Benjamin Peterson <benjamin@python.org> parents: 56594 diff changeset	660
b2d90c1c637d Merged revisions 76230 via svnmerge from Benjamin Peterson <benjamin@python.org> parents: 56594 diff changeset	661 static char *
b2d90c1c637d Merged revisions 76230 via svnmerge from Benjamin Peterson <benjamin@python.org> parents: 56594 diff changeset	662 translate_newlines(const char s, int exec_input, struct tok_state tok) {
61466 cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	663 int skip_next_lf = 0, needed_length = strlen(s) + 2, final_length;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	664 char buf, current;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	665 char c = '\0';
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	666 buf = PyMem_MALLOC(needed_length);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	667 if (buf == NULL) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	668 tok->done = E_NOMEM;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	669 return NULL;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	670 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	671 for (current = buf; *s; s++, current++) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	672 c = *s;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	673 if (skip_next_lf) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	674 skip_next_lf = 0;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	675 if (c == '\n') {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	676 c = *++s;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	677 if (!c)
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	678 break;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	679 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	680 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	681 if (c == '\r') {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	682 skip_next_lf = 1;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	683 c = '\n';
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	684 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	685 *current = c;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	686 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	687 /* If this is exec input, add a newline to the end of the string if
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	688 there isn't one already. */
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	689 if (exec_input && c != '\n') {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	690 *current = '\n';
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	691 current++;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	692 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	693 *current = '\0';
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	694 final_length = current - buf + 1;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	695 if (final_length < needed_length && final_length)
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	696 /* should never fail */
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	697 buf = PyMem_REALLOC(buf, final_length);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	698 return buf;
57206 b2d90c1c637d Merged revisions 76230 via svnmerge from Benjamin Peterson <benjamin@python.org> parents: 56594 diff changeset	699 }
b2d90c1c637d Merged revisions 76230 via svnmerge from Benjamin Peterson <benjamin@python.org> parents: 56594 diff changeset	700
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	701 /* Decode a byte string STR for use as the buffer of TOK.
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	702 Look for encoding declarations inside STR, and record them
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	703 inside TOK. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	704
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	705 static const char *
57206 b2d90c1c637d Merged revisions 76230 via svnmerge from Benjamin Peterson <benjamin@python.org> parents: 56594 diff changeset	706 decode_str(const char input, int single, struct tok_state tok)
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	707 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	708 PyObject* utf8 = NULL;
61466 cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	709 const char *str;
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	710 const char *s;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	711 const char *newl[2] = {NULL, NULL};
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	712 int lineno = 0;
61466 cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	713 tok->input = str = translate_newlines(input, single, tok);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	714 if (str == NULL)
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	715 return NULL;
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	716 tok->enc = NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	717 tok->str = str;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	718 if (!check_bom(buf_getc, buf_ungetc, buf_setreadl, tok))
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	719 return error_ret(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	720 str = tok->str; /* string after BOM if any */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	721 assert(str);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	722 if (tok->enc != NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	723 utf8 = translate_into_utf8(str, tok->enc);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	724 if (utf8 == NULL)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	725 return error_ret(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	726 str = PyBytes_AsString(utf8);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	727 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	728 for (s = str;; s++) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	729 if (*s == '\0') break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	730 else if (*s == '\n') {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	731 assert(lineno < 2);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	732 newl[lineno] = s;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	733 lineno++;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	734 if (lineno == 2) break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	735 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	736 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	737 tok->enc = NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	738 /* need to check line 1 and 2 separately since check_coding_spec
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	739 assumes a single line as input */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	740 if (newl[0]) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	741 if (!check_coding_spec(str, newl[0] - str, tok, buf_setreadl))
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	742 return error_ret(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	743 if (tok->enc == NULL && newl[1]) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	744 if (!check_coding_spec(newl[0]+1, newl[1] - newl[0],
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	745 tok, buf_setreadl))
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	746 return error_ret(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	747 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	748 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	749 if (tok->enc != NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	750 assert(utf8 == NULL);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	751 utf8 = translate_into_utf8(str, tok->enc);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	752 if (utf8 == NULL)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	753 return error_ret(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	754 str = PyBytes_AS_STRING(utf8);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	755 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	756 assert(tok->decoding_buffer == NULL);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	757 tok->decoding_buffer = utf8; /* CAUTION */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	758 return str;
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	759 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	760
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	761 #endif /* PGEN */
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	762
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	763 /* Set up tokenizer for string */
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	764
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	765 struct tok_state *
57206 b2d90c1c637d Merged revisions 76230 via svnmerge from Benjamin Peterson <benjamin@python.org> parents: 56594 diff changeset	766 PyTokenizer_FromString(const char *str, int exec_input)
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	767 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	768 struct tok_state *tok = tok_new();
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	769 if (tok == NULL)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	770 return NULL;
61466 cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	771 str = (char *)decode_str(str, exec_input, tok);
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	772 if (str == NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	773 PyTokenizer_Free(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	774 return NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	775 }
35878 94a409da4bd4 Fix some more memory leaks. Neal Norwitz <nnorwitz@gmail.com> parents: 35751 diff changeset	776
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	777 /* XXX: constify members. */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	778 tok->buf = tok->cur = tok->end = tok->inp = (char*)str;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	779 return tok;
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	780 }
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	781
51956 8fe2988cf550 ignore the coding cookie in compile(), exec(), and eval() if the source is a string #4626 Benjamin Peterson <benjamin@python.org> parents: 50540 diff changeset	782 struct tok_state *
57206 b2d90c1c637d Merged revisions 76230 via svnmerge from Benjamin Peterson <benjamin@python.org> parents: 56594 diff changeset	783 PyTokenizer_FromUTF8(const char *str, int exec_input)
51956 8fe2988cf550 ignore the coding cookie in compile(), exec(), and eval() if the source is a string #4626 Benjamin Peterson <benjamin@python.org> parents: 50540 diff changeset	784 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	785 struct tok_state *tok = tok_new();
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	786 if (tok == NULL)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	787 return NULL;
57206 b2d90c1c637d Merged revisions 76230 via svnmerge from Benjamin Peterson <benjamin@python.org> parents: 56594 diff changeset	788 #ifndef PGEN
61466 cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	789 tok->input = str = translate_newlines(str, exec_input, tok);
57206 b2d90c1c637d Merged revisions 76230 via svnmerge from Benjamin Peterson <benjamin@python.org> parents: 56594 diff changeset	790 #endif
61466 cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	791 if (str == NULL) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	792 PyTokenizer_Free(tok);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	793 return NULL;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	794 }
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	795 tok->decoding_state = STATE_RAW;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	796 tok->read_coding_spec = 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	797 tok->enc = NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	798 tok->str = str;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	799 tok->encoding = (char *)PyMem_MALLOC(6);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	800 if (!tok->encoding) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	801 PyTokenizer_Free(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	802 return NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	803 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	804 strcpy(tok->encoding, "utf-8");
51956 8fe2988cf550 ignore the coding cookie in compile(), exec(), and eval() if the source is a string #4626 Benjamin Peterson <benjamin@python.org> parents: 50540 diff changeset	805
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	806 /* XXX: constify members. */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	807 tok->buf = tok->cur = tok->end = tok->inp = (char*)str;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	808 return tok;
51956 8fe2988cf550 ignore the coding cookie in compile(), exec(), and eval() if the source is a string #4626 Benjamin Peterson <benjamin@python.org> parents: 50540 diff changeset	809 }
8fe2988cf550 ignore the coding cookie in compile(), exec(), and eval() if the source is a string #4626 Benjamin Peterson <benjamin@python.org> parents: 50540 diff changeset	810
326 ff0862e3d63a Completely ignore lines with only a newline token on them, except Guido van Rossum <guido@python.org> parents: 293 diff changeset	811 /* Set up tokenizer for file */
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	812
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	813 struct tok_state *
43225 38f3f9ef193d Convert code from sys.stdin.encoding to UTF-8 in Martin v. Löwis <martin@v.loewis.de> parents: 43018 diff changeset	814 PyTokenizer_FromFile(FILE fp, char enc, char ps1, char ps2)
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	815 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	816 struct tok_state *tok = tok_new();
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	817 if (tok == NULL)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	818 return NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	819 if ((tok->buf = (char *)PyMem_MALLOC(BUFSIZ)) == NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	820 PyTokenizer_Free(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	821 return NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	822 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	823 tok->cur = tok->inp = tok->buf;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	824 tok->end = tok->buf + BUFSIZ;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	825 tok->fp = fp;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	826 tok->prompt = ps1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	827 tok->nextprompt = ps2;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	828 if (enc != NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	829 /* Must copy encoding declaration since it
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	830 gets copied into the parse tree. */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	831 tok->encoding = PyMem_MALLOC(strlen(enc)+1);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	832 if (!tok->encoding) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	833 PyTokenizer_Free(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	834 return NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	835 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	836 strcpy(tok->encoding, enc);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	837 tok->decoding_state = STATE_NORMAL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	838 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	839 return tok;
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	840 }
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	841
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	842
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	843 /* Free a tok_state structure */
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	844
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	845 void
13659 e78153e179fb Mass ANSIfication. Thomas Wouters <thomas@python.org> parents: 13539 diff changeset	846 PyTokenizer_Free(struct tok_state *tok)
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	847 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	848 if (tok->encoding != NULL)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	849 PyMem_FREE(tok->encoding);
24544 0bd521eb714e Make pgen compile with pydebug. Duplicate normalized names, as it may Martin v. Löwis <martin@v.loewis.de> parents: 24543 diff changeset	850 #ifndef PGEN
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	851 Py_XDECREF(tok->decoding_readline);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	852 Py_XDECREF(tok->decoding_buffer);
69147 6e9dc970ac0e Issue #10785: Store the filename as Unicode in the Python parser. Victor Stinner <victor.stinner@haypocalc.com> parents: 68257 diff changeset	853 Py_XDECREF(tok->filename);
24544 0bd521eb714e Make pgen compile with pydebug. Duplicate normalized names, as it may Martin v. Löwis <martin@v.loewis.de> parents: 24543 diff changeset	854 #endif
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	855 if (tok->fp != NULL && tok->buf != NULL)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	856 PyMem_FREE(tok->buf);
61466 cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	857 if (tok->input)
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	858 PyMem_FREE((char *)tok->input);
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	859 PyMem_FREE(tok);
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	860 }
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	861
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	862 /* Get next char, updating state; error code goes into tok->done */
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	863
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	864 static int
13659 e78153e179fb Mass ANSIfication. Thomas Wouters <thomas@python.org> parents: 13539 diff changeset	865 tok_nextc(register struct tok_state *tok)
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	866 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	867 for (;;) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	868 if (tok->cur != tok->inp) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	869 return Py_CHARMASK(tok->cur++); / Fast path */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	870 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	871 if (tok->done != E_OK)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	872 return EOF;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	873 if (tok->fp == NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	874 char *end = strchr(tok->inp, '\n');
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	875 if (end != NULL)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	876 end++;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	877 else {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	878 end = strchr(tok->inp, '\0');
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	879 if (end == tok->inp) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	880 tok->done = E_EOF;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	881 return EOF;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	882 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	883 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	884 if (tok->start == NULL)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	885 tok->buf = tok->cur;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	886 tok->line_start = tok->cur;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	887 tok->lineno++;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	888 tok->inp = end;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	889 return Py_CHARMASK(*tok->cur++);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	890 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	891 if (tok->prompt != NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	892 char *newtok = PyOS_Readline(stdin, stdout, tok->prompt);
43225 38f3f9ef193d Convert code from sys.stdin.encoding to UTF-8 in Martin v. Löwis <martin@v.loewis.de> parents: 43018 diff changeset	893 #ifndef PGEN
67431 0933c3753a71 Issue #10841: set binary mode on files; the parser translates newlines Victor Stinner <victor.stinner@haypocalc.com> parents: 67367 diff changeset	894 if (newtok != NULL) {
0933c3753a71 Issue #10841: set binary mode on files; the parser translates newlines Victor Stinner <victor.stinner@haypocalc.com> parents: 67367 diff changeset	895 char *translated = translate_newlines(newtok, 0, tok);
0933c3753a71 Issue #10841: set binary mode on files; the parser translates newlines Victor Stinner <victor.stinner@haypocalc.com> parents: 67367 diff changeset	896 PyMem_FREE(newtok);
0933c3753a71 Issue #10841: set binary mode on files; the parser translates newlines Victor Stinner <victor.stinner@haypocalc.com> parents: 67367 diff changeset	897 if (translated == NULL)
0933c3753a71 Issue #10841: set binary mode on files; the parser translates newlines Victor Stinner <victor.stinner@haypocalc.com> parents: 67367 diff changeset	898 return EOF;
0933c3753a71 Issue #10841: set binary mode on files; the parser translates newlines Victor Stinner <victor.stinner@haypocalc.com> parents: 67367 diff changeset	899 newtok = translated;
0933c3753a71 Issue #10841: set binary mode on files; the parser translates newlines Victor Stinner <victor.stinner@haypocalc.com> parents: 67367 diff changeset	900 }
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	901 if (tok->encoding && newtok && *newtok) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	902 /* Recode to UTF-8 */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	903 Py_ssize_t buflen;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	904 const char* buf;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	905 PyObject *u = translate_into_utf8(newtok, tok->encoding);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	906 PyMem_FREE(newtok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	907 if (!u) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	908 tok->done = E_DECODE;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	909 return EOF;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	910 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	911 buflen = PyBytes_GET_SIZE(u);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	912 buf = PyBytes_AS_STRING(u);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	913 if (!buf) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	914 Py_DECREF(u);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	915 tok->done = E_DECODE;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	916 return EOF;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	917 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	918 newtok = PyMem_MALLOC(buflen+1);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	919 strcpy(newtok, buf);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	920 Py_DECREF(u);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	921 }
43225 38f3f9ef193d Convert code from sys.stdin.encoding to UTF-8 in Martin v. Löwis <martin@v.loewis.de> parents: 43018 diff changeset	922 #endif
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	923 if (tok->nextprompt != NULL)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	924 tok->prompt = tok->nextprompt;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	925 if (newtok == NULL)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	926 tok->done = E_INTR;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	927 else if (*newtok == '\0') {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	928 PyMem_FREE(newtok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	929 tok->done = E_EOF;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	930 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	931 else if (tok->start != NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	932 size_t start = tok->start - tok->buf;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	933 size_t oldlen = tok->cur - tok->buf;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	934 size_t newlen = oldlen + strlen(newtok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	935 char *buf = tok->buf;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	936 buf = (char *)PyMem_REALLOC(buf, newlen+1);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	937 tok->lineno++;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	938 if (buf == NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	939 PyMem_FREE(tok->buf);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	940 tok->buf = NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	941 PyMem_FREE(newtok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	942 tok->done = E_NOMEM;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	943 return EOF;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	944 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	945 tok->buf = buf;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	946 tok->cur = tok->buf + oldlen;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	947 tok->line_start = tok->cur;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	948 strcpy(tok->buf + oldlen, newtok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	949 PyMem_FREE(newtok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	950 tok->inp = tok->buf + newlen;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	951 tok->end = tok->inp + 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	952 tok->start = tok->buf + start;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	953 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	954 else {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	955 tok->lineno++;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	956 if (tok->buf != NULL)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	957 PyMem_FREE(tok->buf);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	958 tok->buf = newtok;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	959 tok->line_start = tok->buf;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	960 tok->cur = tok->buf;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	961 tok->line_start = tok->buf;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	962 tok->inp = strchr(tok->buf, '\0');
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	963 tok->end = tok->inp + 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	964 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	965 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	966 else {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	967 int done = 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	968 Py_ssize_t cur = 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	969 char *pt;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	970 if (tok->start == NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	971 if (tok->buf == NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	972 tok->buf = (char *)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	973 PyMem_MALLOC(BUFSIZ);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	974 if (tok->buf == NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	975 tok->done = E_NOMEM;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	976 return EOF;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	977 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	978 tok->end = tok->buf + BUFSIZ;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	979 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	980 if (decoding_fgets(tok->buf, (int)(tok->end - tok->buf),
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	981 tok) == NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	982 tok->done = E_EOF;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	983 done = 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	984 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	985 else {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	986 tok->done = E_OK;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	987 tok->inp = strchr(tok->buf, '\0');
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	988 done = tok->inp[-1] == '\n';
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	989 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	990 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	991 else {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	992 cur = tok->cur - tok->buf;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	993 if (decoding_feof(tok)) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	994 tok->done = E_EOF;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	995 done = 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	996 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	997 else
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	998 tok->done = E_OK;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	999 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1000 tok->lineno++;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1001 /* Read until '\n' or EOF */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1002 while (!done) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1003 Py_ssize_t curstart = tok->start == NULL ? -1 :
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1004 tok->start - tok->buf;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1005 Py_ssize_t curvalid = tok->inp - tok->buf;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1006 Py_ssize_t newsize = curvalid + BUFSIZ;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1007 char *newbuf = tok->buf;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1008 newbuf = (char *)PyMem_REALLOC(newbuf,
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1009 newsize);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1010 if (newbuf == NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1011 tok->done = E_NOMEM;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1012 tok->cur = tok->inp;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1013 return EOF;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1014 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1015 tok->buf = newbuf;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1016 tok->inp = tok->buf + curvalid;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1017 tok->end = tok->buf + newsize;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1018 tok->start = curstart < 0 ? NULL :
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1019 tok->buf + curstart;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1020 if (decoding_fgets(tok->inp,
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1021 (int)(tok->end - tok->inp),
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1022 tok) == NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1023 /* Break out early on decoding
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1024 errors, as tok->buf will be NULL
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1025 */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1026 if (tok->decoding_erred)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1027 return EOF;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1028 /* Last line does not end in \n,
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1029 fake one */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1030 strcpy(tok->inp, "\n");
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1031 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1032 tok->inp = strchr(tok->inp, '\0');
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1033 done = tok->inp[-1] == '\n';
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1034 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1035 if (tok->buf != NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1036 tok->cur = tok->buf + cur;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1037 tok->line_start = tok->cur;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1038 /* replace "\r\n" with "\n" */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1039 /* For Mac leave the \r, giving a syntax error */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1040 pt = tok->inp - 2;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1041 if (pt >= tok->buf && *pt == '\r') {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1042 *pt++ = '\n';
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1043 *pt = '\0';
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1044 tok->inp = pt;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1045 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1046 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1047 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1048 if (tok->done != E_OK) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1049 if (tok->prompt != NULL)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1050 PySys_WriteStderr("\n");
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1051 tok->cur = tok->inp;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1052 return EOF;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1053 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1054 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1055 /NOTREACHED/
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1056 }
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1057
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1058
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1059 /* Back-up one character */
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1060
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1061 static void
13659 e78153e179fb Mass ANSIfication. Thomas Wouters <thomas@python.org> parents: 13539 diff changeset	1062 tok_backup(register struct tok_state *tok, register int c)
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1063 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1064 if (c != EOF) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1065 if (--tok->cur < tok->buf)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1066 Py_FatalError("tok_backup: beginning of buffer");
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1067 if (*tok->cur != c)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1068 *tok->cur = c;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1069 }
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1070 }
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1071
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1072
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1073 /* Return the token corresponding to a single character */
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1074
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1075 int
13659 e78153e179fb Mass ANSIfication. Thomas Wouters <thomas@python.org> parents: 13539 diff changeset	1076 PyToken_OneChar(int c)
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1077 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1078 switch (c) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1079 case '(': return LPAR;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1080 case ')': return RPAR;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1081 case '[': return LSQB;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1082 case ']': return RSQB;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1083 case ':': return COLON;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1084 case ',': return COMMA;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1085 case ';': return SEMI;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1086 case '+': return PLUS;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1087 case '-': return MINUS;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1088 case '*': return STAR;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1089 case '/': return SLASH;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1090 case '\|': return VBAR;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1091 case '&': return AMPER;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1092 case '<': return LESS;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1093 case '>': return GREATER;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1094 case '=': return EQUAL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1095 case '.': return DOT;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1096 case '%': return PERCENT;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1097 case '{': return LBRACE;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1098 case '}': return RBRACE;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1099 case '^': return CIRCUMFLEX;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1100 case '~': return TILDE;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1101 case '@': return AT;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1102 default: return OP;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1103 }
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1104 }
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1105
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1106
400 d189a2f681a6 Added 2-char tokens and new versions of comparisons Guido van Rossum <guido@python.org> parents: 326 diff changeset	1107 int
13659 e78153e179fb Mass ANSIfication. Thomas Wouters <thomas@python.org> parents: 13539 diff changeset	1108 PyToken_TwoChars(int c1, int c2)
400 d189a2f681a6 Added 2-char tokens and new versions of comparisons Guido van Rossum <guido@python.org> parents: 326 diff changeset	1109 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1110 switch (c1) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1111 case '=':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1112 switch (c2) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1113 case '=': return EQEQUAL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1114 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1115 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1116 case '!':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1117 switch (c2) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1118 case '=': return NOTEQUAL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1119 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1120 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1121 case '<':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1122 switch (c2) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1123 case '>': return NOTEQUAL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1124 case '=': return LESSEQUAL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1125 case '<': return LEFTSHIFT;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1126 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1127 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1128 case '>':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1129 switch (c2) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1130 case '=': return GREATEREQUAL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1131 case '>': return RIGHTSHIFT;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1132 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1133 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1134 case '+':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1135 switch (c2) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1136 case '=': return PLUSEQUAL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1137 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1138 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1139 case '-':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1140 switch (c2) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1141 case '=': return MINEQUAL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1142 case '>': return RARROW;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1143 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1144 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1145 case '*':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1146 switch (c2) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1147 case '*': return DOUBLESTAR;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1148 case '=': return STAREQUAL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1149 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1150 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1151 case '/':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1152 switch (c2) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1153 case '/': return DOUBLESLASH;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1154 case '=': return SLASHEQUAL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1155 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1156 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1157 case '\|':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1158 switch (c2) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1159 case '=': return VBAREQUAL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1160 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1161 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1162 case '%':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1163 switch (c2) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1164 case '=': return PERCENTEQUAL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1165 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1166 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1167 case '&':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1168 switch (c2) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1169 case '=': return AMPEREQUAL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1170 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1171 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1172 case '^':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1173 switch (c2) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1174 case '=': return CIRCUMFLEXEQUAL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1175 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1176 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1177 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1178 return OP;
400 d189a2f681a6 Added 2-char tokens and new versions of comparisons Guido van Rossum <guido@python.org> parents: 326 diff changeset	1179 }
d189a2f681a6 Added 2-char tokens and new versions of comparisons Guido van Rossum <guido@python.org> parents: 326 diff changeset	1180
14163 1b9fd0cbd914 Support for three-token characters (*=, >>=, <<=) which was written by Thomas Wouters <thomas@python.org>* parents: 13659 diff changeset	1181 int
1b9fd0cbd914 Support for three-token characters (*=, >>=, <<=) which was written by Thomas Wouters <thomas@python.org>* parents: 13659 diff changeset	1182 PyToken_ThreeChars(int c1, int c2, int c3)
1b9fd0cbd914 Support for three-token characters (*=, >>=, <<=) which was written by Thomas Wouters <thomas@python.org>* parents: 13659 diff changeset	1183 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1184 switch (c1) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1185 case '<':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1186 switch (c2) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1187 case '<':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1188 switch (c3) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1189 case '=':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1190 return LEFTSHIFTEQUAL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1191 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1192 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1193 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1194 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1195 case '>':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1196 switch (c2) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1197 case '>':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1198 switch (c3) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1199 case '=':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1200 return RIGHTSHIFTEQUAL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1201 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1202 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1203 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1204 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1205 case '*':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1206 switch (c2) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1207 case '*':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1208 switch (c3) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1209 case '=':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1210 return DOUBLESTAREQUAL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1211 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1212 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1213 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1214 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1215 case '/':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1216 switch (c2) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1217 case '/':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1218 switch (c3) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1219 case '=':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1220 return DOUBLESLASHEQUAL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1221 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1222 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1223 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1224 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1225 case '.':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1226 switch (c2) {
41230 b1a18c7ed5ca Make ELLIPSIS a separate token. This makes it a syntax error to write ". . ." for Ellipsis. Georg Brandl <georg@python.org> parents: 40894 diff changeset	1227 case '.':
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1228 switch (c3) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1229 case '.':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1230 return ELLIPSIS;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1231 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1232 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1233 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1234 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1235 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1236 return OP;
14163 1b9fd0cbd914 Support for three-token characters (*=, >>=, <<=) which was written by Thomas Wouters <thomas@python.org>* parents: 13659 diff changeset	1237 }
400 d189a2f681a6 Added 2-char tokens and new versions of comparisons Guido van Rossum <guido@python.org> parents: 326 diff changeset	1238
7945 dc132cc39573 Add checking for inconsistent tab usage Guido van Rossum <guido@python.org> parents: 7324 diff changeset	1239 static int
13659 e78153e179fb Mass ANSIfication. Thomas Wouters <thomas@python.org> parents: 13539 diff changeset	1240 indenterror(struct tok_state *tok)
7945 dc132cc39573 Add checking for inconsistent tab usage Guido van Rossum <guido@python.org> parents: 7324 diff changeset	1241 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1242 if (tok->alterror) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1243 tok->done = E_TABSPACE;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1244 tok->cur = tok->inp;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1245 return 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1246 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1247 if (tok->altwarning) {
69147 6e9dc970ac0e Issue #10785: Store the filename as Unicode in the Python parser. Victor Stinner <victor.stinner@haypocalc.com> parents: 68257 diff changeset	1248 #ifdef PGEN
6e9dc970ac0e Issue #10785: Store the filename as Unicode in the Python parser. Victor Stinner <victor.stinner@haypocalc.com> parents: 68257 diff changeset	1249 PySys_WriteStderr("inconsistent use of tabs and spaces "
6e9dc970ac0e Issue #10785: Store the filename as Unicode in the Python parser. Victor Stinner <victor.stinner@haypocalc.com> parents: 68257 diff changeset	1250 "in indentation\n");
6e9dc970ac0e Issue #10785: Store the filename as Unicode in the Python parser. Victor Stinner <victor.stinner@haypocalc.com> parents: 68257 diff changeset	1251 #else
6e9dc970ac0e Issue #10785: Store the filename as Unicode in the Python parser. Victor Stinner <victor.stinner@haypocalc.com> parents: 68257 diff changeset	1252 PySys_FormatStderr("%U: inconsistent use of tabs and spaces "
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1253 "in indentation\n", tok->filename);
69147 6e9dc970ac0e Issue #10785: Store the filename as Unicode in the Python parser. Victor Stinner <victor.stinner@haypocalc.com> parents: 68257 diff changeset	1254 #endif
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1255 tok->altwarning = 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1256 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1257 return 0;
7945 dc132cc39573 Add checking for inconsistent tab usage Guido van Rossum <guido@python.org> parents: 7324 diff changeset	1258 }
dc132cc39573 Add checking for inconsistent tab usage Guido van Rossum <guido@python.org> parents: 7324 diff changeset	1259
42597 20477cbfd996 Implement PEP 3131. Add isidentifier to str. Martin v. Löwis <martin@v.loewis.de> parents: 42551 diff changeset	1260 #ifdef PGEN
59865 c2d2e26bcf7f Merged revisions 78872 via svnmerge from Victor Stinner <victor.stinner@haypocalc.com> parents: 59289 diff changeset	1261 #define verify_identifier(tok) 1
42597 20477cbfd996 Implement PEP 3131. Add isidentifier to str. Martin v. Löwis <martin@v.loewis.de> parents: 42551 diff changeset	1262 #else
72475 8beaa9a37387 Implement PEP 393. Martin v. Löwis <martin@v.loewis.de> parents: 69550 diff changeset	1263 /* Verify that the identifier follows PEP 3131.
8beaa9a37387 Implement PEP 393. Martin v. Löwis <martin@v.loewis.de> parents: 69550 diff changeset	1264 All identifier strings are guaranteed to be "ready" unicode objects.
8beaa9a37387 Implement PEP 393. Martin v. Löwis <martin@v.loewis.de> parents: 69550 diff changeset	1265 */
42597 20477cbfd996 Implement PEP 3131. Add isidentifier to str. Martin v. Löwis <martin@v.loewis.de> parents: 42551 diff changeset	1266 static int
59865 c2d2e26bcf7f Merged revisions 78872 via svnmerge from Victor Stinner <victor.stinner@haypocalc.com> parents: 59289 diff changeset	1267 verify_identifier(struct tok_state *tok)
42597 20477cbfd996 Implement PEP 3131. Add isidentifier to str. Martin v. Löwis <martin@v.loewis.de> parents: 42551 diff changeset	1268 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1269 PyObject *s;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1270 int result;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1271 s = PyUnicode_DecodeUTF8(tok->start, tok->cur - tok->start, NULL);
72475 8beaa9a37387 Implement PEP 393. Martin v. Löwis <martin@v.loewis.de> parents: 69550 diff changeset	1272 if (s == NULL \|\| PyUnicode_READY(s) == -1) {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1273 if (PyErr_ExceptionMatches(PyExc_UnicodeDecodeError)) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1274 PyErr_Clear();
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1275 tok->done = E_IDENTIFIER;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1276 } else {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1277 tok->done = E_ERROR;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1278 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1279 return 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1280 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1281 result = PyUnicode_IsIdentifier(s);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1282 Py_DECREF(s);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1283 if (result == 0)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1284 tok->done = E_IDENTIFIER;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1285 return result;
42597 20477cbfd996 Implement PEP 3131. Add isidentifier to str. Martin v. Löwis <martin@v.loewis.de> parents: 42551 diff changeset	1286 }
20477cbfd996 Implement PEP 3131. Add isidentifier to str. Martin v. Löwis <martin@v.loewis.de> parents: 42551 diff changeset	1287 #endif
7945 dc132cc39573 Add checking for inconsistent tab usage Guido van Rossum <guido@python.org> parents: 7324 diff changeset	1288
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1289 /* Get next token, after space stripping etc. */
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1290
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	1291 static int
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	1292 tok_get(register struct tok_state tok, char p_start, char *p_end)
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1293 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1294 register int c;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1295 int blankline, nonascii;
326 ff0862e3d63a Completely ignore lines with only a newline token on them, except Guido van Rossum <guido@python.org> parents: 293 diff changeset	1296
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1297 p_start = p_end = NULL;
326 ff0862e3d63a Completely ignore lines with only a newline token on them, except Guido van Rossum <guido@python.org> parents: 293 diff changeset	1298 nextline:
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1299 tok->start = NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1300 blankline = 0;
326 ff0862e3d63a Completely ignore lines with only a newline token on them, except Guido van Rossum <guido@python.org> parents: 293 diff changeset	1301
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1302 /* Get indentation level */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1303 if (tok->atbol) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1304 register int col = 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1305 register int altcol = 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1306 tok->atbol = 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1307 for (;;) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1308 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1309 if (c == ' ')
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1310 col++, altcol++;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1311 else if (c == '\t') {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1312 col = (col/tok->tabsize + 1) * tok->tabsize;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1313 altcol = (altcol/tok->alttabsize + 1)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1314 * tok->alttabsize;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1315 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1316 else if (c == '\014') /* Control-L (formfeed) */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1317 col = altcol = 0; /* For Emacs users */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1318 else
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1319 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1320 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1321 tok_backup(tok, c);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1322 if (c == '#' \|\| c == '\n') {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1323 /* Lines with only whitespace and/or comments
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1324 shouldn't affect the indentation and are
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1325 not passed to the parser as NEWLINE tokens,
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1326 except totally empty lines in interactive
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1327 mode, which signal the end of a command group. */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1328 if (col == 0 && c == '\n' && tok->prompt != NULL)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1329 blankline = 0; /* Let it through */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1330 else
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1331 blankline = 1; /* Ignore completely */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1332 /* We can't jump back right here since we still
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1333 may need to skip to the end of a comment */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1334 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1335 if (!blankline && tok->level == 0) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1336 if (col == tok->indstack[tok->indent]) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1337 /* No change */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1338 if (altcol != tok->altindstack[tok->indent]) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1339 if (indenterror(tok))
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1340 return ERRORTOKEN;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1341 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1342 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1343 else if (col > tok->indstack[tok->indent]) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1344 /* Indent -- always one */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1345 if (tok->indent+1 >= MAXINDENT) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1346 tok->done = E_TOODEEP;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1347 tok->cur = tok->inp;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1348 return ERRORTOKEN;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1349 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1350 if (altcol <= tok->altindstack[tok->indent]) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1351 if (indenterror(tok))
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1352 return ERRORTOKEN;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1353 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1354 tok->pendin++;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1355 tok->indstack[++tok->indent] = col;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1356 tok->altindstack[tok->indent] = altcol;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1357 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1358 else /* col < tok->indstack[tok->indent] */ {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1359 /* Dedent -- any number, must be consistent */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1360 while (tok->indent > 0 &&
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1361 col < tok->indstack[tok->indent]) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1362 tok->pendin--;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1363 tok->indent--;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1364 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1365 if (col != tok->indstack[tok->indent]) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1366 tok->done = E_DEDENT;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1367 tok->cur = tok->inp;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1368 return ERRORTOKEN;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1369 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1370 if (altcol != tok->altindstack[tok->indent]) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1371 if (indenterror(tok))
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1372 return ERRORTOKEN;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1373 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1374 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1375 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1376 }
37879 f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair Thomas Wouters <thomas@python.org> parents: 37704 diff changeset	1377
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1378 tok->start = tok->cur;
37879 f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair Thomas Wouters <thomas@python.org> parents: 37704 diff changeset	1379
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1380 /* Return pending indents/dedents */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1381 if (tok->pendin != 0) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1382 if (tok->pendin < 0) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1383 tok->pendin++;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1384 return DEDENT;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1385 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1386 else {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1387 tok->pendin--;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1388 return INDENT;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1389 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1390 }
37879 f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair Thomas Wouters <thomas@python.org> parents: 37704 diff changeset	1391
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1392 again:
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1393 tok->start = NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1394 /* Skip spaces */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1395 do {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1396 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1397 } while (c == ' ' \|\| c == '\t' \|\| c == '\014');
37879 f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair Thomas Wouters <thomas@python.org> parents: 37704 diff changeset	1398
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1399 /* Set start of current token */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1400 tok->start = tok->cur - 1;
37879 f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair Thomas Wouters <thomas@python.org> parents: 37704 diff changeset	1401
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1402 /* Skip comment */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1403 if (c == '#')
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1404 while (c != EOF && c != '\n')
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1405 c = tok_nextc(tok);
37879 f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair Thomas Wouters <thomas@python.org> parents: 37704 diff changeset	1406
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1407 /* Check for EOF and errors now */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1408 if (c == EOF) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1409 return tok->done == E_EOF ? ENDMARKER : ERRORTOKEN;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1410 }
37879 f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair Thomas Wouters <thomas@python.org> parents: 37704 diff changeset	1411
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1412 /* Identifier (most frequent token!) */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1413 nonascii = 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1414 if (is_potential_identifier_start(c)) {
77522 8e47e9af826e Issue #15096: Drop support for the ur string prefix Christian Heimes <c.heimes@cheimes.de> parents: 75381 diff changeset	1415 /* Process b"", r"", u"", br"" and rb"" */
75381 ed9497d6a216 Basic support for PEP 414 without docs or tests. Armin Ronacher <armin.ronacher@active-4.com> parents: 74354 diff changeset	1416 int saw_b = 0, saw_r = 0, saw_u = 0;
74354 bbed36370b08 Issue #13748: Raw bytes literals can now be written with the `rb` prefix as well as `br`. Antoine Pitrou <solipsis@pitrou.net> parents: 72925 diff changeset	1417 while (1) {
75381 ed9497d6a216 Basic support for PEP 414 without docs or tests. Armin Ronacher <armin.ronacher@active-4.com> parents: 74354 diff changeset	1418 if (!(saw_b \|\| saw_u) && (c == 'b' \|\| c == 'B'))
74354 bbed36370b08 Issue #13748: Raw bytes literals can now be written with the `rb` prefix as well as `br`. Antoine Pitrou <solipsis@pitrou.net> parents: 72925 diff changeset	1419 saw_b = 1;
75381 ed9497d6a216 Basic support for PEP 414 without docs or tests. Armin Ronacher <armin.ronacher@active-4.com> parents: 74354 diff changeset	1420 /* Since this is a backwards compatibility support literal we don't
ed9497d6a216 Basic support for PEP 414 without docs or tests. Armin Ronacher <armin.ronacher@active-4.com> parents: 74354 diff changeset	1421 want to support it in arbitrary order like byte literals. */
ed9497d6a216 Basic support for PEP 414 without docs or tests. Armin Ronacher <armin.ronacher@active-4.com> parents: 74354 diff changeset	1422 else if (!(saw_b \|\| saw_u \|\| saw_r) && (c == 'u' \|\| c == 'U'))
ed9497d6a216 Basic support for PEP 414 without docs or tests. Armin Ronacher <armin.ronacher@active-4.com> parents: 74354 diff changeset	1423 saw_u = 1;
77522 8e47e9af826e Issue #15096: Drop support for the ur string prefix Christian Heimes <c.heimes@cheimes.de> parents: 75381 diff changeset	1424 /* ur"" and ru"" are not supported */
8e47e9af826e Issue #15096: Drop support for the ur string prefix Christian Heimes <c.heimes@cheimes.de> parents: 75381 diff changeset	1425 else if (!(saw_r \|\| saw_u) && (c == 'r' \|\| c == 'R'))
74354 bbed36370b08 Issue #13748: Raw bytes literals can now be written with the `rb` prefix as well as `br`. Antoine Pitrou <solipsis@pitrou.net> parents: 72925 diff changeset	1426 saw_r = 1;
bbed36370b08 Issue #13748: Raw bytes literals can now be written with the `rb` prefix as well as `br`. Antoine Pitrou <solipsis@pitrou.net> parents: 72925 diff changeset	1427 else
bbed36370b08 Issue #13748: Raw bytes literals can now be written with the `rb` prefix as well as `br`. Antoine Pitrou <solipsis@pitrou.net> parents: 72925 diff changeset	1428 break;
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1429 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1430 if (c == '"' \|\| c == '\'')
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1431 goto letter_quote;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1432 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1433 while (is_potential_identifier_char(c)) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1434 if (c >= 128)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1435 nonascii = 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1436 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1437 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1438 tok_backup(tok, c);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1439 if (nonascii &&
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1440 !verify_identifier(tok)) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1441 tok->done = E_IDENTIFIER;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1442 return ERRORTOKEN;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1443 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1444 *p_start = tok->start;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1445 *p_end = tok->cur;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1446 return NAME;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1447 }
37879 f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair Thomas Wouters <thomas@python.org> parents: 37704 diff changeset	1448
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1449 /* Newline */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1450 if (c == '\n') {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1451 tok->atbol = 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1452 if (blankline \|\| tok->level > 0)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1453 goto nextline;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1454 *p_start = tok->start;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1455 p_end = tok->cur - 1; / Leave '\n' out of the string */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1456 tok->cont_line = 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1457 return NEWLINE;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1458 }
37879 f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair Thomas Wouters <thomas@python.org> parents: 37704 diff changeset	1459
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1460 /* Period or number starting with period? */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1461 if (c == '.') {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1462 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1463 if (isdigit(c)) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1464 goto fraction;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1465 } else if (c == '.') {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1466 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1467 if (c == '.') {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1468 *p_start = tok->start;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1469 *p_end = tok->cur;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1470 return ELLIPSIS;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1471 } else {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1472 tok_backup(tok, c);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1473 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1474 tok_backup(tok, '.');
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1475 } else {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1476 tok_backup(tok, c);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1477 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1478 *p_start = tok->start;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1479 *p_end = tok->cur;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1480 return DOT;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1481 }
3333 4b5bff95804e changes for pow(*) and complex Guido van Rossum <guido@python.org>* parents: 3116 diff changeset	1482
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1483 /* Number */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1484 if (isdigit(c)) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1485 if (c == '0') {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1486 /* Hex, octal or binary -- maybe. */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1487 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1488 if (c == '.')
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1489 goto fraction;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1490 if (c == 'j' \|\| c == 'J')
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1491 goto imaginary;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1492 if (c == 'x' \|\| c == 'X') {
44782 ca2683a67943 Merged revisions 60080-60089,60091-60093 via svnmerge from Georg Brandl <georg@python.org> parents: 44442 diff changeset	1493
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1494 /* Hex */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1495 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1496 if (!isxdigit(c)) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1497 tok->done = E_TOKEN;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1498 tok_backup(tok, c);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1499 return ERRORTOKEN;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1500 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1501 do {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1502 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1503 } while (isxdigit(c));
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1504 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1505 else if (c == 'o' \|\| c == 'O') {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1506 /* Octal */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1507 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1508 if (c < '0' \|\| c >= '8') {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1509 tok->done = E_TOKEN;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1510 tok_backup(tok, c);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1511 return ERRORTOKEN;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1512 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1513 do {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1514 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1515 } while ('0' <= c && c < '8');
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1516 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1517 else if (c == 'b' \|\| c == 'B') {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1518 /* Binary */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1519 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1520 if (c != '0' && c != '1') {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1521 tok->done = E_TOKEN;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1522 tok_backup(tok, c);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1523 return ERRORTOKEN;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1524 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1525 do {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1526 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1527 } while (c == '0' \|\| c == '1');
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1528 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1529 else {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1530 int nonzero = 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1531 /* maybe old-style octal; c is first char of it */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1532 /* in any case, allow '0' as a literal */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1533 while (c == '0')
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1534 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1535 while (isdigit(c)) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1536 nonzero = 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1537 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1538 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1539 if (c == '.')
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1540 goto fraction;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1541 else if (c == 'e' \|\| c == 'E')
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1542 goto exponent;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1543 else if (c == 'j' \|\| c == 'J')
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1544 goto imaginary;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1545 else if (nonzero) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1546 tok->done = E_TOKEN;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1547 tok_backup(tok, c);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1548 return ERRORTOKEN;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1549 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1550 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1551 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1552 else {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1553 /* Decimal */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1554 do {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1555 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1556 } while (isdigit(c));
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1557 {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1558 /* Accept floating point numbers. */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1559 if (c == '.') {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1560 fraction:
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1561 /* Fraction */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1562 do {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1563 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1564 } while (isdigit(c));
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1565 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1566 if (c == 'e' \|\| c == 'E') {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1567 exponent:
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1568 /* Exponent part */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1569 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1570 if (c == '+' \|\| c == '-')
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1571 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1572 if (!isdigit(c)) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1573 tok->done = E_TOKEN;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1574 tok_backup(tok, c);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1575 return ERRORTOKEN;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1576 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1577 do {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1578 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1579 } while (isdigit(c));
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1580 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1581 if (c == 'j' \|\| c == 'J')
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1582 /* Imaginary part */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1583 imaginary:
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1584 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1585 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1586 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1587 tok_backup(tok, c);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1588 *p_start = tok->start;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1589 *p_end = tok->cur;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1590 return NUMBER;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1591 }
5265 3c1788ab9761 Support for alternative string quotes (a"xx", b"xx", c"xx", ...). Guido van Rossum <guido@python.org> parents: 4839 diff changeset	1592
3c1788ab9761 Support for alternative string quotes (a"xx", b"xx", c"xx", ...). Guido van Rossum <guido@python.org> parents: 4839 diff changeset	1593 letter_quote:
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1594 /* String */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1595 if (c == '\'' \|\| c == '"') {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1596 int quote = c;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1597 int quote_size = 1; /* 1 or 3 */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1598 int end_quote_size = 0;
43879 5590157f4fc6 Cleanup of tokenizer.c. Guido van Rossum <guido@python.org> parents: 43873 diff changeset	1599
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1600 /* Find the quote size and start of string */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1601 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1602 if (c == quote) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1603 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1604 if (c == quote)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1605 quote_size = 3;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1606 else
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1607 end_quote_size = 1; /* empty string found */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1608 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1609 if (c != quote)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1610 tok_backup(tok, c);
43879 5590157f4fc6 Cleanup of tokenizer.c. Guido van Rossum <guido@python.org> parents: 43873 diff changeset	1611
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1612 /* Get rest of string */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1613 while (end_quote_size != quote_size) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1614 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1615 if (c == EOF) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1616 if (quote_size == 3)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1617 tok->done = E_EOFS;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1618 else
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1619 tok->done = E_EOLS;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1620 tok->cur = tok->inp;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1621 return ERRORTOKEN;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1622 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1623 if (quote_size == 1 && c == '\n') {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1624 tok->done = E_EOLS;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1625 tok->cur = tok->inp;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1626 return ERRORTOKEN;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1627 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1628 if (c == quote)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1629 end_quote_size += 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1630 else {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1631 end_quote_size = 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1632 if (c == '\\')
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1633 c = tok_nextc(tok); /* skip escaped char */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1634 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1635 }
43879 5590157f4fc6 Cleanup of tokenizer.c. Guido van Rossum <guido@python.org> parents: 43873 diff changeset	1636
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1637 *p_start = tok->start;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1638 *p_end = tok->cur;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1639 return STRING;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1640 }
37879 f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair Thomas Wouters <thomas@python.org> parents: 37704 diff changeset	1641
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1642 /* Line continuation */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1643 if (c == '\\') {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1644 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1645 if (c != '\n') {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1646 tok->done = E_LINECONT;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1647 tok->cur = tok->inp;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1648 return ERRORTOKEN;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1649 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1650 tok->cont_line = 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1651 goto again; /* Read next line */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1652 }
37879 f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair Thomas Wouters <thomas@python.org> parents: 37704 diff changeset	1653
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1654 /* Check for two-character token */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1655 {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1656 int c2 = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1657 int token = PyToken_TwoChars(c, c2);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1658 if (token != OP) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1659 int c3 = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1660 int token3 = PyToken_ThreeChars(c, c2, c3);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1661 if (token3 != OP) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1662 token = token3;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1663 } else {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1664 tok_backup(tok, c3);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1665 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1666 *p_start = tok->start;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1667 *p_end = tok->cur;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1668 return token;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1669 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1670 tok_backup(tok, c2);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1671 }
37879 f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair Thomas Wouters <thomas@python.org> parents: 37704 diff changeset	1672
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1673 /* Keep track of parentheses nesting level */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1674 switch (c) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1675 case '(':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1676 case '[':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1677 case '{':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1678 tok->level++;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1679 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1680 case ')':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1681 case ']':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1682 case '}':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1683 tok->level--;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1684 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1685 }
37879 f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair Thomas Wouters <thomas@python.org> parents: 37704 diff changeset	1686
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1687 /* Punctuation character */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1688 *p_start = tok->start;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1689 *p_end = tok->cur;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1690 return PyToken_OneChar(c);
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1691 }
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1692
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	1693 int
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	1694 PyTokenizer_Get(struct tok_state tok, char p_start, char *p_end)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	1695 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1696 int result = tok_get(tok, p_start, p_end);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1697 if (tok->decoding_erred) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1698 result = ERRORTOKEN;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1699 tok->done = E_DECODE;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1700 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1701 return result;
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	1702 }
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1703
69148 7b8d625eb6e4 Issue #9319: Include the filename in "Non-UTF8 code ..." syntax error. Victor Stinner <victor.stinner@haypocalc.com> parents: 69147 diff changeset	1704 /* Get the encoding of a Python file. Check for the coding cookie and check if
7b8d625eb6e4 Issue #9319: Include the filename in "Non-UTF8 code ..." syntax error. Victor Stinner <victor.stinner@haypocalc.com> parents: 69147 diff changeset	1705 the file starts with a BOM.
43609 d378acdc30ee Fix PyTokenizer_FindEncoding() for OS X 10.4. Turns out that seeking to the Brett Cannon <bcannon@gmail.com> parents: 43607 diff changeset	1706
69148 7b8d625eb6e4 Issue #9319: Include the filename in "Non-UTF8 code ..." syntax error. Victor Stinner <victor.stinner@haypocalc.com> parents: 69147 diff changeset	1707 PyTokenizer_FindEncodingFilename() returns NULL when it can't find the
7b8d625eb6e4 Issue #9319: Include the filename in "Non-UTF8 code ..." syntax error. Victor Stinner <victor.stinner@haypocalc.com> parents: 69147 diff changeset	1708 encoding in the first or second line of the file (in which case the encoding
7b8d625eb6e4 Issue #9319: Include the filename in "Non-UTF8 code ..." syntax error. Victor Stinner <victor.stinner@haypocalc.com> parents: 69147 diff changeset	1709 should be assumed to be UTF-8).
7b8d625eb6e4 Issue #9319: Include the filename in "Non-UTF8 code ..." syntax error. Victor Stinner <victor.stinner@haypocalc.com> parents: 69147 diff changeset	1710
7b8d625eb6e4 Issue #9319: Include the filename in "Non-UTF8 code ..." syntax error. Victor Stinner <victor.stinner@haypocalc.com> parents: 69147 diff changeset	1711 The char* returned is malloc'ed via PyMem_MALLOC() and thus must be freed
7b8d625eb6e4 Issue #9319: Include the filename in "Non-UTF8 code ..." syntax error. Victor Stinner <victor.stinner@haypocalc.com> parents: 69147 diff changeset	1712 by the caller. */
7b8d625eb6e4 Issue #9319: Include the filename in "Non-UTF8 code ..." syntax error. Victor Stinner <victor.stinner@haypocalc.com> parents: 69147 diff changeset	1713
43605 a2bb76d11d66 Patch 1267 by Christian Heimes. Guido van Rossum <guido@python.org> parents: 43523 diff changeset	1714 char *
69148 7b8d625eb6e4 Issue #9319: Include the filename in "Non-UTF8 code ..." syntax error. Victor Stinner <victor.stinner@haypocalc.com> parents: 69147 diff changeset	1715 PyTokenizer_FindEncodingFilename(int fd, PyObject *filename)
43632 0067a6ab4854 Issue 1267, continued. Guido van Rossum <guido@python.org> parents: 43622 diff changeset	1716 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1717 struct tok_state *tok;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1718 FILE *fp;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1719 char p_start =NULL , p_end =NULL , *encoding = NULL;
43605 a2bb76d11d66 Patch 1267 by Christian Heimes. Guido van Rossum <guido@python.org> parents: 43523 diff changeset	1720
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1721 fd = dup(fd);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1722 if (fd < 0) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1723 return NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1724 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1725 fp = fdopen(fd, "r");
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1726 if (fp == NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1727 return NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1728 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1729 tok = PyTokenizer_FromFile(fp, NULL, NULL, NULL);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1730 if (tok == NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1731 fclose(fp);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1732 return NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1733 }
69147 6e9dc970ac0e Issue #10785: Store the filename as Unicode in the Python parser. Victor Stinner <victor.stinner@haypocalc.com> parents: 68257 diff changeset	1734 #ifndef PGEN
69148 7b8d625eb6e4 Issue #9319: Include the filename in "Non-UTF8 code ..." syntax error. Victor Stinner <victor.stinner@haypocalc.com> parents: 69147 diff changeset	1735 if (filename != NULL) {
7b8d625eb6e4 Issue #9319: Include the filename in "Non-UTF8 code ..." syntax error. Victor Stinner <victor.stinner@haypocalc.com> parents: 69147 diff changeset	1736 Py_INCREF(filename);
7b8d625eb6e4 Issue #9319: Include the filename in "Non-UTF8 code ..." syntax error. Victor Stinner <victor.stinner@haypocalc.com> parents: 69147 diff changeset	1737 tok->filename = filename;
7b8d625eb6e4 Issue #9319: Include the filename in "Non-UTF8 code ..." syntax error. Victor Stinner <victor.stinner@haypocalc.com> parents: 69147 diff changeset	1738 }
7b8d625eb6e4 Issue #9319: Include the filename in "Non-UTF8 code ..." syntax error. Victor Stinner <victor.stinner@haypocalc.com> parents: 69147 diff changeset	1739 else {
7b8d625eb6e4 Issue #9319: Include the filename in "Non-UTF8 code ..." syntax error. Victor Stinner <victor.stinner@haypocalc.com> parents: 69147 diff changeset	1740 tok->filename = PyUnicode_FromString("<string>");
7b8d625eb6e4 Issue #9319: Include the filename in "Non-UTF8 code ..." syntax error. Victor Stinner <victor.stinner@haypocalc.com> parents: 69147 diff changeset	1741 if (tok->filename == NULL) {
7b8d625eb6e4 Issue #9319: Include the filename in "Non-UTF8 code ..." syntax error. Victor Stinner <victor.stinner@haypocalc.com> parents: 69147 diff changeset	1742 fclose(fp);
7b8d625eb6e4 Issue #9319: Include the filename in "Non-UTF8 code ..." syntax error. Victor Stinner <victor.stinner@haypocalc.com> parents: 69147 diff changeset	1743 PyTokenizer_Free(tok);
7b8d625eb6e4 Issue #9319: Include the filename in "Non-UTF8 code ..." syntax error. Victor Stinner <victor.stinner@haypocalc.com> parents: 69147 diff changeset	1744 return encoding;
7b8d625eb6e4 Issue #9319: Include the filename in "Non-UTF8 code ..." syntax error. Victor Stinner <victor.stinner@haypocalc.com> parents: 69147 diff changeset	1745 }
7b8d625eb6e4 Issue #9319: Include the filename in "Non-UTF8 code ..." syntax error. Victor Stinner <victor.stinner@haypocalc.com> parents: 69147 diff changeset	1746 }
69147 6e9dc970ac0e Issue #10785: Store the filename as Unicode in the Python parser. Victor Stinner <victor.stinner@haypocalc.com> parents: 68257 diff changeset	1747 #endif
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1748 while (tok->lineno < 2 && tok->done == E_OK) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1749 PyTokenizer_Get(tok, &p_start, &p_end);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1750 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1751 fclose(fp);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1752 if (tok->encoding) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1753 encoding = (char *)PyMem_MALLOC(strlen(tok->encoding) + 1);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1754 if (encoding)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1755 strcpy(encoding, tok->encoding);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1756 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1757 PyTokenizer_Free(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1758 return encoding;
43605 a2bb76d11d66 Patch 1267 by Christian Heimes. Guido van Rossum <guido@python.org> parents: 43523 diff changeset	1759 }
43292 876b7ef4775d Merged revisions 57778-58052 via svnmerge from Thomas Wouters <thomas@python.org> parents: 43225 diff changeset	1760
69148 7b8d625eb6e4 Issue #9319: Include the filename in "Non-UTF8 code ..." syntax error. Victor Stinner <victor.stinner@haypocalc.com> parents: 69147 diff changeset	1761 char *
7b8d625eb6e4 Issue #9319: Include the filename in "Non-UTF8 code ..." syntax error. Victor Stinner <victor.stinner@haypocalc.com> parents: 69147 diff changeset	1762 PyTokenizer_FindEncoding(int fd)
7b8d625eb6e4 Issue #9319: Include the filename in "Non-UTF8 code ..." syntax error. Victor Stinner <victor.stinner@haypocalc.com> parents: 69147 diff changeset	1763 {
7b8d625eb6e4 Issue #9319: Include the filename in "Non-UTF8 code ..." syntax error. Victor Stinner <victor.stinner@haypocalc.com> parents: 69147 diff changeset	1764 return PyTokenizer_FindEncodingFilename(fd, NULL);
7b8d625eb6e4 Issue #9319: Include the filename in "Non-UTF8 code ..." syntax error. Victor Stinner <victor.stinner@haypocalc.com> parents: 69147 diff changeset	1765 }
7b8d625eb6e4 Issue #9319: Include the filename in "Non-UTF8 code ..." syntax error. Victor Stinner <victor.stinner@haypocalc.com> parents: 69147 diff changeset	1766
4839 4567e158d20c Rename DEBUG macro to Py_DEBUG Guido van Rossum <guido@python.org> parents: 4577 diff changeset	1767 #ifdef Py_DEBUG
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1768
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1769 void
13659 e78153e179fb Mass ANSIfication. Thomas Wouters <thomas@python.org> parents: 13539 diff changeset	1770 tok_dump(int type, char start, char end)
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1771 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1772 printf("%s", _PyParser_TokenNames[type]);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1773 if (type == NAME \|\| type == NUMBER \|\| type == STRING \|\| type == OP)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1774 printf("(%.*s)", (int)(end - start), start);
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1775 }
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1776
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1777 #endif

Mercurial > cpython-lru_cache-weakref

annotate Parser/tokenizer.c @ 84007:dbfbdf2b5c19 3.3