cpython-lru_cache-weakref: Parser/tokenizer.c annotate

annotate Parser/tokenizer.c @ 70817:dd775b1561ba

Fix markup.

author	Georg Brandl <georg@python.org>
date	Tue, 14 Jun 2011 21:09:55 +0200
parents	4658762a77cf
children	8beaa9a37387

rev	line source
145 853b736a6a9e Added copyright notice. Guido van Rossum <guido@python.org> parents: 96 diff changeset	1
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	2 /* Tokenizer implementation */
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	3
23029 8291473fde3e Mass checkin of universal newline support. Jack Jansen <jack.jansen@cwi.nl> parents: 19621 diff changeset	4 #include "Python.h"
86 a6934380c6e7 "Compiling" version Guido van Rossum <guido@python.org> parents: 67 diff changeset	5 #include "pgenheaders.h"
a6934380c6e7 "Compiling" version Guido van Rossum <guido@python.org> parents: 67 diff changeset	6
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	7 #include <ctype.h>
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	8 #include <assert.h>
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	9
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	10 #include "tokenizer.h"
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	11 #include "errcode.h"
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	12
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	13 #ifndef PGEN
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	14 #include "unicodeobject.h"
47302 811a80651232 Renamed files bytesobject.[ch] and stringobject.[ch] Christian Heimes <christian@cheimes.de> parents: 47301 diff changeset	15 #include "bytesobject.h"
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	16 #include "fileobject.h"
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	17 #include "codecs.h"
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	18 #include "abstract.h"
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	19 #endif /* PGEN */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	20
42013 cd125fe83051 Make identifiers str (not str8) objects throughout. Martin v. Löwis <martin@v.loewis.de> parents: 41970 diff changeset	21 #define is_potential_identifier_start(c) (\
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	22 (c >= 'a' && c <= 'z')\
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	23 \|\| (c >= 'A' && c <= 'Z')\
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	24 \|\| c == '_'\
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	25 \|\| (c >= 128))
42013 cd125fe83051 Make identifiers str (not str8) objects throughout. Martin v. Löwis <martin@v.loewis.de> parents: 41970 diff changeset	26
cd125fe83051 Make identifiers str (not str8) objects throughout. Martin v. Löwis <martin@v.loewis.de> parents: 41970 diff changeset	27 #define is_potential_identifier_char(c) (\
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	28 (c >= 'a' && c <= 'z')\
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	29 \|\| (c >= 'A' && c <= 'Z')\
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	30 \|\| (c >= '0' && c <= '9')\
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	31 \|\| c == '_'\
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	32 \|\| (c >= 128))
42013 cd125fe83051 Make identifiers str (not str8) objects throughout. Martin v. Löwis <martin@v.loewis.de> parents: 41970 diff changeset	33
25924 dc4a0336a2a3 Patch #512981: Update readline input stream on sys.stdin/out change. Martin v. Löwis <martin@v.loewis.de> parents: 25101 diff changeset	34 extern char PyOS_Readline(FILE , FILE , char );
1835 d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string Guido van Rossum <guido@python.org> parents: 1364 diff changeset	35 /* Return malloc'ed string including trailing \n;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string Guido van Rossum <guido@python.org> parents: 1364 diff changeset	36 empty malloc'ed string for EOF;
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string Guido van Rossum <guido@python.org> parents: 1364 diff changeset	37 NULL if interrupted */
d799889c01e3 * Parser/tokenizer.c: backup over illegal newline in string Guido van Rossum <guido@python.org> parents: 1364 diff changeset	38
678 205b947b1d3c Make tabs always 8 spaces wide -- it's more portable. Guido van Rossum <guido@python.org> parents: 477 diff changeset	39 /* Don't ever change this -- it would break the portability of Python code */
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	40 #define TABSIZE 8
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	41
86 a6934380c6e7 "Compiling" version Guido van Rossum <guido@python.org> parents: 67 diff changeset	42 /* Forward */
13447 a89f7f268de4 Nuke all remaining occurrences of Py_PROTO and Py_FPROTO. Tim Peters <tim.peters@gmail.com> parents: 13256 diff changeset	43 static struct tok_state *tok_new(void);
a89f7f268de4 Nuke all remaining occurrences of Py_PROTO and Py_FPROTO. Tim Peters <tim.peters@gmail.com> parents: 13256 diff changeset	44 static int tok_nextc(struct tok_state *tok);
a89f7f268de4 Nuke all remaining occurrences of Py_PROTO and Py_FPROTO. Tim Peters <tim.peters@gmail.com> parents: 13256 diff changeset	45 static void tok_backup(struct tok_state *tok, int c);
86 a6934380c6e7 "Compiling" version Guido van Rossum <guido@python.org> parents: 67 diff changeset	46
43607 7593436058dd Plug a memory leak where a struct tok_state was not being freed. Brett Cannon <bcannon@gmail.com> parents: 43605 diff changeset	47
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	48 /* Token names */
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	49
5379 1c9a84e0981d Another directory quickly renamed. Guido van Rossum <guido@python.org> parents: 5341 diff changeset	50 char *_PyParser_TokenNames[] = {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	51 "ENDMARKER",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	52 "NAME",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	53 "NUMBER",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	54 "STRING",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	55 "NEWLINE",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	56 "INDENT",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	57 "DEDENT",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	58 "LPAR",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	59 "RPAR",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	60 "LSQB",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	61 "RSQB",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	62 "COLON",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	63 "COMMA",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	64 "SEMI",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	65 "PLUS",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	66 "MINUS",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	67 "STAR",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	68 "SLASH",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	69 "VBAR",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	70 "AMPER",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	71 "LESS",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	72 "GREATER",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	73 "EQUAL",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	74 "DOT",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	75 "PERCENT",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	76 "LBRACE",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	77 "RBRACE",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	78 "EQEQUAL",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	79 "NOTEQUAL",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	80 "LESSEQUAL",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	81 "GREATEREQUAL",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	82 "TILDE",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	83 "CIRCUMFLEX",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	84 "LEFTSHIFT",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	85 "RIGHTSHIFT",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	86 "DOUBLESTAR",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	87 "PLUSEQUAL",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	88 "MINEQUAL",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	89 "STAREQUAL",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	90 "SLASHEQUAL",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	91 "PERCENTEQUAL",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	92 "AMPEREQUAL",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	93 "VBAREQUAL",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	94 "CIRCUMFLEXEQUAL",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	95 "LEFTSHIFTEQUAL",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	96 "RIGHTSHIFTEQUAL",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	97 "DOUBLESTAREQUAL",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	98 "DOUBLESLASH",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	99 "DOUBLESLASHEQUAL",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	100 "AT",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	101 "RARROW",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	102 "ELLIPSIS",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	103 /* This table must match the #defines in token.h! */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	104 "OP",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	105 "<ERRORTOKEN>",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	106 "<N_TOKENS>"
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	107 };
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	108
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	109
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	110 /* Create and initialize a new tok_state structure */
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	111
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	112 static struct tok_state *
13659 e78153e179fb Mass ANSIfication. Thomas Wouters <thomas@python.org> parents: 13539 diff changeset	113 tok_new(void)
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	114 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	115 struct tok_state tok = (struct tok_state )PyMem_MALLOC(
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	116 sizeof(struct tok_state));
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	117 if (tok == NULL)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	118 return NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	119 tok->buf = tok->cur = tok->end = tok->inp = tok->start = NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	120 tok->done = E_OK;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	121 tok->fp = NULL;
61466 cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	122 tok->input = NULL;
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	123 tok->tabsize = TABSIZE;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	124 tok->indent = 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	125 tok->indstack[0] = 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	126 tok->atbol = 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	127 tok->pendin = 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	128 tok->prompt = tok->nextprompt = NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	129 tok->lineno = 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	130 tok->level = 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	131 tok->altwarning = 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	132 tok->alterror = 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	133 tok->alttabsize = 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	134 tok->altindstack[0] = 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	135 tok->decoding_state = STATE_INIT;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	136 tok->decoding_erred = 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	137 tok->read_coding_spec = 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	138 tok->enc = NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	139 tok->encoding = NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	140 tok->cont_line = 0;
24544 0bd521eb714e Make pgen compile with pydebug. Duplicate normalized names, as it may Martin v. Löwis <martin@v.loewis.de> parents: 24543 diff changeset	141 #ifndef PGEN
69147 6e9dc970ac0e Issue #10785: Store the filename as Unicode in the Python parser. Victor Stinner <victor.stinner@haypocalc.com> parents: 68257 diff changeset	142 tok->filename = NULL;
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	143 tok->decoding_readline = NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	144 tok->decoding_buffer = NULL;
24544 0bd521eb714e Make pgen compile with pydebug. Duplicate normalized names, as it may Martin v. Löwis <martin@v.loewis.de> parents: 24543 diff changeset	145 #endif
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	146 return tok;
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	147 }
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	148
57206 b2d90c1c637d Merged revisions 76230 via svnmerge from Benjamin Peterson <benjamin@python.org> parents: 56594 diff changeset	149 static char *
b2d90c1c637d Merged revisions 76230 via svnmerge from Benjamin Peterson <benjamin@python.org> parents: 56594 diff changeset	150 new_string(const char *s, Py_ssize_t len)
b2d90c1c637d Merged revisions 76230 via svnmerge from Benjamin Peterson <benjamin@python.org> parents: 56594 diff changeset	151 {
61466 cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	152 char* result = (char *)PyMem_MALLOC(len + 1);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	153 if (result != NULL) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	154 memcpy(result, s, len);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	155 result[len] = '\0';
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	156 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	157 return result;
57206 b2d90c1c637d Merged revisions 76230 via svnmerge from Benjamin Peterson <benjamin@python.org> parents: 56594 diff changeset	158 }
b2d90c1c637d Merged revisions 76230 via svnmerge from Benjamin Peterson <benjamin@python.org> parents: 56594 diff changeset	159
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	160 #ifdef PGEN
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	161
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	162 static char *
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	163 decoding_fgets(char s, int size, struct tok_state tok)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	164 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	165 return fgets(s, size, tok->fp);
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	166 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	167
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	168 static int
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	169 decoding_feof(struct tok_state *tok)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	170 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	171 return feof(tok->fp);
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	172 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	173
57206 b2d90c1c637d Merged revisions 76230 via svnmerge from Benjamin Peterson <benjamin@python.org> parents: 56594 diff changeset	174 static char *
b2d90c1c637d Merged revisions 76230 via svnmerge from Benjamin Peterson <benjamin@python.org> parents: 56594 diff changeset	175 decode_str(const char str, int exec_input, struct tok_state tok)
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	176 {
61466 cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	177 return new_string(str, strlen(str));
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	178 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	179
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	180 #else /* PGEN */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	181
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	182 static char *
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	183 error_ret(struct tok_state tok) / XXX */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	184 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	185 tok->decoding_erred = 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	186 if (tok->fp != NULL && tok->buf != NULL) /* see PyTokenizer_Free */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	187 PyMem_FREE(tok->buf);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	188 tok->buf = NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	189 return NULL; /* as if it were EOF */
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	190 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	191
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	192
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	193 static char *
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	194 get_normal_name(char s) / for utf-8 and latin-1 */
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	195 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	196 char buf[13];
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	197 int i;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	198 for (i = 0; i < 12; i++) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	199 int c = s[i];
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	200 if (c == '\0')
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	201 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	202 else if (c == '_')
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	203 buf[i] = '-';
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	204 else
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	205 buf[i] = tolower(c);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	206 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	207 buf[i] = '\0';
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	208 if (strcmp(buf, "utf-8") == 0 \|\|
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	209 strncmp(buf, "utf-8-", 6) == 0)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	210 return "utf-8";
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	211 else if (strcmp(buf, "latin-1") == 0 \|\|
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	212 strcmp(buf, "iso-8859-1") == 0 \|\|
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	213 strcmp(buf, "iso-latin-1") == 0 \|\|
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	214 strncmp(buf, "latin-1-", 8) == 0 \|\|
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	215 strncmp(buf, "iso-8859-1-", 11) == 0 \|\|
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	216 strncmp(buf, "iso-latin-1-", 12) == 0)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	217 return "iso-8859-1";
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	218 else
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	219 return s;
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	220 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	221
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	222 /* Return the coding spec in S, or NULL if none is found. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	223
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	224 static char *
36501 7fdc639bc5b4 Merge ssize_t branch. Martin v. Löwis <martin@v.loewis.de> parents: 36090 diff changeset	225 get_coding_spec(const char *s, Py_ssize_t size)
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	226 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	227 Py_ssize_t i;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	228 /* Coding spec must be in a comment, and that comment must be
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	229 * the only statement on the source code line. */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	230 for (i = 0; i < size - 6; i++) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	231 if (s[i] == '#')
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	232 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	233 if (s[i] != ' ' && s[i] != '\t' && s[i] != '\014')
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	234 return NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	235 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	236 for (; i < size - 6; i++) { /* XXX inefficient search */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	237 const char* t = s + i;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	238 if (strncmp(t, "coding", 6) == 0) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	239 const char* begin = NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	240 t += 6;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	241 if (t[0] != ':' && t[0] != '=')
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	242 continue;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	243 do {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	244 t++;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	245 } while (t[0] == '\x20' \|\| t[0] == '\t');
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	246
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	247 begin = t;
61466 cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	248 while (Py_ISALNUM(t[0]) \|\|
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	249 t[0] == '-' \|\| t[0] == '_' \|\| t[0] == '.')
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	250 t++;
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	251
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	252 if (begin < t) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	253 char* r = new_string(begin, t - begin);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	254 char* q = get_normal_name(r);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	255 if (r != q) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	256 PyMem_FREE(r);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	257 r = new_string(q, strlen(q));
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	258 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	259 return r;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	260 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	261 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	262 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	263 return NULL;
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	264 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	265
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	266 /* Check whether the line contains a coding spec. If it does,
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	267 invoke the set_readline function for the new encoding.
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	268 This function receives the tok_state and the new encoding.
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	269 Return 1 on success, 0 on failure. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	270
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	271 static int
36501 7fdc639bc5b4 Merge ssize_t branch. Martin v. Löwis <martin@v.loewis.de> parents: 36090 diff changeset	272 check_coding_spec(const char* line, Py_ssize_t size, struct tok_state *tok,
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	273 int set_readline(struct tok_state , const char ))
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	274 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	275 char * cs;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	276 int r = 1;
25101 32331d26d7cf Removed reliance on gcc/C99 extension. Tim Peters <tim.peters@gmail.com> parents: 25098 diff changeset	277
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	278 if (tok->cont_line)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	279 /* It's a continuation line, so it can't be a coding spec. */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	280 return 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	281 cs = get_coding_spec(line, size);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	282 if (cs != NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	283 tok->read_coding_spec = 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	284 if (tok->encoding == NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	285 assert(tok->decoding_state == STATE_RAW);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	286 if (strcmp(cs, "utf-8") == 0) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	287 tok->encoding = cs;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	288 } else {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	289 r = set_readline(tok, cs);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	290 if (r) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	291 tok->encoding = cs;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	292 tok->decoding_state = STATE_NORMAL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	293 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	294 else
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	295 PyMem_FREE(cs);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	296 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	297 } else { /* then, compare cs with BOM */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	298 r = (strcmp(tok->encoding, cs) == 0);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	299 PyMem_FREE(cs);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	300 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	301 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	302 if (!r) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	303 cs = tok->encoding;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	304 if (!cs)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	305 cs = "with BOM";
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	306 PyErr_Format(PyExc_SyntaxError, "encoding problem: %s", cs);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	307 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	308 return r;
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	309 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	310
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	311 /* See whether the file starts with a BOM. If it does,
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	312 invoke the set_readline function with the new encoding.
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	313 Return 1 on success, 0 on failure. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	314
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	315 static int
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	316 check_bom(int get_char(struct tok_state *),
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	317 void unget_char(int, struct tok_state *),
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	318 int set_readline(struct tok_state , const char ),
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	319 struct tok_state *tok)
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	320 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	321 int ch1, ch2, ch3;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	322 ch1 = get_char(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	323 tok->decoding_state = STATE_RAW;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	324 if (ch1 == EOF) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	325 return 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	326 } else if (ch1 == 0xEF) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	327 ch2 = get_char(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	328 if (ch2 != 0xBB) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	329 unget_char(ch2, tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	330 unget_char(ch1, tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	331 return 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	332 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	333 ch3 = get_char(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	334 if (ch3 != 0xBF) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	335 unget_char(ch3, tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	336 unget_char(ch2, tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	337 unget_char(ch1, tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	338 return 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	339 }
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	340 #if 0
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	341 /* Disable support for UTF-16 BOMs until a decision
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	342 is made whether this needs to be supported. */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	343 } else if (ch1 == 0xFE) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	344 ch2 = get_char(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	345 if (ch2 != 0xFF) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	346 unget_char(ch2, tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	347 unget_char(ch1, tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	348 return 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	349 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	350 if (!set_readline(tok, "utf-16-be"))
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	351 return 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	352 tok->decoding_state = STATE_NORMAL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	353 } else if (ch1 == 0xFF) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	354 ch2 = get_char(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	355 if (ch2 != 0xFE) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	356 unget_char(ch2, tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	357 unget_char(ch1, tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	358 return 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	359 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	360 if (!set_readline(tok, "utf-16-le"))
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	361 return 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	362 tok->decoding_state = STATE_NORMAL;
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	363 #endif
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	364 } else {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	365 unget_char(ch1, tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	366 return 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	367 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	368 if (tok->encoding != NULL)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	369 PyMem_FREE(tok->encoding);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	370 tok->encoding = new_string("utf-8", 5); /* resulting is in utf-8 */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	371 /* No need to set_readline: input is already utf-8 */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	372 return 1;
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	373 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	374
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	375 /* Read a line of text from TOK into S, using the stream in TOK.
35231 763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file Walter Dörwald <walter@livinglogic.de> parents: 34664 diff changeset	376 Return NULL on failure, else S.
37879 f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair Thomas Wouters <thomas@python.org> parents: 37704 diff changeset	377
35231 763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file Walter Dörwald <walter@livinglogic.de> parents: 34664 diff changeset	378 On entry, tok->decoding_buffer will be one of:
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file Walter Dörwald <walter@livinglogic.de> parents: 34664 diff changeset	379 1) NULL: need to call tok->decoding_readline to get a new line
763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file Walter Dörwald <walter@livinglogic.de> parents: 34664 diff changeset	380 2) PyUnicodeObject *: decoding_feof has called tok->decoding_readline and
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	381 stored the result in tok->decoding_buffer
47300 96b34263e448 Renamed PyBytes to PyByteArray Christian Heimes <christian@cheimes.de> parents: 46664 diff changeset	382 3) PyByteArrayObject *: previous call to fp_readl did not have enough room
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	383 (in the s buffer) to copy entire contents of the line read
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	384 by tok->decoding_readline. tok->decoding_buffer has the overflow.
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	385 In this case, fp_readl is called in a loop (with an expanded buffer)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	386 until the buffer ends with a '\n' (or until the end of the file is
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	387 reached): see tok_nextc and its calls to decoding_fgets.
35231 763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file Walter Dörwald <walter@livinglogic.de> parents: 34664 diff changeset	388 */
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	389
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	390 static char *
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	391 fp_readl(char s, int size, struct tok_state tok)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	392 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	393 PyObject* bufobj;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	394 const char *buf;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	395 Py_ssize_t buflen;
35231 763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file Walter Dörwald <walter@livinglogic.de> parents: 34664 diff changeset	396
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	397 /* Ask for one less byte so we can terminate it */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	398 assert(size > 0);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	399 size--;
35231 763e85b107b4 Apply SF patch #1101726: Fix buffer overrun in tokenizer.c when a source file Walter Dörwald <walter@livinglogic.de> parents: 34664 diff changeset	400
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	401 if (tok->decoding_buffer) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	402 bufobj = tok->decoding_buffer;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	403 Py_INCREF(bufobj);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	404 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	405 else
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	406 {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	407 bufobj = PyObject_CallObject(tok->decoding_readline, NULL);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	408 if (bufobj == NULL)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	409 goto error;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	410 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	411 if (PyUnicode_CheckExact(bufobj))
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	412 {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	413 buf = _PyUnicode_AsStringAndSize(bufobj, &buflen);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	414 if (buf == NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	415 goto error;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	416 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	417 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	418 else
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	419 {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	420 buf = PyByteArray_AsString(bufobj);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	421 if (buf == NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	422 goto error;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	423 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	424 buflen = PyByteArray_GET_SIZE(bufobj);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	425 }
43873 1739869cbc6e Correction for issue1134: all source files with a coding spec, except latin-1 Amaury Forgeot d'Arc <amauryfa@gmail.com> parents: 43777 diff changeset	426
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	427 Py_XDECREF(tok->decoding_buffer);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	428 if (buflen > size) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	429 /* Too many chars, the rest goes into tok->decoding_buffer */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	430 tok->decoding_buffer = PyByteArray_FromStringAndSize(buf+size,
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	431 buflen-size);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	432 if (tok->decoding_buffer == NULL)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	433 goto error;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	434 buflen = size;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	435 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	436 else
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	437 tok->decoding_buffer = NULL;
43873 1739869cbc6e Correction for issue1134: all source files with a coding spec, except latin-1 Amaury Forgeot d'Arc <amauryfa@gmail.com> parents: 43777 diff changeset	438
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	439 memcpy(s, buf, buflen);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	440 s[buflen] = '\0';
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	441 if (buflen == 0) /* EOF */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	442 s = NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	443 Py_DECREF(bufobj);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	444 return s;
42551 b6625d2b378a Fix refleaks from execfile('file that contains a # coding: line') Neal Norwitz <nnorwitz@gmail.com> parents: 42548 diff changeset	445
b6625d2b378a Fix refleaks from execfile('file that contains a # coding: line') Neal Norwitz <nnorwitz@gmail.com> parents: 42548 diff changeset	446 error:
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	447 Py_XDECREF(bufobj);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	448 return error_ret(tok);
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	449 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	450
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	451 /* Set the readline function for TOK to a StreamReader's
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	452 readline function. The StreamReader is named ENC.
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	453
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	454 This function is called from check_bom and check_coding_spec.
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	455
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	456 ENC is usually identical to the future value of tok->encoding,
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	457 except for the (currently unsupported) case of UTF-16.
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	458
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	459 Return 1 on success, 0 on failure. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	460
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	461 static int
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	462 fp_setreadl(struct tok_state tok, const char enc)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	463 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	464 PyObject readline = NULL, stream = NULL, *io = NULL;
65311 3d9ef098daf2 Issue #10095: fp_setreadl() doesn't reopen the file, reuse instead the file Victor Stinner <victor.stinner@haypocalc.com> parents: 61466 diff changeset	465 int fd;
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	466
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	467 io = PyImport_ImportModuleNoBlock("io");
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	468 if (io == NULL)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	469 goto cleanup;
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	470
65311 3d9ef098daf2 Issue #10095: fp_setreadl() doesn't reopen the file, reuse instead the file Victor Stinner <victor.stinner@haypocalc.com> parents: 61466 diff changeset	471 fd = fileno(tok->fp);
3d9ef098daf2 Issue #10095: fp_setreadl() doesn't reopen the file, reuse instead the file Victor Stinner <victor.stinner@haypocalc.com> parents: 61466 diff changeset	472 if (lseek(fd, 0, SEEK_SET) == (off_t)-1) {
3d9ef098daf2 Issue #10095: fp_setreadl() doesn't reopen the file, reuse instead the file Victor Stinner <victor.stinner@haypocalc.com> parents: 61466 diff changeset	473 PyErr_SetFromErrnoWithFilename(PyExc_OSError, NULL);
3d9ef098daf2 Issue #10095: fp_setreadl() doesn't reopen the file, reuse instead the file Victor Stinner <victor.stinner@haypocalc.com> parents: 61466 diff changeset	474 goto cleanup;
3d9ef098daf2 Issue #10095: fp_setreadl() doesn't reopen the file, reuse instead the file Victor Stinner <victor.stinner@haypocalc.com> parents: 61466 diff changeset	475 }
3d9ef098daf2 Issue #10095: fp_setreadl() doesn't reopen the file, reuse instead the file Victor Stinner <victor.stinner@haypocalc.com> parents: 61466 diff changeset	476
3d9ef098daf2 Issue #10095: fp_setreadl() doesn't reopen the file, reuse instead the file Victor Stinner <victor.stinner@haypocalc.com> parents: 61466 diff changeset	477 stream = PyObject_CallMethod(io, "open", "isisOOO",
3d9ef098daf2 Issue #10095: fp_setreadl() doesn't reopen the file, reuse instead the file Victor Stinner <victor.stinner@haypocalc.com> parents: 61466 diff changeset	478 fd, "r", -1, enc, Py_None, Py_None, Py_False);
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	479 if (stream == NULL)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	480 goto cleanup;
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	481
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	482 Py_XDECREF(tok->decoding_readline);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	483 readline = PyObject_GetAttrString(stream, "readline");
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	484 tok->decoding_readline = readline;
41970 b3900a51346b tokenizer.c: make coding markup work again. Guido van Rossum <guido@python.org> parents: 41670 diff changeset	485
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	486 /* The file has been reopened; parsing will restart from
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	487 * the beginning of the file, we have to reset the line number.
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	488 * But this function has been called from inside tok_nextc() which
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	489 * will increment lineno before it returns. So we set it -1 so that
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	490 * the next call to tok_nextc() will start with tok->lineno == 0.
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	491 */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	492 tok->lineno = -1;
49484 0b7e023803fb Issues #2384 and #3975: Tracebacks were not correctly printed when the source file Amaury Forgeot d'Arc <amauryfa@gmail.com> parents: 49021 diff changeset	493
41970 b3900a51346b tokenizer.c: make coding markup work again. Guido van Rossum <guido@python.org> parents: 41670 diff changeset	494 cleanup:
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	495 Py_XDECREF(stream);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	496 Py_XDECREF(io);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	497 return readline != NULL;
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	498 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	499
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	500 /* Fetch the next byte from TOK. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	501
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	502 static int fp_getc(struct tok_state *tok) {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	503 return getc(tok->fp);
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	504 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	505
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	506 /* Unfetch the last byte back into TOK. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	507
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	508 static void fp_ungetc(int c, struct tok_state *tok) {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	509 ungetc(c, tok->fp);
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	510 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	511
42403 7e8469617b1c Implement PEP 3120. Martin v. Löwis <martin@v.loewis.de> parents: 42084 diff changeset	512 /* Check whether the characters at s start a valid
7e8469617b1c Implement PEP 3120. Martin v. Löwis <martin@v.loewis.de> parents: 42084 diff changeset	513 UTF-8 sequence. Return the number of characters forming
7e8469617b1c Implement PEP 3120. Martin v. Löwis <martin@v.loewis.de> parents: 42084 diff changeset	514 the sequence if yes, 0 if not. */
7e8469617b1c Implement PEP 3120. Martin v. Löwis <martin@v.loewis.de> parents: 42084 diff changeset	515 static int valid_utf8(const unsigned char* s)
7e8469617b1c Implement PEP 3120. Martin v. Löwis <martin@v.loewis.de> parents: 42084 diff changeset	516 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	517 int expected = 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	518 int length;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	519 if (*s < 0x80)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	520 /* single-byte code */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	521 return 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	522 if (*s < 0xc0)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	523 /* following byte */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	524 return 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	525 if (*s < 0xE0)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	526 expected = 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	527 else if (*s < 0xF0)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	528 expected = 2;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	529 else if (*s < 0xF8)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	530 expected = 3;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	531 else
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	532 return 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	533 length = expected + 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	534 for (; expected; expected--)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	535 if (s[expected] < 0x80 \|\| s[expected] >= 0xC0)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	536 return 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	537 return length;
42403 7e8469617b1c Implement PEP 3120. Martin v. Löwis <martin@v.loewis.de> parents: 42084 diff changeset	538 }
7e8469617b1c Implement PEP 3120. Martin v. Löwis <martin@v.loewis.de> parents: 42084 diff changeset	539
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	540 /* Read a line of input from TOK. Determine encoding
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	541 if necessary. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	542
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	543 static char *
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	544 decoding_fgets(char s, int size, struct tok_state tok)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	545 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	546 char *line = NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	547 int badchar = 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	548 for (;;) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	549 if (tok->decoding_state == STATE_NORMAL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	550 /* We already have a codec associated with
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	551 this input. */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	552 line = fp_readl(s, size, tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	553 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	554 } else if (tok->decoding_state == STATE_RAW) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	555 /* We want a 'raw' read. */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	556 line = Py_UniversalNewlineFgets(s, size,
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	557 tok->fp, NULL);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	558 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	559 } else {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	560 /* We have not yet determined the encoding.
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	561 If an encoding is found, use the file-pointer
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	562 reader functions from now on. */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	563 if (!check_bom(fp_getc, fp_ungetc, fp_setreadl, tok))
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	564 return error_ret(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	565 assert(tok->decoding_state != STATE_INIT);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	566 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	567 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	568 if (line != NULL && tok->lineno < 2 && !tok->read_coding_spec) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	569 if (!check_coding_spec(line, strlen(line), tok, fp_setreadl)) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	570 return error_ret(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	571 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	572 }
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	573 #ifndef PGEN
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	574 /* The default encoding is UTF-8, so make sure we don't have any
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	575 non-UTF-8 sequences in it. */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	576 if (line && !tok->encoding) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	577 unsigned char *c;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	578 int length;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	579 for (c = (unsigned char )line; c; c += length)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	580 if (!(length = valid_utf8(c))) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	581 badchar = *c;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	582 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	583 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	584 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	585 if (badchar) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	586 /* Need to add 1 to the line number, since this line
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	587 has not been counted, yet. */
69550 4658762a77cf Revert bb62908896fe, but keep the test Jesus Cea <jcea@jcea.es> parents: 69548 diff changeset	588 PyErr_Format(PyExc_SyntaxError,
4658762a77cf Revert bb62908896fe, but keep the test Jesus Cea <jcea@jcea.es> parents: 69548 diff changeset	589 "Non-UTF-8 code starting with '\\x%.2x' "
4658762a77cf Revert bb62908896fe, but keep the test Jesus Cea <jcea@jcea.es> parents: 69548 diff changeset	590 "in file %U on line %i, "
4658762a77cf Revert bb62908896fe, but keep the test Jesus Cea <jcea@jcea.es> parents: 69548 diff changeset	591 "but no encoding declared; "
4658762a77cf Revert bb62908896fe, but keep the test Jesus Cea <jcea@jcea.es> parents: 69548 diff changeset	592 "see http://python.org/dev/peps/pep-0263/ for details",
4658762a77cf Revert bb62908896fe, but keep the test Jesus Cea <jcea@jcea.es> parents: 69548 diff changeset	593 badchar, tok->filename, tok->lineno + 1);
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	594 return error_ret(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	595 }
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	596 #endif
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	597 return line;
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	598 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	599
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	600 static int
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	601 decoding_feof(struct tok_state *tok)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	602 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	603 if (tok->decoding_state != STATE_NORMAL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	604 return feof(tok->fp);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	605 } else {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	606 PyObject* buf = tok->decoding_buffer;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	607 if (buf == NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	608 buf = PyObject_CallObject(tok->decoding_readline, NULL);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	609 if (buf == NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	610 error_ret(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	611 return 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	612 } else {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	613 tok->decoding_buffer = buf;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	614 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	615 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	616 return PyObject_Length(buf) == 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	617 }
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	618 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	619
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	620 /* Fetch a byte from TOK, using the string buffer. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	621
37879 f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair Thomas Wouters <thomas@python.org> parents: 37704 diff changeset	622 static int
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair Thomas Wouters <thomas@python.org> parents: 37704 diff changeset	623 buf_getc(struct tok_state *tok) {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	624 return Py_CHARMASK(*tok->str++);
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	625 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	626
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	627 /* Unfetch a byte from TOK, using the string buffer. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	628
37879 f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair Thomas Wouters <thomas@python.org> parents: 37704 diff changeset	629 static void
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair Thomas Wouters <thomas@python.org> parents: 37704 diff changeset	630 buf_ungetc(int c, struct tok_state *tok) {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	631 tok->str--;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	632 assert(Py_CHARMASK(tok->str) == c); / tok->cur may point to read-only segment */
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	633 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	634
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	635 /* Set the readline function for TOK to ENC. For the string-based
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	636 tokenizer, this means to just record the encoding. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	637
37879 f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair Thomas Wouters <thomas@python.org> parents: 37704 diff changeset	638 static int
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair Thomas Wouters <thomas@python.org> parents: 37704 diff changeset	639 buf_setreadl(struct tok_state tok, const char enc) {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	640 tok->enc = enc;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	641 return 1;
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	642 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	643
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	644 /* Return a UTF-8 encoding Python string object from the
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	645 C byte string STR, which is encoded with ENC. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	646
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	647 static PyObject *
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	648 translate_into_utf8(const char* str, const char* enc) {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	649 PyObject *utf8;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	650 PyObject* buf = PyUnicode_Decode(str, strlen(str), enc, NULL);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	651 if (buf == NULL)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	652 return NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	653 utf8 = PyUnicode_AsUTF8String(buf);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	654 Py_DECREF(buf);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	655 return utf8;
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	656 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	657
57206 b2d90c1c637d Merged revisions 76230 via svnmerge from Benjamin Peterson <benjamin@python.org> parents: 56594 diff changeset	658
b2d90c1c637d Merged revisions 76230 via svnmerge from Benjamin Peterson <benjamin@python.org> parents: 56594 diff changeset	659 static char *
b2d90c1c637d Merged revisions 76230 via svnmerge from Benjamin Peterson <benjamin@python.org> parents: 56594 diff changeset	660 translate_newlines(const char s, int exec_input, struct tok_state tok) {
61466 cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	661 int skip_next_lf = 0, needed_length = strlen(s) + 2, final_length;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	662 char buf, current;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	663 char c = '\0';
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	664 buf = PyMem_MALLOC(needed_length);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	665 if (buf == NULL) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	666 tok->done = E_NOMEM;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	667 return NULL;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	668 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	669 for (current = buf; *s; s++, current++) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	670 c = *s;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	671 if (skip_next_lf) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	672 skip_next_lf = 0;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	673 if (c == '\n') {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	674 c = *++s;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	675 if (!c)
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	676 break;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	677 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	678 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	679 if (c == '\r') {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	680 skip_next_lf = 1;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	681 c = '\n';
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	682 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	683 *current = c;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	684 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	685 /* If this is exec input, add a newline to the end of the string if
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	686 there isn't one already. */
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	687 if (exec_input && c != '\n') {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	688 *current = '\n';
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	689 current++;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	690 }
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	691 *current = '\0';
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	692 final_length = current - buf + 1;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	693 if (final_length < needed_length && final_length)
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	694 /* should never fail */
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	695 buf = PyMem_REALLOC(buf, final_length);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	696 return buf;
57206 b2d90c1c637d Merged revisions 76230 via svnmerge from Benjamin Peterson <benjamin@python.org> parents: 56594 diff changeset	697 }
b2d90c1c637d Merged revisions 76230 via svnmerge from Benjamin Peterson <benjamin@python.org> parents: 56594 diff changeset	698
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	699 /* Decode a byte string STR for use as the buffer of TOK.
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	700 Look for encoding declarations inside STR, and record them
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	701 inside TOK. */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	702
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	703 static const char *
57206 b2d90c1c637d Merged revisions 76230 via svnmerge from Benjamin Peterson <benjamin@python.org> parents: 56594 diff changeset	704 decode_str(const char input, int single, struct tok_state tok)
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	705 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	706 PyObject* utf8 = NULL;
61466 cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	707 const char *str;
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	708 const char *s;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	709 const char *newl[2] = {NULL, NULL};
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	710 int lineno = 0;
61466 cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	711 tok->input = str = translate_newlines(input, single, tok);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	712 if (str == NULL)
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	713 return NULL;
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	714 tok->enc = NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	715 tok->str = str;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	716 if (!check_bom(buf_getc, buf_ungetc, buf_setreadl, tok))
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	717 return error_ret(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	718 str = tok->str; /* string after BOM if any */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	719 assert(str);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	720 if (tok->enc != NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	721 utf8 = translate_into_utf8(str, tok->enc);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	722 if (utf8 == NULL)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	723 return error_ret(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	724 str = PyBytes_AsString(utf8);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	725 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	726 for (s = str;; s++) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	727 if (*s == '\0') break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	728 else if (*s == '\n') {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	729 assert(lineno < 2);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	730 newl[lineno] = s;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	731 lineno++;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	732 if (lineno == 2) break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	733 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	734 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	735 tok->enc = NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	736 /* need to check line 1 and 2 separately since check_coding_spec
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	737 assumes a single line as input */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	738 if (newl[0]) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	739 if (!check_coding_spec(str, newl[0] - str, tok, buf_setreadl))
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	740 return error_ret(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	741 if (tok->enc == NULL && newl[1]) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	742 if (!check_coding_spec(newl[0]+1, newl[1] - newl[0],
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	743 tok, buf_setreadl))
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	744 return error_ret(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	745 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	746 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	747 if (tok->enc != NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	748 assert(utf8 == NULL);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	749 utf8 = translate_into_utf8(str, tok->enc);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	750 if (utf8 == NULL)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	751 return error_ret(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	752 str = PyBytes_AS_STRING(utf8);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	753 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	754 assert(tok->decoding_buffer == NULL);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	755 tok->decoding_buffer = utf8; /* CAUTION */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	756 return str;
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	757 }
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	758
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	759 #endif /* PGEN */
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	760
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	761 /* Set up tokenizer for string */
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	762
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	763 struct tok_state *
57206 b2d90c1c637d Merged revisions 76230 via svnmerge from Benjamin Peterson <benjamin@python.org> parents: 56594 diff changeset	764 PyTokenizer_FromString(const char *str, int exec_input)
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	765 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	766 struct tok_state *tok = tok_new();
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	767 if (tok == NULL)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	768 return NULL;
61466 cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	769 str = (char *)decode_str(str, exec_input, tok);
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	770 if (str == NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	771 PyTokenizer_Free(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	772 return NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	773 }
35878 94a409da4bd4 Fix some more memory leaks. Neal Norwitz <nnorwitz@gmail.com> parents: 35751 diff changeset	774
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	775 /* XXX: constify members. */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	776 tok->buf = tok->cur = tok->end = tok->inp = (char*)str;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	777 return tok;
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	778 }
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	779
51956 8fe2988cf550 ignore the coding cookie in compile(), exec(), and eval() if the source is a string #4626 Benjamin Peterson <benjamin@python.org> parents: 50540 diff changeset	780 struct tok_state *
57206 b2d90c1c637d Merged revisions 76230 via svnmerge from Benjamin Peterson <benjamin@python.org> parents: 56594 diff changeset	781 PyTokenizer_FromUTF8(const char *str, int exec_input)
51956 8fe2988cf550 ignore the coding cookie in compile(), exec(), and eval() if the source is a string #4626 Benjamin Peterson <benjamin@python.org> parents: 50540 diff changeset	782 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	783 struct tok_state *tok = tok_new();
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	784 if (tok == NULL)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	785 return NULL;
57206 b2d90c1c637d Merged revisions 76230 via svnmerge from Benjamin Peterson <benjamin@python.org> parents: 56594 diff changeset	786 #ifndef PGEN
61466 cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	787 tok->input = str = translate_newlines(str, exec_input, tok);
57206 b2d90c1c637d Merged revisions 76230 via svnmerge from Benjamin Peterson <benjamin@python.org> parents: 56594 diff changeset	788 #endif
61466 cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	789 if (str == NULL) {
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	790 PyTokenizer_Free(tok);
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	791 return NULL;
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	792 }
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	793 tok->decoding_state = STATE_RAW;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	794 tok->read_coding_spec = 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	795 tok->enc = NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	796 tok->str = str;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	797 tok->encoding = (char *)PyMem_MALLOC(6);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	798 if (!tok->encoding) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	799 PyTokenizer_Free(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	800 return NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	801 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	802 strcpy(tok->encoding, "utf-8");
51956 8fe2988cf550 ignore the coding cookie in compile(), exec(), and eval() if the source is a string #4626 Benjamin Peterson <benjamin@python.org> parents: 50540 diff changeset	803
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	804 /* XXX: constify members. */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	805 tok->buf = tok->cur = tok->end = tok->inp = (char*)str;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	806 return tok;
51956 8fe2988cf550 ignore the coding cookie in compile(), exec(), and eval() if the source is a string #4626 Benjamin Peterson <benjamin@python.org> parents: 50540 diff changeset	807 }
8fe2988cf550 ignore the coding cookie in compile(), exec(), and eval() if the source is a string #4626 Benjamin Peterson <benjamin@python.org> parents: 50540 diff changeset	808
326 ff0862e3d63a Completely ignore lines with only a newline token on them, except Guido van Rossum <guido@python.org> parents: 293 diff changeset	809 /* Set up tokenizer for file */
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	810
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	811 struct tok_state *
43225 38f3f9ef193d Convert code from sys.stdin.encoding to UTF-8 in Martin v. Löwis <martin@v.loewis.de> parents: 43018 diff changeset	812 PyTokenizer_FromFile(FILE fp, char enc, char ps1, char ps2)
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	813 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	814 struct tok_state *tok = tok_new();
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	815 if (tok == NULL)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	816 return NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	817 if ((tok->buf = (char *)PyMem_MALLOC(BUFSIZ)) == NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	818 PyTokenizer_Free(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	819 return NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	820 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	821 tok->cur = tok->inp = tok->buf;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	822 tok->end = tok->buf + BUFSIZ;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	823 tok->fp = fp;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	824 tok->prompt = ps1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	825 tok->nextprompt = ps2;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	826 if (enc != NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	827 /* Must copy encoding declaration since it
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	828 gets copied into the parse tree. */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	829 tok->encoding = PyMem_MALLOC(strlen(enc)+1);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	830 if (!tok->encoding) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	831 PyTokenizer_Free(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	832 return NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	833 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	834 strcpy(tok->encoding, enc);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	835 tok->decoding_state = STATE_NORMAL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	836 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	837 return tok;
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	838 }
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	839
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	840
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	841 /* Free a tok_state structure */
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	842
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	843 void
13659 e78153e179fb Mass ANSIfication. Thomas Wouters <thomas@python.org> parents: 13539 diff changeset	844 PyTokenizer_Free(struct tok_state *tok)
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	845 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	846 if (tok->encoding != NULL)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	847 PyMem_FREE(tok->encoding);
24544 0bd521eb714e Make pgen compile with pydebug. Duplicate normalized names, as it may Martin v. Löwis <martin@v.loewis.de> parents: 24543 diff changeset	848 #ifndef PGEN
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	849 Py_XDECREF(tok->decoding_readline);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	850 Py_XDECREF(tok->decoding_buffer);
69147 6e9dc970ac0e Issue #10785: Store the filename as Unicode in the Python parser. Victor Stinner <victor.stinner@haypocalc.com> parents: 68257 diff changeset	851 Py_XDECREF(tok->filename);
24544 0bd521eb714e Make pgen compile with pydebug. Duplicate normalized names, as it may Martin v. Löwis <martin@v.loewis.de> parents: 24543 diff changeset	852 #endif
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	853 if (tok->fp != NULL && tok->buf != NULL)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	854 PyMem_FREE(tok->buf);
61466 cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	855 if (tok->input)
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 61055 diff changeset	856 PyMem_FREE((char *)tok->input);
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	857 PyMem_FREE(tok);
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	858 }
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	859
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	860 /* Get next char, updating state; error code goes into tok->done */
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	861
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	862 static int
13659 e78153e179fb Mass ANSIfication. Thomas Wouters <thomas@python.org> parents: 13539 diff changeset	863 tok_nextc(register struct tok_state *tok)
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	864 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	865 for (;;) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	866 if (tok->cur != tok->inp) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	867 return Py_CHARMASK(tok->cur++); / Fast path */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	868 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	869 if (tok->done != E_OK)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	870 return EOF;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	871 if (tok->fp == NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	872 char *end = strchr(tok->inp, '\n');
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	873 if (end != NULL)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	874 end++;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	875 else {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	876 end = strchr(tok->inp, '\0');
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	877 if (end == tok->inp) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	878 tok->done = E_EOF;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	879 return EOF;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	880 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	881 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	882 if (tok->start == NULL)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	883 tok->buf = tok->cur;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	884 tok->line_start = tok->cur;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	885 tok->lineno++;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	886 tok->inp = end;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	887 return Py_CHARMASK(*tok->cur++);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	888 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	889 if (tok->prompt != NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	890 char *newtok = PyOS_Readline(stdin, stdout, tok->prompt);
43225 38f3f9ef193d Convert code from sys.stdin.encoding to UTF-8 in Martin v. Löwis <martin@v.loewis.de> parents: 43018 diff changeset	891 #ifndef PGEN
67431 0933c3753a71 Issue #10841: set binary mode on files; the parser translates newlines Victor Stinner <victor.stinner@haypocalc.com> parents: 67367 diff changeset	892 if (newtok != NULL) {
0933c3753a71 Issue #10841: set binary mode on files; the parser translates newlines Victor Stinner <victor.stinner@haypocalc.com> parents: 67367 diff changeset	893 char *translated = translate_newlines(newtok, 0, tok);
0933c3753a71 Issue #10841: set binary mode on files; the parser translates newlines Victor Stinner <victor.stinner@haypocalc.com> parents: 67367 diff changeset	894 PyMem_FREE(newtok);
0933c3753a71 Issue #10841: set binary mode on files; the parser translates newlines Victor Stinner <victor.stinner@haypocalc.com> parents: 67367 diff changeset	895 if (translated == NULL)
0933c3753a71 Issue #10841: set binary mode on files; the parser translates newlines Victor Stinner <victor.stinner@haypocalc.com> parents: 67367 diff changeset	896 return EOF;
0933c3753a71 Issue #10841: set binary mode on files; the parser translates newlines Victor Stinner <victor.stinner@haypocalc.com> parents: 67367 diff changeset	897 newtok = translated;
0933c3753a71 Issue #10841: set binary mode on files; the parser translates newlines Victor Stinner <victor.stinner@haypocalc.com> parents: 67367 diff changeset	898 }
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	899 if (tok->encoding && newtok && *newtok) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	900 /* Recode to UTF-8 */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	901 Py_ssize_t buflen;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	902 const char* buf;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	903 PyObject *u = translate_into_utf8(newtok, tok->encoding);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	904 PyMem_FREE(newtok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	905 if (!u) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	906 tok->done = E_DECODE;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	907 return EOF;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	908 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	909 buflen = PyBytes_GET_SIZE(u);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	910 buf = PyBytes_AS_STRING(u);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	911 if (!buf) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	912 Py_DECREF(u);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	913 tok->done = E_DECODE;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	914 return EOF;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	915 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	916 newtok = PyMem_MALLOC(buflen+1);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	917 strcpy(newtok, buf);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	918 Py_DECREF(u);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	919 }
43225 38f3f9ef193d Convert code from sys.stdin.encoding to UTF-8 in Martin v. Löwis <martin@v.loewis.de> parents: 43018 diff changeset	920 #endif
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	921 if (tok->nextprompt != NULL)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	922 tok->prompt = tok->nextprompt;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	923 if (newtok == NULL)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	924 tok->done = E_INTR;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	925 else if (*newtok == '\0') {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	926 PyMem_FREE(newtok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	927 tok->done = E_EOF;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	928 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	929 else if (tok->start != NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	930 size_t start = tok->start - tok->buf;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	931 size_t oldlen = tok->cur - tok->buf;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	932 size_t newlen = oldlen + strlen(newtok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	933 char *buf = tok->buf;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	934 buf = (char *)PyMem_REALLOC(buf, newlen+1);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	935 tok->lineno++;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	936 if (buf == NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	937 PyMem_FREE(tok->buf);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	938 tok->buf = NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	939 PyMem_FREE(newtok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	940 tok->done = E_NOMEM;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	941 return EOF;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	942 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	943 tok->buf = buf;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	944 tok->cur = tok->buf + oldlen;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	945 tok->line_start = tok->cur;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	946 strcpy(tok->buf + oldlen, newtok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	947 PyMem_FREE(newtok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	948 tok->inp = tok->buf + newlen;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	949 tok->end = tok->inp + 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	950 tok->start = tok->buf + start;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	951 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	952 else {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	953 tok->lineno++;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	954 if (tok->buf != NULL)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	955 PyMem_FREE(tok->buf);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	956 tok->buf = newtok;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	957 tok->line_start = tok->buf;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	958 tok->cur = tok->buf;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	959 tok->line_start = tok->buf;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	960 tok->inp = strchr(tok->buf, '\0');
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	961 tok->end = tok->inp + 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	962 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	963 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	964 else {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	965 int done = 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	966 Py_ssize_t cur = 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	967 char *pt;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	968 if (tok->start == NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	969 if (tok->buf == NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	970 tok->buf = (char *)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	971 PyMem_MALLOC(BUFSIZ);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	972 if (tok->buf == NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	973 tok->done = E_NOMEM;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	974 return EOF;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	975 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	976 tok->end = tok->buf + BUFSIZ;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	977 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	978 if (decoding_fgets(tok->buf, (int)(tok->end - tok->buf),
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	979 tok) == NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	980 tok->done = E_EOF;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	981 done = 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	982 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	983 else {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	984 tok->done = E_OK;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	985 tok->inp = strchr(tok->buf, '\0');
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	986 done = tok->inp[-1] == '\n';
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	987 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	988 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	989 else {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	990 cur = tok->cur - tok->buf;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	991 if (decoding_feof(tok)) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	992 tok->done = E_EOF;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	993 done = 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	994 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	995 else
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	996 tok->done = E_OK;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	997 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	998 tok->lineno++;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	999 /* Read until '\n' or EOF */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1000 while (!done) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1001 Py_ssize_t curstart = tok->start == NULL ? -1 :
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1002 tok->start - tok->buf;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1003 Py_ssize_t curvalid = tok->inp - tok->buf;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1004 Py_ssize_t newsize = curvalid + BUFSIZ;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1005 char *newbuf = tok->buf;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1006 newbuf = (char *)PyMem_REALLOC(newbuf,
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1007 newsize);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1008 if (newbuf == NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1009 tok->done = E_NOMEM;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1010 tok->cur = tok->inp;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1011 return EOF;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1012 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1013 tok->buf = newbuf;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1014 tok->inp = tok->buf + curvalid;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1015 tok->end = tok->buf + newsize;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1016 tok->start = curstart < 0 ? NULL :
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1017 tok->buf + curstart;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1018 if (decoding_fgets(tok->inp,
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1019 (int)(tok->end - tok->inp),
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1020 tok) == NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1021 /* Break out early on decoding
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1022 errors, as tok->buf will be NULL
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1023 */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1024 if (tok->decoding_erred)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1025 return EOF;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1026 /* Last line does not end in \n,
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1027 fake one */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1028 strcpy(tok->inp, "\n");
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1029 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1030 tok->inp = strchr(tok->inp, '\0');
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1031 done = tok->inp[-1] == '\n';
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1032 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1033 if (tok->buf != NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1034 tok->cur = tok->buf + cur;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1035 tok->line_start = tok->cur;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1036 /* replace "\r\n" with "\n" */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1037 /* For Mac leave the \r, giving a syntax error */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1038 pt = tok->inp - 2;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1039 if (pt >= tok->buf && *pt == '\r') {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1040 *pt++ = '\n';
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1041 *pt = '\0';
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1042 tok->inp = pt;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1043 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1044 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1045 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1046 if (tok->done != E_OK) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1047 if (tok->prompt != NULL)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1048 PySys_WriteStderr("\n");
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1049 tok->cur = tok->inp;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1050 return EOF;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1051 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1052 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1053 /NOTREACHED/
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1054 }
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1055
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1056
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1057 /* Back-up one character */
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1058
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1059 static void
13659 e78153e179fb Mass ANSIfication. Thomas Wouters <thomas@python.org> parents: 13539 diff changeset	1060 tok_backup(register struct tok_state *tok, register int c)
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1061 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1062 if (c != EOF) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1063 if (--tok->cur < tok->buf)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1064 Py_FatalError("tok_backup: beginning of buffer");
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1065 if (*tok->cur != c)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1066 *tok->cur = c;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1067 }
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1068 }
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1069
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1070
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1071 /* Return the token corresponding to a single character */
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1072
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1073 int
13659 e78153e179fb Mass ANSIfication. Thomas Wouters <thomas@python.org> parents: 13539 diff changeset	1074 PyToken_OneChar(int c)
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1075 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1076 switch (c) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1077 case '(': return LPAR;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1078 case ')': return RPAR;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1079 case '[': return LSQB;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1080 case ']': return RSQB;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1081 case ':': return COLON;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1082 case ',': return COMMA;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1083 case ';': return SEMI;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1084 case '+': return PLUS;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1085 case '-': return MINUS;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1086 case '*': return STAR;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1087 case '/': return SLASH;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1088 case '\|': return VBAR;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1089 case '&': return AMPER;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1090 case '<': return LESS;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1091 case '>': return GREATER;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1092 case '=': return EQUAL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1093 case '.': return DOT;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1094 case '%': return PERCENT;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1095 case '{': return LBRACE;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1096 case '}': return RBRACE;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1097 case '^': return CIRCUMFLEX;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1098 case '~': return TILDE;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1099 case '@': return AT;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1100 default: return OP;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1101 }
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1102 }
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1103
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1104
400 d189a2f681a6 Added 2-char tokens and new versions of comparisons Guido van Rossum <guido@python.org> parents: 326 diff changeset	1105 int
13659 e78153e179fb Mass ANSIfication. Thomas Wouters <thomas@python.org> parents: 13539 diff changeset	1106 PyToken_TwoChars(int c1, int c2)
400 d189a2f681a6 Added 2-char tokens and new versions of comparisons Guido van Rossum <guido@python.org> parents: 326 diff changeset	1107 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1108 switch (c1) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1109 case '=':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1110 switch (c2) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1111 case '=': return EQEQUAL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1112 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1113 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1114 case '!':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1115 switch (c2) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1116 case '=': return NOTEQUAL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1117 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1118 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1119 case '<':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1120 switch (c2) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1121 case '>': return NOTEQUAL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1122 case '=': return LESSEQUAL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1123 case '<': return LEFTSHIFT;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1124 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1125 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1126 case '>':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1127 switch (c2) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1128 case '=': return GREATEREQUAL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1129 case '>': return RIGHTSHIFT;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1130 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1131 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1132 case '+':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1133 switch (c2) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1134 case '=': return PLUSEQUAL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1135 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1136 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1137 case '-':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1138 switch (c2) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1139 case '=': return MINEQUAL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1140 case '>': return RARROW;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1141 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1142 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1143 case '*':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1144 switch (c2) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1145 case '*': return DOUBLESTAR;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1146 case '=': return STAREQUAL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1147 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1148 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1149 case '/':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1150 switch (c2) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1151 case '/': return DOUBLESLASH;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1152 case '=': return SLASHEQUAL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1153 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1154 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1155 case '\|':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1156 switch (c2) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1157 case '=': return VBAREQUAL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1158 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1159 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1160 case '%':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1161 switch (c2) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1162 case '=': return PERCENTEQUAL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1163 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1164 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1165 case '&':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1166 switch (c2) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1167 case '=': return AMPEREQUAL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1168 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1169 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1170 case '^':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1171 switch (c2) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1172 case '=': return CIRCUMFLEXEQUAL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1173 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1174 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1175 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1176 return OP;
400 d189a2f681a6 Added 2-char tokens and new versions of comparisons Guido van Rossum <guido@python.org> parents: 326 diff changeset	1177 }
d189a2f681a6 Added 2-char tokens and new versions of comparisons Guido van Rossum <guido@python.org> parents: 326 diff changeset	1178
14163 1b9fd0cbd914 Support for three-token characters (*=, >>=, <<=) which was written by Thomas Wouters <thomas@python.org>* parents: 13659 diff changeset	1179 int
1b9fd0cbd914 Support for three-token characters (*=, >>=, <<=) which was written by Thomas Wouters <thomas@python.org>* parents: 13659 diff changeset	1180 PyToken_ThreeChars(int c1, int c2, int c3)
1b9fd0cbd914 Support for three-token characters (*=, >>=, <<=) which was written by Thomas Wouters <thomas@python.org>* parents: 13659 diff changeset	1181 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1182 switch (c1) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1183 case '<':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1184 switch (c2) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1185 case '<':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1186 switch (c3) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1187 case '=':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1188 return LEFTSHIFTEQUAL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1189 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1190 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1191 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1192 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1193 case '>':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1194 switch (c2) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1195 case '>':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1196 switch (c3) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1197 case '=':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1198 return RIGHTSHIFTEQUAL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1199 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1200 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1201 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1202 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1203 case '*':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1204 switch (c2) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1205 case '*':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1206 switch (c3) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1207 case '=':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1208 return DOUBLESTAREQUAL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1209 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1210 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1211 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1212 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1213 case '/':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1214 switch (c2) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1215 case '/':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1216 switch (c3) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1217 case '=':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1218 return DOUBLESLASHEQUAL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1219 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1220 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1221 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1222 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1223 case '.':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1224 switch (c2) {
41230 b1a18c7ed5ca Make ELLIPSIS a separate token. This makes it a syntax error to write ". . ." for Ellipsis. Georg Brandl <georg@python.org> parents: 40894 diff changeset	1225 case '.':
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1226 switch (c3) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1227 case '.':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1228 return ELLIPSIS;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1229 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1230 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1231 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1232 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1233 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1234 return OP;
14163 1b9fd0cbd914 Support for three-token characters (*=, >>=, <<=) which was written by Thomas Wouters <thomas@python.org>* parents: 13659 diff changeset	1235 }
400 d189a2f681a6 Added 2-char tokens and new versions of comparisons Guido van Rossum <guido@python.org> parents: 326 diff changeset	1236
7945 dc132cc39573 Add checking for inconsistent tab usage Guido van Rossum <guido@python.org> parents: 7324 diff changeset	1237 static int
13659 e78153e179fb Mass ANSIfication. Thomas Wouters <thomas@python.org> parents: 13539 diff changeset	1238 indenterror(struct tok_state *tok)
7945 dc132cc39573 Add checking for inconsistent tab usage Guido van Rossum <guido@python.org> parents: 7324 diff changeset	1239 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1240 if (tok->alterror) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1241 tok->done = E_TABSPACE;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1242 tok->cur = tok->inp;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1243 return 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1244 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1245 if (tok->altwarning) {
69147 6e9dc970ac0e Issue #10785: Store the filename as Unicode in the Python parser. Victor Stinner <victor.stinner@haypocalc.com> parents: 68257 diff changeset	1246 #ifdef PGEN
6e9dc970ac0e Issue #10785: Store the filename as Unicode in the Python parser. Victor Stinner <victor.stinner@haypocalc.com> parents: 68257 diff changeset	1247 PySys_WriteStderr("inconsistent use of tabs and spaces "
6e9dc970ac0e Issue #10785: Store the filename as Unicode in the Python parser. Victor Stinner <victor.stinner@haypocalc.com> parents: 68257 diff changeset	1248 "in indentation\n");
6e9dc970ac0e Issue #10785: Store the filename as Unicode in the Python parser. Victor Stinner <victor.stinner@haypocalc.com> parents: 68257 diff changeset	1249 #else
6e9dc970ac0e Issue #10785: Store the filename as Unicode in the Python parser. Victor Stinner <victor.stinner@haypocalc.com> parents: 68257 diff changeset	1250 PySys_FormatStderr("%U: inconsistent use of tabs and spaces "
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1251 "in indentation\n", tok->filename);
69147 6e9dc970ac0e Issue #10785: Store the filename as Unicode in the Python parser. Victor Stinner <victor.stinner@haypocalc.com> parents: 68257 diff changeset	1252 #endif
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1253 tok->altwarning = 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1254 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1255 return 0;
7945 dc132cc39573 Add checking for inconsistent tab usage Guido van Rossum <guido@python.org> parents: 7324 diff changeset	1256 }
dc132cc39573 Add checking for inconsistent tab usage Guido van Rossum <guido@python.org> parents: 7324 diff changeset	1257
42597 20477cbfd996 Implement PEP 3131. Add isidentifier to str. Martin v. Löwis <martin@v.loewis.de> parents: 42551 diff changeset	1258 #ifdef PGEN
59865 c2d2e26bcf7f Merged revisions 78872 via svnmerge from Victor Stinner <victor.stinner@haypocalc.com> parents: 59289 diff changeset	1259 #define verify_identifier(tok) 1
42597 20477cbfd996 Implement PEP 3131. Add isidentifier to str. Martin v. Löwis <martin@v.loewis.de> parents: 42551 diff changeset	1260 #else
20477cbfd996 Implement PEP 3131. Add isidentifier to str. Martin v. Löwis <martin@v.loewis.de> parents: 42551 diff changeset	1261 /* Verify that the identifier follows PEP 3131. */
20477cbfd996 Implement PEP 3131. Add isidentifier to str. Martin v. Löwis <martin@v.loewis.de> parents: 42551 diff changeset	1262 static int
59865 c2d2e26bcf7f Merged revisions 78872 via svnmerge from Victor Stinner <victor.stinner@haypocalc.com> parents: 59289 diff changeset	1263 verify_identifier(struct tok_state *tok)
42597 20477cbfd996 Implement PEP 3131. Add isidentifier to str. Martin v. Löwis <martin@v.loewis.de> parents: 42551 diff changeset	1264 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1265 PyObject *s;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1266 int result;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1267 s = PyUnicode_DecodeUTF8(tok->start, tok->cur - tok->start, NULL);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1268 if (s == NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1269 if (PyErr_ExceptionMatches(PyExc_UnicodeDecodeError)) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1270 PyErr_Clear();
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1271 tok->done = E_IDENTIFIER;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1272 } else {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1273 tok->done = E_ERROR;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1274 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1275 return 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1276 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1277 result = PyUnicode_IsIdentifier(s);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1278 Py_DECREF(s);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1279 if (result == 0)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1280 tok->done = E_IDENTIFIER;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1281 return result;
42597 20477cbfd996 Implement PEP 3131. Add isidentifier to str. Martin v. Löwis <martin@v.loewis.de> parents: 42551 diff changeset	1282 }
20477cbfd996 Implement PEP 3131. Add isidentifier to str. Martin v. Löwis <martin@v.loewis.de> parents: 42551 diff changeset	1283 #endif
7945 dc132cc39573 Add checking for inconsistent tab usage Guido van Rossum <guido@python.org> parents: 7324 diff changeset	1284
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1285 /* Get next token, after space stripping etc. */
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1286
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	1287 static int
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	1288 tok_get(register struct tok_state tok, char p_start, char *p_end)
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1289 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1290 register int c;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1291 int blankline, nonascii;
326 ff0862e3d63a Completely ignore lines with only a newline token on them, except Guido van Rossum <guido@python.org> parents: 293 diff changeset	1292
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1293 p_start = p_end = NULL;
326 ff0862e3d63a Completely ignore lines with only a newline token on them, except Guido van Rossum <guido@python.org> parents: 293 diff changeset	1294 nextline:
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1295 tok->start = NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1296 blankline = 0;
326 ff0862e3d63a Completely ignore lines with only a newline token on them, except Guido van Rossum <guido@python.org> parents: 293 diff changeset	1297
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1298 /* Get indentation level */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1299 if (tok->atbol) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1300 register int col = 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1301 register int altcol = 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1302 tok->atbol = 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1303 for (;;) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1304 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1305 if (c == ' ')
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1306 col++, altcol++;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1307 else if (c == '\t') {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1308 col = (col/tok->tabsize + 1) * tok->tabsize;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1309 altcol = (altcol/tok->alttabsize + 1)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1310 * tok->alttabsize;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1311 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1312 else if (c == '\014') /* Control-L (formfeed) */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1313 col = altcol = 0; /* For Emacs users */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1314 else
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1315 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1316 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1317 tok_backup(tok, c);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1318 if (c == '#' \|\| c == '\n') {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1319 /* Lines with only whitespace and/or comments
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1320 shouldn't affect the indentation and are
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1321 not passed to the parser as NEWLINE tokens,
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1322 except totally empty lines in interactive
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1323 mode, which signal the end of a command group. */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1324 if (col == 0 && c == '\n' && tok->prompt != NULL)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1325 blankline = 0; /* Let it through */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1326 else
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1327 blankline = 1; /* Ignore completely */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1328 /* We can't jump back right here since we still
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1329 may need to skip to the end of a comment */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1330 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1331 if (!blankline && tok->level == 0) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1332 if (col == tok->indstack[tok->indent]) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1333 /* No change */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1334 if (altcol != tok->altindstack[tok->indent]) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1335 if (indenterror(tok))
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1336 return ERRORTOKEN;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1337 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1338 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1339 else if (col > tok->indstack[tok->indent]) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1340 /* Indent -- always one */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1341 if (tok->indent+1 >= MAXINDENT) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1342 tok->done = E_TOODEEP;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1343 tok->cur = tok->inp;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1344 return ERRORTOKEN;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1345 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1346 if (altcol <= tok->altindstack[tok->indent]) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1347 if (indenterror(tok))
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1348 return ERRORTOKEN;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1349 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1350 tok->pendin++;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1351 tok->indstack[++tok->indent] = col;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1352 tok->altindstack[tok->indent] = altcol;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1353 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1354 else /* col < tok->indstack[tok->indent] */ {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1355 /* Dedent -- any number, must be consistent */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1356 while (tok->indent > 0 &&
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1357 col < tok->indstack[tok->indent]) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1358 tok->pendin--;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1359 tok->indent--;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1360 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1361 if (col != tok->indstack[tok->indent]) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1362 tok->done = E_DEDENT;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1363 tok->cur = tok->inp;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1364 return ERRORTOKEN;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1365 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1366 if (altcol != tok->altindstack[tok->indent]) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1367 if (indenterror(tok))
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1368 return ERRORTOKEN;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1369 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1370 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1371 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1372 }
37879 f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair Thomas Wouters <thomas@python.org> parents: 37704 diff changeset	1373
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1374 tok->start = tok->cur;
37879 f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair Thomas Wouters <thomas@python.org> parents: 37704 diff changeset	1375
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1376 /* Return pending indents/dedents */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1377 if (tok->pendin != 0) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1378 if (tok->pendin < 0) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1379 tok->pendin++;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1380 return DEDENT;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1381 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1382 else {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1383 tok->pendin--;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1384 return INDENT;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1385 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1386 }
37879 f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair Thomas Wouters <thomas@python.org> parents: 37704 diff changeset	1387
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1388 again:
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1389 tok->start = NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1390 /* Skip spaces */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1391 do {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1392 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1393 } while (c == ' ' \|\| c == '\t' \|\| c == '\014');
37879 f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair Thomas Wouters <thomas@python.org> parents: 37704 diff changeset	1394
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1395 /* Set start of current token */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1396 tok->start = tok->cur - 1;
37879 f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair Thomas Wouters <thomas@python.org> parents: 37704 diff changeset	1397
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1398 /* Skip comment */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1399 if (c == '#')
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1400 while (c != EOF && c != '\n')
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1401 c = tok_nextc(tok);
37879 f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair Thomas Wouters <thomas@python.org> parents: 37704 diff changeset	1402
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1403 /* Check for EOF and errors now */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1404 if (c == EOF) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1405 return tok->done == E_EOF ? ENDMARKER : ERRORTOKEN;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1406 }
37879 f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair Thomas Wouters <thomas@python.org> parents: 37704 diff changeset	1407
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1408 /* Identifier (most frequent token!) */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1409 nonascii = 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1410 if (is_potential_identifier_start(c)) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1411 /* Process b"", r"" and br"" */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1412 if (c == 'b' \|\| c == 'B') {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1413 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1414 if (c == '"' \|\| c == '\'')
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1415 goto letter_quote;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1416 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1417 if (c == 'r' \|\| c == 'R') {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1418 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1419 if (c == '"' \|\| c == '\'')
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1420 goto letter_quote;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1421 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1422 while (is_potential_identifier_char(c)) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1423 if (c >= 128)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1424 nonascii = 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1425 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1426 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1427 tok_backup(tok, c);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1428 if (nonascii &&
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1429 !verify_identifier(tok)) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1430 tok->done = E_IDENTIFIER;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1431 return ERRORTOKEN;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1432 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1433 *p_start = tok->start;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1434 *p_end = tok->cur;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1435 return NAME;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1436 }
37879 f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair Thomas Wouters <thomas@python.org> parents: 37704 diff changeset	1437
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1438 /* Newline */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1439 if (c == '\n') {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1440 tok->atbol = 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1441 if (blankline \|\| tok->level > 0)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1442 goto nextline;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1443 *p_start = tok->start;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1444 p_end = tok->cur - 1; / Leave '\n' out of the string */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1445 tok->cont_line = 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1446 return NEWLINE;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1447 }
37879 f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair Thomas Wouters <thomas@python.org> parents: 37704 diff changeset	1448
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1449 /* Period or number starting with period? */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1450 if (c == '.') {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1451 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1452 if (isdigit(c)) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1453 goto fraction;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1454 } else if (c == '.') {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1455 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1456 if (c == '.') {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1457 *p_start = tok->start;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1458 *p_end = tok->cur;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1459 return ELLIPSIS;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1460 } else {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1461 tok_backup(tok, c);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1462 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1463 tok_backup(tok, '.');
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1464 } else {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1465 tok_backup(tok, c);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1466 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1467 *p_start = tok->start;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1468 *p_end = tok->cur;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1469 return DOT;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1470 }
3333 4b5bff95804e changes for pow(*) and complex Guido van Rossum <guido@python.org>* parents: 3116 diff changeset	1471
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1472 /* Number */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1473 if (isdigit(c)) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1474 if (c == '0') {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1475 /* Hex, octal or binary -- maybe. */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1476 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1477 if (c == '.')
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1478 goto fraction;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1479 if (c == 'j' \|\| c == 'J')
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1480 goto imaginary;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1481 if (c == 'x' \|\| c == 'X') {
44782 ca2683a67943 Merged revisions 60080-60089,60091-60093 via svnmerge from Georg Brandl <georg@python.org> parents: 44442 diff changeset	1482
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1483 /* Hex */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1484 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1485 if (!isxdigit(c)) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1486 tok->done = E_TOKEN;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1487 tok_backup(tok, c);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1488 return ERRORTOKEN;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1489 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1490 do {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1491 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1492 } while (isxdigit(c));
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1493 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1494 else if (c == 'o' \|\| c == 'O') {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1495 /* Octal */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1496 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1497 if (c < '0' \|\| c >= '8') {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1498 tok->done = E_TOKEN;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1499 tok_backup(tok, c);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1500 return ERRORTOKEN;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1501 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1502 do {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1503 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1504 } while ('0' <= c && c < '8');
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1505 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1506 else if (c == 'b' \|\| c == 'B') {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1507 /* Binary */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1508 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1509 if (c != '0' && c != '1') {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1510 tok->done = E_TOKEN;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1511 tok_backup(tok, c);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1512 return ERRORTOKEN;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1513 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1514 do {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1515 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1516 } while (c == '0' \|\| c == '1');
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1517 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1518 else {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1519 int nonzero = 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1520 /* maybe old-style octal; c is first char of it */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1521 /* in any case, allow '0' as a literal */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1522 while (c == '0')
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1523 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1524 while (isdigit(c)) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1525 nonzero = 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1526 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1527 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1528 if (c == '.')
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1529 goto fraction;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1530 else if (c == 'e' \|\| c == 'E')
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1531 goto exponent;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1532 else if (c == 'j' \|\| c == 'J')
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1533 goto imaginary;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1534 else if (nonzero) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1535 tok->done = E_TOKEN;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1536 tok_backup(tok, c);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1537 return ERRORTOKEN;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1538 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1539 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1540 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1541 else {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1542 /* Decimal */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1543 do {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1544 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1545 } while (isdigit(c));
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1546 {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1547 /* Accept floating point numbers. */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1548 if (c == '.') {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1549 fraction:
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1550 /* Fraction */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1551 do {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1552 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1553 } while (isdigit(c));
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1554 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1555 if (c == 'e' \|\| c == 'E') {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1556 exponent:
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1557 /* Exponent part */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1558 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1559 if (c == '+' \|\| c == '-')
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1560 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1561 if (!isdigit(c)) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1562 tok->done = E_TOKEN;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1563 tok_backup(tok, c);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1564 return ERRORTOKEN;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1565 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1566 do {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1567 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1568 } while (isdigit(c));
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1569 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1570 if (c == 'j' \|\| c == 'J')
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1571 /* Imaginary part */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1572 imaginary:
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1573 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1574 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1575 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1576 tok_backup(tok, c);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1577 *p_start = tok->start;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1578 *p_end = tok->cur;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1579 return NUMBER;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1580 }
5265 3c1788ab9761 Support for alternative string quotes (a"xx", b"xx", c"xx", ...). Guido van Rossum <guido@python.org> parents: 4839 diff changeset	1581
3c1788ab9761 Support for alternative string quotes (a"xx", b"xx", c"xx", ...). Guido van Rossum <guido@python.org> parents: 4839 diff changeset	1582 letter_quote:
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1583 /* String */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1584 if (c == '\'' \|\| c == '"') {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1585 int quote = c;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1586 int quote_size = 1; /* 1 or 3 */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1587 int end_quote_size = 0;
43879 5590157f4fc6 Cleanup of tokenizer.c. Guido van Rossum <guido@python.org> parents: 43873 diff changeset	1588
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1589 /* Find the quote size and start of string */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1590 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1591 if (c == quote) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1592 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1593 if (c == quote)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1594 quote_size = 3;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1595 else
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1596 end_quote_size = 1; /* empty string found */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1597 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1598 if (c != quote)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1599 tok_backup(tok, c);
43879 5590157f4fc6 Cleanup of tokenizer.c. Guido van Rossum <guido@python.org> parents: 43873 diff changeset	1600
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1601 /* Get rest of string */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1602 while (end_quote_size != quote_size) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1603 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1604 if (c == EOF) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1605 if (quote_size == 3)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1606 tok->done = E_EOFS;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1607 else
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1608 tok->done = E_EOLS;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1609 tok->cur = tok->inp;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1610 return ERRORTOKEN;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1611 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1612 if (quote_size == 1 && c == '\n') {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1613 tok->done = E_EOLS;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1614 tok->cur = tok->inp;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1615 return ERRORTOKEN;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1616 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1617 if (c == quote)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1618 end_quote_size += 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1619 else {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1620 end_quote_size = 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1621 if (c == '\\')
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1622 c = tok_nextc(tok); /* skip escaped char */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1623 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1624 }
43879 5590157f4fc6 Cleanup of tokenizer.c. Guido van Rossum <guido@python.org> parents: 43873 diff changeset	1625
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1626 *p_start = tok->start;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1627 *p_end = tok->cur;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1628 return STRING;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1629 }
37879 f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair Thomas Wouters <thomas@python.org> parents: 37704 diff changeset	1630
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1631 /* Line continuation */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1632 if (c == '\\') {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1633 c = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1634 if (c != '\n') {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1635 tok->done = E_LINECONT;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1636 tok->cur = tok->inp;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1637 return ERRORTOKEN;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1638 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1639 tok->cont_line = 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1640 goto again; /* Read next line */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1641 }
37879 f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair Thomas Wouters <thomas@python.org> parents: 37704 diff changeset	1642
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1643 /* Check for two-character token */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1644 {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1645 int c2 = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1646 int token = PyToken_TwoChars(c, c2);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1647 if (token != OP) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1648 int c3 = tok_nextc(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1649 int token3 = PyToken_ThreeChars(c, c2, c3);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1650 if (token3 != OP) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1651 token = token3;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1652 } else {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1653 tok_backup(tok, c3);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1654 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1655 *p_start = tok->start;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1656 *p_end = tok->cur;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1657 return token;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1658 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1659 tok_backup(tok, c2);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1660 }
37879 f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair Thomas Wouters <thomas@python.org> parents: 37704 diff changeset	1661
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1662 /* Keep track of parentheses nesting level */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1663 switch (c) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1664 case '(':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1665 case '[':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1666 case '{':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1667 tok->level++;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1668 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1669 case ')':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1670 case ']':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1671 case '}':
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1672 tok->level--;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1673 break;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1674 }
37879 f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair Thomas Wouters <thomas@python.org> parents: 37704 diff changeset	1675
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1676 /* Punctuation character */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1677 *p_start = tok->start;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1678 *p_end = tok->cur;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1679 return PyToken_OneChar(c);
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1680 }
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1681
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	1682 int
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	1683 PyTokenizer_Get(struct tok_state tok, char p_start, char *p_end)
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	1684 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1685 int result = tok_get(tok, p_start, p_end);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1686 if (tok->decoding_erred) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1687 result = ERRORTOKEN;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1688 tok->done = E_DECODE;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1689 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1690 return result;
24539 06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263. Martin v. Löwis <martin@v.loewis.de> parents: 23029 diff changeset	1691 }
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1692
69148 7b8d625eb6e4 Issue #9319: Include the filename in "Non-UTF8 code ..." syntax error. Victor Stinner <victor.stinner@haypocalc.com> parents: 69147 diff changeset	1693 /* Get the encoding of a Python file. Check for the coding cookie and check if
7b8d625eb6e4 Issue #9319: Include the filename in "Non-UTF8 code ..." syntax error. Victor Stinner <victor.stinner@haypocalc.com> parents: 69147 diff changeset	1694 the file starts with a BOM.
43609 d378acdc30ee Fix PyTokenizer_FindEncoding() for OS X 10.4. Turns out that seeking to the Brett Cannon <bcannon@gmail.com> parents: 43607 diff changeset	1695
69148 7b8d625eb6e4 Issue #9319: Include the filename in "Non-UTF8 code ..." syntax error. Victor Stinner <victor.stinner@haypocalc.com> parents: 69147 diff changeset	1696 PyTokenizer_FindEncodingFilename() returns NULL when it can't find the
7b8d625eb6e4 Issue #9319: Include the filename in "Non-UTF8 code ..." syntax error. Victor Stinner <victor.stinner@haypocalc.com> parents: 69147 diff changeset	1697 encoding in the first or second line of the file (in which case the encoding
7b8d625eb6e4 Issue #9319: Include the filename in "Non-UTF8 code ..." syntax error. Victor Stinner <victor.stinner@haypocalc.com> parents: 69147 diff changeset	1698 should be assumed to be UTF-8).
7b8d625eb6e4 Issue #9319: Include the filename in "Non-UTF8 code ..." syntax error. Victor Stinner <victor.stinner@haypocalc.com> parents: 69147 diff changeset	1699
7b8d625eb6e4 Issue #9319: Include the filename in "Non-UTF8 code ..." syntax error. Victor Stinner <victor.stinner@haypocalc.com> parents: 69147 diff changeset	1700 The char* returned is malloc'ed via PyMem_MALLOC() and thus must be freed
7b8d625eb6e4 Issue #9319: Include the filename in "Non-UTF8 code ..." syntax error. Victor Stinner <victor.stinner@haypocalc.com> parents: 69147 diff changeset	1701 by the caller. */
7b8d625eb6e4 Issue #9319: Include the filename in "Non-UTF8 code ..." syntax error. Victor Stinner <victor.stinner@haypocalc.com> parents: 69147 diff changeset	1702
43605 a2bb76d11d66 Patch 1267 by Christian Heimes. Guido van Rossum <guido@python.org> parents: 43523 diff changeset	1703 char *
69148 7b8d625eb6e4 Issue #9319: Include the filename in "Non-UTF8 code ..." syntax error. Victor Stinner <victor.stinner@haypocalc.com> parents: 69147 diff changeset	1704 PyTokenizer_FindEncodingFilename(int fd, PyObject *filename)
43632 0067a6ab4854 Issue 1267, continued. Guido van Rossum <guido@python.org> parents: 43622 diff changeset	1705 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1706 struct tok_state *tok;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1707 FILE *fp;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1708 char p_start =NULL , p_end =NULL , *encoding = NULL;
43605 a2bb76d11d66 Patch 1267 by Christian Heimes. Guido van Rossum <guido@python.org> parents: 43523 diff changeset	1709
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1710 fd = dup(fd);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1711 if (fd < 0) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1712 return NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1713 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1714 fp = fdopen(fd, "r");
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1715 if (fp == NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1716 return NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1717 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1718 tok = PyTokenizer_FromFile(fp, NULL, NULL, NULL);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1719 if (tok == NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1720 fclose(fp);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1721 return NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1722 }
69147 6e9dc970ac0e Issue #10785: Store the filename as Unicode in the Python parser. Victor Stinner <victor.stinner@haypocalc.com> parents: 68257 diff changeset	1723 #ifndef PGEN
69148 7b8d625eb6e4 Issue #9319: Include the filename in "Non-UTF8 code ..." syntax error. Victor Stinner <victor.stinner@haypocalc.com> parents: 69147 diff changeset	1724 if (filename != NULL) {
7b8d625eb6e4 Issue #9319: Include the filename in "Non-UTF8 code ..." syntax error. Victor Stinner <victor.stinner@haypocalc.com> parents: 69147 diff changeset	1725 Py_INCREF(filename);
7b8d625eb6e4 Issue #9319: Include the filename in "Non-UTF8 code ..." syntax error. Victor Stinner <victor.stinner@haypocalc.com> parents: 69147 diff changeset	1726 tok->filename = filename;
7b8d625eb6e4 Issue #9319: Include the filename in "Non-UTF8 code ..." syntax error. Victor Stinner <victor.stinner@haypocalc.com> parents: 69147 diff changeset	1727 }
7b8d625eb6e4 Issue #9319: Include the filename in "Non-UTF8 code ..." syntax error. Victor Stinner <victor.stinner@haypocalc.com> parents: 69147 diff changeset	1728 else {
7b8d625eb6e4 Issue #9319: Include the filename in "Non-UTF8 code ..." syntax error. Victor Stinner <victor.stinner@haypocalc.com> parents: 69147 diff changeset	1729 tok->filename = PyUnicode_FromString("<string>");
7b8d625eb6e4 Issue #9319: Include the filename in "Non-UTF8 code ..." syntax error. Victor Stinner <victor.stinner@haypocalc.com> parents: 69147 diff changeset	1730 if (tok->filename == NULL) {
7b8d625eb6e4 Issue #9319: Include the filename in "Non-UTF8 code ..." syntax error. Victor Stinner <victor.stinner@haypocalc.com> parents: 69147 diff changeset	1731 fclose(fp);
7b8d625eb6e4 Issue #9319: Include the filename in "Non-UTF8 code ..." syntax error. Victor Stinner <victor.stinner@haypocalc.com> parents: 69147 diff changeset	1732 PyTokenizer_Free(tok);
7b8d625eb6e4 Issue #9319: Include the filename in "Non-UTF8 code ..." syntax error. Victor Stinner <victor.stinner@haypocalc.com> parents: 69147 diff changeset	1733 return encoding;
7b8d625eb6e4 Issue #9319: Include the filename in "Non-UTF8 code ..." syntax error. Victor Stinner <victor.stinner@haypocalc.com> parents: 69147 diff changeset	1734 }
7b8d625eb6e4 Issue #9319: Include the filename in "Non-UTF8 code ..." syntax error. Victor Stinner <victor.stinner@haypocalc.com> parents: 69147 diff changeset	1735 }
69147 6e9dc970ac0e Issue #10785: Store the filename as Unicode in the Python parser. Victor Stinner <victor.stinner@haypocalc.com> parents: 68257 diff changeset	1736 #endif
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1737 while (tok->lineno < 2 && tok->done == E_OK) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1738 PyTokenizer_Get(tok, &p_start, &p_end);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1739 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1740 fclose(fp);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1741 if (tok->encoding) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1742 encoding = (char *)PyMem_MALLOC(strlen(tok->encoding) + 1);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1743 if (encoding)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1744 strcpy(encoding, tok->encoding);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1745 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1746 PyTokenizer_Free(tok);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1747 return encoding;
43605 a2bb76d11d66 Patch 1267 by Christian Heimes. Guido van Rossum <guido@python.org> parents: 43523 diff changeset	1748 }
43292 876b7ef4775d Merged revisions 57778-58052 via svnmerge from Thomas Wouters <thomas@python.org> parents: 43225 diff changeset	1749
69148 7b8d625eb6e4 Issue #9319: Include the filename in "Non-UTF8 code ..." syntax error. Victor Stinner <victor.stinner@haypocalc.com> parents: 69147 diff changeset	1750 char *
7b8d625eb6e4 Issue #9319: Include the filename in "Non-UTF8 code ..." syntax error. Victor Stinner <victor.stinner@haypocalc.com> parents: 69147 diff changeset	1751 PyTokenizer_FindEncoding(int fd)
7b8d625eb6e4 Issue #9319: Include the filename in "Non-UTF8 code ..." syntax error. Victor Stinner <victor.stinner@haypocalc.com> parents: 69147 diff changeset	1752 {
7b8d625eb6e4 Issue #9319: Include the filename in "Non-UTF8 code ..." syntax error. Victor Stinner <victor.stinner@haypocalc.com> parents: 69147 diff changeset	1753 return PyTokenizer_FindEncodingFilename(fd, NULL);
7b8d625eb6e4 Issue #9319: Include the filename in "Non-UTF8 code ..." syntax error. Victor Stinner <victor.stinner@haypocalc.com> parents: 69147 diff changeset	1754 }
7b8d625eb6e4 Issue #9319: Include the filename in "Non-UTF8 code ..." syntax error. Victor Stinner <victor.stinner@haypocalc.com> parents: 69147 diff changeset	1755
4839 4567e158d20c Rename DEBUG macro to Py_DEBUG Guido van Rossum <guido@python.org> parents: 4577 diff changeset	1756 #ifdef Py_DEBUG
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1757
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1758 void
13659 e78153e179fb Mass ANSIfication. Thomas Wouters <thomas@python.org> parents: 13539 diff changeset	1759 tok_dump(int type, char start, char end)
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1760 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1761 printf("%s", _PyParser_TokenNames[type]);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1762 if (type == NAME \|\| type == NUMBER \|\| type == STRING \|\| type == OP)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59865 diff changeset	1763 printf("(%.*s)", (int)(end - start), start);
6 fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1764 }
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1765
fc6fcd7df4f7 Initial revision Guido van Rossum <guido@python.org> parents: diff changeset	1766 #endif

Mercurial > cpython-lru_cache-weakref

annotate Parser/tokenizer.c @ 70817:dd775b1561ba