Mercurial > cpython-lru_cache-weakref

annotate Parser/tokenizer.h @ 79691:05c8928362ee

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Merge __next__ method link fixes with 3.3.
author Ezio Melotti <ezio.melotti@gmail.com>
date Fri, 12 Oct 2012 13:46:37 +0300
parents 7b8d625eb6e4
children 3055e61f1e66
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
1329
519c25039c6a * Added support for X11 modules.
Guido van Rossum <guido@python.org>
parents: 1284
diff changeset
1 #ifndef Py_TOKENIZER_H
519c25039c6a * Added support for X11 modules.
Guido van Rossum <guido@python.org>
parents: 1284
diff changeset
2 #define Py_TOKENIZER_H
519c25039c6a * Added support for X11 modules.
Guido van Rossum <guido@python.org>
parents: 1284
diff changeset
3 #ifdef __cplusplus
519c25039c6a * Added support for X11 modules.
Guido van Rossum <guido@python.org>
parents: 1284
diff changeset
4 extern "C" {
519c25039c6a * Added support for X11 modules.
Guido van Rossum <guido@python.org>
parents: 1284
diff changeset
5 #endif
519c25039c6a * Added support for X11 modules.
Guido van Rossum <guido@python.org>
parents: 1284
diff changeset
6
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 14375
diff changeset
7 #include "object.h"
145
853b736a6a9e Added copyright notice.
Guido van Rossum <guido@python.org>
parents: 6
diff changeset
8
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
9 /* Tokenizer interface */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
10
61467
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51956
diff changeset
11 #include "token.h" /* For token types */
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
12
61467
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51956
diff changeset
13 #define MAXINDENT 100 /* Max indentation level */
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
14
43421
f31d415554cd Use an enum for decoding_state. It makes the code a little more
Neil Schemenauer <nascheme@enme.ucalgary.ca>
parents: 43292
diff changeset
15 enum decoding_state {
61467
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51956
diff changeset
16 STATE_INIT,
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51956
diff changeset
17 STATE_RAW,
66475
9bead909d8ef Merged revisions 85820,85823,85825,85840,85843-85845,85849-85851,85855,85867,85875,85907-85908,85911,85914 via svnmerge from
Georg Brandl <georg@python.org>
parents: 65536
diff changeset
18 STATE_NORMAL /* have a codec associated with input */
43421
f31d415554cd Use an enum for decoding_state. It makes the code a little more
Neil Schemenauer <nascheme@enme.ucalgary.ca>
parents: 43292
diff changeset
19 };
f31d415554cd Use an enum for decoding_state. It makes the code a little more
Neil Schemenauer <nascheme@enme.ucalgary.ca>
parents: 43292
diff changeset
20
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
21 /* Tokenizer state */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
22 struct tok_state {
61467
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51956
diff changeset
23 /* Input state; buf <= cur <= inp <= end */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51956
diff changeset
24 /* NB an entire line is held in the buffer */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51956
diff changeset
25 char *buf; /* Input buffer, or NULL; malloc'ed if fp != NULL */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51956
diff changeset
26 char *cur; /* Next character in buffer */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51956
diff changeset
27 char *inp; /* End of data in buffer */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51956
diff changeset
28 char *end; /* End of input buffer if buf != NULL */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51956
diff changeset
29 char *start; /* Start of current token if not NULL */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51956
diff changeset
30 int done; /* E_OK normally, E_EOF at EOF, otherwise error code */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51956
diff changeset
31 /* NB If done != E_OK, cur must be == inp!!! */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51956
diff changeset
32 FILE *fp; /* Rest of input; NULL if tokenizing a string */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51956
diff changeset
33 int tabsize; /* Tab spacing */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51956
diff changeset
34 int indent; /* Current indentation index */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51956
diff changeset
35 int indstack[MAXINDENT]; /* Stack of indents */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51956
diff changeset
36 int atbol; /* Nonzero if at begin of new line */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51956
diff changeset
37 int pendin; /* Pending indents (if > 0) or dedents (if < 0) */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51956
diff changeset
38 char *prompt, *nextprompt; /* For interactive prompting */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51956
diff changeset
39 int lineno; /* Current line number */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51956
diff changeset
40 int level; /* () [] {} Parentheses nesting level */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51956
diff changeset
41 /* Used to allow free continuations inside them */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51956
diff changeset
42 /* Stuff for checking on different tab sizes */
69147
6e9dc970ac0e Issue #10785: Store the filename as Unicode in the Python parser.
Victor Stinner <victor.stinner@haypocalc.com>
parents: 68257
diff changeset
43 #ifndef PGEN
6e9dc970ac0e Issue #10785: Store the filename as Unicode in the Python parser.
Victor Stinner <victor.stinner@haypocalc.com>
parents: 68257
diff changeset
44 /* pgen doesn't have access to Python codecs, it cannot decode the input
6e9dc970ac0e Issue #10785: Store the filename as Unicode in the Python parser.
Victor Stinner <victor.stinner@haypocalc.com>
parents: 68257
diff changeset
45 filename. The bytes filename might be kept, but it is only used by
6e9dc970ac0e Issue #10785: Store the filename as Unicode in the Python parser.
Victor Stinner <victor.stinner@haypocalc.com>
parents: 68257
diff changeset
46 indenterror() and it is not really needed: pgen only compiles one file
6e9dc970ac0e Issue #10785: Store the filename as Unicode in the Python parser.
Victor Stinner <victor.stinner@haypocalc.com>
parents: 68257
diff changeset
47 (Grammar/Grammar). */
6e9dc970ac0e Issue #10785: Store the filename as Unicode in the Python parser.
Victor Stinner <victor.stinner@haypocalc.com>
parents: 68257
diff changeset
48 PyObject *filename;
6e9dc970ac0e Issue #10785: Store the filename as Unicode in the Python parser.
Victor Stinner <victor.stinner@haypocalc.com>
parents: 68257
diff changeset
49 #endif
61467
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51956
diff changeset
50 int altwarning; /* Issue warning if alternate tabs don't match */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51956
diff changeset
51 int alterror; /* Issue error if alternate tabs don't match */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51956
diff changeset
52 int alttabsize; /* Alternate tab spacing */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51956
diff changeset
53 int altindstack[MAXINDENT]; /* Stack of alternate indents */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51956
diff changeset
54 /* Stuff for PEP 0263 */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51956
diff changeset
55 enum decoding_state decoding_state;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51956
diff changeset
56 int decoding_erred; /* whether erred in decoding */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51956
diff changeset
57 int read_coding_spec; /* whether 'coding:...' has been read */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51956
diff changeset
58 char *encoding; /* Source encoding. */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51956
diff changeset
59 int cont_line; /* whether we are in a continuation line. */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51956
diff changeset
60 const char* line_start; /* pointer to start of current line */
24544
0bd521eb714e Make pgen compile with pydebug. Duplicate normalized names, as it may
Martin v. Löwis <martin@v.loewis.de>
parents: 24539
diff changeset
61 #ifndef PGEN
65311
3d9ef098daf2 Issue #10095: fp_setreadl() doesn't reopen the file, reuse instead the file
Victor Stinner <victor.stinner@haypocalc.com>
parents: 61466
diff changeset
62 PyObject *decoding_readline; /* open(...).readline */
61467
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51956
diff changeset
63 PyObject *decoding_buffer;
24544
0bd521eb714e Make pgen compile with pydebug. Duplicate normalized names, as it may
Martin v. Löwis <martin@v.loewis.de>
parents: 24539
diff changeset
64 #endif
61467
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51956
diff changeset
65 const char* enc; /* Encoding for the current str. */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51956
diff changeset
66 const char* str;
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 57206
diff changeset
67 const char* input; /* Tokenizer's newline translated copy of the string. */
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
68 };
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
69
57206
b2d90c1c637d Merged revisions 76230 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 51956
diff changeset
70 extern struct tok_state *PyTokenizer_FromString(const char *, int);
b2d90c1c637d Merged revisions 76230 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 51956
diff changeset
71 extern struct tok_state *PyTokenizer_FromUTF8(const char *, int);
43225
38f3f9ef193d Convert code from sys.stdin.encoding to UTF-8 in
Martin v. Löwis <martin@v.loewis.de>
parents: 36786
diff changeset
72 extern struct tok_state *PyTokenizer_FromFile(FILE *, char*,
61467
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51956
diff changeset
73 char *, char *);
13447
a89f7f268de4 Nuke all remaining occurrences of Py_PROTO and Py_FPROTO.
Tim Peters <tim.peters@gmail.com>
parents: 13256
diff changeset
74 extern void PyTokenizer_Free(struct tok_state *);
a89f7f268de4 Nuke all remaining occurrences of Py_PROTO and Py_FPROTO.
Tim Peters <tim.peters@gmail.com>
parents: 13256
diff changeset
75 extern int PyTokenizer_Get(struct tok_state *, char **, char **);
61467
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51956
diff changeset
76 extern char * PyTokenizer_RestoreEncoding(struct tok_state* tok,
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51956
diff changeset
77 int len, int *offset);
1329
519c25039c6a * Added support for X11 modules.
Guido van Rossum <guido@python.org>
parents: 1284
diff changeset
78
519c25039c6a * Added support for X11 modules.
Guido van Rossum <guido@python.org>
parents: 1284
diff changeset
79 #ifdef __cplusplus
519c25039c6a * Added support for X11 modules.
Guido van Rossum <guido@python.org>
parents: 1284
diff changeset
80 }
519c25039c6a * Added support for X11 modules.
Guido van Rossum <guido@python.org>
parents: 1284
diff changeset
81 #endif
519c25039c6a * Added support for X11 modules.
Guido van Rossum <guido@python.org>
parents: 1284
diff changeset
82 #endif /* !Py_TOKENIZER_H */