Mercurial > cpython-lru_cache-weakref

annotate Parser/tokenizer.h @ 80661:94de77bd0b4b 2.7

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Issue #16194: document imp.load_dynamic problems when read several extension modules from the same binary object file.
author Andrew Svetlov <andrew.svetlov@gmail.com>
date Thu, 29 Nov 2012 19:51:28 +0200
parents b36cb4602e21
children
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
1329
519c25039c6a * Added support for X11 modules.
Guido van Rossum <guido@python.org>
parents: 1284
diff changeset
1 #ifndef Py_TOKENIZER_H
519c25039c6a * Added support for X11 modules.
Guido van Rossum <guido@python.org>
parents: 1284
diff changeset
2 #define Py_TOKENIZER_H
519c25039c6a * Added support for X11 modules.
Guido van Rossum <guido@python.org>
parents: 1284
diff changeset
3 #ifdef __cplusplus
519c25039c6a * Added support for X11 modules.
Guido van Rossum <guido@python.org>
parents: 1284
diff changeset
4 extern "C" {
519c25039c6a * Added support for X11 modules.
Guido van Rossum <guido@python.org>
parents: 1284
diff changeset
5 #endif
519c25039c6a * Added support for X11 modules.
Guido van Rossum <guido@python.org>
parents: 1284
diff changeset
6
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 14375
diff changeset
7 #include "object.h"
145
853b736a6a9e Added copyright notice.
Guido van Rossum <guido@python.org>
parents: 6
diff changeset
8
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
9 /* Tokenizer interface */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
10
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 44575
diff changeset
11 #include "token.h" /* For token types */
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
12
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 44575
diff changeset
13 #define MAXINDENT 100 /* Max indentation level */
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
14
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
15 /* Tokenizer state */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
16 struct tok_state {
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 44575
diff changeset
17 /* Input state; buf <= cur <= inp <= end */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 44575
diff changeset
18 /* NB an entire line is held in the buffer */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 44575
diff changeset
19 char *buf; /* Input buffer, or NULL; malloc'ed if fp != NULL */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 44575
diff changeset
20 char *cur; /* Next character in buffer */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 44575
diff changeset
21 char *inp; /* End of data in buffer */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 44575
diff changeset
22 char *end; /* End of input buffer if buf != NULL */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 44575
diff changeset
23 char *start; /* Start of current token if not NULL */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 44575
diff changeset
24 int done; /* E_OK normally, E_EOF at EOF, otherwise error code */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 44575
diff changeset
25 /* NB If done != E_OK, cur must be == inp!!! */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 44575
diff changeset
26 FILE *fp; /* Rest of input; NULL if tokenizing a string */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 44575
diff changeset
27 int tabsize; /* Tab spacing */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 44575
diff changeset
28 int indent; /* Current indentation index */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 44575
diff changeset
29 int indstack[MAXINDENT]; /* Stack of indents */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 44575
diff changeset
30 int atbol; /* Nonzero if at begin of new line */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 44575
diff changeset
31 int pendin; /* Pending indents (if > 0) or dedents (if < 0) */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 44575
diff changeset
32 char *prompt, *nextprompt; /* For interactive prompting */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 44575
diff changeset
33 int lineno; /* Current line number */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 44575
diff changeset
34 int level; /* () [] {} Parentheses nesting level */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 44575
diff changeset
35 /* Used to allow free continuations inside them */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 44575
diff changeset
36 /* Stuff for checking on different tab sizes */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 44575
diff changeset
37 const char *filename; /* For error messages */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 44575
diff changeset
38 int altwarning; /* Issue warning if alternate tabs don't match */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 44575
diff changeset
39 int alterror; /* Issue error if alternate tabs don't match */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 44575
diff changeset
40 int alttabsize; /* Alternate tab spacing */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 44575
diff changeset
41 int altindstack[MAXINDENT]; /* Stack of alternate indents */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 44575
diff changeset
42 /* Stuff for PEP 0263 */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 44575
diff changeset
43 int decoding_state; /* -1:decoding, 0:init, 1:raw */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 44575
diff changeset
44 int decoding_erred; /* whether erred in decoding */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 44575
diff changeset
45 int read_coding_spec; /* whether 'coding:...' has been read */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 44575
diff changeset
46 char *encoding;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 44575
diff changeset
47 int cont_line; /* whether we are in a continuation line. */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 44575
diff changeset
48 const char* line_start; /* pointer to start of current line */
24544
0bd521eb714e Make pgen compile with pydebug. Duplicate normalized names, as it may
Martin v. Löwis <martin@v.loewis.de>
parents: 24539
diff changeset
49 #ifndef PGEN
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 44575
diff changeset
50 PyObject *decoding_readline; /* codecs.open(...).readline */
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 44575
diff changeset
51 PyObject *decoding_buffer;
24544
0bd521eb714e Make pgen compile with pydebug. Duplicate normalized names, as it may
Martin v. Löwis <martin@v.loewis.de>
parents: 24539
diff changeset
52 #endif
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 44575
diff changeset
53 const char* enc;
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 44575
diff changeset
54 const char* str;
61463
fd25030223b0 Untabify C files. Will watch buildbots.
Antoine Pitrou <solipsis@pitrou.net>
parents: 57204
diff changeset
55 const char* input; /* Tokenizer's newline translated copy of the string. */
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
56 };
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
57
57204
89f42938d5cc fix several compile() issues by translating newlines in the tokenizer
Benjamin Peterson <benjamin@python.org>
parents: 44575
diff changeset
58 extern struct tok_state *PyTokenizer_FromString(const char *, int);
13447
a89f7f268de4 Nuke all remaining occurrences of Py_PROTO and Py_FPROTO.
Tim Peters <tim.peters@gmail.com>
parents: 13256
diff changeset
59 extern struct tok_state *PyTokenizer_FromFile(FILE *, char *, char *);
a89f7f268de4 Nuke all remaining occurrences of Py_PROTO and Py_FPROTO.
Tim Peters <tim.peters@gmail.com>
parents: 13256
diff changeset
60 extern void PyTokenizer_Free(struct tok_state *);
a89f7f268de4 Nuke all remaining occurrences of Py_PROTO and Py_FPROTO.
Tim Peters <tim.peters@gmail.com>
parents: 13256
diff changeset
61 extern int PyTokenizer_Get(struct tok_state *, char **, char **);
44575
cf36e01e83f2 Fix #define ordering.
Georg Brandl <georg@python.org>
parents: 44572
diff changeset
62 #if defined(PGEN) || defined(Py_USING_UNICODE)
61465
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 44575
diff changeset
63 extern char * PyTokenizer_RestoreEncoding(struct tok_state* tok,
e38335d4df91 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 44575
diff changeset
64 int len, int *offset);
44572
db5e01c33eb7 Make Python compile with --disable-unicode.
Georg Brandl <georg@python.org>
parents: 43229
diff changeset
65 #endif
1329
519c25039c6a * Added support for X11 modules.
Guido van Rossum <guido@python.org>
parents: 1284
diff changeset
66
519c25039c6a * Added support for X11 modules.
Guido van Rossum <guido@python.org>
parents: 1284
diff changeset
67 #ifdef __cplusplus
519c25039c6a * Added support for X11 modules.
Guido van Rossum <guido@python.org>
parents: 1284
diff changeset
68 }
519c25039c6a * Added support for X11 modules.
Guido van Rossum <guido@python.org>
parents: 1284
diff changeset
69 #endif
519c25039c6a * Added support for X11 modules.
Guido van Rossum <guido@python.org>
parents: 1284
diff changeset
70 #endif /* !Py_TOKENIZER_H */