Mercurial > cpython-lru_cache-weakref

annotate Parser/tokenizer.h @ 58765:2ba7677bfa95 legacy-trunk

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
testCopysign was defined twice in test_math; combine the definitions
author Mark Dickinson <dickinsm@gmail.com>
date Sat, 06 Feb 2010 23:11:25 +0000
parents 89f42938d5cc
children fd25030223b0
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
1329
519c25039c6a * Added support for X11 modules.
Guido van Rossum <guido@python.org>
parents: 1284
diff changeset
1 #ifndef Py_TOKENIZER_H
519c25039c6a * Added support for X11 modules.
Guido van Rossum <guido@python.org>
parents: 1284
diff changeset
2 #define Py_TOKENIZER_H
519c25039c6a * Added support for X11 modules.
Guido van Rossum <guido@python.org>
parents: 1284
diff changeset
3 #ifdef __cplusplus
519c25039c6a * Added support for X11 modules.
Guido van Rossum <guido@python.org>
parents: 1284
diff changeset
4 extern "C" {
519c25039c6a * Added support for X11 modules.
Guido van Rossum <guido@python.org>
parents: 1284
diff changeset
5 #endif
519c25039c6a * Added support for X11 modules.
Guido van Rossum <guido@python.org>
parents: 1284
diff changeset
6
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 14375
diff changeset
7 #include "object.h"
145
853b736a6a9e Added copyright notice.
Guido van Rossum <guido@python.org>
parents: 6
diff changeset
8
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
9 /* Tokenizer interface */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
10
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
11 #include "token.h" /* For token types */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
12
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
13 #define MAXINDENT 100 /* Max indentation level */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
14
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
15 /* Tokenizer state */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
16 struct tok_state {
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
17 /* Input state; buf <= cur <= inp <= end */
1282
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
18 /* NB an entire line is held in the buffer */
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
19 char *buf; /* Input buffer, or NULL; malloc'ed if fp != NULL */
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
20 char *cur; /* Next character in buffer */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
21 char *inp; /* End of data in buffer */
1282
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
22 char *end; /* End of input buffer if buf != NULL */
1848
b724599401ff Merge back to main trunk
Guido van Rossum <guido@python.org>
parents: 1441
diff changeset
23 char *start; /* Start of current token if not NULL */
1282
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
24 int done; /* E_OK normally, E_EOF at EOF, otherwise error code */
e0464e7115ca * pythonrun.c: Print exception type+arg *after* stack trace instead of
Guido van Rossum <guido@python.org>
parents: 1257
diff changeset
25 /* NB If done != E_OK, cur must be == inp!!! */
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
26 FILE *fp; /* Rest of input; NULL if tokenizing a string */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
27 int tabsize; /* Tab spacing */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
28 int indent; /* Current indentation index */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
29 int indstack[MAXINDENT]; /* Stack of indents */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
30 int atbol; /* Nonzero if at begin of new line */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
31 int pendin; /* Pending indents (if > 0) or dedents (if < 0) */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
32 char *prompt, *nextprompt; /* For interactive prompting */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
33 int lineno; /* Current line number */
1284
dbce72afe261 * selectmodule.c: fix (another!) two memory leaks -- this time in list2set
Guido van Rossum <guido@python.org>
parents: 1282
diff changeset
34 int level; /* () [] {} Parentheses nesting level */
dbce72afe261 * selectmodule.c: fix (another!) two memory leaks -- this time in list2set
Guido van Rossum <guido@python.org>
parents: 1282
diff changeset
35 /* Used to allow free continuations inside them */
7945
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 5379
diff changeset
36 /* Stuff for checking on different tab sizes */
26601
cc05c2c8660e Constify filenames and scripts. Fixes #651362.
Martin v. Löwis <martin@v.loewis.de>
parents: 25098
diff changeset
37 const char *filename; /* For error messages */
7945
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 5379
diff changeset
38 int altwarning; /* Issue warning if alternate tabs don't match */
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 5379
diff changeset
39 int alterror; /* Issue error if alternate tabs don't match */
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 5379
diff changeset
40 int alttabsize; /* Alternate tab spacing */
dc132cc39573 Add checking for inconsistent tab usage
Guido van Rossum <guido@python.org>
parents: 5379
diff changeset
41 int altindstack[MAXINDENT]; /* Stack of alternate indents */
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 14375
diff changeset
42 /* Stuff for PEP 0263 */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 14375
diff changeset
43 int decoding_state; /* -1:decoding, 0:init, 1:raw */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 14375
diff changeset
44 int decoding_erred; /* whether erred in decoding */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 14375
diff changeset
45 int read_coding_spec; /* whether 'coding:...' has been read */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 14375
diff changeset
46 char *encoding;
25098
c02a668488d0 Ignore encoding declarations inside strings. Fixes #603509.
Martin v. Löwis <martin@v.loewis.de>
parents: 24544
diff changeset
47 int cont_line; /* whether we are in a continuation line. */
36786
57322cb2ca11 Patch #1440601: Add col_offset attribute to AST nodes.
Martin v. Löwis <martin@v.loewis.de>
parents: 36749
diff changeset
48 const char* line_start; /* pointer to start of current line */
24544
0bd521eb714e Make pgen compile with pydebug. Duplicate normalized names, as it may
Martin v. Löwis <martin@v.loewis.de>
parents: 24539
diff changeset
49 #ifndef PGEN
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 14375
diff changeset
50 PyObject *decoding_readline; /* codecs.open(...).readline */
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 14375
diff changeset
51 PyObject *decoding_buffer;
24544
0bd521eb714e Make pgen compile with pydebug. Duplicate normalized names, as it may
Martin v. Löwis <martin@v.loewis.de>
parents: 24539
diff changeset
52 #endif
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 14375
diff changeset
53 const char* enc;
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 14375
diff changeset
54 const char* str;
57204
89f42938d5cc fix several compile() issues by translating newlines in the tokenizer
Benjamin Peterson <benjamin@python.org>
parents: 44575
diff changeset
55 const char* input; /* Tokenizer's newline translated copy of the string. */
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
56 };
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
57
57204
89f42938d5cc fix several compile() issues by translating newlines in the tokenizer
Benjamin Peterson <benjamin@python.org>
parents: 44575
diff changeset
58 extern struct tok_state *PyTokenizer_FromString(const char *, int);
13447
a89f7f268de4 Nuke all remaining occurrences of Py_PROTO and Py_FPROTO.
Tim Peters <tim.peters@gmail.com>
parents: 13256
diff changeset
59 extern struct tok_state *PyTokenizer_FromFile(FILE *, char *, char *);
a89f7f268de4 Nuke all remaining occurrences of Py_PROTO and Py_FPROTO.
Tim Peters <tim.peters@gmail.com>
parents: 13256
diff changeset
60 extern void PyTokenizer_Free(struct tok_state *);
a89f7f268de4 Nuke all remaining occurrences of Py_PROTO and Py_FPROTO.
Tim Peters <tim.peters@gmail.com>
parents: 13256
diff changeset
61 extern int PyTokenizer_Get(struct tok_state *, char **, char **);
44575
cf36e01e83f2 Fix #define ordering.
Georg Brandl <georg@python.org>
parents: 44572
diff changeset
62 #if defined(PGEN) || defined(Py_USING_UNICODE)
43229
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 36786
diff changeset
63 extern char * PyTokenizer_RestoreEncoding(struct tok_state* tok,
6a048ff30488 Patch #1031213: Decode source line in SyntaxErrors back to its original
Martin v. Löwis <martin@v.loewis.de>
parents: 36786
diff changeset
64 int len, int *offset);
44572
db5e01c33eb7 Make Python compile with --disable-unicode.
Georg Brandl <georg@python.org>
parents: 43229
diff changeset
65 #endif
1329
519c25039c6a * Added support for X11 modules.
Guido van Rossum <guido@python.org>
parents: 1284
diff changeset
66
519c25039c6a * Added support for X11 modules.
Guido van Rossum <guido@python.org>
parents: 1284
diff changeset
67 #ifdef __cplusplus
519c25039c6a * Added support for X11 modules.
Guido van Rossum <guido@python.org>
parents: 1284
diff changeset
68 }
519c25039c6a * Added support for X11 modules.
Guido van Rossum <guido@python.org>
parents: 1284
diff changeset
69 #endif
519c25039c6a * Added support for X11 modules.
Guido van Rossum <guido@python.org>
parents: 1284
diff changeset
70 #endif /* !Py_TOKENIZER_H */