Mercurial > cpython-lru_cache-weakref

annotate Parser/tokenizer.h @ 87690:66c1c9f32567 default tip

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Add use_weakref option to functools.lru_cache The use_weakref option makes lru_cache create weak references to the function arguments when creating the cache key, which prevents them from staying in the memory forever even after all other references to them are gone. This is a proof of concept for http://bugs.python.org/issue19859
author Radomir Dopieralski <devel@sheep.art.pl>
date Tue, 03 Dec 2013 16:02:48 +0100
parents 3055e61f1e66
children
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
1329
519c25039c6a * Added support for X11 modules.
Guido van Rossum <guido@python.org>
parents: 1284
diff changeset
1 #ifndef Py_TOKENIZER_H
519c25039c6a * Added support for X11 modules.
Guido van Rossum <guido@python.org>
parents: 1284
diff changeset
2 #define Py_TOKENIZER_H
519c25039c6a * Added support for X11 modules.
Guido van Rossum <guido@python.org>
parents: 1284
diff changeset
3 #ifdef __cplusplus
519c25039c6a * Added support for X11 modules.
Guido van Rossum <guido@python.org>
parents: 1284
diff changeset
4 extern "C" {
519c25039c6a * Added support for X11 modules.
Guido van Rossum <guido@python.org>
parents: 1284
diff changeset
5 #endif
519c25039c6a * Added support for X11 modules.
Guido van Rossum <guido@python.org>
parents: 1284
diff changeset
6
24539
06f1e2fd05bc Patch #534304: Implement phase 1 of PEP 263.
Martin v. Löwis <martin@v.loewis.de>
parents: 14375
diff changeset
7 #include "object.h"
145
853b736a6a9e Added copyright notice.
Guido van Rossum <guido@python.org>
parents: 6
diff changeset
8
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
9 /* Tokenizer interface */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
10
61467
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51956
diff changeset
11 #include "token.h" /* For token types */
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
12
61467
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51956
diff changeset
13 #define MAXINDENT 100 /* Max indentation level */
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
14
43421
f31d415554cd Use an enum for decoding_state. It makes the code a little more
Neil Schemenauer <nascheme@enme.ucalgary.ca>
parents: 43292
diff changeset
15 enum decoding_state {
61467
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51956
diff changeset
16 STATE_INIT,
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51956
diff changeset
17 STATE_RAW,
66475
9bead909d8ef Merged revisions 85820,85823,85825,85840,85843-85845,85849-85851,85855,85867,85875,85907-85908,85911,85914 via svnmerge from
Georg Brandl <georg@python.org>
parents: 65536
diff changeset
18 STATE_NORMAL /* have a codec associated with input */
43421
f31d415554cd Use an enum for decoding_state. It makes the code a little more
Neil Schemenauer <nascheme@enme.ucalgary.ca>
parents: 43292
diff changeset
19 };
f31d415554cd Use an enum for decoding_state. It makes the code a little more
Neil Schemenauer <nascheme@enme.ucalgary.ca>
parents: 43292
diff changeset
20
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
21 /* Tokenizer state */
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
22 struct tok_state {
61467
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51956
diff changeset
23 /* Input state; buf <= cur <= inp <= end */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51956
diff changeset
24 /* NB an entire line is held in the buffer */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51956
diff changeset
25 char *buf; /* Input buffer, or NULL; malloc'ed if fp != NULL */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51956
diff changeset
26 char *cur; /* Next character in buffer */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51956
diff changeset
27 char *inp; /* End of data in buffer */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51956
diff changeset
28 char *end; /* End of input buffer if buf != NULL */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51956
diff changeset
29 char *start; /* Start of current token if not NULL */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51956
diff changeset
30 int done; /* E_OK normally, E_EOF at EOF, otherwise error code */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51956
diff changeset
31 /* NB If done != E_OK, cur must be == inp!!! */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51956
diff changeset
32 FILE *fp; /* Rest of input; NULL if tokenizing a string */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51956
diff changeset
33 int tabsize; /* Tab spacing */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51956
diff changeset
34 int indent; /* Current indentation index */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51956
diff changeset
35 int indstack[MAXINDENT]; /* Stack of indents */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51956
diff changeset
36 int atbol; /* Nonzero if at begin of new line */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51956
diff changeset
37 int pendin; /* Pending indents (if > 0) or dedents (if < 0) */
86484
3055e61f1e66 Issue #1772673: The type of `char*` arguments now changed to `const char*`.
Serhiy Storchaka <storchaka@gmail.com>
parents: 69148
diff changeset
38 const char *prompt, *nextprompt; /* For interactive prompting */
61467
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51956
diff changeset
39 int lineno; /* Current line number */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51956
diff changeset
40 int level; /* () [] {} Parentheses nesting level */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51956
diff changeset
41 /* Used to allow free continuations inside them */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51956
diff changeset
42 /* Stuff for checking on different tab sizes */
69147
6e9dc970ac0e Issue #10785: Store the filename as Unicode in the Python parser.
Victor Stinner <victor.stinner@haypocalc.com>
parents: 68257
diff changeset
43 #ifndef PGEN
6e9dc970ac0e Issue #10785: Store the filename as Unicode in the Python parser.
Victor Stinner <victor.stinner@haypocalc.com>
parents: 68257
diff changeset
44 /* pgen doesn't have access to Python codecs, it cannot decode the input
6e9dc970ac0e Issue #10785: Store the filename as Unicode in the Python parser.
Victor Stinner <victor.stinner@haypocalc.com>
parents: 68257
diff changeset
45 filename. The bytes filename might be kept, but it is only used by
6e9dc970ac0e Issue #10785: Store the filename as Unicode in the Python parser.
Victor Stinner <victor.stinner@haypocalc.com>
parents: 68257
diff changeset
46 indenterror() and it is not really needed: pgen only compiles one file
6e9dc970ac0e Issue #10785: Store the filename as Unicode in the Python parser.
Victor Stinner <victor.stinner@haypocalc.com>
parents: 68257
diff changeset
47 (Grammar/Grammar). */
6e9dc970ac0e Issue #10785: Store the filename as Unicode in the Python parser.
Victor Stinner <victor.stinner@haypocalc.com>
parents: 68257
diff changeset
48 PyObject *filename;
6e9dc970ac0e Issue #10785: Store the filename as Unicode in the Python parser.
Victor Stinner <victor.stinner@haypocalc.com>
parents: 68257
diff changeset
49 #endif
61467
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51956
diff changeset
50 int altwarning; /* Issue warning if alternate tabs don't match */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51956
diff changeset
51 int alterror; /* Issue error if alternate tabs don't match */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51956
diff changeset
52 int alttabsize; /* Alternate tab spacing */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51956
diff changeset
53 int altindstack[MAXINDENT]; /* Stack of alternate indents */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51956
diff changeset
54 /* Stuff for PEP 0263 */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51956
diff changeset
55 enum decoding_state decoding_state;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51956
diff changeset
56 int decoding_erred; /* whether erred in decoding */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51956
diff changeset
57 int read_coding_spec; /* whether 'coding:...' has been read */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51956
diff changeset
58 char *encoding; /* Source encoding. */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51956
diff changeset
59 int cont_line; /* whether we are in a continuation line. */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51956
diff changeset
60 const char* line_start; /* pointer to start of current line */
24544
0bd521eb714e Make pgen compile with pydebug. Duplicate normalized names, as it may
Martin v. Löwis <martin@v.loewis.de>
parents: 24539
diff changeset
61 #ifndef PGEN
65311
3d9ef098daf2 Issue #10095: fp_setreadl() doesn't reopen the file, reuse instead the file
Victor Stinner <victor.stinner@haypocalc.com>
parents: 61466
diff changeset
62 PyObject *decoding_readline; /* open(...).readline */
61467
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51956
diff changeset
63 PyObject *decoding_buffer;
24544
0bd521eb714e Make pgen compile with pydebug. Duplicate normalized names, as it may
Martin v. Löwis <martin@v.loewis.de>
parents: 24539
diff changeset
64 #endif
61467
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51956
diff changeset
65 const char* enc; /* Encoding for the current str. */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51956
diff changeset
66 const char* str;
61466
cab14be0ada1 Recorded merge of revisions 81029 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 57206
diff changeset
67 const char* input; /* Tokenizer's newline translated copy of the string. */
6
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
68 };
fc6fcd7df4f7 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
69
57206
b2d90c1c637d Merged revisions 76230 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 51956
diff changeset
70 extern struct tok_state *PyTokenizer_FromString(const char *, int);
b2d90c1c637d Merged revisions 76230 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 51956
diff changeset
71 extern struct tok_state *PyTokenizer_FromUTF8(const char *, int);
86484
3055e61f1e66 Issue #1772673: The type of `char*` arguments now changed to `const char*`.
Serhiy Storchaka <storchaka@gmail.com>
parents: 69148
diff changeset
72 extern struct tok_state *PyTokenizer_FromFile(FILE *, const char*,
3055e61f1e66 Issue #1772673: The type of `char*` arguments now changed to `const char*`.
Serhiy Storchaka <storchaka@gmail.com>
parents: 69148
diff changeset
73 const char *, const char *);
13447
a89f7f268de4 Nuke all remaining occurrences of Py_PROTO and Py_FPROTO.
Tim Peters <tim.peters@gmail.com>
parents: 13256
diff changeset
74 extern void PyTokenizer_Free(struct tok_state *);
a89f7f268de4 Nuke all remaining occurrences of Py_PROTO and Py_FPROTO.
Tim Peters <tim.peters@gmail.com>
parents: 13256
diff changeset
75 extern int PyTokenizer_Get(struct tok_state *, char **, char **);
61467
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51956
diff changeset
76 extern char * PyTokenizer_RestoreEncoding(struct tok_state* tok,
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51956
diff changeset
77 int len, int *offset);
1329
519c25039c6a * Added support for X11 modules.
Guido van Rossum <guido@python.org>
parents: 1284
diff changeset
78
519c25039c6a * Added support for X11 modules.
Guido van Rossum <guido@python.org>
parents: 1284
diff changeset
79 #ifdef __cplusplus
519c25039c6a * Added support for X11 modules.
Guido van Rossum <guido@python.org>
parents: 1284
diff changeset
80 }
519c25039c6a * Added support for X11 modules.
Guido van Rossum <guido@python.org>
parents: 1284
diff changeset
81 #endif
519c25039c6a * Added support for X11 modules.
Guido van Rossum <guido@python.org>
parents: 1284
diff changeset
82 #endif /* !Py_TOKENIZER_H */