Mercurial > cpython-lru_cache-weakref
annotate Lib/tokenize.py @ 4566:6403c94de50d legacy-trunk
Added safeguard against failure in __del__.
| author | Guido van Rossum <guido@python.org> |
|---|---|
| date | Wed, 27 Nov 1996 19:39:58 +0000 |
| parents | b9fcfb99a3e5 |
| children | 31b0e5f0badf |
| rev | line source |
|---|---|
| 558 | 1 # This module compiles a regular expression that recognizes Python tokens. |
| 2 # It is designed to match the working of the Python tokenizer exactly. | |
| 3 # It takes care of everything except indentation; | |
| 4 # note that un-escaped newlines are tokens, too. | |
| 5 # tokenprog.regs[3] gives the location of the token without whitespace | |
| 6 # It also defines various subexpressions, but doesn't compile them. | |
| 7 # See the function test() below for an example of how to use. | |
| 8 | |
| 9 import regex | |
| 10 | |
| 11 # Note: to get a quoted backslash in a regexp, it must be quadrupled. | |
| 12 | |
| 13 Ignore = '[ \t]*\(\\\\\n[ \t]*\)*\(#.*\)?' | |
| 14 | |
| 15 Name = '[a-zA-Z_][a-zA-Z0-9_]*' | |
| 16 | |
| 17 Hexnumber = '0[xX][0-9a-fA-F]*[lL]?' | |
| 18 Octnumber = '0[0-7]*[lL]?' | |
| 19 Decnumber = '[1-9][0-9]*[lL]?' | |
| 20 Intnumber = Hexnumber + '\|' + Octnumber + '\|' + Decnumber | |
| 21 Exponent = '[eE][-+]?[0-9]+' | |
| 22 Pointfloat = '\([0-9]+\.[0-9]*\|\.[0-9]+\)\(' + Exponent + '\)?' | |
| 23 Expfloat = '[0-9]+' + Exponent | |
| 24 Floatnumber = Pointfloat + '\|' + Expfloat | |
|
705
1e8e3ab67b45
Change the order in which Floatnumber and Intnumber are tried
Guido van Rossum <guido@python.org>
parents:
558
diff
changeset
|
25 Number = Floatnumber + '\|' + Intnumber |
| 558 | 26 |
|
1407
b9fcfb99a3e5
* test_select.py: (some) tests for built-in select module
Guido van Rossum <guido@python.org>
parents:
705
diff
changeset
|
27 String = '\'\(\\\\.\|[^\\\n\']\)*\'' + '\|' + '"\(\\\\.\|[^\\\n"]\)*"' |
|
b9fcfb99a3e5
* test_select.py: (some) tests for built-in select module
Guido van Rossum <guido@python.org>
parents:
705
diff
changeset
|
28 # Note: this module *recognizes* double quotes, but for backward |
|
b9fcfb99a3e5
* test_select.py: (some) tests for built-in select module
Guido van Rossum <guido@python.org>
parents:
705
diff
changeset
|
29 # compatibility, it doesn't *use* them! |
| 558 | 30 |
| 31 Operator = '~\|\+\|-\|\*\|/\|%\|\^\|&\||\|<<\|>>\|==\|<=\|<>\|!=\|>=\|=\|<\|>' | |
| 32 Bracket = '[][(){}]' | |
| 33 Special = '[:;.,`\n]' | |
| 34 Funny = Operator + '\|' + Bracket + '\|' + Special | |
| 35 | |
| 36 PlainToken = Name + '\|' + Number + '\|' + String + '\|' + Funny | |
| 37 | |
| 38 Token = Ignore + '\(' + PlainToken + '\)' | |
| 39 | |
| 40 try: | |
| 41 save_syntax = regex.set_syntax(0) # Use default syntax | |
| 42 tokenprog = regex.compile(Token) | |
| 43 finally: | |
|
705
1e8e3ab67b45
Change the order in which Floatnumber and Intnumber are tried
Guido van Rossum <guido@python.org>
parents:
558
diff
changeset
|
44 if save_syntax != 0: |
|
1e8e3ab67b45
Change the order in which Floatnumber and Intnumber are tried
Guido van Rossum <guido@python.org>
parents:
558
diff
changeset
|
45 dummy = regex.set_syntax(save_syntax) # Restore original syntax |
| 558 | 46 |
| 47 | |
| 48 def test(file): | |
| 49 f = open(file, 'r') | |
| 50 while 1: | |
| 51 line = f.readline() | |
| 52 if not line: break | |
| 53 i, n = 0, len(line) | |
| 54 while i < n: | |
| 55 j = tokenprog.match(line, i) | |
| 56 if j < 0: | |
| 57 print 'No token at', `line[i:i+20]` + '...' | |
| 58 i = i+1 | |
| 59 else: | |
| 60 i = i+j | |
| 61 a, b = tokenprog.regs[3] | |
| 62 if a < b: | |
| 63 print 'Token:', `line[a:b]` |