Mercurial > cpython-lru_cache-weakref
annotate Lib/test/test_tokenize.py @ 87690:66c1c9f32567 default tip
Add use_weakref option to functools.lru_cache
The use_weakref option makes lru_cache create weak references to the function
arguments when creating the cache key, which prevents them from staying in the
memory forever even after all other references to them are gone.
This is a proof of concept for http://bugs.python.org/issue19859
| author | Radomir Dopieralski <devel@sheep.art.pl> |
|---|---|
| date | Tue, 03 Dec 2013 16:02:48 +0100 |
| parents | 2dfe8262093c |
| children |
| rev | line source |
|---|---|
|
45773
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
1 doctests = """ |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
2 Tests for the tokenize module. |
|
40450
cc992d75d5b3
Four months of trunk changes (including a few releases...)
Thomas Wouters <thomas@python.org>
parents:
37316
diff
changeset
|
3 |
|
45773
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
4 The tests can be really simple. Given a small fragment of source |
|
45802
de6b2834b8af
Added tests for tokenize, blocked corresponding checkin from trunk.
Eric Smith <eric@trueblade.com>
parents:
45777
diff
changeset
|
5 code, print out a table with tokens. The ENDMARK is omitted for |
|
40450
cc992d75d5b3
Four months of trunk changes (including a few releases...)
Thomas Wouters <thomas@python.org>
parents:
37316
diff
changeset
|
6 brevity. |
|
cc992d75d5b3
Four months of trunk changes (including a few releases...)
Thomas Wouters <thomas@python.org>
parents:
37316
diff
changeset
|
7 |
|
45773
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
8 >>> dump_tokens("1 + 1") |
|
45888
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
9 ENCODING 'utf-8' (0, 0) (0, 0) |
|
45773
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
10 NUMBER '1' (1, 0) (1, 1) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
11 OP '+' (1, 2) (1, 3) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
12 NUMBER '1' (1, 4) (1, 5) |
|
40450
cc992d75d5b3
Four months of trunk changes (including a few releases...)
Thomas Wouters <thomas@python.org>
parents:
37316
diff
changeset
|
13 |
|
45773
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
14 >>> dump_tokens("if False:\\n" |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
15 ... " # NL\\n" |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
16 ... " True = False # NEWLINE\\n") |
|
45888
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
17 ENCODING 'utf-8' (0, 0) (0, 0) |
|
45773
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
18 NAME 'if' (1, 0) (1, 2) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
19 NAME 'False' (1, 3) (1, 8) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
20 OP ':' (1, 8) (1, 9) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
21 NEWLINE '\\n' (1, 9) (1, 10) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
22 COMMENT '# NL' (2, 4) (2, 8) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
23 NL '\\n' (2, 8) (2, 9) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
24 INDENT ' ' (3, 0) (3, 4) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
25 NAME 'True' (3, 4) (3, 8) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
26 OP '=' (3, 9) (3, 10) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
27 NAME 'False' (3, 11) (3, 16) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
28 COMMENT '# NEWLINE' (3, 17) (3, 26) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
29 NEWLINE '\\n' (3, 26) (3, 27) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
30 DEDENT '' (4, 0) (4, 0) |
|
40450
cc992d75d5b3
Four months of trunk changes (including a few releases...)
Thomas Wouters <thomas@python.org>
parents:
37316
diff
changeset
|
31 |
|
45773
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
32 >>> indent_error_file = \""" |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
33 ... def k(x): |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
34 ... x += 2 |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
35 ... x += 5 |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
36 ... \""" |
|
45888
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
37 >>> readline = BytesIO(indent_error_file.encode('utf-8')).readline |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
38 >>> for tok in tokenize(readline): pass |
|
45773
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
39 Traceback (most recent call last): |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
40 ... |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
41 IndentationError: unindent does not match any outer indentation level |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
42 |
|
62620
ebfb6bc082d9
Merged revisions 82356 via svnmerge from
Mark Dickinson <dickinsm@gmail.com>
parents:
59700
diff
changeset
|
43 There are some standard formatting practices that are easy to get right. |
|
40450
cc992d75d5b3
Four months of trunk changes (including a few releases...)
Thomas Wouters <thomas@python.org>
parents:
37316
diff
changeset
|
44 |
|
45773
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
45 >>> roundtrip("if x == 1:\\n" |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
46 ... " print(x)\\n") |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
47 True |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
48 |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
49 >>> roundtrip("# This is a comment\\n# This also") |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
50 True |
|
40450
cc992d75d5b3
Four months of trunk changes (including a few releases...)
Thomas Wouters <thomas@python.org>
parents:
37316
diff
changeset
|
51 |
|
cc992d75d5b3
Four months of trunk changes (including a few releases...)
Thomas Wouters <thomas@python.org>
parents:
37316
diff
changeset
|
52 Some people use different formatting conventions, which makes |
|
45773
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
53 untokenize a little trickier. Note that this test involves trailing |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
54 whitespace after the colon. Note that we use hex escapes to make the |
|
45888
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
55 two trailing blanks apparent in the expected output. |
|
45773
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
56 |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
57 >>> roundtrip("if x == 1 : \\n" |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
58 ... " print(x)\\n") |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
59 True |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
60 |
|
47201
9c930a32cebb
#2621 rename test.test_support to test.support
Benjamin Peterson <benjamin@python.org>
parents:
46978
diff
changeset
|
61 >>> f = support.findfile("tokenize_tests.txt") |
|
45888
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
62 >>> roundtrip(open(f, 'rb')) |
|
45773
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
63 True |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
64 |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
65 >>> roundtrip("if x == 1:\\n" |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
66 ... " # A comment by itself.\\n" |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
67 ... " print(x) # Comment here, too.\\n" |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
68 ... " # Another comment.\\n" |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
69 ... "after_if = True\\n") |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
70 True |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
71 |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
72 >>> roundtrip("if (x # The comments need to go in the right place\\n" |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
73 ... " == 1):\\n" |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
74 ... " print('x==1')\\n") |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
75 True |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
76 |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
77 >>> roundtrip("class Test: # A comment here\\n" |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
78 ... " # A comment with weird indent\\n" |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
79 ... " after_com = 5\\n" |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
80 ... " def x(m): return m*5 # a one liner\\n" |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
81 ... " def y(m): # A whitespace after the colon\\n" |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
82 ... " return y*4 # 3-space indent\\n") |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
83 True |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
84 |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
85 Some error-handling code |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
86 |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
87 >>> roundtrip("try: import somemodule\\n" |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
88 ... "except ImportError: # comment\\n" |
|
46145
da5597a5c195
Merged revisions 61964-61979 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
45888
diff
changeset
|
89 ... " print('Can not import' # comment2\\n)" |
|
46978
55d8c35322a7
Convert a lot of print statements to print functions in docstrings,
Neal Norwitz <nnorwitz@gmail.com>
parents:
46145
diff
changeset
|
90 ... "else: print('Loaded')\\n") |
|
45773
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
91 True |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
92 |
|
45802
de6b2834b8af
Added tests for tokenize, blocked corresponding checkin from trunk.
Eric Smith <eric@trueblade.com>
parents:
45777
diff
changeset
|
93 Balancing continuation |
|
45773
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
94 |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
95 >>> roundtrip("a = (3,4, \\n" |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
96 ... "5,6)\\n" |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
97 ... "y = [3, 4,\\n" |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
98 ... "5]\\n" |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
99 ... "z = {'a': 5,\\n" |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
100 ... "'b':15, 'c':True}\\n" |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
101 ... "x = len(y) + 5 - a[\\n" |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
102 ... "3] - a[2]\\n" |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
103 ... "+ len(z) - z[\\n" |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
104 ... "'b']\\n") |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
105 True |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
106 |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
107 Ordinary integers and binary operators |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
108 |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
109 >>> dump_tokens("0xff <= 255") |
|
45888
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
110 ENCODING 'utf-8' (0, 0) (0, 0) |
|
45773
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
111 NUMBER '0xff' (1, 0) (1, 4) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
112 OP '<=' (1, 5) (1, 7) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
113 NUMBER '255' (1, 8) (1, 11) |
|
45802
de6b2834b8af
Added tests for tokenize, blocked corresponding checkin from trunk.
Eric Smith <eric@trueblade.com>
parents:
45777
diff
changeset
|
114 >>> dump_tokens("0b10 <= 255") |
|
45888
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
115 ENCODING 'utf-8' (0, 0) (0, 0) |
|
45802
de6b2834b8af
Added tests for tokenize, blocked corresponding checkin from trunk.
Eric Smith <eric@trueblade.com>
parents:
45777
diff
changeset
|
116 NUMBER '0b10' (1, 0) (1, 4) |
|
de6b2834b8af
Added tests for tokenize, blocked corresponding checkin from trunk.
Eric Smith <eric@trueblade.com>
parents:
45777
diff
changeset
|
117 OP '<=' (1, 5) (1, 7) |
|
de6b2834b8af
Added tests for tokenize, blocked corresponding checkin from trunk.
Eric Smith <eric@trueblade.com>
parents:
45777
diff
changeset
|
118 NUMBER '255' (1, 8) (1, 11) |
|
de6b2834b8af
Added tests for tokenize, blocked corresponding checkin from trunk.
Eric Smith <eric@trueblade.com>
parents:
45777
diff
changeset
|
119 >>> dump_tokens("0o123 <= 0O123") |
|
45888
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
120 ENCODING 'utf-8' (0, 0) (0, 0) |
|
45802
de6b2834b8af
Added tests for tokenize, blocked corresponding checkin from trunk.
Eric Smith <eric@trueblade.com>
parents:
45777
diff
changeset
|
121 NUMBER '0o123' (1, 0) (1, 5) |
|
de6b2834b8af
Added tests for tokenize, blocked corresponding checkin from trunk.
Eric Smith <eric@trueblade.com>
parents:
45777
diff
changeset
|
122 OP '<=' (1, 6) (1, 8) |
|
de6b2834b8af
Added tests for tokenize, blocked corresponding checkin from trunk.
Eric Smith <eric@trueblade.com>
parents:
45777
diff
changeset
|
123 NUMBER '0O123' (1, 9) (1, 14) |
|
45777
d0c677ddd6b8
Fix doctests in test_tokenize by converting Python 2.x syntax
Mark Dickinson <dickinsm@gmail.com>
parents:
45773
diff
changeset
|
124 >>> dump_tokens("1234567 > ~0x15") |
|
45888
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
125 ENCODING 'utf-8' (0, 0) (0, 0) |
|
45777
d0c677ddd6b8
Fix doctests in test_tokenize by converting Python 2.x syntax
Mark Dickinson <dickinsm@gmail.com>
parents:
45773
diff
changeset
|
126 NUMBER '1234567' (1, 0) (1, 7) |
|
d0c677ddd6b8
Fix doctests in test_tokenize by converting Python 2.x syntax
Mark Dickinson <dickinsm@gmail.com>
parents:
45773
diff
changeset
|
127 OP '>' (1, 8) (1, 9) |
|
d0c677ddd6b8
Fix doctests in test_tokenize by converting Python 2.x syntax
Mark Dickinson <dickinsm@gmail.com>
parents:
45773
diff
changeset
|
128 OP '~' (1, 10) (1, 11) |
|
d0c677ddd6b8
Fix doctests in test_tokenize by converting Python 2.x syntax
Mark Dickinson <dickinsm@gmail.com>
parents:
45773
diff
changeset
|
129 NUMBER '0x15' (1, 11) (1, 15) |
|
d0c677ddd6b8
Fix doctests in test_tokenize by converting Python 2.x syntax
Mark Dickinson <dickinsm@gmail.com>
parents:
45773
diff
changeset
|
130 >>> dump_tokens("2134568 != 1231515") |
|
45888
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
131 ENCODING 'utf-8' (0, 0) (0, 0) |
|
45773
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
132 NUMBER '2134568' (1, 0) (1, 7) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
133 OP '!=' (1, 8) (1, 10) |
|
45777
d0c677ddd6b8
Fix doctests in test_tokenize by converting Python 2.x syntax
Mark Dickinson <dickinsm@gmail.com>
parents:
45773
diff
changeset
|
134 NUMBER '1231515' (1, 11) (1, 18) |
|
d0c677ddd6b8
Fix doctests in test_tokenize by converting Python 2.x syntax
Mark Dickinson <dickinsm@gmail.com>
parents:
45773
diff
changeset
|
135 >>> dump_tokens("(-124561-1) & 200000000") |
|
45888
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
136 ENCODING 'utf-8' (0, 0) (0, 0) |
|
45773
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
137 OP '(' (1, 0) (1, 1) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
138 OP '-' (1, 1) (1, 2) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
139 NUMBER '124561' (1, 2) (1, 8) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
140 OP '-' (1, 8) (1, 9) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
141 NUMBER '1' (1, 9) (1, 10) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
142 OP ')' (1, 10) (1, 11) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
143 OP '&' (1, 12) (1, 13) |
|
45777
d0c677ddd6b8
Fix doctests in test_tokenize by converting Python 2.x syntax
Mark Dickinson <dickinsm@gmail.com>
parents:
45773
diff
changeset
|
144 NUMBER '200000000' (1, 14) (1, 23) |
|
45773
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
145 >>> dump_tokens("0xdeadbeef != -1") |
|
45888
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
146 ENCODING 'utf-8' (0, 0) (0, 0) |
|
45773
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
147 NUMBER '0xdeadbeef' (1, 0) (1, 10) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
148 OP '!=' (1, 11) (1, 13) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
149 OP '-' (1, 14) (1, 15) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
150 NUMBER '1' (1, 15) (1, 16) |
|
45777
d0c677ddd6b8
Fix doctests in test_tokenize by converting Python 2.x syntax
Mark Dickinson <dickinsm@gmail.com>
parents:
45773
diff
changeset
|
151 >>> dump_tokens("0xdeadc0de & 12345") |
|
45888
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
152 ENCODING 'utf-8' (0, 0) (0, 0) |
|
45773
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
153 NUMBER '0xdeadc0de' (1, 0) (1, 10) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
154 OP '&' (1, 11) (1, 12) |
|
45777
d0c677ddd6b8
Fix doctests in test_tokenize by converting Python 2.x syntax
Mark Dickinson <dickinsm@gmail.com>
parents:
45773
diff
changeset
|
155 NUMBER '12345' (1, 13) (1, 18) |
|
45773
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
156 >>> dump_tokens("0xFF & 0x15 | 1234") |
|
45888
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
157 ENCODING 'utf-8' (0, 0) (0, 0) |
|
45773
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
158 NUMBER '0xFF' (1, 0) (1, 4) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
159 OP '&' (1, 5) (1, 6) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
160 NUMBER '0x15' (1, 7) (1, 11) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
161 OP '|' (1, 12) (1, 13) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
162 NUMBER '1234' (1, 14) (1, 18) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
163 |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
164 Long integers |
|
40450
cc992d75d5b3
Four months of trunk changes (including a few releases...)
Thomas Wouters <thomas@python.org>
parents:
37316
diff
changeset
|
165 |
|
45777
d0c677ddd6b8
Fix doctests in test_tokenize by converting Python 2.x syntax
Mark Dickinson <dickinsm@gmail.com>
parents:
45773
diff
changeset
|
166 >>> dump_tokens("x = 0") |
|
45888
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
167 ENCODING 'utf-8' (0, 0) (0, 0) |
|
45773
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
168 NAME 'x' (1, 0) (1, 1) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
169 OP '=' (1, 2) (1, 3) |
|
45777
d0c677ddd6b8
Fix doctests in test_tokenize by converting Python 2.x syntax
Mark Dickinson <dickinsm@gmail.com>
parents:
45773
diff
changeset
|
170 NUMBER '0' (1, 4) (1, 5) |
|
45773
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
171 >>> dump_tokens("x = 0xfffffffffff") |
|
45888
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
172 ENCODING 'utf-8' (0, 0) (0, 0) |
|
45773
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
173 NAME 'x' (1, 0) (1, 1) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
174 OP '=' (1, 2) (1, 3) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
175 NUMBER '0xffffffffff (1, 4) (1, 17) |
|
45777
d0c677ddd6b8
Fix doctests in test_tokenize by converting Python 2.x syntax
Mark Dickinson <dickinsm@gmail.com>
parents:
45773
diff
changeset
|
176 >>> dump_tokens("x = 123141242151251616110") |
|
45888
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
177 ENCODING 'utf-8' (0, 0) (0, 0) |
|
45773
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
178 NAME 'x' (1, 0) (1, 1) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
179 OP '=' (1, 2) (1, 3) |
|
45777
d0c677ddd6b8
Fix doctests in test_tokenize by converting Python 2.x syntax
Mark Dickinson <dickinsm@gmail.com>
parents:
45773
diff
changeset
|
180 NUMBER '123141242151 (1, 4) (1, 25) |
|
d0c677ddd6b8
Fix doctests in test_tokenize by converting Python 2.x syntax
Mark Dickinson <dickinsm@gmail.com>
parents:
45773
diff
changeset
|
181 >>> dump_tokens("x = -15921590215012591") |
|
45888
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
182 ENCODING 'utf-8' (0, 0) (0, 0) |
|
45773
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
183 NAME 'x' (1, 0) (1, 1) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
184 OP '=' (1, 2) (1, 3) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
185 OP '-' (1, 4) (1, 5) |
|
45777
d0c677ddd6b8
Fix doctests in test_tokenize by converting Python 2.x syntax
Mark Dickinson <dickinsm@gmail.com>
parents:
45773
diff
changeset
|
186 NUMBER '159215902150 (1, 5) (1, 22) |
|
45773
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
187 |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
188 Floating point numbers |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
189 |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
190 >>> dump_tokens("x = 3.14159") |
|
45888
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
191 ENCODING 'utf-8' (0, 0) (0, 0) |
|
45773
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
192 NAME 'x' (1, 0) (1, 1) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
193 OP '=' (1, 2) (1, 3) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
194 NUMBER '3.14159' (1, 4) (1, 11) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
195 >>> dump_tokens("x = 314159.") |
|
45888
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
196 ENCODING 'utf-8' (0, 0) (0, 0) |
|
45773
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
197 NAME 'x' (1, 0) (1, 1) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
198 OP '=' (1, 2) (1, 3) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
199 NUMBER '314159.' (1, 4) (1, 11) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
200 >>> dump_tokens("x = .314159") |
|
45888
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
201 ENCODING 'utf-8' (0, 0) (0, 0) |
|
45773
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
202 NAME 'x' (1, 0) (1, 1) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
203 OP '=' (1, 2) (1, 3) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
204 NUMBER '.314159' (1, 4) (1, 11) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
205 >>> dump_tokens("x = 3e14159") |
|
45888
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
206 ENCODING 'utf-8' (0, 0) (0, 0) |
|
45773
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
207 NAME 'x' (1, 0) (1, 1) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
208 OP '=' (1, 2) (1, 3) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
209 NUMBER '3e14159' (1, 4) (1, 11) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
210 >>> dump_tokens("x = 3E123") |
|
45888
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
211 ENCODING 'utf-8' (0, 0) (0, 0) |
|
45773
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
212 NAME 'x' (1, 0) (1, 1) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
213 OP '=' (1, 2) (1, 3) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
214 NUMBER '3E123' (1, 4) (1, 9) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
215 >>> dump_tokens("x+y = 3e-1230") |
|
45888
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
216 ENCODING 'utf-8' (0, 0) (0, 0) |
|
45773
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
217 NAME 'x' (1, 0) (1, 1) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
218 OP '+' (1, 1) (1, 2) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
219 NAME 'y' (1, 2) (1, 3) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
220 OP '=' (1, 4) (1, 5) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
221 NUMBER '3e-1230' (1, 6) (1, 13) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
222 >>> dump_tokens("x = 3.14e159") |
|
45888
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
223 ENCODING 'utf-8' (0, 0) (0, 0) |
|
45773
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
224 NAME 'x' (1, 0) (1, 1) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
225 OP '=' (1, 2) (1, 3) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
226 NUMBER '3.14e159' (1, 4) (1, 12) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
227 |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
228 String literals |
|
40450
cc992d75d5b3
Four months of trunk changes (including a few releases...)
Thomas Wouters <thomas@python.org>
parents:
37316
diff
changeset
|
229 |
|
45773
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
230 >>> dump_tokens("x = ''; y = \\\"\\\"") |
|
45888
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
231 ENCODING 'utf-8' (0, 0) (0, 0) |
|
45773
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
232 NAME 'x' (1, 0) (1, 1) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
233 OP '=' (1, 2) (1, 3) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
234 STRING "''" (1, 4) (1, 6) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
235 OP ';' (1, 6) (1, 7) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
236 NAME 'y' (1, 8) (1, 9) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
237 OP '=' (1, 10) (1, 11) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
238 STRING '""' (1, 12) (1, 14) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
239 >>> dump_tokens("x = '\\\"'; y = \\\"'\\\"") |
|
45888
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
240 ENCODING 'utf-8' (0, 0) (0, 0) |
|
45773
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
241 NAME 'x' (1, 0) (1, 1) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
242 OP '=' (1, 2) (1, 3) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
243 STRING '\\'"\\'' (1, 4) (1, 7) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
244 OP ';' (1, 7) (1, 8) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
245 NAME 'y' (1, 9) (1, 10) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
246 OP '=' (1, 11) (1, 12) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
247 STRING '"\\'"' (1, 13) (1, 16) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
248 >>> dump_tokens("x = \\\"doesn't \\\"shrink\\\", does it\\\"") |
|
45888
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
249 ENCODING 'utf-8' (0, 0) (0, 0) |
|
45773
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
250 NAME 'x' (1, 0) (1, 1) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
251 OP '=' (1, 2) (1, 3) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
252 STRING '"doesn\\'t "' (1, 4) (1, 14) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
253 NAME 'shrink' (1, 14) (1, 20) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
254 STRING '", does it"' (1, 20) (1, 31) |
|
45777
d0c677ddd6b8
Fix doctests in test_tokenize by converting Python 2.x syntax
Mark Dickinson <dickinsm@gmail.com>
parents:
45773
diff
changeset
|
255 >>> dump_tokens("x = 'abc' + 'ABC'") |
|
45888
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
256 ENCODING 'utf-8' (0, 0) (0, 0) |
|
45773
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
257 NAME 'x' (1, 0) (1, 1) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
258 OP '=' (1, 2) (1, 3) |
|
45777
d0c677ddd6b8
Fix doctests in test_tokenize by converting Python 2.x syntax
Mark Dickinson <dickinsm@gmail.com>
parents:
45773
diff
changeset
|
259 STRING "'abc'" (1, 4) (1, 9) |
|
d0c677ddd6b8
Fix doctests in test_tokenize by converting Python 2.x syntax
Mark Dickinson <dickinsm@gmail.com>
parents:
45773
diff
changeset
|
260 OP '+' (1, 10) (1, 11) |
|
d0c677ddd6b8
Fix doctests in test_tokenize by converting Python 2.x syntax
Mark Dickinson <dickinsm@gmail.com>
parents:
45773
diff
changeset
|
261 STRING "'ABC'" (1, 12) (1, 17) |
|
45773
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
262 >>> dump_tokens('y = "ABC" + "ABC"') |
|
45888
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
263 ENCODING 'utf-8' (0, 0) (0, 0) |
|
45773
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
264 NAME 'y' (1, 0) (1, 1) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
265 OP '=' (1, 2) (1, 3) |
|
45777
d0c677ddd6b8
Fix doctests in test_tokenize by converting Python 2.x syntax
Mark Dickinson <dickinsm@gmail.com>
parents:
45773
diff
changeset
|
266 STRING '"ABC"' (1, 4) (1, 9) |
|
d0c677ddd6b8
Fix doctests in test_tokenize by converting Python 2.x syntax
Mark Dickinson <dickinsm@gmail.com>
parents:
45773
diff
changeset
|
267 OP '+' (1, 10) (1, 11) |
|
d0c677ddd6b8
Fix doctests in test_tokenize by converting Python 2.x syntax
Mark Dickinson <dickinsm@gmail.com>
parents:
45773
diff
changeset
|
268 STRING '"ABC"' (1, 12) (1, 17) |
|
45773
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
269 >>> dump_tokens("x = r'abc' + r'ABC' + R'ABC' + R'ABC'") |
|
45888
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
270 ENCODING 'utf-8' (0, 0) (0, 0) |
|
45773
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
271 NAME 'x' (1, 0) (1, 1) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
272 OP '=' (1, 2) (1, 3) |
|
45777
d0c677ddd6b8
Fix doctests in test_tokenize by converting Python 2.x syntax
Mark Dickinson <dickinsm@gmail.com>
parents:
45773
diff
changeset
|
273 STRING "r'abc'" (1, 4) (1, 10) |
|
d0c677ddd6b8
Fix doctests in test_tokenize by converting Python 2.x syntax
Mark Dickinson <dickinsm@gmail.com>
parents:
45773
diff
changeset
|
274 OP '+' (1, 11) (1, 12) |
|
d0c677ddd6b8
Fix doctests in test_tokenize by converting Python 2.x syntax
Mark Dickinson <dickinsm@gmail.com>
parents:
45773
diff
changeset
|
275 STRING "r'ABC'" (1, 13) (1, 19) |
|
d0c677ddd6b8
Fix doctests in test_tokenize by converting Python 2.x syntax
Mark Dickinson <dickinsm@gmail.com>
parents:
45773
diff
changeset
|
276 OP '+' (1, 20) (1, 21) |
|
d0c677ddd6b8
Fix doctests in test_tokenize by converting Python 2.x syntax
Mark Dickinson <dickinsm@gmail.com>
parents:
45773
diff
changeset
|
277 STRING "R'ABC'" (1, 22) (1, 28) |
|
d0c677ddd6b8
Fix doctests in test_tokenize by converting Python 2.x syntax
Mark Dickinson <dickinsm@gmail.com>
parents:
45773
diff
changeset
|
278 OP '+' (1, 29) (1, 30) |
|
d0c677ddd6b8
Fix doctests in test_tokenize by converting Python 2.x syntax
Mark Dickinson <dickinsm@gmail.com>
parents:
45773
diff
changeset
|
279 STRING "R'ABC'" (1, 31) (1, 37) |
|
45773
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
280 >>> dump_tokens('y = r"abc" + r"ABC" + R"ABC" + R"ABC"') |
|
45888
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
281 ENCODING 'utf-8' (0, 0) (0, 0) |
|
45773
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
282 NAME 'y' (1, 0) (1, 1) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
283 OP '=' (1, 2) (1, 3) |
|
45777
d0c677ddd6b8
Fix doctests in test_tokenize by converting Python 2.x syntax
Mark Dickinson <dickinsm@gmail.com>
parents:
45773
diff
changeset
|
284 STRING 'r"abc"' (1, 4) (1, 10) |
|
d0c677ddd6b8
Fix doctests in test_tokenize by converting Python 2.x syntax
Mark Dickinson <dickinsm@gmail.com>
parents:
45773
diff
changeset
|
285 OP '+' (1, 11) (1, 12) |
|
d0c677ddd6b8
Fix doctests in test_tokenize by converting Python 2.x syntax
Mark Dickinson <dickinsm@gmail.com>
parents:
45773
diff
changeset
|
286 STRING 'r"ABC"' (1, 13) (1, 19) |
|
d0c677ddd6b8
Fix doctests in test_tokenize by converting Python 2.x syntax
Mark Dickinson <dickinsm@gmail.com>
parents:
45773
diff
changeset
|
287 OP '+' (1, 20) (1, 21) |
|
d0c677ddd6b8
Fix doctests in test_tokenize by converting Python 2.x syntax
Mark Dickinson <dickinsm@gmail.com>
parents:
45773
diff
changeset
|
288 STRING 'R"ABC"' (1, 22) (1, 28) |
|
d0c677ddd6b8
Fix doctests in test_tokenize by converting Python 2.x syntax
Mark Dickinson <dickinsm@gmail.com>
parents:
45773
diff
changeset
|
289 OP '+' (1, 29) (1, 30) |
|
d0c677ddd6b8
Fix doctests in test_tokenize by converting Python 2.x syntax
Mark Dickinson <dickinsm@gmail.com>
parents:
45773
diff
changeset
|
290 STRING 'R"ABC"' (1, 31) (1, 37) |
|
45773
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
291 |
|
77475
115b0cb52c6c
Issue #15054: Fix incorrect tokenization of 'b' string literals.
Meador Inge <meadori@gmail.com>
parents:
76430
diff
changeset
|
292 >>> dump_tokens("u'abc' + U'abc'") |
|
115b0cb52c6c
Issue #15054: Fix incorrect tokenization of 'b' string literals.
Meador Inge <meadori@gmail.com>
parents:
76430
diff
changeset
|
293 ENCODING 'utf-8' (0, 0) (0, 0) |
|
115b0cb52c6c
Issue #15054: Fix incorrect tokenization of 'b' string literals.
Meador Inge <meadori@gmail.com>
parents:
76430
diff
changeset
|
294 STRING "u'abc'" (1, 0) (1, 6) |
|
115b0cb52c6c
Issue #15054: Fix incorrect tokenization of 'b' string literals.
Meador Inge <meadori@gmail.com>
parents:
76430
diff
changeset
|
295 OP '+' (1, 7) (1, 8) |
|
115b0cb52c6c
Issue #15054: Fix incorrect tokenization of 'b' string literals.
Meador Inge <meadori@gmail.com>
parents:
76430
diff
changeset
|
296 STRING "U'abc'" (1, 9) (1, 15) |
|
115b0cb52c6c
Issue #15054: Fix incorrect tokenization of 'b' string literals.
Meador Inge <meadori@gmail.com>
parents:
76430
diff
changeset
|
297 >>> dump_tokens('u"abc" + U"abc"') |
|
115b0cb52c6c
Issue #15054: Fix incorrect tokenization of 'b' string literals.
Meador Inge <meadori@gmail.com>
parents:
76430
diff
changeset
|
298 ENCODING 'utf-8' (0, 0) (0, 0) |
|
115b0cb52c6c
Issue #15054: Fix incorrect tokenization of 'b' string literals.
Meador Inge <meadori@gmail.com>
parents:
76430
diff
changeset
|
299 STRING 'u"abc"' (1, 0) (1, 6) |
|
115b0cb52c6c
Issue #15054: Fix incorrect tokenization of 'b' string literals.
Meador Inge <meadori@gmail.com>
parents:
76430
diff
changeset
|
300 OP '+' (1, 7) (1, 8) |
|
115b0cb52c6c
Issue #15054: Fix incorrect tokenization of 'b' string literals.
Meador Inge <meadori@gmail.com>
parents:
76430
diff
changeset
|
301 STRING 'U"abc"' (1, 9) (1, 15) |
|
115b0cb52c6c
Issue #15054: Fix incorrect tokenization of 'b' string literals.
Meador Inge <meadori@gmail.com>
parents:
76430
diff
changeset
|
302 |
|
115b0cb52c6c
Issue #15054: Fix incorrect tokenization of 'b' string literals.
Meador Inge <meadori@gmail.com>
parents:
76430
diff
changeset
|
303 >>> dump_tokens("b'abc' + B'abc'") |
|
115b0cb52c6c
Issue #15054: Fix incorrect tokenization of 'b' string literals.
Meador Inge <meadori@gmail.com>
parents:
76430
diff
changeset
|
304 ENCODING 'utf-8' (0, 0) (0, 0) |
|
115b0cb52c6c
Issue #15054: Fix incorrect tokenization of 'b' string literals.
Meador Inge <meadori@gmail.com>
parents:
76430
diff
changeset
|
305 STRING "b'abc'" (1, 0) (1, 6) |
|
115b0cb52c6c
Issue #15054: Fix incorrect tokenization of 'b' string literals.
Meador Inge <meadori@gmail.com>
parents:
76430
diff
changeset
|
306 OP '+' (1, 7) (1, 8) |
|
115b0cb52c6c
Issue #15054: Fix incorrect tokenization of 'b' string literals.
Meador Inge <meadori@gmail.com>
parents:
76430
diff
changeset
|
307 STRING "B'abc'" (1, 9) (1, 15) |
|
115b0cb52c6c
Issue #15054: Fix incorrect tokenization of 'b' string literals.
Meador Inge <meadori@gmail.com>
parents:
76430
diff
changeset
|
308 >>> dump_tokens('b"abc" + B"abc"') |
|
115b0cb52c6c
Issue #15054: Fix incorrect tokenization of 'b' string literals.
Meador Inge <meadori@gmail.com>
parents:
76430
diff
changeset
|
309 ENCODING 'utf-8' (0, 0) (0, 0) |
|
115b0cb52c6c
Issue #15054: Fix incorrect tokenization of 'b' string literals.
Meador Inge <meadori@gmail.com>
parents:
76430
diff
changeset
|
310 STRING 'b"abc"' (1, 0) (1, 6) |
|
115b0cb52c6c
Issue #15054: Fix incorrect tokenization of 'b' string literals.
Meador Inge <meadori@gmail.com>
parents:
76430
diff
changeset
|
311 OP '+' (1, 7) (1, 8) |
|
115b0cb52c6c
Issue #15054: Fix incorrect tokenization of 'b' string literals.
Meador Inge <meadori@gmail.com>
parents:
76430
diff
changeset
|
312 STRING 'B"abc"' (1, 9) (1, 15) |
|
115b0cb52c6c
Issue #15054: Fix incorrect tokenization of 'b' string literals.
Meador Inge <meadori@gmail.com>
parents:
76430
diff
changeset
|
313 >>> dump_tokens("br'abc' + bR'abc' + Br'abc' + BR'abc'") |
|
115b0cb52c6c
Issue #15054: Fix incorrect tokenization of 'b' string literals.
Meador Inge <meadori@gmail.com>
parents:
76430
diff
changeset
|
314 ENCODING 'utf-8' (0, 0) (0, 0) |
|
115b0cb52c6c
Issue #15054: Fix incorrect tokenization of 'b' string literals.
Meador Inge <meadori@gmail.com>
parents:
76430
diff
changeset
|
315 STRING "br'abc'" (1, 0) (1, 7) |
|
115b0cb52c6c
Issue #15054: Fix incorrect tokenization of 'b' string literals.
Meador Inge <meadori@gmail.com>
parents:
76430
diff
changeset
|
316 OP '+' (1, 8) (1, 9) |
|
115b0cb52c6c
Issue #15054: Fix incorrect tokenization of 'b' string literals.
Meador Inge <meadori@gmail.com>
parents:
76430
diff
changeset
|
317 STRING "bR'abc'" (1, 10) (1, 17) |
|
115b0cb52c6c
Issue #15054: Fix incorrect tokenization of 'b' string literals.
Meador Inge <meadori@gmail.com>
parents:
76430
diff
changeset
|
318 OP '+' (1, 18) (1, 19) |
|
115b0cb52c6c
Issue #15054: Fix incorrect tokenization of 'b' string literals.
Meador Inge <meadori@gmail.com>
parents:
76430
diff
changeset
|
319 STRING "Br'abc'" (1, 20) (1, 27) |
|
115b0cb52c6c
Issue #15054: Fix incorrect tokenization of 'b' string literals.
Meador Inge <meadori@gmail.com>
parents:
76430
diff
changeset
|
320 OP '+' (1, 28) (1, 29) |
|
115b0cb52c6c
Issue #15054: Fix incorrect tokenization of 'b' string literals.
Meador Inge <meadori@gmail.com>
parents:
76430
diff
changeset
|
321 STRING "BR'abc'" (1, 30) (1, 37) |
|
115b0cb52c6c
Issue #15054: Fix incorrect tokenization of 'b' string literals.
Meador Inge <meadori@gmail.com>
parents:
76430
diff
changeset
|
322 >>> dump_tokens('br"abc" + bR"abc" + Br"abc" + BR"abc"') |
|
115b0cb52c6c
Issue #15054: Fix incorrect tokenization of 'b' string literals.
Meador Inge <meadori@gmail.com>
parents:
76430
diff
changeset
|
323 ENCODING 'utf-8' (0, 0) (0, 0) |
|
115b0cb52c6c
Issue #15054: Fix incorrect tokenization of 'b' string literals.
Meador Inge <meadori@gmail.com>
parents:
76430
diff
changeset
|
324 STRING 'br"abc"' (1, 0) (1, 7) |
|
115b0cb52c6c
Issue #15054: Fix incorrect tokenization of 'b' string literals.
Meador Inge <meadori@gmail.com>
parents:
76430
diff
changeset
|
325 OP '+' (1, 8) (1, 9) |
|
115b0cb52c6c
Issue #15054: Fix incorrect tokenization of 'b' string literals.
Meador Inge <meadori@gmail.com>
parents:
76430
diff
changeset
|
326 STRING 'bR"abc"' (1, 10) (1, 17) |
|
115b0cb52c6c
Issue #15054: Fix incorrect tokenization of 'b' string literals.
Meador Inge <meadori@gmail.com>
parents:
76430
diff
changeset
|
327 OP '+' (1, 18) (1, 19) |
|
115b0cb52c6c
Issue #15054: Fix incorrect tokenization of 'b' string literals.
Meador Inge <meadori@gmail.com>
parents:
76430
diff
changeset
|
328 STRING 'Br"abc"' (1, 20) (1, 27) |
|
115b0cb52c6c
Issue #15054: Fix incorrect tokenization of 'b' string literals.
Meador Inge <meadori@gmail.com>
parents:
76430
diff
changeset
|
329 OP '+' (1, 28) (1, 29) |
|
115b0cb52c6c
Issue #15054: Fix incorrect tokenization of 'b' string literals.
Meador Inge <meadori@gmail.com>
parents:
76430
diff
changeset
|
330 STRING 'BR"abc"' (1, 30) (1, 37) |
|
115b0cb52c6c
Issue #15054: Fix incorrect tokenization of 'b' string literals.
Meador Inge <meadori@gmail.com>
parents:
76430
diff
changeset
|
331 >>> dump_tokens("rb'abc' + rB'abc' + Rb'abc' + RB'abc'") |
|
115b0cb52c6c
Issue #15054: Fix incorrect tokenization of 'b' string literals.
Meador Inge <meadori@gmail.com>
parents:
76430
diff
changeset
|
332 ENCODING 'utf-8' (0, 0) (0, 0) |
|
115b0cb52c6c
Issue #15054: Fix incorrect tokenization of 'b' string literals.
Meador Inge <meadori@gmail.com>
parents:
76430
diff
changeset
|
333 STRING "rb'abc'" (1, 0) (1, 7) |
|
115b0cb52c6c
Issue #15054: Fix incorrect tokenization of 'b' string literals.
Meador Inge <meadori@gmail.com>
parents:
76430
diff
changeset
|
334 OP '+' (1, 8) (1, 9) |
|
115b0cb52c6c
Issue #15054: Fix incorrect tokenization of 'b' string literals.
Meador Inge <meadori@gmail.com>
parents:
76430
diff
changeset
|
335 STRING "rB'abc'" (1, 10) (1, 17) |
|
115b0cb52c6c
Issue #15054: Fix incorrect tokenization of 'b' string literals.
Meador Inge <meadori@gmail.com>
parents:
76430
diff
changeset
|
336 OP '+' (1, 18) (1, 19) |
|
115b0cb52c6c
Issue #15054: Fix incorrect tokenization of 'b' string literals.
Meador Inge <meadori@gmail.com>
parents:
76430
diff
changeset
|
337 STRING "Rb'abc'" (1, 20) (1, 27) |
|
115b0cb52c6c
Issue #15054: Fix incorrect tokenization of 'b' string literals.
Meador Inge <meadori@gmail.com>
parents:
76430
diff
changeset
|
338 OP '+' (1, 28) (1, 29) |
|
115b0cb52c6c
Issue #15054: Fix incorrect tokenization of 'b' string literals.
Meador Inge <meadori@gmail.com>
parents:
76430
diff
changeset
|
339 STRING "RB'abc'" (1, 30) (1, 37) |
|
115b0cb52c6c
Issue #15054: Fix incorrect tokenization of 'b' string literals.
Meador Inge <meadori@gmail.com>
parents:
76430
diff
changeset
|
340 >>> dump_tokens('rb"abc" + rB"abc" + Rb"abc" + RB"abc"') |
|
115b0cb52c6c
Issue #15054: Fix incorrect tokenization of 'b' string literals.
Meador Inge <meadori@gmail.com>
parents:
76430
diff
changeset
|
341 ENCODING 'utf-8' (0, 0) (0, 0) |
|
115b0cb52c6c
Issue #15054: Fix incorrect tokenization of 'b' string literals.
Meador Inge <meadori@gmail.com>
parents:
76430
diff
changeset
|
342 STRING 'rb"abc"' (1, 0) (1, 7) |
|
115b0cb52c6c
Issue #15054: Fix incorrect tokenization of 'b' string literals.
Meador Inge <meadori@gmail.com>
parents:
76430
diff
changeset
|
343 OP '+' (1, 8) (1, 9) |
|
115b0cb52c6c
Issue #15054: Fix incorrect tokenization of 'b' string literals.
Meador Inge <meadori@gmail.com>
parents:
76430
diff
changeset
|
344 STRING 'rB"abc"' (1, 10) (1, 17) |
|
115b0cb52c6c
Issue #15054: Fix incorrect tokenization of 'b' string literals.
Meador Inge <meadori@gmail.com>
parents:
76430
diff
changeset
|
345 OP '+' (1, 18) (1, 19) |
|
115b0cb52c6c
Issue #15054: Fix incorrect tokenization of 'b' string literals.
Meador Inge <meadori@gmail.com>
parents:
76430
diff
changeset
|
346 STRING 'Rb"abc"' (1, 20) (1, 27) |
|
115b0cb52c6c
Issue #15054: Fix incorrect tokenization of 'b' string literals.
Meador Inge <meadori@gmail.com>
parents:
76430
diff
changeset
|
347 OP '+' (1, 28) (1, 29) |
|
115b0cb52c6c
Issue #15054: Fix incorrect tokenization of 'b' string literals.
Meador Inge <meadori@gmail.com>
parents:
76430
diff
changeset
|
348 STRING 'RB"abc"' (1, 30) (1, 37) |
|
115b0cb52c6c
Issue #15054: Fix incorrect tokenization of 'b' string literals.
Meador Inge <meadori@gmail.com>
parents:
76430
diff
changeset
|
349 |
|
45773
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
350 Operators |
|
40450
cc992d75d5b3
Four months of trunk changes (including a few releases...)
Thomas Wouters <thomas@python.org>
parents:
37316
diff
changeset
|
351 |
|
45773
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
352 >>> dump_tokens("def d22(a, b, c=2, d=2, *k): pass") |
|
45888
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
353 ENCODING 'utf-8' (0, 0) (0, 0) |
|
45773
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
354 NAME 'def' (1, 0) (1, 3) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
355 NAME 'd22' (1, 4) (1, 7) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
356 OP '(' (1, 7) (1, 8) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
357 NAME 'a' (1, 8) (1, 9) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
358 OP ',' (1, 9) (1, 10) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
359 NAME 'b' (1, 11) (1, 12) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
360 OP ',' (1, 12) (1, 13) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
361 NAME 'c' (1, 14) (1, 15) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
362 OP '=' (1, 15) (1, 16) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
363 NUMBER '2' (1, 16) (1, 17) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
364 OP ',' (1, 17) (1, 18) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
365 NAME 'd' (1, 19) (1, 20) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
366 OP '=' (1, 20) (1, 21) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
367 NUMBER '2' (1, 21) (1, 22) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
368 OP ',' (1, 22) (1, 23) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
369 OP '*' (1, 24) (1, 25) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
370 NAME 'k' (1, 25) (1, 26) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
371 OP ')' (1, 26) (1, 27) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
372 OP ':' (1, 27) (1, 28) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
373 NAME 'pass' (1, 29) (1, 33) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
374 >>> dump_tokens("def d01v_(a=1, *k, **w): pass") |
|
45888
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
375 ENCODING 'utf-8' (0, 0) (0, 0) |
|
45773
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
376 NAME 'def' (1, 0) (1, 3) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
377 NAME 'd01v_' (1, 4) (1, 9) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
378 OP '(' (1, 9) (1, 10) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
379 NAME 'a' (1, 10) (1, 11) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
380 OP '=' (1, 11) (1, 12) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
381 NUMBER '1' (1, 12) (1, 13) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
382 OP ',' (1, 13) (1, 14) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
383 OP '*' (1, 15) (1, 16) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
384 NAME 'k' (1, 16) (1, 17) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
385 OP ',' (1, 17) (1, 18) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
386 OP '**' (1, 19) (1, 21) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
387 NAME 'w' (1, 21) (1, 22) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
388 OP ')' (1, 22) (1, 23) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
389 OP ':' (1, 23) (1, 24) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
390 NAME 'pass' (1, 25) (1, 29) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
391 |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
392 Comparison |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
393 |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
394 >>> dump_tokens("if 1 < 1 > 1 == 1 >= 5 <= 0x15 <= 0x12 != " + |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
395 ... "1 and 5 in 1 not in 1 is 1 or 5 is not 1: pass") |
|
45888
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
396 ENCODING 'utf-8' (0, 0) (0, 0) |
|
45773
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
397 NAME 'if' (1, 0) (1, 2) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
398 NUMBER '1' (1, 3) (1, 4) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
399 OP '<' (1, 5) (1, 6) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
400 NUMBER '1' (1, 7) (1, 8) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
401 OP '>' (1, 9) (1, 10) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
402 NUMBER '1' (1, 11) (1, 12) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
403 OP '==' (1, 13) (1, 15) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
404 NUMBER '1' (1, 16) (1, 17) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
405 OP '>=' (1, 18) (1, 20) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
406 NUMBER '5' (1, 21) (1, 22) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
407 OP '<=' (1, 23) (1, 25) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
408 NUMBER '0x15' (1, 26) (1, 30) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
409 OP '<=' (1, 31) (1, 33) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
410 NUMBER '0x12' (1, 34) (1, 38) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
411 OP '!=' (1, 39) (1, 41) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
412 NUMBER '1' (1, 42) (1, 43) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
413 NAME 'and' (1, 44) (1, 47) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
414 NUMBER '5' (1, 48) (1, 49) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
415 NAME 'in' (1, 50) (1, 52) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
416 NUMBER '1' (1, 53) (1, 54) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
417 NAME 'not' (1, 55) (1, 58) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
418 NAME 'in' (1, 59) (1, 61) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
419 NUMBER '1' (1, 62) (1, 63) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
420 NAME 'is' (1, 64) (1, 66) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
421 NUMBER '1' (1, 67) (1, 68) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
422 NAME 'or' (1, 69) (1, 71) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
423 NUMBER '5' (1, 72) (1, 73) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
424 NAME 'is' (1, 74) (1, 76) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
425 NAME 'not' (1, 77) (1, 80) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
426 NUMBER '1' (1, 81) (1, 82) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
427 OP ':' (1, 82) (1, 83) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
428 NAME 'pass' (1, 84) (1, 88) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
429 |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
430 Shift |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
431 |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
432 >>> dump_tokens("x = 1 << 1 >> 5") |
|
45888
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
433 ENCODING 'utf-8' (0, 0) (0, 0) |
|
45773
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
434 NAME 'x' (1, 0) (1, 1) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
435 OP '=' (1, 2) (1, 3) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
436 NUMBER '1' (1, 4) (1, 5) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
437 OP '<<' (1, 6) (1, 8) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
438 NUMBER '1' (1, 9) (1, 10) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
439 OP '>>' (1, 11) (1, 13) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
440 NUMBER '5' (1, 14) (1, 15) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
441 |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
442 Additive |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
443 |
|
45777
d0c677ddd6b8
Fix doctests in test_tokenize by converting Python 2.x syntax
Mark Dickinson <dickinsm@gmail.com>
parents:
45773
diff
changeset
|
444 >>> dump_tokens("x = 1 - y + 15 - 1 + 0x124 + z + a[5]") |
|
45888
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
445 ENCODING 'utf-8' (0, 0) (0, 0) |
|
45773
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
446 NAME 'x' (1, 0) (1, 1) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
447 OP '=' (1, 2) (1, 3) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
448 NUMBER '1' (1, 4) (1, 5) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
449 OP '-' (1, 6) (1, 7) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
450 NAME 'y' (1, 8) (1, 9) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
451 OP '+' (1, 10) (1, 11) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
452 NUMBER '15' (1, 12) (1, 14) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
453 OP '-' (1, 15) (1, 16) |
|
45777
d0c677ddd6b8
Fix doctests in test_tokenize by converting Python 2.x syntax
Mark Dickinson <dickinsm@gmail.com>
parents:
45773
diff
changeset
|
454 NUMBER '1' (1, 17) (1, 18) |
|
d0c677ddd6b8
Fix doctests in test_tokenize by converting Python 2.x syntax
Mark Dickinson <dickinsm@gmail.com>
parents:
45773
diff
changeset
|
455 OP '+' (1, 19) (1, 20) |
|
d0c677ddd6b8
Fix doctests in test_tokenize by converting Python 2.x syntax
Mark Dickinson <dickinsm@gmail.com>
parents:
45773
diff
changeset
|
456 NUMBER '0x124' (1, 21) (1, 26) |
|
d0c677ddd6b8
Fix doctests in test_tokenize by converting Python 2.x syntax
Mark Dickinson <dickinsm@gmail.com>
parents:
45773
diff
changeset
|
457 OP '+' (1, 27) (1, 28) |
|
d0c677ddd6b8
Fix doctests in test_tokenize by converting Python 2.x syntax
Mark Dickinson <dickinsm@gmail.com>
parents:
45773
diff
changeset
|
458 NAME 'z' (1, 29) (1, 30) |
|
d0c677ddd6b8
Fix doctests in test_tokenize by converting Python 2.x syntax
Mark Dickinson <dickinsm@gmail.com>
parents:
45773
diff
changeset
|
459 OP '+' (1, 31) (1, 32) |
|
d0c677ddd6b8
Fix doctests in test_tokenize by converting Python 2.x syntax
Mark Dickinson <dickinsm@gmail.com>
parents:
45773
diff
changeset
|
460 NAME 'a' (1, 33) (1, 34) |
|
d0c677ddd6b8
Fix doctests in test_tokenize by converting Python 2.x syntax
Mark Dickinson <dickinsm@gmail.com>
parents:
45773
diff
changeset
|
461 OP '[' (1, 34) (1, 35) |
|
d0c677ddd6b8
Fix doctests in test_tokenize by converting Python 2.x syntax
Mark Dickinson <dickinsm@gmail.com>
parents:
45773
diff
changeset
|
462 NUMBER '5' (1, 35) (1, 36) |
|
d0c677ddd6b8
Fix doctests in test_tokenize by converting Python 2.x syntax
Mark Dickinson <dickinsm@gmail.com>
parents:
45773
diff
changeset
|
463 OP ']' (1, 36) (1, 37) |
|
40450
cc992d75d5b3
Four months of trunk changes (including a few releases...)
Thomas Wouters <thomas@python.org>
parents:
37316
diff
changeset
|
464 |
|
45773
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
465 Multiplicative |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
466 |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
467 >>> dump_tokens("x = 1//1*1/5*12%0x12") |
|
45888
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
468 ENCODING 'utf-8' (0, 0) (0, 0) |
|
45773
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
469 NAME 'x' (1, 0) (1, 1) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
470 OP '=' (1, 2) (1, 3) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
471 NUMBER '1' (1, 4) (1, 5) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
472 OP '//' (1, 5) (1, 7) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
473 NUMBER '1' (1, 7) (1, 8) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
474 OP '*' (1, 8) (1, 9) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
475 NUMBER '1' (1, 9) (1, 10) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
476 OP '/' (1, 10) (1, 11) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
477 NUMBER '5' (1, 11) (1, 12) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
478 OP '*' (1, 12) (1, 13) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
479 NUMBER '12' (1, 13) (1, 15) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
480 OP '%' (1, 15) (1, 16) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
481 NUMBER '0x12' (1, 16) (1, 20) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
482 |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
483 Unary |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
484 |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
485 >>> dump_tokens("~1 ^ 1 & 1 |1 ^ -1") |
|
45888
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
486 ENCODING 'utf-8' (0, 0) (0, 0) |
|
45773
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
487 OP '~' (1, 0) (1, 1) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
488 NUMBER '1' (1, 1) (1, 2) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
489 OP '^' (1, 3) (1, 4) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
490 NUMBER '1' (1, 5) (1, 6) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
491 OP '&' (1, 7) (1, 8) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
492 NUMBER '1' (1, 9) (1, 10) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
493 OP '|' (1, 11) (1, 12) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
494 NUMBER '1' (1, 12) (1, 13) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
495 OP '^' (1, 14) (1, 15) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
496 OP '-' (1, 16) (1, 17) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
497 NUMBER '1' (1, 17) (1, 18) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
498 >>> dump_tokens("-1*1/1+1*1//1 - ---1**1") |
|
45888
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
499 ENCODING 'utf-8' (0, 0) (0, 0) |
|
45773
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
500 OP '-' (1, 0) (1, 1) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
501 NUMBER '1' (1, 1) (1, 2) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
502 OP '*' (1, 2) (1, 3) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
503 NUMBER '1' (1, 3) (1, 4) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
504 OP '/' (1, 4) (1, 5) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
505 NUMBER '1' (1, 5) (1, 6) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
506 OP '+' (1, 6) (1, 7) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
507 NUMBER '1' (1, 7) (1, 8) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
508 OP '*' (1, 8) (1, 9) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
509 NUMBER '1' (1, 9) (1, 10) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
510 OP '//' (1, 10) (1, 12) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
511 NUMBER '1' (1, 12) (1, 13) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
512 OP '-' (1, 14) (1, 15) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
513 OP '-' (1, 16) (1, 17) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
514 OP '-' (1, 17) (1, 18) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
515 OP '-' (1, 18) (1, 19) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
516 NUMBER '1' (1, 19) (1, 20) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
517 OP '**' (1, 20) (1, 22) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
518 NUMBER '1' (1, 22) (1, 23) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
519 |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
520 Selector |
|
40450
cc992d75d5b3
Four months of trunk changes (including a few releases...)
Thomas Wouters <thomas@python.org>
parents:
37316
diff
changeset
|
521 |
|
45773
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
522 >>> dump_tokens("import sys, time\\nx = sys.modules['time'].time()") |
|
45888
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
523 ENCODING 'utf-8' (0, 0) (0, 0) |
|
45773
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
524 NAME 'import' (1, 0) (1, 6) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
525 NAME 'sys' (1, 7) (1, 10) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
526 OP ',' (1, 10) (1, 11) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
527 NAME 'time' (1, 12) (1, 16) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
528 NEWLINE '\\n' (1, 16) (1, 17) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
529 NAME 'x' (2, 0) (2, 1) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
530 OP '=' (2, 2) (2, 3) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
531 NAME 'sys' (2, 4) (2, 7) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
532 OP '.' (2, 7) (2, 8) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
533 NAME 'modules' (2, 8) (2, 15) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
534 OP '[' (2, 15) (2, 16) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
535 STRING "'time'" (2, 16) (2, 22) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
536 OP ']' (2, 22) (2, 23) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
537 OP '.' (2, 23) (2, 24) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
538 NAME 'time' (2, 24) (2, 28) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
539 OP '(' (2, 28) (2, 29) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
540 OP ')' (2, 29) (2, 30) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
541 |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
542 Methods |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
543 |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
544 >>> dump_tokens("@staticmethod\\ndef foo(x,y): pass") |
|
45888
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
545 ENCODING 'utf-8' (0, 0) (0, 0) |
|
45773
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
546 OP '@' (1, 0) (1, 1) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
547 NAME 'staticmethod (1, 1) (1, 13) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
548 NEWLINE '\\n' (1, 13) (1, 14) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
549 NAME 'def' (2, 0) (2, 3) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
550 NAME 'foo' (2, 4) (2, 7) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
551 OP '(' (2, 7) (2, 8) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
552 NAME 'x' (2, 8) (2, 9) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
553 OP ',' (2, 9) (2, 10) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
554 NAME 'y' (2, 10) (2, 11) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
555 OP ')' (2, 11) (2, 12) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
556 OP ':' (2, 12) (2, 13) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
557 NAME 'pass' (2, 14) (2, 18) |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
558 |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
559 Backslash means line continuation, except for comments |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
560 |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
561 >>> roundtrip("x=1+\\\\n" |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
562 ... "1\\n" |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
563 ... "# This is a comment\\\\n" |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
564 ... "# This also\\n") |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
565 True |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
566 >>> roundtrip("# Comment \\\\nx = 0") |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
567 True |
|
46145
da5597a5c195
Merged revisions 61964-61979 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
45888
diff
changeset
|
568 |
|
da5597a5c195
Merged revisions 61964-61979 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
45888
diff
changeset
|
569 Two string literals on the same line |
|
da5597a5c195
Merged revisions 61964-61979 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
45888
diff
changeset
|
570 |
|
da5597a5c195
Merged revisions 61964-61979 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
45888
diff
changeset
|
571 >>> roundtrip("'' ''") |
|
da5597a5c195
Merged revisions 61964-61979 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
45888
diff
changeset
|
572 True |
|
da5597a5c195
Merged revisions 61964-61979 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
45888
diff
changeset
|
573 |
|
da5597a5c195
Merged revisions 61964-61979 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
45888
diff
changeset
|
574 Test roundtrip on random python modules. |
|
65317
b83f0a667e7c
Merged revisions 85482 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
64318
diff
changeset
|
575 pass the '-ucpu' option to process the full directory. |
|
46145
da5597a5c195
Merged revisions 61964-61979 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
45888
diff
changeset
|
576 |
|
da5597a5c195
Merged revisions 61964-61979 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
45888
diff
changeset
|
577 >>> import random |
|
da5597a5c195
Merged revisions 61964-61979 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
45888
diff
changeset
|
578 >>> tempdir = os.path.dirname(f) or os.curdir |
|
da5597a5c195
Merged revisions 61964-61979 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
45888
diff
changeset
|
579 >>> testfiles = glob.glob(os.path.join(tempdir, "test*.py")) |
|
da5597a5c195
Merged revisions 61964-61979 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
45888
diff
changeset
|
580 |
|
71853
c13abed5d764
tokenize is just broken on test_pep3131.py
Benjamin Peterson <benjamin@python.org>
parents:
71426
diff
changeset
|
581 tokenize is broken on test_pep3131.py because regular expressions are broken on |
|
c13abed5d764
tokenize is just broken on test_pep3131.py
Benjamin Peterson <benjamin@python.org>
parents:
71426
diff
changeset
|
582 the obscure unicode identifiers in it. *sigh* |
|
c13abed5d764
tokenize is just broken on test_pep3131.py
Benjamin Peterson <benjamin@python.org>
parents:
71426
diff
changeset
|
583 >>> testfiles.remove(os.path.join(tempdir, "test_pep3131.py")) |
|
65317
b83f0a667e7c
Merged revisions 85482 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents:
64318
diff
changeset
|
584 >>> if not support.is_resource_enabled("cpu"): |
|
46145
da5597a5c195
Merged revisions 61964-61979 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
45888
diff
changeset
|
585 ... testfiles = random.sample(testfiles, 10) |
|
da5597a5c195
Merged revisions 61964-61979 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
45888
diff
changeset
|
586 ... |
|
da5597a5c195
Merged revisions 61964-61979 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
45888
diff
changeset
|
587 >>> for testfile in testfiles: |
|
da5597a5c195
Merged revisions 61964-61979 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
45888
diff
changeset
|
588 ... if not roundtrip(open(testfile, 'rb')): |
|
da5597a5c195
Merged revisions 61964-61979 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
45888
diff
changeset
|
589 ... print("Roundtrip failed for file %s" % testfile) |
|
da5597a5c195
Merged revisions 61964-61979 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
45888
diff
changeset
|
590 ... break |
|
da5597a5c195
Merged revisions 61964-61979 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
45888
diff
changeset
|
591 ... else: True |
|
da5597a5c195
Merged revisions 61964-61979 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
45888
diff
changeset
|
592 True |
|
57210
a0951f11be3d
Merged revisions 76235 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
56434
diff
changeset
|
593 |
|
a0951f11be3d
Merged revisions 76235 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
56434
diff
changeset
|
594 Evil tabs |
|
64318
a492f1ac0575
Merged revisions 84364 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
57210
diff
changeset
|
595 |
|
57210
a0951f11be3d
Merged revisions 76235 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
56434
diff
changeset
|
596 >>> dump_tokens("def f():\\n\\tif x\\n \\tpass") |
|
a0951f11be3d
Merged revisions 76235 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
56434
diff
changeset
|
597 ENCODING 'utf-8' (0, 0) (0, 0) |
|
a0951f11be3d
Merged revisions 76235 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
56434
diff
changeset
|
598 NAME 'def' (1, 0) (1, 3) |
|
a0951f11be3d
Merged revisions 76235 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
56434
diff
changeset
|
599 NAME 'f' (1, 4) (1, 5) |
|
a0951f11be3d
Merged revisions 76235 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
56434
diff
changeset
|
600 OP '(' (1, 5) (1, 6) |
|
a0951f11be3d
Merged revisions 76235 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
56434
diff
changeset
|
601 OP ')' (1, 6) (1, 7) |
|
a0951f11be3d
Merged revisions 76235 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
56434
diff
changeset
|
602 OP ':' (1, 7) (1, 8) |
|
a0951f11be3d
Merged revisions 76235 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
56434
diff
changeset
|
603 NEWLINE '\\n' (1, 8) (1, 9) |
|
a0951f11be3d
Merged revisions 76235 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
56434
diff
changeset
|
604 INDENT '\\t' (2, 0) (2, 1) |
|
a0951f11be3d
Merged revisions 76235 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
56434
diff
changeset
|
605 NAME 'if' (2, 1) (2, 3) |
|
a0951f11be3d
Merged revisions 76235 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
56434
diff
changeset
|
606 NAME 'x' (2, 4) (2, 5) |
|
a0951f11be3d
Merged revisions 76235 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
56434
diff
changeset
|
607 NEWLINE '\\n' (2, 5) (2, 6) |
|
a0951f11be3d
Merged revisions 76235 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
56434
diff
changeset
|
608 INDENT ' \\t' (3, 0) (3, 9) |
|
a0951f11be3d
Merged revisions 76235 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
56434
diff
changeset
|
609 NAME 'pass' (3, 9) (3, 13) |
|
a0951f11be3d
Merged revisions 76235 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
56434
diff
changeset
|
610 DEDENT '' (4, 0) (4, 0) |
|
a0951f11be3d
Merged revisions 76235 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
56434
diff
changeset
|
611 DEDENT '' (4, 0) (4, 0) |
|
64318
a492f1ac0575
Merged revisions 84364 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
57210
diff
changeset
|
612 |
|
a492f1ac0575
Merged revisions 84364 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
57210
diff
changeset
|
613 Non-ascii identifiers |
|
a492f1ac0575
Merged revisions 84364 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
57210
diff
changeset
|
614 |
|
a492f1ac0575
Merged revisions 84364 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
57210
diff
changeset
|
615 >>> dump_tokens("Örter = 'places'\\ngrün = 'green'") |
|
a492f1ac0575
Merged revisions 84364 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
57210
diff
changeset
|
616 ENCODING 'utf-8' (0, 0) (0, 0) |
|
a492f1ac0575
Merged revisions 84364 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
57210
diff
changeset
|
617 NAME 'Örter' (1, 0) (1, 5) |
|
a492f1ac0575
Merged revisions 84364 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
57210
diff
changeset
|
618 OP '=' (1, 6) (1, 7) |
|
a492f1ac0575
Merged revisions 84364 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
57210
diff
changeset
|
619 STRING "'places'" (1, 8) (1, 16) |
|
a492f1ac0575
Merged revisions 84364 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
57210
diff
changeset
|
620 NEWLINE '\\n' (1, 16) (1, 17) |
|
a492f1ac0575
Merged revisions 84364 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
57210
diff
changeset
|
621 NAME 'grün' (2, 0) (2, 4) |
|
a492f1ac0575
Merged revisions 84364 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
57210
diff
changeset
|
622 OP '=' (2, 5) (2, 6) |
|
a492f1ac0575
Merged revisions 84364 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
57210
diff
changeset
|
623 STRING "'green'" (2, 7) (2, 14) |
|
75383
2822765e48a7
Updated tokenize to support the inverse byte literals new in 3.3
Armin Ronacher <armin.ronacher@active-4.com>
parents:
74519
diff
changeset
|
624 |
|
2822765e48a7
Updated tokenize to support the inverse byte literals new in 3.3
Armin Ronacher <armin.ronacher@active-4.com>
parents:
74519
diff
changeset
|
625 Legacy unicode literals: |
|
2822765e48a7
Updated tokenize to support the inverse byte literals new in 3.3
Armin Ronacher <armin.ronacher@active-4.com>
parents:
74519
diff
changeset
|
626 |
|
77522
8e47e9af826e
Issue #15096: Drop support for the ur string prefix
Christian Heimes <c.heimes@cheimes.de>
parents:
77475
diff
changeset
|
627 >>> dump_tokens("Örter = u'places'\\ngrün = U'green'") |
|
75383
2822765e48a7
Updated tokenize to support the inverse byte literals new in 3.3
Armin Ronacher <armin.ronacher@active-4.com>
parents:
74519
diff
changeset
|
628 ENCODING 'utf-8' (0, 0) (0, 0) |
|
2822765e48a7
Updated tokenize to support the inverse byte literals new in 3.3
Armin Ronacher <armin.ronacher@active-4.com>
parents:
74519
diff
changeset
|
629 NAME 'Örter' (1, 0) (1, 5) |
|
2822765e48a7
Updated tokenize to support the inverse byte literals new in 3.3
Armin Ronacher <armin.ronacher@active-4.com>
parents:
74519
diff
changeset
|
630 OP '=' (1, 6) (1, 7) |
|
2822765e48a7
Updated tokenize to support the inverse byte literals new in 3.3
Armin Ronacher <armin.ronacher@active-4.com>
parents:
74519
diff
changeset
|
631 STRING "u'places'" (1, 8) (1, 17) |
|
2822765e48a7
Updated tokenize to support the inverse byte literals new in 3.3
Armin Ronacher <armin.ronacher@active-4.com>
parents:
74519
diff
changeset
|
632 NEWLINE '\\n' (1, 17) (1, 18) |
|
2822765e48a7
Updated tokenize to support the inverse byte literals new in 3.3
Armin Ronacher <armin.ronacher@active-4.com>
parents:
74519
diff
changeset
|
633 NAME 'grün' (2, 0) (2, 4) |
|
2822765e48a7
Updated tokenize to support the inverse byte literals new in 3.3
Armin Ronacher <armin.ronacher@active-4.com>
parents:
74519
diff
changeset
|
634 OP '=' (2, 5) (2, 6) |
|
77522
8e47e9af826e
Issue #15096: Drop support for the ur string prefix
Christian Heimes <c.heimes@cheimes.de>
parents:
77475
diff
changeset
|
635 STRING "U'green'" (2, 7) (2, 15) |
|
40450
cc992d75d5b3
Four months of trunk changes (including a few releases...)
Thomas Wouters <thomas@python.org>
parents:
37316
diff
changeset
|
636 """ |
|
cc992d75d5b3
Four months of trunk changes (including a few releases...)
Thomas Wouters <thomas@python.org>
parents:
37316
diff
changeset
|
637 |
|
47201
9c930a32cebb
#2621 rename test.test_support to test.support
Benjamin Peterson <benjamin@python.org>
parents:
46978
diff
changeset
|
638 from test import support |
|
45888
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
639 from tokenize import (tokenize, _tokenize, untokenize, NUMBER, NAME, OP, |
|
74519
f4976fa6e830
Issue #2134: Add support for tokenize.TokenInfo.exact_type.
Meador Inge <meadori@gmail.com>
parents:
72485
diff
changeset
|
640 STRING, ENDMARKER, ENCODING, tok_name, detect_encoding, |
|
66097
306339c31758
Issue #10335: Add tokenize.open(), detect the file encoding using
Victor Stinner <victor.stinner@haypocalc.com>
parents:
65772
diff
changeset
|
641 open as tokenize_open) |
|
45888
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
642 from io import BytesIO |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
643 from unittest import TestCase |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
644 import os, sys, glob |
|
74519
f4976fa6e830
Issue #2134: Add support for tokenize.TokenInfo.exact_type.
Meador Inge <meadori@gmail.com>
parents:
72485
diff
changeset
|
645 import token |
|
35119
04868959f1fe
SF bug #1224621: tokenize module does not detect inconsistent dedents
Raymond Hettinger <python@rcn.com>
parents:
35058
diff
changeset
|
646 |
|
40450
cc992d75d5b3
Four months of trunk changes (including a few releases...)
Thomas Wouters <thomas@python.org>
parents:
37316
diff
changeset
|
647 def dump_tokens(s): |
|
cc992d75d5b3
Four months of trunk changes (including a few releases...)
Thomas Wouters <thomas@python.org>
parents:
37316
diff
changeset
|
648 """Print out the tokens in s in a table format. |
|
cc992d75d5b3
Four months of trunk changes (including a few releases...)
Thomas Wouters <thomas@python.org>
parents:
37316
diff
changeset
|
649 |
|
cc992d75d5b3
Four months of trunk changes (including a few releases...)
Thomas Wouters <thomas@python.org>
parents:
37316
diff
changeset
|
650 The ENDMARKER is omitted. |
|
cc992d75d5b3
Four months of trunk changes (including a few releases...)
Thomas Wouters <thomas@python.org>
parents:
37316
diff
changeset
|
651 """ |
|
45888
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
652 f = BytesIO(s.encode('utf-8')) |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
653 for type, token, start, end, line in tokenize(f.readline): |
|
40450
cc992d75d5b3
Four months of trunk changes (including a few releases...)
Thomas Wouters <thomas@python.org>
parents:
37316
diff
changeset
|
654 if type == ENDMARKER: |
|
cc992d75d5b3
Four months of trunk changes (including a few releases...)
Thomas Wouters <thomas@python.org>
parents:
37316
diff
changeset
|
655 break |
|
cc992d75d5b3
Four months of trunk changes (including a few releases...)
Thomas Wouters <thomas@python.org>
parents:
37316
diff
changeset
|
656 type = tok_name[type] |
|
45773
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
657 print("%(type)-10.10s %(token)-13.13r %(start)s %(end)s" % locals()) |
|
40450
cc992d75d5b3
Four months of trunk changes (including a few releases...)
Thomas Wouters <thomas@python.org>
parents:
37316
diff
changeset
|
658 |
|
45888
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
659 def roundtrip(f): |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
660 """ |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
661 Test roundtrip for `untokenize`. `f` is an open file or a string. |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
662 The source code in f is tokenized, converted back to source code via |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
663 tokenize.untokenize(), and tokenized again from the latter. The test |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
664 fails if the second tokenization doesn't match the first. |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
665 """ |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
666 if isinstance(f, str): |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
667 f = BytesIO(f.encode('utf-8')) |
|
65773
8d8243483228
Merged revisions 85990 via svnmerge from
Brian Curtin <brian.curtin@gmail.com>
parents:
65317
diff
changeset
|
668 try: |
|
8d8243483228
Merged revisions 85990 via svnmerge from
Brian Curtin <brian.curtin@gmail.com>
parents:
65317
diff
changeset
|
669 token_list = list(tokenize(f.readline)) |
|
8d8243483228
Merged revisions 85990 via svnmerge from
Brian Curtin <brian.curtin@gmail.com>
parents:
65317
diff
changeset
|
670 finally: |
|
8d8243483228
Merged revisions 85990 via svnmerge from
Brian Curtin <brian.curtin@gmail.com>
parents:
65317
diff
changeset
|
671 f.close() |
|
45888
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
672 tokens1 = [tok[:2] for tok in token_list] |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
673 new_bytes = untokenize(tokens1) |
|
72485
83f43b58c988
#13012: use splitlines(keepends=True/False) instead of splitlines(0/1).
Ezio Melotti <ezio.melotti@gmail.com>
parents:
71853
diff
changeset
|
674 readline = (line for line in new_bytes.splitlines(keepends=True)).__next__ |
|
45888
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
675 tokens2 = [tok[:2] for tok in tokenize(readline)] |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
676 return tokens1 == tokens2 |
|
40450
cc992d75d5b3
Four months of trunk changes (including a few releases...)
Thomas Wouters <thomas@python.org>
parents:
37316
diff
changeset
|
677 |
|
37316
049606933328
Repaired a number of errors in this test:
Tim Peters <tim.peters@gmail.com>
parents:
35119
diff
changeset
|
678 # This is an example from the docs, set up as a doctest. |
|
35058
8076424d455d
Add untokenize() function to allow full round-trip tokenization.
Raymond Hettinger <python@rcn.com>
parents:
29086
diff
changeset
|
679 def decistmt(s): |
|
8076424d455d
Add untokenize() function to allow full round-trip tokenization.
Raymond Hettinger <python@rcn.com>
parents:
29086
diff
changeset
|
680 """Substitute Decimals for floats in a string of statements. |
|
8076424d455d
Add untokenize() function to allow full round-trip tokenization.
Raymond Hettinger <python@rcn.com>
parents:
29086
diff
changeset
|
681 |
|
8076424d455d
Add untokenize() function to allow full round-trip tokenization.
Raymond Hettinger <python@rcn.com>
parents:
29086
diff
changeset
|
682 >>> from decimal import Decimal |
|
40788
e9a6234b26af
* Remove PRINT_ITEM(_TO), PRINT_NEWLINE(_TO) opcodes.
Georg Brandl <georg@python.org>
parents:
40775
diff
changeset
|
683 >>> s = 'print(+21.3e-5*-.1234/81.7)' |
|
35058
8076424d455d
Add untokenize() function to allow full round-trip tokenization.
Raymond Hettinger <python@rcn.com>
parents:
29086
diff
changeset
|
684 >>> decistmt(s) |
|
40788
e9a6234b26af
* Remove PRINT_ITEM(_TO), PRINT_NEWLINE(_TO) opcodes.
Georg Brandl <georg@python.org>
parents:
40775
diff
changeset
|
685 "print (+Decimal ('21.3e-5')*-Decimal ('.1234')/Decimal ('81.7'))" |
|
35058
8076424d455d
Add untokenize() function to allow full round-trip tokenization.
Raymond Hettinger <python@rcn.com>
parents:
29086
diff
changeset
|
686 |
|
37316
049606933328
Repaired a number of errors in this test:
Tim Peters <tim.peters@gmail.com>
parents:
35119
diff
changeset
|
687 The format of the exponent is inherited from the platform C library. |
|
049606933328
Repaired a number of errors in this test:
Tim Peters <tim.peters@gmail.com>
parents:
35119
diff
changeset
|
688 Known cases are "e-007" (Windows) and "e-07" (not Windows). Since |
|
63750
eafe07d40eb7
Issue #9337: Make float.__str__ identical to float.__repr__.
Mark Dickinson <dickinsm@gmail.com>
parents:
62620
diff
changeset
|
689 we're only showing 11 digits, and the 12th isn't close to 5, the |
|
37316
049606933328
Repaired a number of errors in this test:
Tim Peters <tim.peters@gmail.com>
parents:
35119
diff
changeset
|
690 rest of the output should be platform-independent. |
|
049606933328
Repaired a number of errors in this test:
Tim Peters <tim.peters@gmail.com>
parents:
35119
diff
changeset
|
691 |
|
049606933328
Repaired a number of errors in this test:
Tim Peters <tim.peters@gmail.com>
parents:
35119
diff
changeset
|
692 >>> exec(s) #doctest: +ELLIPSIS |
|
63750
eafe07d40eb7
Issue #9337: Make float.__str__ identical to float.__repr__.
Mark Dickinson <dickinsm@gmail.com>
parents:
62620
diff
changeset
|
693 -3.2171603427...e-0...7 |
|
37316
049606933328
Repaired a number of errors in this test:
Tim Peters <tim.peters@gmail.com>
parents:
35119
diff
changeset
|
694 |
|
049606933328
Repaired a number of errors in this test:
Tim Peters <tim.peters@gmail.com>
parents:
35119
diff
changeset
|
695 Output from calculations with Decimal should be identical across all |
|
049606933328
Repaired a number of errors in this test:
Tim Peters <tim.peters@gmail.com>
parents:
35119
diff
changeset
|
696 platforms. |
|
049606933328
Repaired a number of errors in this test:
Tim Peters <tim.peters@gmail.com>
parents:
35119
diff
changeset
|
697 |
|
35058
8076424d455d
Add untokenize() function to allow full round-trip tokenization.
Raymond Hettinger <python@rcn.com>
parents:
29086
diff
changeset
|
698 >>> exec(decistmt(s)) |
|
8076424d455d
Add untokenize() function to allow full round-trip tokenization.
Raymond Hettinger <python@rcn.com>
parents:
29086
diff
changeset
|
699 -3.217160342717258261933904529E-7 |
|
37316
049606933328
Repaired a number of errors in this test:
Tim Peters <tim.peters@gmail.com>
parents:
35119
diff
changeset
|
700 """ |
|
35058
8076424d455d
Add untokenize() function to allow full round-trip tokenization.
Raymond Hettinger <python@rcn.com>
parents:
29086
diff
changeset
|
701 result = [] |
|
45888
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
702 g = tokenize(BytesIO(s.encode('utf-8')).readline) # tokenize the string |
|
35058
8076424d455d
Add untokenize() function to allow full round-trip tokenization.
Raymond Hettinger <python@rcn.com>
parents:
29086
diff
changeset
|
703 for toknum, tokval, _, _, _ in g: |
|
8076424d455d
Add untokenize() function to allow full round-trip tokenization.
Raymond Hettinger <python@rcn.com>
parents:
29086
diff
changeset
|
704 if toknum == NUMBER and '.' in tokval: # replace NUMBER tokens |
|
8076424d455d
Add untokenize() function to allow full round-trip tokenization.
Raymond Hettinger <python@rcn.com>
parents:
29086
diff
changeset
|
705 result.extend([ |
|
8076424d455d
Add untokenize() function to allow full round-trip tokenization.
Raymond Hettinger <python@rcn.com>
parents:
29086
diff
changeset
|
706 (NAME, 'Decimal'), |
|
8076424d455d
Add untokenize() function to allow full round-trip tokenization.
Raymond Hettinger <python@rcn.com>
parents:
29086
diff
changeset
|
707 (OP, '('), |
|
8076424d455d
Add untokenize() function to allow full round-trip tokenization.
Raymond Hettinger <python@rcn.com>
parents:
29086
diff
changeset
|
708 (STRING, repr(tokval)), |
|
8076424d455d
Add untokenize() function to allow full round-trip tokenization.
Raymond Hettinger <python@rcn.com>
parents:
29086
diff
changeset
|
709 (OP, ')') |
|
8076424d455d
Add untokenize() function to allow full round-trip tokenization.
Raymond Hettinger <python@rcn.com>
parents:
29086
diff
changeset
|
710 ]) |
|
8076424d455d
Add untokenize() function to allow full round-trip tokenization.
Raymond Hettinger <python@rcn.com>
parents:
29086
diff
changeset
|
711 else: |
|
8076424d455d
Add untokenize() function to allow full round-trip tokenization.
Raymond Hettinger <python@rcn.com>
parents:
29086
diff
changeset
|
712 result.append((toknum, tokval)) |
|
45888
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
713 return untokenize(result).decode('utf-8') |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
714 |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
715 |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
716 class TestTokenizerAdheresToPep0263(TestCase): |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
717 """ |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
718 Test that tokenizer adheres to the coding behaviour stipulated in PEP 0263. |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
719 """ |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
720 |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
721 def _testFile(self, filename): |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
722 path = os.path.join(os.path.dirname(__file__), filename) |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
723 return roundtrip(open(path, 'rb')) |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
724 |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
725 def test_utf8_coding_cookie_and_no_utf8_bom(self): |
|
71426
0c254698e0ed
Issue #12587: Correct faulty test file and reference in test_tokenize.
Ned Deily <nad@acm.org>
parents:
68257
diff
changeset
|
726 f = 'tokenize_tests-utf8-coding-cookie-and-no-utf8-bom-sig.txt' |
|
45888
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
727 self.assertTrue(self._testFile(f)) |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
728 |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
729 def test_latin1_coding_cookie_and_utf8_bom(self): |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
730 """ |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
731 As per PEP 0263, if a file starts with a utf-8 BOM signature, the only |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
732 allowed encoding for the comment is 'utf-8'. The text file used in |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
733 this test starts with a BOM signature, but specifies latin1 as the |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
734 coding, so verify that a SyntaxError is raised, which matches the |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
735 behaviour of the interpreter when it encounters a similar condition. |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
736 """ |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
737 f = 'tokenize_tests-latin1-coding-cookie-and-utf8-bom-sig.txt' |
|
55593
0bb232078b8c
Merged revisions 73715 via svnmerge from
Georg Brandl <georg@python.org>
parents:
50127
diff
changeset
|
738 self.assertRaises(SyntaxError, self._testFile, f) |
|
45888
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
739 |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
740 def test_no_coding_cookie_and_utf8_bom(self): |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
741 f = 'tokenize_tests-no-coding-cookie-and-utf8-bom-sig-only.txt' |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
742 self.assertTrue(self._testFile(f)) |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
743 |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
744 def test_utf8_coding_cookie_and_utf8_bom(self): |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
745 f = 'tokenize_tests-utf8-coding-cookie-and-utf8-bom-sig.txt' |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
746 self.assertTrue(self._testFile(f)) |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
747 |
|
77961
5020afc0b7c9
Issue #14990: tokenize: correctly fail with SyntaxError on invalid encoding declaration.
Florent Xicluna <florent.xicluna@gmail.com>
parents:
76425
diff
changeset
|
748 def test_bad_coding_cookie(self): |
|
5020afc0b7c9
Issue #14990: tokenize: correctly fail with SyntaxError on invalid encoding declaration.
Florent Xicluna <florent.xicluna@gmail.com>
parents:
76425
diff
changeset
|
749 self.assertRaises(SyntaxError, self._testFile, 'bad_coding.py') |
|
5020afc0b7c9
Issue #14990: tokenize: correctly fail with SyntaxError on invalid encoding declaration.
Florent Xicluna <florent.xicluna@gmail.com>
parents:
76425
diff
changeset
|
750 self.assertRaises(SyntaxError, self._testFile, 'bad_coding2.py') |
|
5020afc0b7c9
Issue #14990: tokenize: correctly fail with SyntaxError on invalid encoding declaration.
Florent Xicluna <florent.xicluna@gmail.com>
parents:
76425
diff
changeset
|
751 |
|
45888
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
752 |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
753 class Test_Tokenize(TestCase): |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
754 |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
755 def test__tokenize_decodes_with_specified_encoding(self): |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
756 literal = '"ЉЊЈЁЂ"' |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
757 line = literal.encode('utf-8') |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
758 first = False |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
759 def readline(): |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
760 nonlocal first |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
761 if not first: |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
762 first = True |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
763 return line |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
764 else: |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
765 return b'' |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
766 |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
767 # skip the initial encoding token and the end token |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
768 tokens = list(_tokenize(readline, encoding='utf-8'))[1:-1] |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
769 expected_tokens = [(3, '"ЉЊЈЁЂ"', (1, 0), (1, 7), '"ЉЊЈЁЂ"')] |
|
66340
2beb25369289
Merged revisions 86596 via svnmerge from
Ezio Melotti <ezio.melotti@gmail.com>
parents:
65773
diff
changeset
|
770 self.assertEqual(tokens, expected_tokens, |
|
2beb25369289
Merged revisions 86596 via svnmerge from
Ezio Melotti <ezio.melotti@gmail.com>
parents:
65773
diff
changeset
|
771 "bytes not decoded with encoding") |
|
45888
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
772 |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
773 def test__tokenize_does_not_decode_with_encoding_none(self): |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
774 literal = '"ЉЊЈЁЂ"' |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
775 first = False |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
776 def readline(): |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
777 nonlocal first |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
778 if not first: |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
779 first = True |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
780 return literal |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
781 else: |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
782 return b'' |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
783 |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
784 # skip the end token |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
785 tokens = list(_tokenize(readline, encoding=None))[:-1] |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
786 expected_tokens = [(3, '"ЉЊЈЁЂ"', (1, 0), (1, 7), '"ЉЊЈЁЂ"')] |
|
66340
2beb25369289
Merged revisions 86596 via svnmerge from
Ezio Melotti <ezio.melotti@gmail.com>
parents:
65773
diff
changeset
|
787 self.assertEqual(tokens, expected_tokens, |
|
2beb25369289
Merged revisions 86596 via svnmerge from
Ezio Melotti <ezio.melotti@gmail.com>
parents:
65773
diff
changeset
|
788 "string not tokenized when encoding is None") |
|
45888
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
789 |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
790 |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
791 class TestDetectEncoding(TestCase): |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
792 |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
793 def get_readline(self, lines): |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
794 index = 0 |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
795 def readline(): |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
796 nonlocal index |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
797 if index == len(lines): |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
798 raise StopIteration |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
799 line = lines[index] |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
800 index += 1 |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
801 return line |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
802 return readline |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
803 |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
804 def test_no_bom_no_encoding_cookie(self): |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
805 lines = ( |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
806 b'# something\n', |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
807 b'print(something)\n', |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
808 b'do_something(else)\n' |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
809 ) |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
810 encoding, consumed_lines = detect_encoding(self.get_readline(lines)) |
|
66340
2beb25369289
Merged revisions 86596 via svnmerge from
Ezio Melotti <ezio.melotti@gmail.com>
parents:
65773
diff
changeset
|
811 self.assertEqual(encoding, 'utf-8') |
|
2beb25369289
Merged revisions 86596 via svnmerge from
Ezio Melotti <ezio.melotti@gmail.com>
parents:
65773
diff
changeset
|
812 self.assertEqual(consumed_lines, list(lines[:2])) |
|
45888
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
813 |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
814 def test_bom_no_cookie(self): |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
815 lines = ( |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
816 b'\xef\xbb\xbf# something\n', |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
817 b'print(something)\n', |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
818 b'do_something(else)\n' |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
819 ) |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
820 encoding, consumed_lines = detect_encoding(self.get_readline(lines)) |
|
66310
9fe76c26ea9f
#9424: Replace deprecated assert* methods in the Python test suite.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
66098
diff
changeset
|
821 self.assertEqual(encoding, 'utf-8-sig') |
|
66340
2beb25369289
Merged revisions 86596 via svnmerge from
Ezio Melotti <ezio.melotti@gmail.com>
parents:
65773
diff
changeset
|
822 self.assertEqual(consumed_lines, |
|
2beb25369289
Merged revisions 86596 via svnmerge from
Ezio Melotti <ezio.melotti@gmail.com>
parents:
65773
diff
changeset
|
823 [b'# something\n', b'print(something)\n']) |
|
45888
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
824 |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
825 def test_cookie_first_line_no_bom(self): |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
826 lines = ( |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
827 b'# -*- coding: latin-1 -*-\n', |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
828 b'print(something)\n', |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
829 b'do_something(else)\n' |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
830 ) |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
831 encoding, consumed_lines = detect_encoding(self.get_readline(lines)) |
|
66340
2beb25369289
Merged revisions 86596 via svnmerge from
Ezio Melotti <ezio.melotti@gmail.com>
parents:
65773
diff
changeset
|
832 self.assertEqual(encoding, 'iso-8859-1') |
|
2beb25369289
Merged revisions 86596 via svnmerge from
Ezio Melotti <ezio.melotti@gmail.com>
parents:
65773
diff
changeset
|
833 self.assertEqual(consumed_lines, [b'# -*- coding: latin-1 -*-\n']) |
|
45888
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
834 |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
835 def test_matched_bom_and_cookie_first_line(self): |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
836 lines = ( |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
837 b'\xef\xbb\xbf# coding=utf-8\n', |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
838 b'print(something)\n', |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
839 b'do_something(else)\n' |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
840 ) |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
841 encoding, consumed_lines = detect_encoding(self.get_readline(lines)) |
|
66310
9fe76c26ea9f
#9424: Replace deprecated assert* methods in the Python test suite.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
66098
diff
changeset
|
842 self.assertEqual(encoding, 'utf-8-sig') |
|
66340
2beb25369289
Merged revisions 86596 via svnmerge from
Ezio Melotti <ezio.melotti@gmail.com>
parents:
65773
diff
changeset
|
843 self.assertEqual(consumed_lines, [b'# coding=utf-8\n']) |
|
45888
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
844 |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
845 def test_mismatched_bom_and_cookie_first_line_raises_syntaxerror(self): |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
846 lines = ( |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
847 b'\xef\xbb\xbf# vim: set fileencoding=ascii :\n', |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
848 b'print(something)\n', |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
849 b'do_something(else)\n' |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
850 ) |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
851 readline = self.get_readline(lines) |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
852 self.assertRaises(SyntaxError, detect_encoding, readline) |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
853 |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
854 def test_cookie_second_line_no_bom(self): |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
855 lines = ( |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
856 b'#! something\n', |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
857 b'# vim: set fileencoding=ascii :\n', |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
858 b'print(something)\n', |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
859 b'do_something(else)\n' |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
860 ) |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
861 encoding, consumed_lines = detect_encoding(self.get_readline(lines)) |
|
66340
2beb25369289
Merged revisions 86596 via svnmerge from
Ezio Melotti <ezio.melotti@gmail.com>
parents:
65773
diff
changeset
|
862 self.assertEqual(encoding, 'ascii') |
|
45888
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
863 expected = [b'#! something\n', b'# vim: set fileencoding=ascii :\n'] |
|
66340
2beb25369289
Merged revisions 86596 via svnmerge from
Ezio Melotti <ezio.melotti@gmail.com>
parents:
65773
diff
changeset
|
864 self.assertEqual(consumed_lines, expected) |
|
45888
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
865 |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
866 def test_matched_bom_and_cookie_second_line(self): |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
867 lines = ( |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
868 b'\xef\xbb\xbf#! something\n', |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
869 b'f# coding=utf-8\n', |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
870 b'print(something)\n', |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
871 b'do_something(else)\n' |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
872 ) |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
873 encoding, consumed_lines = detect_encoding(self.get_readline(lines)) |
|
66310
9fe76c26ea9f
#9424: Replace deprecated assert* methods in the Python test suite.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
66098
diff
changeset
|
874 self.assertEqual(encoding, 'utf-8-sig') |
|
66340
2beb25369289
Merged revisions 86596 via svnmerge from
Ezio Melotti <ezio.melotti@gmail.com>
parents:
65773
diff
changeset
|
875 self.assertEqual(consumed_lines, |
|
2beb25369289
Merged revisions 86596 via svnmerge from
Ezio Melotti <ezio.melotti@gmail.com>
parents:
65773
diff
changeset
|
876 [b'#! something\n', b'f# coding=utf-8\n']) |
|
45888
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
877 |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
878 def test_mismatched_bom_and_cookie_second_line_raises_syntaxerror(self): |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
879 lines = ( |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
880 b'\xef\xbb\xbf#! something\n', |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
881 b'# vim: set fileencoding=ascii :\n', |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
882 b'print(something)\n', |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
883 b'do_something(else)\n' |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
884 ) |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
885 readline = self.get_readline(lines) |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
886 self.assertRaises(SyntaxError, detect_encoding, readline) |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
887 |
|
56434
12cd94d39874
Merged revisions 75299 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
55593
diff
changeset
|
888 def test_latin1_normalization(self): |
|
12cd94d39874
Merged revisions 75299 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
55593
diff
changeset
|
889 # See get_normal_name() in tokenizer.c. |
|
12cd94d39874
Merged revisions 75299 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
55593
diff
changeset
|
890 encodings = ("latin-1", "iso-8859-1", "iso-latin-1", "latin-1-unix", |
|
12cd94d39874
Merged revisions 75299 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
55593
diff
changeset
|
891 "iso-8859-1-unix", "iso-latin-1-mac") |
|
12cd94d39874
Merged revisions 75299 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
55593
diff
changeset
|
892 for encoding in encodings: |
|
12cd94d39874
Merged revisions 75299 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
55593
diff
changeset
|
893 for rep in ("-", "_"): |
|
12cd94d39874
Merged revisions 75299 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
55593
diff
changeset
|
894 enc = encoding.replace("-", rep) |
|
12cd94d39874
Merged revisions 75299 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
55593
diff
changeset
|
895 lines = (b"#!/usr/bin/python\n", |
|
12cd94d39874
Merged revisions 75299 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
55593
diff
changeset
|
896 b"# coding: " + enc.encode("ascii") + b"\n", |
|
12cd94d39874
Merged revisions 75299 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
55593
diff
changeset
|
897 b"print(things)\n", |
|
12cd94d39874
Merged revisions 75299 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
55593
diff
changeset
|
898 b"do_something += 4\n") |
|
12cd94d39874
Merged revisions 75299 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
55593
diff
changeset
|
899 rl = self.get_readline(lines) |
|
12cd94d39874
Merged revisions 75299 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
55593
diff
changeset
|
900 found, consumed_lines = detect_encoding(rl) |
|
66340
2beb25369289
Merged revisions 86596 via svnmerge from
Ezio Melotti <ezio.melotti@gmail.com>
parents:
65773
diff
changeset
|
901 self.assertEqual(found, "iso-8859-1") |
|
56434
12cd94d39874
Merged revisions 75299 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
55593
diff
changeset
|
902 |
|
76425
b07488490001
Issue #14629: Raise SyntaxError in tokenizer.detect_encoding
Martin v. Löwis <martin@v.loewis.de>
parents:
71853
diff
changeset
|
903 def test_syntaxerror_latin1(self): |
|
b07488490001
Issue #14629: Raise SyntaxError in tokenizer.detect_encoding
Martin v. Löwis <martin@v.loewis.de>
parents:
71853
diff
changeset
|
904 # Issue 14629: need to raise SyntaxError if the first |
|
b07488490001
Issue #14629: Raise SyntaxError in tokenizer.detect_encoding
Martin v. Löwis <martin@v.loewis.de>
parents:
71853
diff
changeset
|
905 # line(s) have non-UTF-8 characters |
|
b07488490001
Issue #14629: Raise SyntaxError in tokenizer.detect_encoding
Martin v. Löwis <martin@v.loewis.de>
parents:
71853
diff
changeset
|
906 lines = ( |
|
b07488490001
Issue #14629: Raise SyntaxError in tokenizer.detect_encoding
Martin v. Löwis <martin@v.loewis.de>
parents:
71853
diff
changeset
|
907 b'print("\xdf")', # Latin-1: LATIN SMALL LETTER SHARP S |
|
b07488490001
Issue #14629: Raise SyntaxError in tokenizer.detect_encoding
Martin v. Löwis <martin@v.loewis.de>
parents:
71853
diff
changeset
|
908 ) |
|
b07488490001
Issue #14629: Raise SyntaxError in tokenizer.detect_encoding
Martin v. Löwis <martin@v.loewis.de>
parents:
71853
diff
changeset
|
909 readline = self.get_readline(lines) |
|
b07488490001
Issue #14629: Raise SyntaxError in tokenizer.detect_encoding
Martin v. Löwis <martin@v.loewis.de>
parents:
71853
diff
changeset
|
910 self.assertRaises(SyntaxError, detect_encoding, readline) |
|
b07488490001
Issue #14629: Raise SyntaxError in tokenizer.detect_encoding
Martin v. Löwis <martin@v.loewis.de>
parents:
71853
diff
changeset
|
911 |
|
b07488490001
Issue #14629: Raise SyntaxError in tokenizer.detect_encoding
Martin v. Löwis <martin@v.loewis.de>
parents:
71853
diff
changeset
|
912 |
|
56434
12cd94d39874
Merged revisions 75299 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
55593
diff
changeset
|
913 def test_utf8_normalization(self): |
|
12cd94d39874
Merged revisions 75299 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
55593
diff
changeset
|
914 # See get_normal_name() in tokenizer.c. |
|
12cd94d39874
Merged revisions 75299 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
55593
diff
changeset
|
915 encodings = ("utf-8", "utf-8-mac", "utf-8-unix") |
|
12cd94d39874
Merged revisions 75299 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
55593
diff
changeset
|
916 for encoding in encodings: |
|
12cd94d39874
Merged revisions 75299 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
55593
diff
changeset
|
917 for rep in ("-", "_"): |
|
12cd94d39874
Merged revisions 75299 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
55593
diff
changeset
|
918 enc = encoding.replace("-", rep) |
|
12cd94d39874
Merged revisions 75299 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
55593
diff
changeset
|
919 lines = (b"#!/usr/bin/python\n", |
|
12cd94d39874
Merged revisions 75299 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
55593
diff
changeset
|
920 b"# coding: " + enc.encode("ascii") + b"\n", |
|
12cd94d39874
Merged revisions 75299 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
55593
diff
changeset
|
921 b"1 + 3\n") |
|
12cd94d39874
Merged revisions 75299 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
55593
diff
changeset
|
922 rl = self.get_readline(lines) |
|
12cd94d39874
Merged revisions 75299 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
55593
diff
changeset
|
923 found, consumed_lines = detect_encoding(rl) |
|
66340
2beb25369289
Merged revisions 86596 via svnmerge from
Ezio Melotti <ezio.melotti@gmail.com>
parents:
65773
diff
changeset
|
924 self.assertEqual(found, "utf-8") |
|
56434
12cd94d39874
Merged revisions 75299 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
55593
diff
changeset
|
925 |
|
45888
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
926 def test_short_files(self): |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
927 readline = self.get_readline((b'print(something)\n',)) |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
928 encoding, consumed_lines = detect_encoding(readline) |
|
66340
2beb25369289
Merged revisions 86596 via svnmerge from
Ezio Melotti <ezio.melotti@gmail.com>
parents:
65773
diff
changeset
|
929 self.assertEqual(encoding, 'utf-8') |
|
2beb25369289
Merged revisions 86596 via svnmerge from
Ezio Melotti <ezio.melotti@gmail.com>
parents:
65773
diff
changeset
|
930 self.assertEqual(consumed_lines, [b'print(something)\n']) |
|
45888
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
931 |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
932 encoding, consumed_lines = detect_encoding(self.get_readline(())) |
|
66340
2beb25369289
Merged revisions 86596 via svnmerge from
Ezio Melotti <ezio.melotti@gmail.com>
parents:
65773
diff
changeset
|
933 self.assertEqual(encoding, 'utf-8') |
|
2beb25369289
Merged revisions 86596 via svnmerge from
Ezio Melotti <ezio.melotti@gmail.com>
parents:
65773
diff
changeset
|
934 self.assertEqual(consumed_lines, []) |
|
45888
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
935 |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
936 readline = self.get_readline((b'\xef\xbb\xbfprint(something)\n',)) |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
937 encoding, consumed_lines = detect_encoding(readline) |
|
66310
9fe76c26ea9f
#9424: Replace deprecated assert* methods in the Python test suite.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
66098
diff
changeset
|
938 self.assertEqual(encoding, 'utf-8-sig') |
|
66340
2beb25369289
Merged revisions 86596 via svnmerge from
Ezio Melotti <ezio.melotti@gmail.com>
parents:
65773
diff
changeset
|
939 self.assertEqual(consumed_lines, [b'print(something)\n']) |
|
45888
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
940 |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
941 readline = self.get_readline((b'\xef\xbb\xbf',)) |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
942 encoding, consumed_lines = detect_encoding(readline) |
|
66310
9fe76c26ea9f
#9424: Replace deprecated assert* methods in the Python test suite.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
66098
diff
changeset
|
943 self.assertEqual(encoding, 'utf-8-sig') |
|
66340
2beb25369289
Merged revisions 86596 via svnmerge from
Ezio Melotti <ezio.melotti@gmail.com>
parents:
65773
diff
changeset
|
944 self.assertEqual(consumed_lines, []) |
|
45888
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
945 |
|
50127
df6649951410
Merged revisions 67711 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
47201
diff
changeset
|
946 readline = self.get_readline((b'# coding: bad\n',)) |
|
df6649951410
Merged revisions 67711 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
47201
diff
changeset
|
947 self.assertRaises(SyntaxError, detect_encoding, readline) |
|
45888
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
948 |
|
85731
2dfe8262093c
Issue #18873: The tokenize module, IDLE, 2to3, and the findnocoding.py script
Serhiy Storchaka <storchaka@gmail.com>
parents:
80192
diff
changeset
|
949 def test_false_encoding(self): |
|
2dfe8262093c
Issue #18873: The tokenize module, IDLE, 2to3, and the findnocoding.py script
Serhiy Storchaka <storchaka@gmail.com>
parents:
80192
diff
changeset
|
950 # Issue 18873: "Encoding" detected in non-comment lines |
|
2dfe8262093c
Issue #18873: The tokenize module, IDLE, 2to3, and the findnocoding.py script
Serhiy Storchaka <storchaka@gmail.com>
parents:
80192
diff
changeset
|
951 readline = self.get_readline((b'print("#coding=fake")',)) |
|
2dfe8262093c
Issue #18873: The tokenize module, IDLE, 2to3, and the findnocoding.py script
Serhiy Storchaka <storchaka@gmail.com>
parents:
80192
diff
changeset
|
952 encoding, consumed_lines = detect_encoding(readline) |
|
2dfe8262093c
Issue #18873: The tokenize module, IDLE, 2to3, and the findnocoding.py script
Serhiy Storchaka <storchaka@gmail.com>
parents:
80192
diff
changeset
|
953 self.assertEqual(encoding, 'utf-8') |
|
2dfe8262093c
Issue #18873: The tokenize module, IDLE, 2to3, and the findnocoding.py script
Serhiy Storchaka <storchaka@gmail.com>
parents:
80192
diff
changeset
|
954 self.assertEqual(consumed_lines, [b'print("#coding=fake")']) |
|
2dfe8262093c
Issue #18873: The tokenize module, IDLE, 2to3, and the findnocoding.py script
Serhiy Storchaka <storchaka@gmail.com>
parents:
80192
diff
changeset
|
955 |
|
66097
306339c31758
Issue #10335: Add tokenize.open(), detect the file encoding using
Victor Stinner <victor.stinner@haypocalc.com>
parents:
65772
diff
changeset
|
956 def test_open(self): |
|
306339c31758
Issue #10335: Add tokenize.open(), detect the file encoding using
Victor Stinner <victor.stinner@haypocalc.com>
parents:
65772
diff
changeset
|
957 filename = support.TESTFN + '.py' |
|
306339c31758
Issue #10335: Add tokenize.open(), detect the file encoding using
Victor Stinner <victor.stinner@haypocalc.com>
parents:
65772
diff
changeset
|
958 self.addCleanup(support.unlink, filename) |
|
306339c31758
Issue #10335: Add tokenize.open(), detect the file encoding using
Victor Stinner <victor.stinner@haypocalc.com>
parents:
65772
diff
changeset
|
959 |
|
306339c31758
Issue #10335: Add tokenize.open(), detect the file encoding using
Victor Stinner <victor.stinner@haypocalc.com>
parents:
65772
diff
changeset
|
960 # test coding cookie |
|
306339c31758
Issue #10335: Add tokenize.open(), detect the file encoding using
Victor Stinner <victor.stinner@haypocalc.com>
parents:
65772
diff
changeset
|
961 for encoding in ('iso-8859-15', 'utf-8'): |
|
306339c31758
Issue #10335: Add tokenize.open(), detect the file encoding using
Victor Stinner <victor.stinner@haypocalc.com>
parents:
65772
diff
changeset
|
962 with open(filename, 'w', encoding=encoding) as fp: |
|
306339c31758
Issue #10335: Add tokenize.open(), detect the file encoding using
Victor Stinner <victor.stinner@haypocalc.com>
parents:
65772
diff
changeset
|
963 print("# coding: %s" % encoding, file=fp) |
|
306339c31758
Issue #10335: Add tokenize.open(), detect the file encoding using
Victor Stinner <victor.stinner@haypocalc.com>
parents:
65772
diff
changeset
|
964 print("print('euro:\u20ac')", file=fp) |
|
306339c31758
Issue #10335: Add tokenize.open(), detect the file encoding using
Victor Stinner <victor.stinner@haypocalc.com>
parents:
65772
diff
changeset
|
965 with tokenize_open(filename) as fp: |
|
66098
323d43e8953a
test_tokenize: use self.assertEqual() instead of plain assert
Victor Stinner <victor.stinner@haypocalc.com>
parents:
66097
diff
changeset
|
966 self.assertEqual(fp.encoding, encoding) |
|
323d43e8953a
test_tokenize: use self.assertEqual() instead of plain assert
Victor Stinner <victor.stinner@haypocalc.com>
parents:
66097
diff
changeset
|
967 self.assertEqual(fp.mode, 'r') |
|
66097
306339c31758
Issue #10335: Add tokenize.open(), detect the file encoding using
Victor Stinner <victor.stinner@haypocalc.com>
parents:
65772
diff
changeset
|
968 |
|
306339c31758
Issue #10335: Add tokenize.open(), detect the file encoding using
Victor Stinner <victor.stinner@haypocalc.com>
parents:
65772
diff
changeset
|
969 # test BOM (no coding cookie) |
|
306339c31758
Issue #10335: Add tokenize.open(), detect the file encoding using
Victor Stinner <victor.stinner@haypocalc.com>
parents:
65772
diff
changeset
|
970 with open(filename, 'w', encoding='utf-8-sig') as fp: |
|
306339c31758
Issue #10335: Add tokenize.open(), detect the file encoding using
Victor Stinner <victor.stinner@haypocalc.com>
parents:
65772
diff
changeset
|
971 print("print('euro:\u20ac')", file=fp) |
|
306339c31758
Issue #10335: Add tokenize.open(), detect the file encoding using
Victor Stinner <victor.stinner@haypocalc.com>
parents:
65772
diff
changeset
|
972 with tokenize_open(filename) as fp: |
|
66098
323d43e8953a
test_tokenize: use self.assertEqual() instead of plain assert
Victor Stinner <victor.stinner@haypocalc.com>
parents:
66097
diff
changeset
|
973 self.assertEqual(fp.encoding, 'utf-8-sig') |
|
323d43e8953a
test_tokenize: use self.assertEqual() instead of plain assert
Victor Stinner <victor.stinner@haypocalc.com>
parents:
66097
diff
changeset
|
974 self.assertEqual(fp.mode, 'r') |
|
66097
306339c31758
Issue #10335: Add tokenize.open(), detect the file encoding using
Victor Stinner <victor.stinner@haypocalc.com>
parents:
65772
diff
changeset
|
975 |
|
76430
1b57de8a8383
Issue #14629: Mention the filename in SyntaxError exceptions from
Brett Cannon <brett@python.org>
parents:
76426
diff
changeset
|
976 def test_filename_in_exception(self): |
|
1b57de8a8383
Issue #14629: Mention the filename in SyntaxError exceptions from
Brett Cannon <brett@python.org>
parents:
76426
diff
changeset
|
977 # When possible, include the file name in the exception. |
|
1b57de8a8383
Issue #14629: Mention the filename in SyntaxError exceptions from
Brett Cannon <brett@python.org>
parents:
76426
diff
changeset
|
978 path = 'some_file_path' |
|
1b57de8a8383
Issue #14629: Mention the filename in SyntaxError exceptions from
Brett Cannon <brett@python.org>
parents:
76426
diff
changeset
|
979 lines = ( |
|
1b57de8a8383
Issue #14629: Mention the filename in SyntaxError exceptions from
Brett Cannon <brett@python.org>
parents:
76426
diff
changeset
|
980 b'print("\xdf")', # Latin-1: LATIN SMALL LETTER SHARP S |
|
1b57de8a8383
Issue #14629: Mention the filename in SyntaxError exceptions from
Brett Cannon <brett@python.org>
parents:
76426
diff
changeset
|
981 ) |
|
1b57de8a8383
Issue #14629: Mention the filename in SyntaxError exceptions from
Brett Cannon <brett@python.org>
parents:
76426
diff
changeset
|
982 class Bunk: |
|
1b57de8a8383
Issue #14629: Mention the filename in SyntaxError exceptions from
Brett Cannon <brett@python.org>
parents:
76426
diff
changeset
|
983 def __init__(self, lines, path): |
|
1b57de8a8383
Issue #14629: Mention the filename in SyntaxError exceptions from
Brett Cannon <brett@python.org>
parents:
76426
diff
changeset
|
984 self.name = path |
|
1b57de8a8383
Issue #14629: Mention the filename in SyntaxError exceptions from
Brett Cannon <brett@python.org>
parents:
76426
diff
changeset
|
985 self._lines = lines |
|
1b57de8a8383
Issue #14629: Mention the filename in SyntaxError exceptions from
Brett Cannon <brett@python.org>
parents:
76426
diff
changeset
|
986 self._index = 0 |
|
1b57de8a8383
Issue #14629: Mention the filename in SyntaxError exceptions from
Brett Cannon <brett@python.org>
parents:
76426
diff
changeset
|
987 |
|
1b57de8a8383
Issue #14629: Mention the filename in SyntaxError exceptions from
Brett Cannon <brett@python.org>
parents:
76426
diff
changeset
|
988 def readline(self): |
|
1b57de8a8383
Issue #14629: Mention the filename in SyntaxError exceptions from
Brett Cannon <brett@python.org>
parents:
76426
diff
changeset
|
989 if self._index == len(lines): |
|
1b57de8a8383
Issue #14629: Mention the filename in SyntaxError exceptions from
Brett Cannon <brett@python.org>
parents:
76426
diff
changeset
|
990 raise StopIteration |
|
1b57de8a8383
Issue #14629: Mention the filename in SyntaxError exceptions from
Brett Cannon <brett@python.org>
parents:
76426
diff
changeset
|
991 line = lines[self._index] |
|
1b57de8a8383
Issue #14629: Mention the filename in SyntaxError exceptions from
Brett Cannon <brett@python.org>
parents:
76426
diff
changeset
|
992 self._index += 1 |
|
1b57de8a8383
Issue #14629: Mention the filename in SyntaxError exceptions from
Brett Cannon <brett@python.org>
parents:
76426
diff
changeset
|
993 return line |
|
1b57de8a8383
Issue #14629: Mention the filename in SyntaxError exceptions from
Brett Cannon <brett@python.org>
parents:
76426
diff
changeset
|
994 |
|
1b57de8a8383
Issue #14629: Mention the filename in SyntaxError exceptions from
Brett Cannon <brett@python.org>
parents:
76426
diff
changeset
|
995 with self.assertRaises(SyntaxError): |
|
1b57de8a8383
Issue #14629: Mention the filename in SyntaxError exceptions from
Brett Cannon <brett@python.org>
parents:
76426
diff
changeset
|
996 ins = Bunk(lines, path) |
|
1b57de8a8383
Issue #14629: Mention the filename in SyntaxError exceptions from
Brett Cannon <brett@python.org>
parents:
76426
diff
changeset
|
997 # Make sure lacking a name isn't an issue. |
|
1b57de8a8383
Issue #14629: Mention the filename in SyntaxError exceptions from
Brett Cannon <brett@python.org>
parents:
76426
diff
changeset
|
998 del ins.name |
|
1b57de8a8383
Issue #14629: Mention the filename in SyntaxError exceptions from
Brett Cannon <brett@python.org>
parents:
76426
diff
changeset
|
999 detect_encoding(ins.readline) |
|
1b57de8a8383
Issue #14629: Mention the filename in SyntaxError exceptions from
Brett Cannon <brett@python.org>
parents:
76426
diff
changeset
|
1000 with self.assertRaisesRegex(SyntaxError, '.*{}'.format(path)): |
|
1b57de8a8383
Issue #14629: Mention the filename in SyntaxError exceptions from
Brett Cannon <brett@python.org>
parents:
76426
diff
changeset
|
1001 ins = Bunk(lines, path) |
|
1b57de8a8383
Issue #14629: Mention the filename in SyntaxError exceptions from
Brett Cannon <brett@python.org>
parents:
76426
diff
changeset
|
1002 detect_encoding(ins.readline) |
|
1b57de8a8383
Issue #14629: Mention the filename in SyntaxError exceptions from
Brett Cannon <brett@python.org>
parents:
76426
diff
changeset
|
1003 |
|
1b57de8a8383
Issue #14629: Mention the filename in SyntaxError exceptions from
Brett Cannon <brett@python.org>
parents:
76426
diff
changeset
|
1004 |
|
45888
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
1005 class TestTokenize(TestCase): |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
1006 |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
1007 def test_tokenize(self): |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
1008 import tokenize as tokenize_module |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
1009 encoding = object() |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
1010 encoding_used = None |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
1011 def mock_detect_encoding(readline): |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
1012 return encoding, ['first', 'second'] |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
1013 |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
1014 def mock__tokenize(readline, encoding): |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
1015 nonlocal encoding_used |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
1016 encoding_used = encoding |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
1017 out = [] |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
1018 while True: |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
1019 next_line = readline() |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
1020 if next_line: |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
1021 out.append(next_line) |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
1022 continue |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
1023 return out |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
1024 |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
1025 counter = 0 |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
1026 def mock_readline(): |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
1027 nonlocal counter |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
1028 counter += 1 |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
1029 if counter == 5: |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
1030 return b'' |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
1031 return counter |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
1032 |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
1033 orig_detect_encoding = tokenize_module.detect_encoding |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
1034 orig__tokenize = tokenize_module._tokenize |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
1035 tokenize_module.detect_encoding = mock_detect_encoding |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
1036 tokenize_module._tokenize = mock__tokenize |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
1037 try: |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
1038 results = tokenize(mock_readline) |
|
66340
2beb25369289
Merged revisions 86596 via svnmerge from
Ezio Melotti <ezio.melotti@gmail.com>
parents:
65773
diff
changeset
|
1039 self.assertEqual(list(results), ['first', 'second', 1, 2, 3, 4]) |
|
45888
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
1040 finally: |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
1041 tokenize_module.detect_encoding = orig_detect_encoding |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
1042 tokenize_module._tokenize = orig__tokenize |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
1043 |
|
51e24512e305
- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been
Trent Nelson <trent.nelson@snakebite.org>
parents:
45802
diff
changeset
|
1044 self.assertTrue(encoding_used, encoding) |
|
35058
8076424d455d
Add untokenize() function to allow full round-trip tokenization.
Raymond Hettinger <python@rcn.com>
parents:
29086
diff
changeset
|
1045 |
|
74519
f4976fa6e830
Issue #2134: Add support for tokenize.TokenInfo.exact_type.
Meador Inge <meadori@gmail.com>
parents:
72485
diff
changeset
|
1046 def assertExactTypeEqual(self, opstr, *optypes): |
|
f4976fa6e830
Issue #2134: Add support for tokenize.TokenInfo.exact_type.
Meador Inge <meadori@gmail.com>
parents:
72485
diff
changeset
|
1047 tokens = list(tokenize(BytesIO(opstr.encode('utf-8')).readline)) |
|
f4976fa6e830
Issue #2134: Add support for tokenize.TokenInfo.exact_type.
Meador Inge <meadori@gmail.com>
parents:
72485
diff
changeset
|
1048 num_optypes = len(optypes) |
|
f4976fa6e830
Issue #2134: Add support for tokenize.TokenInfo.exact_type.
Meador Inge <meadori@gmail.com>
parents:
72485
diff
changeset
|
1049 self.assertEqual(len(tokens), 2 + num_optypes) |
|
f4976fa6e830
Issue #2134: Add support for tokenize.TokenInfo.exact_type.
Meador Inge <meadori@gmail.com>
parents:
72485
diff
changeset
|
1050 self.assertEqual(token.tok_name[tokens[0].exact_type], |
|
f4976fa6e830
Issue #2134: Add support for tokenize.TokenInfo.exact_type.
Meador Inge <meadori@gmail.com>
parents:
72485
diff
changeset
|
1051 token.tok_name[ENCODING]) |
|
f4976fa6e830
Issue #2134: Add support for tokenize.TokenInfo.exact_type.
Meador Inge <meadori@gmail.com>
parents:
72485
diff
changeset
|
1052 for i in range(num_optypes): |
|
f4976fa6e830
Issue #2134: Add support for tokenize.TokenInfo.exact_type.
Meador Inge <meadori@gmail.com>
parents:
72485
diff
changeset
|
1053 self.assertEqual(token.tok_name[tokens[i + 1].exact_type], |
|
f4976fa6e830
Issue #2134: Add support for tokenize.TokenInfo.exact_type.
Meador Inge <meadori@gmail.com>
parents:
72485
diff
changeset
|
1054 token.tok_name[optypes[i]]) |
|
f4976fa6e830
Issue #2134: Add support for tokenize.TokenInfo.exact_type.
Meador Inge <meadori@gmail.com>
parents:
72485
diff
changeset
|
1055 self.assertEqual(token.tok_name[tokens[1 + num_optypes].exact_type], |
|
f4976fa6e830
Issue #2134: Add support for tokenize.TokenInfo.exact_type.
Meador Inge <meadori@gmail.com>
parents:
72485
diff
changeset
|
1056 token.tok_name[token.ENDMARKER]) |
|
f4976fa6e830
Issue #2134: Add support for tokenize.TokenInfo.exact_type.
Meador Inge <meadori@gmail.com>
parents:
72485
diff
changeset
|
1057 |
|
f4976fa6e830
Issue #2134: Add support for tokenize.TokenInfo.exact_type.
Meador Inge <meadori@gmail.com>
parents:
72485
diff
changeset
|
1058 def test_exact_type(self): |
|
f4976fa6e830
Issue #2134: Add support for tokenize.TokenInfo.exact_type.
Meador Inge <meadori@gmail.com>
parents:
72485
diff
changeset
|
1059 self.assertExactTypeEqual('()', token.LPAR, token.RPAR) |
|
f4976fa6e830
Issue #2134: Add support for tokenize.TokenInfo.exact_type.
Meador Inge <meadori@gmail.com>
parents:
72485
diff
changeset
|
1060 self.assertExactTypeEqual('[]', token.LSQB, token.RSQB) |
|
f4976fa6e830
Issue #2134: Add support for tokenize.TokenInfo.exact_type.
Meador Inge <meadori@gmail.com>
parents:
72485
diff
changeset
|
1061 self.assertExactTypeEqual(':', token.COLON) |
|
f4976fa6e830
Issue #2134: Add support for tokenize.TokenInfo.exact_type.
Meador Inge <meadori@gmail.com>
parents:
72485
diff
changeset
|
1062 self.assertExactTypeEqual(',', token.COMMA) |
|
f4976fa6e830
Issue #2134: Add support for tokenize.TokenInfo.exact_type.
Meador Inge <meadori@gmail.com>
parents:
72485
diff
changeset
|
1063 self.assertExactTypeEqual(';', token.SEMI) |
|
f4976fa6e830
Issue #2134: Add support for tokenize.TokenInfo.exact_type.
Meador Inge <meadori@gmail.com>
parents:
72485
diff
changeset
|
1064 self.assertExactTypeEqual('+', token.PLUS) |
|
f4976fa6e830
Issue #2134: Add support for tokenize.TokenInfo.exact_type.
Meador Inge <meadori@gmail.com>
parents:
72485
diff
changeset
|
1065 self.assertExactTypeEqual('-', token.MINUS) |
|
f4976fa6e830
Issue #2134: Add support for tokenize.TokenInfo.exact_type.
Meador Inge <meadori@gmail.com>
parents:
72485
diff
changeset
|
1066 self.assertExactTypeEqual('*', token.STAR) |
|
f4976fa6e830
Issue #2134: Add support for tokenize.TokenInfo.exact_type.
Meador Inge <meadori@gmail.com>
parents:
72485
diff
changeset
|
1067 self.assertExactTypeEqual('/', token.SLASH) |
|
f4976fa6e830
Issue #2134: Add support for tokenize.TokenInfo.exact_type.
Meador Inge <meadori@gmail.com>
parents:
72485
diff
changeset
|
1068 self.assertExactTypeEqual('|', token.VBAR) |
|
f4976fa6e830
Issue #2134: Add support for tokenize.TokenInfo.exact_type.
Meador Inge <meadori@gmail.com>
parents:
72485
diff
changeset
|
1069 self.assertExactTypeEqual('&', token.AMPER) |
|
f4976fa6e830
Issue #2134: Add support for tokenize.TokenInfo.exact_type.
Meador Inge <meadori@gmail.com>
parents:
72485
diff
changeset
|
1070 self.assertExactTypeEqual('<', token.LESS) |
|
f4976fa6e830
Issue #2134: Add support for tokenize.TokenInfo.exact_type.
Meador Inge <meadori@gmail.com>
parents:
72485
diff
changeset
|
1071 self.assertExactTypeEqual('>', token.GREATER) |
|
f4976fa6e830
Issue #2134: Add support for tokenize.TokenInfo.exact_type.
Meador Inge <meadori@gmail.com>
parents:
72485
diff
changeset
|
1072 self.assertExactTypeEqual('=', token.EQUAL) |
|
f4976fa6e830
Issue #2134: Add support for tokenize.TokenInfo.exact_type.
Meador Inge <meadori@gmail.com>
parents:
72485
diff
changeset
|
1073 self.assertExactTypeEqual('.', token.DOT) |
|
f4976fa6e830
Issue #2134: Add support for tokenize.TokenInfo.exact_type.
Meador Inge <meadori@gmail.com>
parents:
72485
diff
changeset
|
1074 self.assertExactTypeEqual('%', token.PERCENT) |
|
f4976fa6e830
Issue #2134: Add support for tokenize.TokenInfo.exact_type.
Meador Inge <meadori@gmail.com>
parents:
72485
diff
changeset
|
1075 self.assertExactTypeEqual('{}', token.LBRACE, token.RBRACE) |
|
f4976fa6e830
Issue #2134: Add support for tokenize.TokenInfo.exact_type.
Meador Inge <meadori@gmail.com>
parents:
72485
diff
changeset
|
1076 self.assertExactTypeEqual('==', token.EQEQUAL) |
|
f4976fa6e830
Issue #2134: Add support for tokenize.TokenInfo.exact_type.
Meador Inge <meadori@gmail.com>
parents:
72485
diff
changeset
|
1077 self.assertExactTypeEqual('!=', token.NOTEQUAL) |
|
f4976fa6e830
Issue #2134: Add support for tokenize.TokenInfo.exact_type.
Meador Inge <meadori@gmail.com>
parents:
72485
diff
changeset
|
1078 self.assertExactTypeEqual('<=', token.LESSEQUAL) |
|
f4976fa6e830
Issue #2134: Add support for tokenize.TokenInfo.exact_type.
Meador Inge <meadori@gmail.com>
parents:
72485
diff
changeset
|
1079 self.assertExactTypeEqual('>=', token.GREATEREQUAL) |
|
f4976fa6e830
Issue #2134: Add support for tokenize.TokenInfo.exact_type.
Meador Inge <meadori@gmail.com>
parents:
72485
diff
changeset
|
1080 self.assertExactTypeEqual('~', token.TILDE) |
|
f4976fa6e830
Issue #2134: Add support for tokenize.TokenInfo.exact_type.
Meador Inge <meadori@gmail.com>
parents:
72485
diff
changeset
|
1081 self.assertExactTypeEqual('^', token.CIRCUMFLEX) |
|
f4976fa6e830
Issue #2134: Add support for tokenize.TokenInfo.exact_type.
Meador Inge <meadori@gmail.com>
parents:
72485
diff
changeset
|
1082 self.assertExactTypeEqual('<<', token.LEFTSHIFT) |
|
f4976fa6e830
Issue #2134: Add support for tokenize.TokenInfo.exact_type.
Meador Inge <meadori@gmail.com>
parents:
72485
diff
changeset
|
1083 self.assertExactTypeEqual('>>', token.RIGHTSHIFT) |
|
f4976fa6e830
Issue #2134: Add support for tokenize.TokenInfo.exact_type.
Meador Inge <meadori@gmail.com>
parents:
72485
diff
changeset
|
1084 self.assertExactTypeEqual('**', token.DOUBLESTAR) |
|
f4976fa6e830
Issue #2134: Add support for tokenize.TokenInfo.exact_type.
Meador Inge <meadori@gmail.com>
parents:
72485
diff
changeset
|
1085 self.assertExactTypeEqual('+=', token.PLUSEQUAL) |
|
f4976fa6e830
Issue #2134: Add support for tokenize.TokenInfo.exact_type.
Meador Inge <meadori@gmail.com>
parents:
72485
diff
changeset
|
1086 self.assertExactTypeEqual('-=', token.MINEQUAL) |
|
f4976fa6e830
Issue #2134: Add support for tokenize.TokenInfo.exact_type.
Meador Inge <meadori@gmail.com>
parents:
72485
diff
changeset
|
1087 self.assertExactTypeEqual('*=', token.STAREQUAL) |
|
f4976fa6e830
Issue #2134: Add support for tokenize.TokenInfo.exact_type.
Meador Inge <meadori@gmail.com>
parents:
72485
diff
changeset
|
1088 self.assertExactTypeEqual('/=', token.SLASHEQUAL) |
|
f4976fa6e830
Issue #2134: Add support for tokenize.TokenInfo.exact_type.
Meador Inge <meadori@gmail.com>
parents:
72485
diff
changeset
|
1089 self.assertExactTypeEqual('%=', token.PERCENTEQUAL) |
|
f4976fa6e830
Issue #2134: Add support for tokenize.TokenInfo.exact_type.
Meador Inge <meadori@gmail.com>
parents:
72485
diff
changeset
|
1090 self.assertExactTypeEqual('&=', token.AMPEREQUAL) |
|
f4976fa6e830
Issue #2134: Add support for tokenize.TokenInfo.exact_type.
Meador Inge <meadori@gmail.com>
parents:
72485
diff
changeset
|
1091 self.assertExactTypeEqual('|=', token.VBAREQUAL) |
|
f4976fa6e830
Issue #2134: Add support for tokenize.TokenInfo.exact_type.
Meador Inge <meadori@gmail.com>
parents:
72485
diff
changeset
|
1092 self.assertExactTypeEqual('^=', token.CIRCUMFLEXEQUAL) |
|
f4976fa6e830
Issue #2134: Add support for tokenize.TokenInfo.exact_type.
Meador Inge <meadori@gmail.com>
parents:
72485
diff
changeset
|
1093 self.assertExactTypeEqual('^=', token.CIRCUMFLEXEQUAL) |
|
f4976fa6e830
Issue #2134: Add support for tokenize.TokenInfo.exact_type.
Meador Inge <meadori@gmail.com>
parents:
72485
diff
changeset
|
1094 self.assertExactTypeEqual('<<=', token.LEFTSHIFTEQUAL) |
|
f4976fa6e830
Issue #2134: Add support for tokenize.TokenInfo.exact_type.
Meador Inge <meadori@gmail.com>
parents:
72485
diff
changeset
|
1095 self.assertExactTypeEqual('>>=', token.RIGHTSHIFTEQUAL) |
|
f4976fa6e830
Issue #2134: Add support for tokenize.TokenInfo.exact_type.
Meador Inge <meadori@gmail.com>
parents:
72485
diff
changeset
|
1096 self.assertExactTypeEqual('**=', token.DOUBLESTAREQUAL) |
|
f4976fa6e830
Issue #2134: Add support for tokenize.TokenInfo.exact_type.
Meador Inge <meadori@gmail.com>
parents:
72485
diff
changeset
|
1097 self.assertExactTypeEqual('//', token.DOUBLESLASH) |
|
f4976fa6e830
Issue #2134: Add support for tokenize.TokenInfo.exact_type.
Meador Inge <meadori@gmail.com>
parents:
72485
diff
changeset
|
1098 self.assertExactTypeEqual('//=', token.DOUBLESLASHEQUAL) |
|
f4976fa6e830
Issue #2134: Add support for tokenize.TokenInfo.exact_type.
Meador Inge <meadori@gmail.com>
parents:
72485
diff
changeset
|
1099 self.assertExactTypeEqual('@', token.AT) |
|
f4976fa6e830
Issue #2134: Add support for tokenize.TokenInfo.exact_type.
Meador Inge <meadori@gmail.com>
parents:
72485
diff
changeset
|
1100 |
|
f4976fa6e830
Issue #2134: Add support for tokenize.TokenInfo.exact_type.
Meador Inge <meadori@gmail.com>
parents:
72485
diff
changeset
|
1101 self.assertExactTypeEqual('a**2+b**2==c**2', |
|
f4976fa6e830
Issue #2134: Add support for tokenize.TokenInfo.exact_type.
Meador Inge <meadori@gmail.com>
parents:
72485
diff
changeset
|
1102 NAME, token.DOUBLESTAR, NUMBER, |
|
f4976fa6e830
Issue #2134: Add support for tokenize.TokenInfo.exact_type.
Meador Inge <meadori@gmail.com>
parents:
72485
diff
changeset
|
1103 token.PLUS, |
|
f4976fa6e830
Issue #2134: Add support for tokenize.TokenInfo.exact_type.
Meador Inge <meadori@gmail.com>
parents:
72485
diff
changeset
|
1104 NAME, token.DOUBLESTAR, NUMBER, |
|
f4976fa6e830
Issue #2134: Add support for tokenize.TokenInfo.exact_type.
Meador Inge <meadori@gmail.com>
parents:
72485
diff
changeset
|
1105 token.EQEQUAL, |
|
f4976fa6e830
Issue #2134: Add support for tokenize.TokenInfo.exact_type.
Meador Inge <meadori@gmail.com>
parents:
72485
diff
changeset
|
1106 NAME, token.DOUBLESTAR, NUMBER) |
|
f4976fa6e830
Issue #2134: Add support for tokenize.TokenInfo.exact_type.
Meador Inge <meadori@gmail.com>
parents:
72485
diff
changeset
|
1107 self.assertExactTypeEqual('{1, 2, 3}', |
|
f4976fa6e830
Issue #2134: Add support for tokenize.TokenInfo.exact_type.
Meador Inge <meadori@gmail.com>
parents:
72485
diff
changeset
|
1108 token.LBRACE, |
|
f4976fa6e830
Issue #2134: Add support for tokenize.TokenInfo.exact_type.
Meador Inge <meadori@gmail.com>
parents:
72485
diff
changeset
|
1109 token.NUMBER, token.COMMA, |
|
f4976fa6e830
Issue #2134: Add support for tokenize.TokenInfo.exact_type.
Meador Inge <meadori@gmail.com>
parents:
72485
diff
changeset
|
1110 token.NUMBER, token.COMMA, |
|
f4976fa6e830
Issue #2134: Add support for tokenize.TokenInfo.exact_type.
Meador Inge <meadori@gmail.com>
parents:
72485
diff
changeset
|
1111 token.NUMBER, |
|
f4976fa6e830
Issue #2134: Add support for tokenize.TokenInfo.exact_type.
Meador Inge <meadori@gmail.com>
parents:
72485
diff
changeset
|
1112 token.RBRACE) |
|
f4976fa6e830
Issue #2134: Add support for tokenize.TokenInfo.exact_type.
Meador Inge <meadori@gmail.com>
parents:
72485
diff
changeset
|
1113 self.assertExactTypeEqual('^(x & 0x1)', |
|
f4976fa6e830
Issue #2134: Add support for tokenize.TokenInfo.exact_type.
Meador Inge <meadori@gmail.com>
parents:
72485
diff
changeset
|
1114 token.CIRCUMFLEX, |
|
f4976fa6e830
Issue #2134: Add support for tokenize.TokenInfo.exact_type.
Meador Inge <meadori@gmail.com>
parents:
72485
diff
changeset
|
1115 token.LPAR, |
|
f4976fa6e830
Issue #2134: Add support for tokenize.TokenInfo.exact_type.
Meador Inge <meadori@gmail.com>
parents:
72485
diff
changeset
|
1116 token.NAME, token.AMPER, token.NUMBER, |
|
f4976fa6e830
Issue #2134: Add support for tokenize.TokenInfo.exact_type.
Meador Inge <meadori@gmail.com>
parents:
72485
diff
changeset
|
1117 token.RPAR) |
|
37316
049606933328
Repaired a number of errors in this test:
Tim Peters <tim.peters@gmail.com>
parents:
35119
diff
changeset
|
1118 |
| 80192 | 1119 def test_pathological_trailing_whitespace(self): |
| 1120 # See http://bugs.python.org/issue16152 | |
| 1121 self.assertExactTypeEqual('@ ', token.AT) | |
|
37316
049606933328
Repaired a number of errors in this test:
Tim Peters <tim.peters@gmail.com>
parents:
35119
diff
changeset
|
1122 |
|
45773
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
1123 __test__ = {"doctests" : doctests, 'decistmt': decistmt} |
|
37316
049606933328
Repaired a number of errors in this test:
Tim Peters <tim.peters@gmail.com>
parents:
35119
diff
changeset
|
1124 |
|
45773
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
1125 def test_main(): |
|
832c820e9d14
Merged revisions 61239-61249,61252-61257,61260-61264,61269-61275,61278-61279,61285-61286,61288-61290,61298,61303-61305,61312-61314,61317,61329,61332,61344,61350-61351,61363-61376,61378-61379,61382-61383,61387-61388,61392,61395-61396,61402-61403 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
43833
diff
changeset
|
1126 from test import test_tokenize |
|
47201
9c930a32cebb
#2621 rename test.test_support to test.support
Benjamin Peterson <benjamin@python.org>
parents:
46978
diff
changeset
|
1127 support.run_doctest(test_tokenize, True) |
|
9c930a32cebb
#2621 rename test.test_support to test.support
Benjamin Peterson <benjamin@python.org>
parents:
46978
diff
changeset
|
1128 support.run_unittest(TestTokenizerAdheresToPep0263) |
|
9c930a32cebb
#2621 rename test.test_support to test.support
Benjamin Peterson <benjamin@python.org>
parents:
46978
diff
changeset
|
1129 support.run_unittest(Test_Tokenize) |
|
9c930a32cebb
#2621 rename test.test_support to test.support
Benjamin Peterson <benjamin@python.org>
parents:
46978
diff
changeset
|
1130 support.run_unittest(TestDetectEncoding) |
|
9c930a32cebb
#2621 rename test.test_support to test.support
Benjamin Peterson <benjamin@python.org>
parents:
46978
diff
changeset
|
1131 support.run_unittest(TestTokenize) |
|
40529
54cd47788811
PEP 3107 - Function Annotations thanks to Tony Lownds
Neal Norwitz <nnorwitz@gmail.com>
parents:
40450
diff
changeset
|
1132 |
|
37316
049606933328
Repaired a number of errors in this test:
Tim Peters <tim.peters@gmail.com>
parents:
35119
diff
changeset
|
1133 if __name__ == "__main__": |
|
049606933328
Repaired a number of errors in this test:
Tim Peters <tim.peters@gmail.com>
parents:
35119
diff
changeset
|
1134 test_main() |