Mercurial > cpython-lru_cache-weakref

annotate Lib/json/scanner.py @ 58925:eecf1bef052d 2.6

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Merged revisions 78207 via svnmerge from svn+ssh://pythondev@svn.python.org/python/trunk ........ r78207 | ezio.melotti | 2010-02-17 01:26:09 +0200 (Wed, 17 Feb 2010) | 1 line #7930: fix stripid ........
author Ezio Melotti <ezio.melotti@gmail.com>
date Tue, 16 Feb 2010 23:29:44 +0000
parents 4ded35754cb7
children d95e5add3ca4 f686aced02a3
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
46676
4ded35754cb7 Add the 'json' package. Code taken from simplejson 1.9 and contributed by Bob
Brett Cannon <bcannon@gmail.com>
parents:
diff changeset
1 """Iterator based sre token scanner
4ded35754cb7 Add the 'json' package. Code taken from simplejson 1.9 and contributed by Bob
Brett Cannon <bcannon@gmail.com>
parents:
diff changeset
2
4ded35754cb7 Add the 'json' package. Code taken from simplejson 1.9 and contributed by Bob
Brett Cannon <bcannon@gmail.com>
parents:
diff changeset
3 """
4ded35754cb7 Add the 'json' package. Code taken from simplejson 1.9 and contributed by Bob
Brett Cannon <bcannon@gmail.com>
parents:
diff changeset
4
4ded35754cb7 Add the 'json' package. Code taken from simplejson 1.9 and contributed by Bob
Brett Cannon <bcannon@gmail.com>
parents:
diff changeset
5 import re
4ded35754cb7 Add the 'json' package. Code taken from simplejson 1.9 and contributed by Bob
Brett Cannon <bcannon@gmail.com>
parents:
diff changeset
6 import sre_parse
4ded35754cb7 Add the 'json' package. Code taken from simplejson 1.9 and contributed by Bob
Brett Cannon <bcannon@gmail.com>
parents:
diff changeset
7 import sre_compile
4ded35754cb7 Add the 'json' package. Code taken from simplejson 1.9 and contributed by Bob
Brett Cannon <bcannon@gmail.com>
parents:
diff changeset
8 import sre_constants
4ded35754cb7 Add the 'json' package. Code taken from simplejson 1.9 and contributed by Bob
Brett Cannon <bcannon@gmail.com>
parents:
diff changeset
9
4ded35754cb7 Add the 'json' package. Code taken from simplejson 1.9 and contributed by Bob
Brett Cannon <bcannon@gmail.com>
parents:
diff changeset
10 from re import VERBOSE, MULTILINE, DOTALL
4ded35754cb7 Add the 'json' package. Code taken from simplejson 1.9 and contributed by Bob
Brett Cannon <bcannon@gmail.com>
parents:
diff changeset
11 from sre_constants import BRANCH, SUBPATTERN
4ded35754cb7 Add the 'json' package. Code taken from simplejson 1.9 and contributed by Bob
Brett Cannon <bcannon@gmail.com>
parents:
diff changeset
12
4ded35754cb7 Add the 'json' package. Code taken from simplejson 1.9 and contributed by Bob
Brett Cannon <bcannon@gmail.com>
parents:
diff changeset
13 __all__ = ['Scanner', 'pattern']
4ded35754cb7 Add the 'json' package. Code taken from simplejson 1.9 and contributed by Bob
Brett Cannon <bcannon@gmail.com>
parents:
diff changeset
14
4ded35754cb7 Add the 'json' package. Code taken from simplejson 1.9 and contributed by Bob
Brett Cannon <bcannon@gmail.com>
parents:
diff changeset
15 FLAGS = (VERBOSE | MULTILINE | DOTALL)
4ded35754cb7 Add the 'json' package. Code taken from simplejson 1.9 and contributed by Bob
Brett Cannon <bcannon@gmail.com>
parents:
diff changeset
16
4ded35754cb7 Add the 'json' package. Code taken from simplejson 1.9 and contributed by Bob
Brett Cannon <bcannon@gmail.com>
parents:
diff changeset
17 class Scanner(object):
4ded35754cb7 Add the 'json' package. Code taken from simplejson 1.9 and contributed by Bob
Brett Cannon <bcannon@gmail.com>
parents:
diff changeset
18 def __init__(self, lexicon, flags=FLAGS):
4ded35754cb7 Add the 'json' package. Code taken from simplejson 1.9 and contributed by Bob
Brett Cannon <bcannon@gmail.com>
parents:
diff changeset
19 self.actions = [None]
4ded35754cb7 Add the 'json' package. Code taken from simplejson 1.9 and contributed by Bob
Brett Cannon <bcannon@gmail.com>
parents:
diff changeset
20 # Combine phrases into a compound pattern
4ded35754cb7 Add the 'json' package. Code taken from simplejson 1.9 and contributed by Bob
Brett Cannon <bcannon@gmail.com>
parents:
diff changeset
21 s = sre_parse.Pattern()
4ded35754cb7 Add the 'json' package. Code taken from simplejson 1.9 and contributed by Bob
Brett Cannon <bcannon@gmail.com>
parents:
diff changeset
22 s.flags = flags
4ded35754cb7 Add the 'json' package. Code taken from simplejson 1.9 and contributed by Bob
Brett Cannon <bcannon@gmail.com>
parents:
diff changeset
23 p = []
4ded35754cb7 Add the 'json' package. Code taken from simplejson 1.9 and contributed by Bob
Brett Cannon <bcannon@gmail.com>
parents:
diff changeset
24 for idx, token in enumerate(lexicon):
4ded35754cb7 Add the 'json' package. Code taken from simplejson 1.9 and contributed by Bob
Brett Cannon <bcannon@gmail.com>
parents:
diff changeset
25 phrase = token.pattern
4ded35754cb7 Add the 'json' package. Code taken from simplejson 1.9 and contributed by Bob
Brett Cannon <bcannon@gmail.com>
parents:
diff changeset
26 try:
4ded35754cb7 Add the 'json' package. Code taken from simplejson 1.9 and contributed by Bob
Brett Cannon <bcannon@gmail.com>
parents:
diff changeset
27 subpattern = sre_parse.SubPattern(s,
4ded35754cb7 Add the 'json' package. Code taken from simplejson 1.9 and contributed by Bob
Brett Cannon <bcannon@gmail.com>
parents:
diff changeset
28 [(SUBPATTERN, (idx + 1, sre_parse.parse(phrase, flags)))])
4ded35754cb7 Add the 'json' package. Code taken from simplejson 1.9 and contributed by Bob
Brett Cannon <bcannon@gmail.com>
parents:
diff changeset
29 except sre_constants.error:
4ded35754cb7 Add the 'json' package. Code taken from simplejson 1.9 and contributed by Bob
Brett Cannon <bcannon@gmail.com>
parents:
diff changeset
30 raise
4ded35754cb7 Add the 'json' package. Code taken from simplejson 1.9 and contributed by Bob
Brett Cannon <bcannon@gmail.com>
parents:
diff changeset
31 p.append(subpattern)
4ded35754cb7 Add the 'json' package. Code taken from simplejson 1.9 and contributed by Bob
Brett Cannon <bcannon@gmail.com>
parents:
diff changeset
32 self.actions.append(token)
4ded35754cb7 Add the 'json' package. Code taken from simplejson 1.9 and contributed by Bob
Brett Cannon <bcannon@gmail.com>
parents:
diff changeset
33
4ded35754cb7 Add the 'json' package. Code taken from simplejson 1.9 and contributed by Bob
Brett Cannon <bcannon@gmail.com>
parents:
diff changeset
34 s.groups = len(p) + 1 # NOTE(guido): Added to make SRE validation work
4ded35754cb7 Add the 'json' package. Code taken from simplejson 1.9 and contributed by Bob
Brett Cannon <bcannon@gmail.com>
parents:
diff changeset
35 p = sre_parse.SubPattern(s, [(BRANCH, (None, p))])
4ded35754cb7 Add the 'json' package. Code taken from simplejson 1.9 and contributed by Bob
Brett Cannon <bcannon@gmail.com>
parents:
diff changeset
36 self.scanner = sre_compile.compile(p)
4ded35754cb7 Add the 'json' package. Code taken from simplejson 1.9 and contributed by Bob
Brett Cannon <bcannon@gmail.com>
parents:
diff changeset
37
4ded35754cb7 Add the 'json' package. Code taken from simplejson 1.9 and contributed by Bob
Brett Cannon <bcannon@gmail.com>
parents:
diff changeset
38 def iterscan(self, string, idx=0, context=None):
4ded35754cb7 Add the 'json' package. Code taken from simplejson 1.9 and contributed by Bob
Brett Cannon <bcannon@gmail.com>
parents:
diff changeset
39 """Yield match, end_idx for each match
4ded35754cb7 Add the 'json' package. Code taken from simplejson 1.9 and contributed by Bob
Brett Cannon <bcannon@gmail.com>
parents:
diff changeset
40
4ded35754cb7 Add the 'json' package. Code taken from simplejson 1.9 and contributed by Bob
Brett Cannon <bcannon@gmail.com>
parents:
diff changeset
41 """
4ded35754cb7 Add the 'json' package. Code taken from simplejson 1.9 and contributed by Bob
Brett Cannon <bcannon@gmail.com>
parents:
diff changeset
42 match = self.scanner.scanner(string, idx).match
4ded35754cb7 Add the 'json' package. Code taken from simplejson 1.9 and contributed by Bob
Brett Cannon <bcannon@gmail.com>
parents:
diff changeset
43 actions = self.actions
4ded35754cb7 Add the 'json' package. Code taken from simplejson 1.9 and contributed by Bob
Brett Cannon <bcannon@gmail.com>
parents:
diff changeset
44 lastend = idx
4ded35754cb7 Add the 'json' package. Code taken from simplejson 1.9 and contributed by Bob
Brett Cannon <bcannon@gmail.com>
parents:
diff changeset
45 end = len(string)
4ded35754cb7 Add the 'json' package. Code taken from simplejson 1.9 and contributed by Bob
Brett Cannon <bcannon@gmail.com>
parents:
diff changeset
46 while True:
4ded35754cb7 Add the 'json' package. Code taken from simplejson 1.9 and contributed by Bob
Brett Cannon <bcannon@gmail.com>
parents:
diff changeset
47 m = match()
4ded35754cb7 Add the 'json' package. Code taken from simplejson 1.9 and contributed by Bob
Brett Cannon <bcannon@gmail.com>
parents:
diff changeset
48 if m is None:
4ded35754cb7 Add the 'json' package. Code taken from simplejson 1.9 and contributed by Bob
Brett Cannon <bcannon@gmail.com>
parents:
diff changeset
49 break
4ded35754cb7 Add the 'json' package. Code taken from simplejson 1.9 and contributed by Bob
Brett Cannon <bcannon@gmail.com>
parents:
diff changeset
50 matchbegin, matchend = m.span()
4ded35754cb7 Add the 'json' package. Code taken from simplejson 1.9 and contributed by Bob
Brett Cannon <bcannon@gmail.com>
parents:
diff changeset
51 if lastend == matchend:
4ded35754cb7 Add the 'json' package. Code taken from simplejson 1.9 and contributed by Bob
Brett Cannon <bcannon@gmail.com>
parents:
diff changeset
52 break
4ded35754cb7 Add the 'json' package. Code taken from simplejson 1.9 and contributed by Bob
Brett Cannon <bcannon@gmail.com>
parents:
diff changeset
53 action = actions[m.lastindex]
4ded35754cb7 Add the 'json' package. Code taken from simplejson 1.9 and contributed by Bob
Brett Cannon <bcannon@gmail.com>
parents:
diff changeset
54 if action is not None:
4ded35754cb7 Add the 'json' package. Code taken from simplejson 1.9 and contributed by Bob
Brett Cannon <bcannon@gmail.com>
parents:
diff changeset
55 rval, next_pos = action(m, context)
4ded35754cb7 Add the 'json' package. Code taken from simplejson 1.9 and contributed by Bob
Brett Cannon <bcannon@gmail.com>
parents:
diff changeset
56 if next_pos is not None and next_pos != matchend:
4ded35754cb7 Add the 'json' package. Code taken from simplejson 1.9 and contributed by Bob
Brett Cannon <bcannon@gmail.com>
parents:
diff changeset
57 # "fast forward" the scanner
4ded35754cb7 Add the 'json' package. Code taken from simplejson 1.9 and contributed by Bob
Brett Cannon <bcannon@gmail.com>
parents:
diff changeset
58 matchend = next_pos
4ded35754cb7 Add the 'json' package. Code taken from simplejson 1.9 and contributed by Bob
Brett Cannon <bcannon@gmail.com>
parents:
diff changeset
59 match = self.scanner.scanner(string, matchend).match
4ded35754cb7 Add the 'json' package. Code taken from simplejson 1.9 and contributed by Bob
Brett Cannon <bcannon@gmail.com>
parents:
diff changeset
60 yield rval, matchend
4ded35754cb7 Add the 'json' package. Code taken from simplejson 1.9 and contributed by Bob
Brett Cannon <bcannon@gmail.com>
parents:
diff changeset
61 lastend = matchend
4ded35754cb7 Add the 'json' package. Code taken from simplejson 1.9 and contributed by Bob
Brett Cannon <bcannon@gmail.com>
parents:
diff changeset
62
4ded35754cb7 Add the 'json' package. Code taken from simplejson 1.9 and contributed by Bob
Brett Cannon <bcannon@gmail.com>
parents:
diff changeset
63
4ded35754cb7 Add the 'json' package. Code taken from simplejson 1.9 and contributed by Bob
Brett Cannon <bcannon@gmail.com>
parents:
diff changeset
64 def pattern(pattern, flags=FLAGS):
4ded35754cb7 Add the 'json' package. Code taken from simplejson 1.9 and contributed by Bob
Brett Cannon <bcannon@gmail.com>
parents:
diff changeset
65 def decorator(fn):
4ded35754cb7 Add the 'json' package. Code taken from simplejson 1.9 and contributed by Bob
Brett Cannon <bcannon@gmail.com>
parents:
diff changeset
66 fn.pattern = pattern
4ded35754cb7 Add the 'json' package. Code taken from simplejson 1.9 and contributed by Bob
Brett Cannon <bcannon@gmail.com>
parents:
diff changeset
67 fn.regex = re.compile(pattern, flags)
4ded35754cb7 Add the 'json' package. Code taken from simplejson 1.9 and contributed by Bob
Brett Cannon <bcannon@gmail.com>
parents:
diff changeset
68 return fn
4ded35754cb7 Add the 'json' package. Code taken from simplejson 1.9 and contributed by Bob
Brett Cannon <bcannon@gmail.com>
parents:
diff changeset
69 return decorator