Mercurial > cpython-lru_cache-weakref

annotate Lib/json/scanner.py @ 78069:a8aa77458e12 3.2

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Add c_size_t and c_ssize_t to the ctypes summary table.
author Antoine Pitrou <solipsis@pitrou.net>
date Thu, 12 Jul 2012 20:31:50 +0200
parents b279611146d7
children 17b4bf9e9f43
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
53667
f686aced02a3 port simplejson upgrade from the trunk #4136
Benjamin Peterson <benjamin@python.org>
parents: 46737
diff changeset
1 """JSON token scanner
46737
9f45979558a9 Merged revisions 62734,62736,62748,62769 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
diff changeset
2 """
9f45979558a9 Merged revisions 62734,62736,62748,62769 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
diff changeset
3 import re
53667
f686aced02a3 port simplejson upgrade from the trunk #4136
Benjamin Peterson <benjamin@python.org>
parents: 46737
diff changeset
4 try:
f686aced02a3 port simplejson upgrade from the trunk #4136
Benjamin Peterson <benjamin@python.org>
parents: 46737
diff changeset
5 from _json import make_scanner as c_make_scanner
f686aced02a3 port simplejson upgrade from the trunk #4136
Benjamin Peterson <benjamin@python.org>
parents: 46737
diff changeset
6 except ImportError:
f686aced02a3 port simplejson upgrade from the trunk #4136
Benjamin Peterson <benjamin@python.org>
parents: 46737
diff changeset
7 c_make_scanner = None
46737
9f45979558a9 Merged revisions 62734,62736,62748,62769 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
diff changeset
8
53667
f686aced02a3 port simplejson upgrade from the trunk #4136
Benjamin Peterson <benjamin@python.org>
parents: 46737
diff changeset
9 __all__ = ['make_scanner']
46737
9f45979558a9 Merged revisions 62734,62736,62748,62769 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
diff changeset
10
53667
f686aced02a3 port simplejson upgrade from the trunk #4136
Benjamin Peterson <benjamin@python.org>
parents: 46737
diff changeset
11 NUMBER_RE = re.compile(
f686aced02a3 port simplejson upgrade from the trunk #4136
Benjamin Peterson <benjamin@python.org>
parents: 46737
diff changeset
12 r'(-?(?:0|[1-9]\d*))(\.\d+)?([eE][-+]?\d+)?',
f686aced02a3 port simplejson upgrade from the trunk #4136
Benjamin Peterson <benjamin@python.org>
parents: 46737
diff changeset
13 (re.VERBOSE | re.MULTILINE | re.DOTALL))
46737
9f45979558a9 Merged revisions 62734,62736,62748,62769 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
diff changeset
14
53667
f686aced02a3 port simplejson upgrade from the trunk #4136
Benjamin Peterson <benjamin@python.org>
parents: 46737
diff changeset
15 def py_make_scanner(context):
f686aced02a3 port simplejson upgrade from the trunk #4136
Benjamin Peterson <benjamin@python.org>
parents: 46737
diff changeset
16 parse_object = context.parse_object
f686aced02a3 port simplejson upgrade from the trunk #4136
Benjamin Peterson <benjamin@python.org>
parents: 46737
diff changeset
17 parse_array = context.parse_array
f686aced02a3 port simplejson upgrade from the trunk #4136
Benjamin Peterson <benjamin@python.org>
parents: 46737
diff changeset
18 parse_string = context.parse_string
f686aced02a3 port simplejson upgrade from the trunk #4136
Benjamin Peterson <benjamin@python.org>
parents: 46737
diff changeset
19 match_number = NUMBER_RE.match
f686aced02a3 port simplejson upgrade from the trunk #4136
Benjamin Peterson <benjamin@python.org>
parents: 46737
diff changeset
20 strict = context.strict
f686aced02a3 port simplejson upgrade from the trunk #4136
Benjamin Peterson <benjamin@python.org>
parents: 46737
diff changeset
21 parse_float = context.parse_float
f686aced02a3 port simplejson upgrade from the trunk #4136
Benjamin Peterson <benjamin@python.org>
parents: 46737
diff changeset
22 parse_int = context.parse_int
f686aced02a3 port simplejson upgrade from the trunk #4136
Benjamin Peterson <benjamin@python.org>
parents: 46737
diff changeset
23 parse_constant = context.parse_constant
f686aced02a3 port simplejson upgrade from the trunk #4136
Benjamin Peterson <benjamin@python.org>
parents: 46737
diff changeset
24 object_hook = context.object_hook
69317
7019fc1a9663 #9233: Fix json to work properly even when _json is not available.
Ezio Melotti
parents: 53667
diff changeset
25 object_pairs_hook = context.object_pairs_hook
64443
c3ad883b940b Issue #7451: Improve decoding performance of JSON objects, and reduce
Antoine Pitrou <solipsis@pitrou.net>
parents: 53667
diff changeset
26 memo = context.memo
53667
f686aced02a3 port simplejson upgrade from the trunk #4136
Benjamin Peterson <benjamin@python.org>
parents: 46737
diff changeset
27
f686aced02a3 port simplejson upgrade from the trunk #4136
Benjamin Peterson <benjamin@python.org>
parents: 46737
diff changeset
28 def _scan_once(string, idx):
f686aced02a3 port simplejson upgrade from the trunk #4136
Benjamin Peterson <benjamin@python.org>
parents: 46737
diff changeset
29 try:
f686aced02a3 port simplejson upgrade from the trunk #4136
Benjamin Peterson <benjamin@python.org>
parents: 46737
diff changeset
30 nextchar = string[idx]
f686aced02a3 port simplejson upgrade from the trunk #4136
Benjamin Peterson <benjamin@python.org>
parents: 46737
diff changeset
31 except IndexError:
f686aced02a3 port simplejson upgrade from the trunk #4136
Benjamin Peterson <benjamin@python.org>
parents: 46737
diff changeset
32 raise StopIteration
46737
9f45979558a9 Merged revisions 62734,62736,62748,62769 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
diff changeset
33
53667
f686aced02a3 port simplejson upgrade from the trunk #4136
Benjamin Peterson <benjamin@python.org>
parents: 46737
diff changeset
34 if nextchar == '"':
f686aced02a3 port simplejson upgrade from the trunk #4136
Benjamin Peterson <benjamin@python.org>
parents: 46737
diff changeset
35 return parse_string(string, idx + 1, strict)
f686aced02a3 port simplejson upgrade from the trunk #4136
Benjamin Peterson <benjamin@python.org>
parents: 46737
diff changeset
36 elif nextchar == '{':
f686aced02a3 port simplejson upgrade from the trunk #4136
Benjamin Peterson <benjamin@python.org>
parents: 46737
diff changeset
37 return parse_object((string, idx + 1), strict,
64443
c3ad883b940b Issue #7451: Improve decoding performance of JSON objects, and reduce
Antoine Pitrou <solipsis@pitrou.net>
parents: 53667
diff changeset
38 _scan_once, object_hook, object_pairs_hook, memo)
53667
f686aced02a3 port simplejson upgrade from the trunk #4136
Benjamin Peterson <benjamin@python.org>
parents: 46737
diff changeset
39 elif nextchar == '[':
f686aced02a3 port simplejson upgrade from the trunk #4136
Benjamin Peterson <benjamin@python.org>
parents: 46737
diff changeset
40 return parse_array((string, idx + 1), _scan_once)
f686aced02a3 port simplejson upgrade from the trunk #4136
Benjamin Peterson <benjamin@python.org>
parents: 46737
diff changeset
41 elif nextchar == 'n' and string[idx:idx + 4] == 'null':
f686aced02a3 port simplejson upgrade from the trunk #4136
Benjamin Peterson <benjamin@python.org>
parents: 46737
diff changeset
42 return None, idx + 4
f686aced02a3 port simplejson upgrade from the trunk #4136
Benjamin Peterson <benjamin@python.org>
parents: 46737
diff changeset
43 elif nextchar == 't' and string[idx:idx + 4] == 'true':
f686aced02a3 port simplejson upgrade from the trunk #4136
Benjamin Peterson <benjamin@python.org>
parents: 46737
diff changeset
44 return True, idx + 4
f686aced02a3 port simplejson upgrade from the trunk #4136
Benjamin Peterson <benjamin@python.org>
parents: 46737
diff changeset
45 elif nextchar == 'f' and string[idx:idx + 5] == 'false':
f686aced02a3 port simplejson upgrade from the trunk #4136
Benjamin Peterson <benjamin@python.org>
parents: 46737
diff changeset
46 return False, idx + 5
46737
9f45979558a9 Merged revisions 62734,62736,62748,62769 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
diff changeset
47
53667
f686aced02a3 port simplejson upgrade from the trunk #4136
Benjamin Peterson <benjamin@python.org>
parents: 46737
diff changeset
48 m = match_number(string, idx)
f686aced02a3 port simplejson upgrade from the trunk #4136
Benjamin Peterson <benjamin@python.org>
parents: 46737
diff changeset
49 if m is not None:
f686aced02a3 port simplejson upgrade from the trunk #4136
Benjamin Peterson <benjamin@python.org>
parents: 46737
diff changeset
50 integer, frac, exp = m.groups()
f686aced02a3 port simplejson upgrade from the trunk #4136
Benjamin Peterson <benjamin@python.org>
parents: 46737
diff changeset
51 if frac or exp:
f686aced02a3 port simplejson upgrade from the trunk #4136
Benjamin Peterson <benjamin@python.org>
parents: 46737
diff changeset
52 res = parse_float(integer + (frac or '') + (exp or ''))
f686aced02a3 port simplejson upgrade from the trunk #4136
Benjamin Peterson <benjamin@python.org>
parents: 46737
diff changeset
53 else:
f686aced02a3 port simplejson upgrade from the trunk #4136
Benjamin Peterson <benjamin@python.org>
parents: 46737
diff changeset
54 res = parse_int(integer)
f686aced02a3 port simplejson upgrade from the trunk #4136
Benjamin Peterson <benjamin@python.org>
parents: 46737
diff changeset
55 return res, m.end()
f686aced02a3 port simplejson upgrade from the trunk #4136
Benjamin Peterson <benjamin@python.org>
parents: 46737
diff changeset
56 elif nextchar == 'N' and string[idx:idx + 3] == 'NaN':
f686aced02a3 port simplejson upgrade from the trunk #4136
Benjamin Peterson <benjamin@python.org>
parents: 46737
diff changeset
57 return parse_constant('NaN'), idx + 3
f686aced02a3 port simplejson upgrade from the trunk #4136
Benjamin Peterson <benjamin@python.org>
parents: 46737
diff changeset
58 elif nextchar == 'I' and string[idx:idx + 8] == 'Infinity':
f686aced02a3 port simplejson upgrade from the trunk #4136
Benjamin Peterson <benjamin@python.org>
parents: 46737
diff changeset
59 return parse_constant('Infinity'), idx + 8
f686aced02a3 port simplejson upgrade from the trunk #4136
Benjamin Peterson <benjamin@python.org>
parents: 46737
diff changeset
60 elif nextchar == '-' and string[idx:idx + 9] == '-Infinity':
f686aced02a3 port simplejson upgrade from the trunk #4136
Benjamin Peterson <benjamin@python.org>
parents: 46737
diff changeset
61 return parse_constant('-Infinity'), idx + 9
f686aced02a3 port simplejson upgrade from the trunk #4136
Benjamin Peterson <benjamin@python.org>
parents: 46737
diff changeset
62 else:
f686aced02a3 port simplejson upgrade from the trunk #4136
Benjamin Peterson <benjamin@python.org>
parents: 46737
diff changeset
63 raise StopIteration
46737
9f45979558a9 Merged revisions 62734,62736,62748,62769 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
diff changeset
64
64443
c3ad883b940b Issue #7451: Improve decoding performance of JSON objects, and reduce
Antoine Pitrou <solipsis@pitrou.net>
parents: 53667
diff changeset
65 def scan_once(string, idx):
c3ad883b940b Issue #7451: Improve decoding performance of JSON objects, and reduce
Antoine Pitrou <solipsis@pitrou.net>
parents: 53667
diff changeset
66 try:
c3ad883b940b Issue #7451: Improve decoding performance of JSON objects, and reduce
Antoine Pitrou <solipsis@pitrou.net>
parents: 53667
diff changeset
67 return _scan_once(string, idx)
c3ad883b940b Issue #7451: Improve decoding performance of JSON objects, and reduce
Antoine Pitrou <solipsis@pitrou.net>
parents: 53667
diff changeset
68 finally:
c3ad883b940b Issue #7451: Improve decoding performance of JSON objects, and reduce
Antoine Pitrou <solipsis@pitrou.net>
parents: 53667
diff changeset
69 memo.clear()
c3ad883b940b Issue #7451: Improve decoding performance of JSON objects, and reduce
Antoine Pitrou <solipsis@pitrou.net>
parents: 53667
diff changeset
70
53667
f686aced02a3 port simplejson upgrade from the trunk #4136
Benjamin Peterson <benjamin@python.org>
parents: 46737
diff changeset
71 return _scan_once
46737
9f45979558a9 Merged revisions 62734,62736,62748,62769 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents:
diff changeset
72
53667
f686aced02a3 port simplejson upgrade from the trunk #4136
Benjamin Peterson <benjamin@python.org>
parents: 46737
diff changeset
73 make_scanner = c_make_scanner or py_make_scanner