Mercurial > cpython-lru_cache-weakref

annotate Tools/scripts/parse_html5_entities.py @ 87690:66c1c9f32567 default tip

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Add use_weakref option to functools.lru_cache The use_weakref option makes lru_cache create weak references to the function arguments when creating the cache key, which prevents them from staying in the memory forever even after all other references to them are gone. This is a proof of concept for http://bugs.python.org/issue19859
author Radomir Dopieralski <devel@sheep.art.pl>
date Tue, 03 Dec 2013 16:02:48 +0100
parents dd8b969d7459
children
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
79915
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
1 #!/usr/bin/env python3
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
2 """
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
3 Utility for parsing HTML5 entity definitions available from:
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
4
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
5 http://dev.w3.org/html5/spec/entities.json
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
6
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
7 Written by Ezio Melotti and Iuliia Proskurnia.
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
8
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
9 """
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
10
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
11 import os
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
12 import sys
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
13 import json
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
14 from urllib.request import urlopen
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
15 from html.entities import html5
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
16
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
17 entities_url = 'http://dev.w3.org/html5/spec/entities.json'
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
18
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
19 def get_json(url):
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
20 """Download the json file from the url and returns a decoded object."""
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
21 with urlopen(url) as f:
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
22 data = f.read().decode('utf-8')
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
23 return json.loads(data)
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
24
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
25 def create_dict(entities):
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
26 """Create the html5 dict from the decoded json object."""
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
27 new_html5 = {}
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
28 for name, value in entities.items():
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
29 new_html5[name.lstrip('&')] = value['characters']
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
30 return new_html5
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
31
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
32 def compare_dicts(old, new):
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
33 """Compare the old and new dicts and print the differences."""
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
34 added = new.keys() - old.keys()
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
35 if added:
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
36 print('{} entitie(s) have been added:'.format(len(added)))
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
37 for name in sorted(added):
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
38 print(' {!r}: {!r}'.format(name, new[name]))
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
39 removed = old.keys() - new.keys()
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
40 if removed:
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
41 print('{} entitie(s) have been removed:'.format(len(removed)))
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
42 for name in sorted(removed):
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
43 print(' {!r}: {!r}'.format(name, old[name]))
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
44 changed = set()
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
45 for name in (old.keys() & new.keys()):
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
46 if old[name] != new[name]:
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
47 changed.add((name, old[name], new[name]))
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
48 if changed:
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
49 print('{} entitie(s) have been modified:'.format(len(changed)))
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
50 for item in sorted(changed):
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
51 print(' {!r}: {!r} -> {!r}'.format(*item))
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
52
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
53 def write_items(entities, file=sys.stdout):
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
54 """Write the items of the dictionary in the specified file."""
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
55 # The keys in the generated dictionary should be sorted
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
56 # in a case-insensitive way, however, when two keys are equal,
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
57 # the uppercase version should come first so that the result
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
58 # looks like: ['Aacute', 'aacute', 'Aacute;', 'aacute;', ...]
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
59 # To do this we first sort in a case-sensitive way (so all the
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
60 # uppercase chars come first) and then sort with key=str.lower.
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
61 # Since the sorting is stable the uppercase keys will eventually
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
62 # be before their equivalent lowercase version.
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
63 keys = sorted(entities.keys())
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
64 keys = sorted(keys, key=str.lower)
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
65 print('html5 = {', file=file)
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
66 for name in keys:
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
67 print(' {!r}: {!a},'.format(name, entities[name]), file=file)
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
68 print('}', file=file)
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
69
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
70
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
71 if __name__ == '__main__':
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
72 # without args print a diff between html.entities.html5 and new_html5
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
73 # with --create print the new html5 dict
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
74 # with --patch patch the Lib/html/entities.py file
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
75 new_html5 = create_dict(get_json(entities_url))
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
76 if '--create' in sys.argv:
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
77 print('# map the HTML5 named character references to the '
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
78 'equivalent Unicode character(s)')
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
79 print('# Generated by {}. Do not edit manually.'.format(__file__))
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
80 write_items(new_html5)
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
81 elif '--patch' in sys.argv:
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
82 fname = 'Lib/html/entities.py'
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
83 temp_fname = fname + '.temp'
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
84 with open(fname) as f1, open(temp_fname, 'w') as f2:
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
85 skip = False
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
86 for line in f1:
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
87 if line.startswith('html5 = {'):
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
88 write_items(new_html5, file=f2)
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
89 skip = True
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
90 continue
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
91 if skip:
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
92 # skip the old items until the }
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
93 if line.startswith('}'):
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
94 skip = False
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
95 continue
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
96 f2.write(line)
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
97 os.remove(fname)
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
98 os.rename(temp_fname, fname)
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
99 else:
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
100 if html5 == new_html5:
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
101 print('The current dictionary is updated.')
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
102 else:
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
103 compare_dicts(html5, new_html5)
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
104 print('Run "./python {0} --patch" to update Lib/html/entities.html '
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
105 'or "./python {0} --create" to see the generated ' 'dictionary.'.format(__file__))