Mercurial > cpython-lru_cache-weakref

annotate Tools/scripts/parse_html5_entities.py @ 81343:3a7cd8efc19f

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
#16852: merge with 3.3.
author Ezio Melotti <ezio.melotti@gmail.com>
date Thu, 10 Jan 2013 03:29:45 +0200
parents dd8b969d7459
children
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
79915
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
1 #!/usr/bin/env python3
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
2 """
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
3 Utility for parsing HTML5 entity definitions available from:
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
4
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
5 http://dev.w3.org/html5/spec/entities.json
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
6
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
7 Written by Ezio Melotti and Iuliia Proskurnia.
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
8
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
9 """
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
10
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
11 import os
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
12 import sys
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
13 import json
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
14 from urllib.request import urlopen
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
15 from html.entities import html5
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
16
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
17 entities_url = 'http://dev.w3.org/html5/spec/entities.json'
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
18
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
19 def get_json(url):
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
20 """Download the json file from the url and returns a decoded object."""
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
21 with urlopen(url) as f:
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
22 data = f.read().decode('utf-8')
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
23 return json.loads(data)
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
24
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
25 def create_dict(entities):
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
26 """Create the html5 dict from the decoded json object."""
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
27 new_html5 = {}
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
28 for name, value in entities.items():
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
29 new_html5[name.lstrip('&')] = value['characters']
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
30 return new_html5
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
31
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
32 def compare_dicts(old, new):
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
33 """Compare the old and new dicts and print the differences."""
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
34 added = new.keys() - old.keys()
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
35 if added:
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
36 print('{} entitie(s) have been added:'.format(len(added)))
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
37 for name in sorted(added):
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
38 print(' {!r}: {!r}'.format(name, new[name]))
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
39 removed = old.keys() - new.keys()
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
40 if removed:
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
41 print('{} entitie(s) have been removed:'.format(len(removed)))
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
42 for name in sorted(removed):
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
43 print(' {!r}: {!r}'.format(name, old[name]))
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
44 changed = set()
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
45 for name in (old.keys() & new.keys()):
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
46 if old[name] != new[name]:
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
47 changed.add((name, old[name], new[name]))
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
48 if changed:
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
49 print('{} entitie(s) have been modified:'.format(len(changed)))
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
50 for item in sorted(changed):
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
51 print(' {!r}: {!r} -> {!r}'.format(*item))
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
52
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
53 def write_items(entities, file=sys.stdout):
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
54 """Write the items of the dictionary in the specified file."""
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
55 # The keys in the generated dictionary should be sorted
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
56 # in a case-insensitive way, however, when two keys are equal,
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
57 # the uppercase version should come first so that the result
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
58 # looks like: ['Aacute', 'aacute', 'Aacute;', 'aacute;', ...]
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
59 # To do this we first sort in a case-sensitive way (so all the
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
60 # uppercase chars come first) and then sort with key=str.lower.
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
61 # Since the sorting is stable the uppercase keys will eventually
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
62 # be before their equivalent lowercase version.
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
63 keys = sorted(entities.keys())
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
64 keys = sorted(keys, key=str.lower)
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
65 print('html5 = {', file=file)
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
66 for name in keys:
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
67 print(' {!r}: {!a},'.format(name, entities[name]), file=file)
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
68 print('}', file=file)
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
69
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
70
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
71 if __name__ == '__main__':
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
72 # without args print a diff between html.entities.html5 and new_html5
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
73 # with --create print the new html5 dict
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
74 # with --patch patch the Lib/html/entities.py file
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
75 new_html5 = create_dict(get_json(entities_url))
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
76 if '--create' in sys.argv:
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
77 print('# map the HTML5 named character references to the '
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
78 'equivalent Unicode character(s)')
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
79 print('# Generated by {}. Do not edit manually.'.format(__file__))
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
80 write_items(new_html5)
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
81 elif '--patch' in sys.argv:
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
82 fname = 'Lib/html/entities.py'
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
83 temp_fname = fname + '.temp'
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
84 with open(fname) as f1, open(temp_fname, 'w') as f2:
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
85 skip = False
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
86 for line in f1:
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
87 if line.startswith('html5 = {'):
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
88 write_items(new_html5, file=f2)
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
89 skip = True
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
90 continue
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
91 if skip:
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
92 # skip the old items until the }
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
93 if line.startswith('}'):
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
94 skip = False
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
95 continue
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
96 f2.write(line)
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
97 os.remove(fname)
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
98 os.rename(temp_fname, fname)
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
99 else:
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
100 if html5 == new_html5:
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
101 print('The current dictionary is updated.')
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
102 else:
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
103 compare_dicts(html5, new_html5)
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
104 print('Run "./python {0} --patch" to update Lib/html/entities.html '
dd8b969d7459 #16245: add a script to generate the html.entities.html5 dict.
Ezio Melotti <ezio.melotti@gmail.com>
parents:
diff changeset
105 'or "./python {0} --create" to see the generated ' 'dictionary.'.format(__file__))