Mercurial > cpython-lru_cache-weakref

annotate Lib/HTMLParser.py @ 80661:94de77bd0b4b 2.7

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Issue #16194: document imp.load_dynamic problems when read several extension modules from the same binary object file.
author Andrew Svetlov <andrew.svetlov@gmail.com>
date Thu, 29 Nov 2012 19:51:28 +0200
parents 36c901fcfcda
children 695f988824bb
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
47177
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
1 """A parser for HTML and XHTML."""
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
2
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
3 # This file is based on sgmllib.py, but the API is slightly different.
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
4
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
5 # XXX There should be a way to distinguish between PCDATA (parsed
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
6 # character data -- the normal case), RCDATA (replaceable character
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
7 # data -- only char and entity references and end tags are special)
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
8 # and CDATA (character data -- only end tags are special).
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
9
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
10
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
11 import markupbase
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
12 import re
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
13
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
14 # Regular expressions used for parsing
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
15
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
16 interesting_normal = re.compile('[&<]')
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
17 incomplete = re.compile('&[a-zA-Z#]')
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
18
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
19 entityref = re.compile('&([a-zA-Z][-.a-zA-Z0-9]*)[^a-zA-Z0-9]')
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
20 charref = re.compile('&#(?:[0-9]+|[xX][0-9a-fA-F]+)[^0-9a-fA-F]')
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
21
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
22 starttagopen = re.compile('<[a-zA-Z]')
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
23 piclose = re.compile('>')
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
24 commentclose = re.compile(r'--\s*>')
76413
36c901fcfcda #14538: HTMLParser can now parse correctly start tags that contain a bare /.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 75116
diff changeset
25 tagfind = re.compile('([a-zA-Z][-.a-zA-Z0-9:_]*)(?:\s|/(?!>))*')
74909
a349448474ea #13993: HTMLParser is now able to handle broken end tags.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 74908
diff changeset
26 # see http://www.w3.org/TR/html5/tokenization.html#tag-open-state
a349448474ea #13993: HTMLParser is now able to handle broken end tags.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 74908
diff changeset
27 # and http://www.w3.org/TR/html5/tokenization.html#tag-name-state
a349448474ea #13993: HTMLParser is now able to handle broken end tags.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 74908
diff changeset
28 tagfind_tolerant = re.compile('[a-zA-Z][^\t\n\r\f />\x00]*')
73545
3c3009f63700 #1745761, #755670, #13357, #12629, #1200313: improve attribute handling in HTMLParser.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 73273
diff changeset
29
47177
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
30 attrfind = re.compile(
76413
36c901fcfcda #14538: HTMLParser can now parse correctly start tags that contain a bare /.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 75116
diff changeset
31 r'((?<=[\'"\s/])[^\s/>][^\s/=>]*)(\s*=+\s*'
75116
37c824c3efe8 HTMLParser is now able to handle slashes in the start tag.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 74946
diff changeset
32 r'(\'[^\']*\'|"[^"]*"|(?![\'"])[^>\s]*))?(?:\s|/(?!>))*')
47177
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
33
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
34 locatestarttagend = re.compile(r"""
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
35 <[a-zA-Z][-.a-zA-Z0-9:_]* # tag name
75116
37c824c3efe8 HTMLParser is now able to handle slashes in the start tag.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 74946
diff changeset
36 (?:[\s/]* # optional whitespace before attribute name
37c824c3efe8 HTMLParser is now able to handle slashes in the start tag.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 74946
diff changeset
37 (?:(?<=['"\s/])[^\s/>][^\s/=>]* # attribute name
73545
3c3009f63700 #1745761, #755670, #13357, #12629, #1200313: improve attribute handling in HTMLParser.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 73273
diff changeset
38 (?:\s*=+\s* # value indicator
47177
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
39 (?:'[^']*' # LITA-enclosed value
73545
3c3009f63700 #1745761, #755670, #13357, #12629, #1200313: improve attribute handling in HTMLParser.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 73273
diff changeset
40 |"[^"]*" # LIT-enclosed value
3c3009f63700 #1745761, #755670, #13357, #12629, #1200313: improve attribute handling in HTMLParser.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 73273
diff changeset
41 |(?!['"])[^>\s]* # bare value
47177
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
42 )
75116
37c824c3efe8 HTMLParser is now able to handle slashes in the start tag.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 74946
diff changeset
43 )?(?:\s|/(?!>))*
73545
3c3009f63700 #1745761, #755670, #13357, #12629, #1200313: improve attribute handling in HTMLParser.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 73273
diff changeset
44 )*
3c3009f63700 #1745761, #755670, #13357, #12629, #1200313: improve attribute handling in HTMLParser.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 73273
diff changeset
45 )?
47177
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
46 \s* # trailing whitespace
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
47 """, re.VERBOSE)
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
48 endendtag = re.compile('>')
73273
0a5eb57d5876 #670664: Fix HTMLParser to correctly handle the content of ``<script>...</script>`` and ``<style>...</style>``.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 70423
diff changeset
49 # the HTML 5 spec, section 8.1.2.2, doesn't allow spaces between
0a5eb57d5876 #670664: Fix HTMLParser to correctly handle the content of ``<script>...</script>`` and ``<style>...</style>``.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 70423
diff changeset
50 # </ and the tag name, so maybe this should be fixed
47177
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
51 endtagfind = re.compile('</\s*([a-zA-Z][-.a-zA-Z0-9:_]*)\s*>')
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
52
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
53
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
54 class HTMLParseError(Exception):
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
55 """Exception raised for all parse errors."""
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
56
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
57 def __init__(self, msg, position=(None, None)):
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
58 assert msg
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
59 self.msg = msg
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
60 self.lineno = position[0]
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
61 self.offset = position[1]
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
62
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
63 def __str__(self):
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
64 result = self.msg
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
65 if self.lineno is not None:
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
66 result = result + ", at line %d" % self.lineno
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
67 if self.offset is not None:
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
68 result = result + ", column %d" % (self.offset + 1)
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
69 return result
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
70
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
71
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
72 class HTMLParser(markupbase.ParserBase):
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
73 """Find tags and other markup and call handler functions.
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
74
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
75 Usage:
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
76 p = HTMLParser()
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
77 p.feed(data)
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
78 ...
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
79 p.close()
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
80
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
81 Start tags are handled by calling self.handle_starttag() or
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
82 self.handle_startendtag(); end tags by self.handle_endtag(). The
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
83 data between tags is passed from the parser to the derived class
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
84 by calling self.handle_data() with the data as argument (the data
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
85 may be split up in arbitrary chunks). Entity references are
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
86 passed by calling self.handle_entityref() with the entity
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
87 reference as the argument. Numeric character references are
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
88 passed to self.handle_charref() with the string containing the
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
89 reference as the argument.
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
90 """
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
91
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
92 CDATA_CONTENT_ELEMENTS = ("script", "style")
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
93
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
94
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
95 def __init__(self):
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
96 """Initialize and reset this instance."""
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
97 self.reset()
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
98
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
99 def reset(self):
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
100 """Reset this instance. Loses all unprocessed data."""
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
101 self.rawdata = ''
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
102 self.lasttag = '???'
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
103 self.interesting = interesting_normal
73273
0a5eb57d5876 #670664: Fix HTMLParser to correctly handle the content of ``<script>...</script>`` and ``<style>...</style>``.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 70423
diff changeset
104 self.cdata_elem = None
47177
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
105 markupbase.ParserBase.reset(self)
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
106
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
107 def feed(self, data):
70423
02a4535c0a6c Fix display of html.parser.HTMLParser.feed docstrin
Éric Araujo <merwok@netwok.org>
parents: 69160
diff changeset
108 r"""Feed data to the parser.
47177
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
109
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
110 Call this as often as you want, with as little or as much text
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
111 as you want (may include '\n').
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
112 """
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
113 self.rawdata = self.rawdata + data
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
114 self.goahead(0)
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
115
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
116 def close(self):
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
117 """Handle any buffered data."""
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
118 self.goahead(1)
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
119
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
120 def error(self, message):
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
121 raise HTMLParseError(message, self.getpos())
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
122
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
123 __starttag_text = None
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
124
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
125 def get_starttag_text(self):
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
126 """Return full source of start tag: '<...>'."""
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
127 return self.__starttag_text
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
128
73273
0a5eb57d5876 #670664: Fix HTMLParser to correctly handle the content of ``<script>...</script>`` and ``<style>...</style>``.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 70423
diff changeset
129 def set_cdata_mode(self, elem):
0a5eb57d5876 #670664: Fix HTMLParser to correctly handle the content of ``<script>...</script>`` and ``<style>...</style>``.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 70423
diff changeset
130 self.cdata_elem = elem.lower()
73608
91163aa3d5b4 #13358: HTMLParser now calls handle_data only once for each CDATA.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 73545
diff changeset
131 self.interesting = re.compile(r'</\s*%s\s*>' % self.cdata_elem, re.I)
47177
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
132
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
133 def clear_cdata_mode(self):
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
134 self.interesting = interesting_normal
73273
0a5eb57d5876 #670664: Fix HTMLParser to correctly handle the content of ``<script>...</script>`` and ``<style>...</style>``.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 70423
diff changeset
135 self.cdata_elem = None
47177
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
136
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
137 # Internal -- handle data as far as reasonable. May leave state
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
138 # and data to be processed by a subsequent call. If 'end' is
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
139 # true, force handling all data as if followed by EOF marker.
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
140 def goahead(self, end):
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
141 rawdata = self.rawdata
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
142 i = 0
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
143 n = len(rawdata)
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
144 while i < n:
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
145 match = self.interesting.search(rawdata, i) # < or &
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
146 if match:
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
147 j = match.start()
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
148 else:
73608
91163aa3d5b4 #13358: HTMLParser now calls handle_data only once for each CDATA.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 73545
diff changeset
149 if self.cdata_elem:
91163aa3d5b4 #13358: HTMLParser now calls handle_data only once for each CDATA.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 73545
diff changeset
150 break
47177
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
151 j = n
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
152 if i < j: self.handle_data(rawdata[i:j])
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
153 i = self.updatepos(i, j)
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
154 if i == n: break
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
155 startswith = rawdata.startswith
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
156 if startswith('<', i):
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
157 if starttagopen.match(rawdata, i): # < + letter
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
158 k = self.parse_starttag(i)
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
159 elif startswith("</", i):
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
160 k = self.parse_endtag(i)
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
161 elif startswith("<!--", i):
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
162 k = self.parse_comment(i)
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
163 elif startswith("<?", i):
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
164 k = self.parse_pi(i)
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
165 elif startswith("<!", i):
74908
333e3acf2008 #13960: HTMLParser is now able to handle broken comments.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 73608
diff changeset
166 k = self.parse_html_declaration(i)
47177
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
167 elif (i + 1) < n:
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
168 self.handle_data("<")
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
169 k = i + 1
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
170 else:
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
171 break
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
172 if k < 0:
74945
11a31eb5da93 #13987: HTMLParser is now able to handle EOFs in the middle of a construct.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 74915
diff changeset
173 if not end:
11a31eb5da93 #13987: HTMLParser is now able to handle EOFs in the middle of a construct.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 74915
diff changeset
174 break
11a31eb5da93 #13987: HTMLParser is now able to handle EOFs in the middle of a construct.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 74915
diff changeset
175 k = rawdata.find('>', i + 1)
11a31eb5da93 #13987: HTMLParser is now able to handle EOFs in the middle of a construct.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 74915
diff changeset
176 if k < 0:
11a31eb5da93 #13987: HTMLParser is now able to handle EOFs in the middle of a construct.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 74915
diff changeset
177 k = rawdata.find('<', i + 1)
11a31eb5da93 #13987: HTMLParser is now able to handle EOFs in the middle of a construct.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 74915
diff changeset
178 if k < 0:
11a31eb5da93 #13987: HTMLParser is now able to handle EOFs in the middle of a construct.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 74915
diff changeset
179 k = i + 1
11a31eb5da93 #13987: HTMLParser is now able to handle EOFs in the middle of a construct.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 74915
diff changeset
180 else:
11a31eb5da93 #13987: HTMLParser is now able to handle EOFs in the middle of a construct.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 74915
diff changeset
181 k += 1
11a31eb5da93 #13987: HTMLParser is now able to handle EOFs in the middle of a construct.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 74915
diff changeset
182 self.handle_data(rawdata[i:k])
47177
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
183 i = self.updatepos(i, k)
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
184 elif startswith("&#", i):
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
185 match = charref.match(rawdata, i)
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
186 if match:
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
187 name = match.group()[2:-1]
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
188 self.handle_charref(name)
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
189 k = match.end()
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
190 if not startswith(';', k-1):
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
191 k = k - 1
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
192 i = self.updatepos(i, k)
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
193 continue
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
194 else:
61885
ebda6803f8d5 Issue #6662: Fix parsing of malformatted charref (&#bad;)
Victor Stinner <victor.stinner@haypocalc.com>
parents: 47177
diff changeset
195 if ";" in rawdata[i:]: #bail by consuming &#
ebda6803f8d5 Issue #6662: Fix parsing of malformatted charref (&#bad;)
Victor Stinner <victor.stinner@haypocalc.com>
parents: 47177
diff changeset
196 self.handle_data(rawdata[0:2])
ebda6803f8d5 Issue #6662: Fix parsing of malformatted charref (&#bad;)
Victor Stinner <victor.stinner@haypocalc.com>
parents: 47177
diff changeset
197 i = self.updatepos(i, 2)
47177
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
198 break
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
199 elif startswith('&', i):
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
200 match = entityref.match(rawdata, i)
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
201 if match:
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
202 name = match.group(1)
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
203 self.handle_entityref(name)
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
204 k = match.end()
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
205 if not startswith(';', k-1):
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
206 k = k - 1
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
207 i = self.updatepos(i, k)
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
208 continue
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
209 match = incomplete.match(rawdata, i)
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
210 if match:
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
211 # match.group() will contain at least 2 chars
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
212 if end and match.group() == rawdata[i:]:
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
213 self.error("EOF in middle of entity or char ref")
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
214 # incomplete
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
215 break
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
216 elif (i + 1) < n:
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
217 # not the end of the buffer, and can't be confused
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
218 # with some other construct
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
219 self.handle_data("&")
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
220 i = self.updatepos(i, i + 1)
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
221 else:
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
222 break
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
223 else:
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
224 assert 0, "interesting.search() lied"
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
225 # end while
73608
91163aa3d5b4 #13358: HTMLParser now calls handle_data only once for each CDATA.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 73545
diff changeset
226 if end and i < n and not self.cdata_elem:
47177
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
227 self.handle_data(rawdata[i:n])
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
228 i = self.updatepos(i, n)
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
229 self.rawdata = rawdata[i:]
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
230
74908
333e3acf2008 #13960: HTMLParser is now able to handle broken comments.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 73608
diff changeset
231 # Internal -- parse html declarations, return length or -1 if not terminated
333e3acf2008 #13960: HTMLParser is now able to handle broken comments.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 73608
diff changeset
232 # See w3.org/TR/html5/tokenization.html#markup-declaration-open-state
333e3acf2008 #13960: HTMLParser is now able to handle broken comments.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 73608
diff changeset
233 # See also parse_declaration in _markupbase
333e3acf2008 #13960: HTMLParser is now able to handle broken comments.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 73608
diff changeset
234 def parse_html_declaration(self, i):
333e3acf2008 #13960: HTMLParser is now able to handle broken comments.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 73608
diff changeset
235 rawdata = self.rawdata
333e3acf2008 #13960: HTMLParser is now able to handle broken comments.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 73608
diff changeset
236 if rawdata[i:i+2] != '<!':
333e3acf2008 #13960: HTMLParser is now able to handle broken comments.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 73608
diff changeset
237 self.error('unexpected call to parse_html_declaration()')
333e3acf2008 #13960: HTMLParser is now able to handle broken comments.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 73608
diff changeset
238 if rawdata[i:i+4] == '<!--':
74915
4743a3a1e669 Fix an index, add more tests, avoid raising errors for unknown declarations, and clean up comments.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 74909
diff changeset
239 # this case is actually already handled in goahead()
74908
333e3acf2008 #13960: HTMLParser is now able to handle broken comments.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 73608
diff changeset
240 return self.parse_comment(i)
333e3acf2008 #13960: HTMLParser is now able to handle broken comments.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 73608
diff changeset
241 elif rawdata[i:i+3] == '<![':
333e3acf2008 #13960: HTMLParser is now able to handle broken comments.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 73608
diff changeset
242 return self.parse_marked_section(i)
333e3acf2008 #13960: HTMLParser is now able to handle broken comments.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 73608
diff changeset
243 elif rawdata[i:i+9].lower() == '<!doctype':
333e3acf2008 #13960: HTMLParser is now able to handle broken comments.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 73608
diff changeset
244 # find the closing >
74915
4743a3a1e669 Fix an index, add more tests, avoid raising errors for unknown declarations, and clean up comments.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 74909
diff changeset
245 gtpos = rawdata.find('>', i+9)
74908
333e3acf2008 #13960: HTMLParser is now able to handle broken comments.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 73608
diff changeset
246 if gtpos == -1:
333e3acf2008 #13960: HTMLParser is now able to handle broken comments.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 73608
diff changeset
247 return -1
333e3acf2008 #13960: HTMLParser is now able to handle broken comments.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 73608
diff changeset
248 self.handle_decl(rawdata[i+2:gtpos])
333e3acf2008 #13960: HTMLParser is now able to handle broken comments.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 73608
diff changeset
249 return gtpos+1
333e3acf2008 #13960: HTMLParser is now able to handle broken comments.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 73608
diff changeset
250 else:
333e3acf2008 #13960: HTMLParser is now able to handle broken comments.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 73608
diff changeset
251 return self.parse_bogus_comment(i)
333e3acf2008 #13960: HTMLParser is now able to handle broken comments.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 73608
diff changeset
252
333e3acf2008 #13960: HTMLParser is now able to handle broken comments.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 73608
diff changeset
253 # Internal -- parse bogus comment, return length or -1 if not terminated
333e3acf2008 #13960: HTMLParser is now able to handle broken comments.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 73608
diff changeset
254 # see http://www.w3.org/TR/html5/tokenization.html#bogus-comment-state
333e3acf2008 #13960: HTMLParser is now able to handle broken comments.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 73608
diff changeset
255 def parse_bogus_comment(self, i, report=1):
333e3acf2008 #13960: HTMLParser is now able to handle broken comments.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 73608
diff changeset
256 rawdata = self.rawdata
74909
a349448474ea #13993: HTMLParser is now able to handle broken end tags.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 74908
diff changeset
257 if rawdata[i:i+2] not in ('<!', '</'):
74908
333e3acf2008 #13960: HTMLParser is now able to handle broken comments.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 73608
diff changeset
258 self.error('unexpected call to parse_comment()')
333e3acf2008 #13960: HTMLParser is now able to handle broken comments.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 73608
diff changeset
259 pos = rawdata.find('>', i+2)
333e3acf2008 #13960: HTMLParser is now able to handle broken comments.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 73608
diff changeset
260 if pos == -1:
333e3acf2008 #13960: HTMLParser is now able to handle broken comments.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 73608
diff changeset
261 return -1
333e3acf2008 #13960: HTMLParser is now able to handle broken comments.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 73608
diff changeset
262 if report:
333e3acf2008 #13960: HTMLParser is now able to handle broken comments.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 73608
diff changeset
263 self.handle_comment(rawdata[i+2:pos])
333e3acf2008 #13960: HTMLParser is now able to handle broken comments.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 73608
diff changeset
264 return pos + 1
333e3acf2008 #13960: HTMLParser is now able to handle broken comments.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 73608
diff changeset
265
47177
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
266 # Internal -- parse processing instr, return end or -1 if not terminated
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
267 def parse_pi(self, i):
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
268 rawdata = self.rawdata
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
269 assert rawdata[i:i+2] == '<?', 'unexpected call to parse_pi()'
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
270 match = piclose.search(rawdata, i+2) # >
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
271 if not match:
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
272 return -1
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
273 j = match.start()
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
274 self.handle_pi(rawdata[i+2: j])
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
275 j = match.end()
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
276 return j
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
277
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
278 # Internal -- handle starttag, return end or -1 if not terminated
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
279 def parse_starttag(self, i):
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
280 self.__starttag_text = None
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
281 endpos = self.check_for_whole_start_tag(i)
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
282 if endpos < 0:
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
283 return endpos
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
284 rawdata = self.rawdata
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
285 self.__starttag_text = rawdata[i:endpos]
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
286
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
287 # Now parse the data between i+1 and j into a tag and attrs
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
288 attrs = []
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
289 match = tagfind.match(rawdata, i+1)
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
290 assert match, 'unexpected call to parse_starttag()'
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
291 k = match.end()
76413
36c901fcfcda #14538: HTMLParser can now parse correctly start tags that contain a bare /.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 75116
diff changeset
292 self.lasttag = tag = match.group(1).lower()
47177
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
293
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
294 while k < endpos:
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
295 m = attrfind.match(rawdata, k)
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
296 if not m:
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
297 break
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
298 attrname, rest, attrvalue = m.group(1, 2, 3)
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
299 if not rest:
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
300 attrvalue = None
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
301 elif attrvalue[:1] == '\'' == attrvalue[-1:] or \
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
302 attrvalue[:1] == '"' == attrvalue[-1:]:
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
303 attrvalue = attrvalue[1:-1]
73545
3c3009f63700 #1745761, #755670, #13357, #12629, #1200313: improve attribute handling in HTMLParser.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 73273
diff changeset
304 if attrvalue:
47177
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
305 attrvalue = self.unescape(attrvalue)
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
306 attrs.append((attrname.lower(), attrvalue))
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
307 k = m.end()
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
308
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
309 end = rawdata[k:endpos].strip()
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
310 if end not in (">", "/>"):
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
311 lineno, offset = self.getpos()
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
312 if "\n" in self.__starttag_text:
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
313 lineno = lineno + self.__starttag_text.count("\n")
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
314 offset = len(self.__starttag_text) \
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
315 - self.__starttag_text.rfind("\n")
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
316 else:
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
317 offset = offset + len(self.__starttag_text)
74946
3d7904e3f4b9 #13987: HTMLParser is now able to handle malformed start tags.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 74945
diff changeset
318 self.handle_data(rawdata[i:endpos])
3d7904e3f4b9 #13987: HTMLParser is now able to handle malformed start tags.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 74945
diff changeset
319 return endpos
47177
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
320 if end.endswith('/>'):
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
321 # XHTML-style empty tag: <span attr="value" />
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
322 self.handle_startendtag(tag, attrs)
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
323 else:
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
324 self.handle_starttag(tag, attrs)
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
325 if tag in self.CDATA_CONTENT_ELEMENTS:
73273
0a5eb57d5876 #670664: Fix HTMLParser to correctly handle the content of ``<script>...</script>`` and ``<style>...</style>``.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 70423
diff changeset
326 self.set_cdata_mode(tag)
47177
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
327 return endpos
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
328
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
329 # Internal -- check to see if we have a complete starttag; return end
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
330 # or -1 if incomplete.
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
331 def check_for_whole_start_tag(self, i):
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
332 rawdata = self.rawdata
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
333 m = locatestarttagend.match(rawdata, i)
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
334 if m:
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
335 j = m.end()
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
336 next = rawdata[j:j+1]
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
337 if next == ">":
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
338 return j + 1
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
339 if next == "/":
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
340 if rawdata.startswith("/>", j):
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
341 return j + 2
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
342 if rawdata.startswith("/", j):
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
343 # buffer boundary
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
344 return -1
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
345 # else bogus input
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
346 self.updatepos(i, j + 1)
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
347 self.error("malformed empty start tag")
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
348 if next == "":
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
349 # end of input
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
350 return -1
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
351 if next in ("abcdefghijklmnopqrstuvwxyz=/"
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
352 "ABCDEFGHIJKLMNOPQRSTUVWXYZ"):
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
353 # end of input in or before attribute value, or we have the
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
354 # '/' from a '/>' ending
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
355 return -1
74946
3d7904e3f4b9 #13987: HTMLParser is now able to handle malformed start tags.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 74945
diff changeset
356 if j > i:
3d7904e3f4b9 #13987: HTMLParser is now able to handle malformed start tags.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 74945
diff changeset
357 return j
3d7904e3f4b9 #13987: HTMLParser is now able to handle malformed start tags.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 74945
diff changeset
358 else:
3d7904e3f4b9 #13987: HTMLParser is now able to handle malformed start tags.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 74945
diff changeset
359 return i + 1
47177
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
360 raise AssertionError("we should not get here!")
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
361
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
362 # Internal -- parse endtag, return end or -1 if incomplete
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
363 def parse_endtag(self, i):
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
364 rawdata = self.rawdata
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
365 assert rawdata[i:i+2] == "</", "unexpected call to parse_endtag"
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
366 match = endendtag.search(rawdata, i+1) # >
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
367 if not match:
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
368 return -1
74909
a349448474ea #13993: HTMLParser is now able to handle broken end tags.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 74908
diff changeset
369 gtpos = match.end()
47177
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
370 match = endtagfind.match(rawdata, i) # </ + tag + >
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
371 if not match:
73273
0a5eb57d5876 #670664: Fix HTMLParser to correctly handle the content of ``<script>...</script>`` and ``<style>...</style>``.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 70423
diff changeset
372 if self.cdata_elem is not None:
74909
a349448474ea #13993: HTMLParser is now able to handle broken end tags.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 74908
diff changeset
373 self.handle_data(rawdata[i:gtpos])
a349448474ea #13993: HTMLParser is now able to handle broken end tags.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 74908
diff changeset
374 return gtpos
a349448474ea #13993: HTMLParser is now able to handle broken end tags.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 74908
diff changeset
375 # find the name: w3.org/TR/html5/tokenization.html#tag-name-state
a349448474ea #13993: HTMLParser is now able to handle broken end tags.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 74908
diff changeset
376 namematch = tagfind_tolerant.match(rawdata, i+2)
a349448474ea #13993: HTMLParser is now able to handle broken end tags.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 74908
diff changeset
377 if not namematch:
a349448474ea #13993: HTMLParser is now able to handle broken end tags.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 74908
diff changeset
378 # w3.org/TR/html5/tokenization.html#end-tag-open-state
a349448474ea #13993: HTMLParser is now able to handle broken end tags.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 74908
diff changeset
379 if rawdata[i:i+3] == '</>':
a349448474ea #13993: HTMLParser is now able to handle broken end tags.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 74908
diff changeset
380 return i+3
a349448474ea #13993: HTMLParser is now able to handle broken end tags.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 74908
diff changeset
381 else:
a349448474ea #13993: HTMLParser is now able to handle broken end tags.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 74908
diff changeset
382 return self.parse_bogus_comment(i)
a349448474ea #13993: HTMLParser is now able to handle broken end tags.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 74908
diff changeset
383 tagname = namematch.group().lower()
a349448474ea #13993: HTMLParser is now able to handle broken end tags.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 74908
diff changeset
384 # consume and ignore other stuff between the name and the >
a349448474ea #13993: HTMLParser is now able to handle broken end tags.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 74908
diff changeset
385 # Note: this is not 100% correct, since we might have things like
a349448474ea #13993: HTMLParser is now able to handle broken end tags.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 74908
diff changeset
386 # </tag attr=">">, but looking for > after tha name should cover
a349448474ea #13993: HTMLParser is now able to handle broken end tags.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 74908
diff changeset
387 # most of the cases and is much simpler
a349448474ea #13993: HTMLParser is now able to handle broken end tags.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 74908
diff changeset
388 gtpos = rawdata.find('>', namematch.end())
a349448474ea #13993: HTMLParser is now able to handle broken end tags.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 74908
diff changeset
389 self.handle_endtag(tagname)
a349448474ea #13993: HTMLParser is now able to handle broken end tags.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 74908
diff changeset
390 return gtpos+1
73273
0a5eb57d5876 #670664: Fix HTMLParser to correctly handle the content of ``<script>...</script>`` and ``<style>...</style>``.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 70423
diff changeset
391
0a5eb57d5876 #670664: Fix HTMLParser to correctly handle the content of ``<script>...</script>`` and ``<style>...</style>``.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 70423
diff changeset
392 elem = match.group(1).lower() # script or style
0a5eb57d5876 #670664: Fix HTMLParser to correctly handle the content of ``<script>...</script>`` and ``<style>...</style>``.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 70423
diff changeset
393 if self.cdata_elem is not None:
0a5eb57d5876 #670664: Fix HTMLParser to correctly handle the content of ``<script>...</script>`` and ``<style>...</style>``.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 70423
diff changeset
394 if elem != self.cdata_elem:
74909
a349448474ea #13993: HTMLParser is now able to handle broken end tags.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 74908
diff changeset
395 self.handle_data(rawdata[i:gtpos])
a349448474ea #13993: HTMLParser is now able to handle broken end tags.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 74908
diff changeset
396 return gtpos
73273
0a5eb57d5876 #670664: Fix HTMLParser to correctly handle the content of ``<script>...</script>`` and ``<style>...</style>``.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 70423
diff changeset
397
0a5eb57d5876 #670664: Fix HTMLParser to correctly handle the content of ``<script>...</script>`` and ``<style>...</style>``.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 70423
diff changeset
398 self.handle_endtag(elem)
47177
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
399 self.clear_cdata_mode()
74909
a349448474ea #13993: HTMLParser is now able to handle broken end tags.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 74908
diff changeset
400 return gtpos
47177
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
401
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
402 # Overridable -- finish processing of start+end tag: <tag.../>
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
403 def handle_startendtag(self, tag, attrs):
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
404 self.handle_starttag(tag, attrs)
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
405 self.handle_endtag(tag)
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
406
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
407 # Overridable -- handle start tag
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
408 def handle_starttag(self, tag, attrs):
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
409 pass
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
410
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
411 # Overridable -- handle end tag
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
412 def handle_endtag(self, tag):
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
413 pass
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
414
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
415 # Overridable -- handle character reference
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
416 def handle_charref(self, name):
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
417 pass
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
418
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
419 # Overridable -- handle entity reference
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
420 def handle_entityref(self, name):
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
421 pass
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
422
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
423 # Overridable -- handle data
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
424 def handle_data(self, data):
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
425 pass
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
426
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
427 # Overridable -- handle comment
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
428 def handle_comment(self, data):
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
429 pass
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
430
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
431 # Overridable -- handle declaration
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
432 def handle_decl(self, decl):
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
433 pass
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
434
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
435 # Overridable -- handle processing instruction
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
436 def handle_pi(self, data):
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
437 pass
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
438
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
439 def unknown_decl(self, data):
74915
4743a3a1e669 Fix an index, add more tests, avoid raising errors for unknown declarations, and clean up comments.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 74909
diff changeset
440 pass
47177
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
441
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
442 # Internal -- helper to remove special character quoting
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
443 entitydefs = None
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
444 def unescape(self, s):
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
445 if '&' not in s:
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
446 return s
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
447 def replaceEntities(s):
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
448 s = s.groups()[0]
67188
05cccd4257a2 Fix Issue10759 - HTMLParser.unescape() to handle malform charrefs.
Senthil Kumaran <orsenthil@gmail.com>
parents: 61885
diff changeset
449 try:
05cccd4257a2 Fix Issue10759 - HTMLParser.unescape() to handle malform charrefs.
Senthil Kumaran <orsenthil@gmail.com>
parents: 61885
diff changeset
450 if s[0] == "#":
05cccd4257a2 Fix Issue10759 - HTMLParser.unescape() to handle malform charrefs.
Senthil Kumaran <orsenthil@gmail.com>
parents: 61885
diff changeset
451 s = s[1:]
05cccd4257a2 Fix Issue10759 - HTMLParser.unescape() to handle malform charrefs.
Senthil Kumaran <orsenthil@gmail.com>
parents: 61885
diff changeset
452 if s[0] in ['x','X']:
05cccd4257a2 Fix Issue10759 - HTMLParser.unescape() to handle malform charrefs.
Senthil Kumaran <orsenthil@gmail.com>
parents: 61885
diff changeset
453 c = int(s[1:], 16)
05cccd4257a2 Fix Issue10759 - HTMLParser.unescape() to handle malform charrefs.
Senthil Kumaran <orsenthil@gmail.com>
parents: 61885
diff changeset
454 else:
05cccd4257a2 Fix Issue10759 - HTMLParser.unescape() to handle malform charrefs.
Senthil Kumaran <orsenthil@gmail.com>
parents: 61885
diff changeset
455 c = int(s)
05cccd4257a2 Fix Issue10759 - HTMLParser.unescape() to handle malform charrefs.
Senthil Kumaran <orsenthil@gmail.com>
parents: 61885
diff changeset
456 return unichr(c)
05cccd4257a2 Fix Issue10759 - HTMLParser.unescape() to handle malform charrefs.
Senthil Kumaran <orsenthil@gmail.com>
parents: 61885
diff changeset
457 except ValueError:
05cccd4257a2 Fix Issue10759 - HTMLParser.unescape() to handle malform charrefs.
Senthil Kumaran <orsenthil@gmail.com>
parents: 61885
diff changeset
458 return '&#'+s+';'
47177
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
459 else:
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
460 # Cannot use name2codepoint directly, because HTMLParser supports apos,
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
461 # which is not part of HTML 4
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
462 import htmlentitydefs
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
463 if HTMLParser.entitydefs is None:
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
464 entitydefs = HTMLParser.entitydefs = {'apos':u"'"}
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
465 for k, v in htmlentitydefs.name2codepoint.iteritems():
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
466 entitydefs[k] = unichr(v)
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
467 try:
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
468 return self.entitydefs[s]
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
469 except KeyError:
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
470 return '&'+s+';'
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
471
454d38c1f47e revert creation of the html.entities and html.parser modules
Fred Drake <fdrake@acm.org>
parents:
diff changeset
472 return re.sub(r"&(#?[xX]?(?:[0-9a-fA-F]+|\w{1,8}));", replaceEntities, s)