Mercurial > wikicreole
changeset 21:cb1ca7fd2f76
reorganize the sources to accomodate library growth
| author | Radomir Dopieralski <creole@sheep.art.pl> |
|---|---|
| date | Tue, 01 Jun 2010 21:59:03 +0200 |
| parents | 9afb2558bf2a |
| children | 173702c38d36 |
| files | .hgignore MANIFEST creole.py creole/__init__.py creole/document.py creole/html_emitter.py creole/parser.py creole/rules.py creole2html.py examples/creole2html.py examples/headings.py setup.py test.py tests/test_html_emitter.py |
| diffstat | 14 files changed, 746 insertions(+), 738 deletions(-) [+] |
line wrap: on
line diff
--- a/.hgignore Sat Jan 16 23:11:50 2010 +0100 +++ b/.hgignore Tue Jun 01 21:59:03 2010 +0200 @@ -0,0 +1,7 @@ +syntax: glob +*.pyc +*~ +.*.swp +build/* +dist/* +*.DS_Store
--- a/MANIFEST Sat Jan 16 23:11:50 2010 +0100 +++ b/MANIFEST Tue Jun 01 21:59:03 2010 +0200 @@ -1,5 +1,11 @@ -creole.py +creole/__init__.py +creole/document.py +creole/parser.py +creole/rules.py +creole/html_emitter.py +tests/test_html_emitter.py +examples/headings.py +examples/creole2html.py setup.py -creole2html.py -test.py COPYING +MANIFEST
--- a/creole.py Sat Jan 16 23:11:50 2010 +0100 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,430 +0,0 @@ -# -*- coding: iso-8859-1 -*- -""" - Creole wiki markup parser - - See http://wikicreole.org/ for latest specs. - - Notes: - * No markup allowed in headings. - Creole 1.0 does not require us to support this. - * No markup allowed in table headings. - Creole 1.0 does not require us to support this. - * No (non-bracketed) generic url recognition: this is "mission impossible" - except if you want to risk lots of false positives. Only known protocols - are recognized. - * We do not allow ":" before "//" italic markup to avoid urls with - unrecognized schemes (like wtf://server/path) triggering italic rendering - for the rest of the paragraph. - - @copyright: 2007 MoinMoin:RadomirDopieralski (creole 0.5 implementation), - 2007 MoinMoin:ThomasWaldmann (updates) - @license: GNU GPL, see COPYING for details. - @license: BSD, see COPYING for details. -""" - -import re -import sys - -__version__ = '1.1' - - -class Rules: - """Hold all the rules for generating regular expressions.""" - - # For the inline elements: - proto = r'http|https|ftp|nntp|news|mailto|telnet|file|irc' - link = r'''(?P<link> - \[\[ - (?P<link_target>.+?) \s* - ([|] \s* (?P<link_text>.+?) \s*)? - ]] - )''' - image = r'''(?P<image> - {{ - (?P<image_target>.+?) \s* - ([|] \s* (?P<image_text>.+?) \s*)? - }} - )''' - macro = r'''(?P<macro> - << - (?P<macro_name> \w+) - (\( (?P<macro_args> .*?) \))? \s* - ([|] \s* (?P<macro_text> .+?) \s* )? - >> - )''' - code = r'(?P<code> {{{ (?P<code_text>.*?) }}} )' - emph = r'(?P<emph> (?<!:)// )' # there must be no : in front of the // - # avoids italic rendering in urls with - # unknown protocols - strong = r'(?P<strong> \*\* )' - linebreak = r'(?P<break> \\\\ )' - escape = r'(?P<escape> ~ (?P<escaped_char>\S) )' - char = r'(?P<char> . )' - - # For the block elements: - separator = r'(?P<separator> ^ \s* ---- \s* $ )' # horizontal line - line = r'(?P<line> ^ \s* $ )' # empty line that separates paragraphs - head = r'''(?P<head> - ^ \s* - (?P<head_head>=+) \s* - (?P<head_text> .*? ) \s* - (?P<head_tail>=*) \s* - $ - )''' - text = r'(?P<text> .+ )' - list = r'''(?P<list> - ^ [ \t]* ([*][^*\#]|[\#][^\#*]).* $ - ( \n[ \t]* [*\#]+.* $ )* - )''' # Matches the whole list, separate items are parsed later. The - # list *must* start with a single bullet. - item = r'''(?P<item> - ^ \s* - (?P<item_head> [\#*]+) \s* - (?P<item_text> .*?) - $ - )''' # Matches single list items - pre = r'''(?P<pre> - ^{{{ \s* $ - (\n)? - (?P<pre_text> - ([\#]!(?P<pre_kind>\w*?)(\s+.*)?$)? - (.|\n)+? - ) - (\n)? - ^}}} \s*$ - )''' - pre_escape = r' ^(?P<indent>\s*) ~ (?P<rest> \}\}\} \s*) $' - table = r'''(?P<table> - ^ \s* - [|].*? \s* - [|]? \s* - $ - )''' - - # For splitting table cells: - cell = r''' - \| \s* - ( - (?P<head> [=][^|]+ ) | - (?P<cell> ( %s | [^|])+ ) - ) \s* - ''' % '|'.join([link, macro, image, code]) - - def __init__(self, bloglike_lines=False, url_protocols=None, - wiki_words=False): - c = re.compile - # For pre escaping, in creole 1.0 done with ~: - self.pre_escape_re = c(self.pre_escape, re.M | re.X) - # for link descriptions - self.link_re = c('|'.join([self.image, self.linebreak, - self.char]), re.X | re.U) - # for list items - self.item_re = c(self.item, re.X | re.U | re.M) - # for table cells - self.cell_re = c(self.cell, re.X | re.U) - - # For block elements: - if bloglike_lines: - self.text = r'(?P<text> .+ ) (?P<break> (?<!\\)$\n(?!\s*$) )?' - self.block_re = c('|'.join([self.line, self.head, self.separator, - self.pre, self.list, self.table, - self.text]), re.X | re.U | re.M) - - # For inline elements: - if url_protocols is not None: - self.proto = '|'.join(re.escape(p) for p in url_protocols) - self.url = r'''(?P<url> - (^ | (?<=\s | [.,:;!?()/=])) - (?P<escaped_url>~)? - (?P<url_target> (?P<url_proto> %s ):\S+? ) - ($ | (?=\s | [,.:;!?()] (\s | $))))''' % self.proto - inline_elements = [self.link, self.url, self.macro, - self.code, self.image, self.strong, - self.emph, self.linebreak, - self.escape, self.char] - if wiki_words: - import unicodedata - up_case = u''.join(unichr(i) for i in xrange(sys.maxunicode) - if unicodedata.category(unichr(i))=='Lu') - self.wiki = ur'''(?P<wiki>[%s]\w+[%s]\w+)''' % (up_case, up_case) - inline_elements.insert(3, self.wiki) - self.inline_re = c('|'.join(inline_elements), re.X | re.U) - -class Parser: - """ - Parse the raw text and create a document object - that can be converted into output using Emitter. - - A separate instance should be created for parsing a new document. - The first parameter is the raw text to be parsed. An optional second - argument is the Rules object to use. You can customize the parsing - rules to enable optional features or extend the parser. - """ - - def __init__(self, raw, rules=None): - self.rules = rules or Rules() - self.raw = raw - self.root = DocNode('document', None) - self.cur = self.root # The most recent document node - self.text = None # The node to add inline characters to - - def _upto(self, node, kinds): - """ - Look up the tree to the first occurence - of one of the listed kinds of nodes or root. - Start at the node node. - """ - while node.parent is not None and not node.kind in kinds: - node = node.parent - return node - - # The _*_repl methods called for matches in regexps. Sometimes the - # same method needs several names, because of group names in regexps. - - def _url_repl(self, groups): - """Handle raw urls in text.""" - - if not groups.get('escaped_url'): - # this url is NOT escaped - target = groups.get('url_target', '') - node = DocNode('link', self.cur) - node.content = target - DocNode('text', node, node.content) - self.text = None - else: - # this url is escaped, we render it as text - if self.text is None: - self.text = DocNode('text', self.cur, u'') - self.text.content += groups.get('url_target') - _url_target_repl = _url_repl - _url_proto_repl = _url_repl - _escaped_url = _url_repl - - def _link_repl(self, groups): - """Handle all kinds of links.""" - - target = groups.get('link_target', '') - text = (groups.get('link_text', '') or '').strip() - parent = self.cur - self.cur = DocNode('link', self.cur) - self.cur.content = target - self.text = None - re.sub(self.rules.link_re, self._replace, text) - self.cur = parent - self.text = None - _link_target_repl = _link_repl - _link_text_repl = _link_repl - - def _wiki_repl(self, groups): - """Handle WikiWord links, if enabled.""" - - text = groups.get('wiki', '') - node = DocNode('link', self.cur) - node.content = text - DocNode('text', node, node.content) - self.text = None - - def _macro_repl(self, groups): - """Handles macros using the placeholder syntax.""" - - name = groups.get('macro_name', '') - text = (groups.get('macro_text', '') or '').strip() - node = DocNode('macro', self.cur, name) - node.args = groups.get('macro_args', '') or '' - DocNode('text', node, text or name) - self.text = None - _macro_name_repl = _macro_repl - _macro_args_repl = _macro_repl - _macro_text_repl = _macro_repl - - def _image_repl(self, groups): - """Handles images and attachemnts included in the page.""" - - target = groups.get('image_target', '').strip() - text = (groups.get('image_text', '') or '').strip() - node = DocNode("image", self.cur, target) - DocNode('text', node, text or node.content) - self.text = None - _image_target_repl = _image_repl - _image_text_repl = _image_repl - - def _separator_repl(self, groups): - self.cur = self._upto(self.cur, ('document', 'section', 'blockquote')) - DocNode('separator', self.cur) - - def _item_repl(self, groups): - bullet = groups.get('item_head', u'') - text = groups.get('item_text', u'') - if bullet[-1] == '#': - kind = 'number_list' - else: - kind = 'bullet_list' - level = len(bullet) - lst = self.cur - # Find a list of the same kind and level up the tree - while (lst and - not (lst.kind in ('number_list', 'bullet_list') and - lst.level == level) and - not lst.kind in ('document', 'section', 'blockquote')): - lst = lst.parent - if lst and lst.kind == kind: - self.cur = lst - else: - # Create a new level of list - self.cur = self._upto(self.cur, - ('list_item', 'document', 'section', 'blockquote')) - self.cur = DocNode(kind, self.cur) - self.cur.level = level - self.cur = DocNode('list_item', self.cur) - self.parse_inline(text) - self.text = None - _item_text_repl = _item_repl - _item_head_repl = _item_repl - - def _list_repl(self, groups): - text = groups.get('list', u'') - self.rules.item_re.sub(self._replace, text) - - def _head_repl(self, groups): - self.cur = self._upto(self.cur, ('document', 'section', 'blockquote')) - node = DocNode('header', self.cur, groups.get('head_text', '').strip()) - node.level = len(groups.get('head_head', ' ')) - _head_head_repl = _head_repl - _head_text_repl = _head_repl - - def _text_repl(self, groups): - text = groups.get('text', '') - if self.cur.kind in ('table', 'table_row', 'bullet_list', - 'number_list'): - self.cur = self._upto(self.cur, - ('document', 'section', 'blockquote')) - if self.cur.kind in ('document', 'section', 'blockquote'): - self.cur = DocNode('paragraph', self.cur) - else: - text = u' ' + text - self.parse_inline(text) - if groups.get('break') and self.cur.kind in ('paragraph', - 'emphasis', 'strong', 'code'): - DocNode('break', self.cur, '') - self.text = None - _break_repl = _text_repl - - def _table_repl(self, groups): - row = groups.get('table', '|').strip() - self.cur = self._upto(self.cur, ( - 'table', 'document', 'section', 'blockquote')) - if self.cur.kind != 'table': - self.cur = DocNode('table', self.cur) - tb = self.cur - tr = DocNode('table_row', tb) - - text = '' - for m in self.rules.cell_re.finditer(row): - cell = m.group('cell') - if cell: - self.cur = DocNode('table_cell', tr) - self.text = None - self.parse_inline(cell) - else: - cell = m.group('head') - self.cur = DocNode('table_head', tr) - self.text = DocNode('text', self.cur, u'') - self.text.content = cell.strip('=') - self.cur = tb - self.text = None - - def _pre_repl(self, groups): - self.cur = self._upto(self.cur, ('document', 'section', 'blockquote')) - kind = groups.get('pre_kind', None) - text = groups.get('pre_text', u'') - def remove_tilde(m): - return m.group('indent') + m.group('rest') - text = self.rules.pre_escape_re.sub(remove_tilde, text) - node = DocNode('preformatted', self.cur, text) - node.sect = kind or '' - self.text = None - _pre_text_repl = _pre_repl - _pre_head_repl = _pre_repl - _pre_kind_repl = _pre_repl - - def _line_repl(self, groups): - self.cur = self._upto(self.cur, ('document', 'section', 'blockquote')) - - def _code_repl(self, groups): - DocNode('code', self.cur, groups.get('code_text', u'').strip()) - self.text = None - _code_text_repl = _code_repl - _code_head_repl = _code_repl - - def _emph_repl(self, groups): - if self.cur.kind != 'emphasis': - self.cur = DocNode('emphasis', self.cur) - else: - self.cur = self._upto(self.cur, ('emphasis', )).parent - self.text = None - - def _strong_repl(self, groups): - if self.cur.kind != 'strong': - self.cur = DocNode('strong', self.cur) - else: - self.cur = self._upto(self.cur, ('strong', )).parent - self.text = None - - def _break_repl(self, groups): - DocNode('break', self.cur, None) - self.text = None - - def _escape_repl(self, groups): - if self.text is None: - self.text = DocNode('text', self.cur, u'') - self.text.content += groups.get('escaped_char', u'') - - def _char_repl(self, groups): - if self.text is None: - self.text = DocNode('text', self.cur, u'') - self.text.content += groups.get('char', u'') - - def _replace(self, match): - """Invoke appropriate _*_repl method. Called for every matched group.""" - - groups = match.groupdict() - for name, text in groups.iteritems(): - if text is not None: - replace = getattr(self, '_%s_repl' % name) - replace(groups) - return - - def parse_inline(self, raw): - """Recognize inline elements inside blocks.""" - - re.sub(self.rules.inline_re, self._replace, raw) - - def parse_block(self, raw): - """Recognize block elements.""" - - re.sub(self.rules.block_re, self._replace, raw) - - def parse(self): - """Parse the text given as self.raw and return DOM tree.""" - - self.parse_block(self.raw) - return self.root - -#################### Helper classes - -### The document model - -class DocNode: - """ - A node in the document. - """ - - def __init__(self, kind='', parent=None, content=None): - self.children = [] - self.parent = parent - self.kind = kind - self.content = content - if self.parent is not None: - self.parent.children.append(self) - -
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/creole/__init__.py Tue Jun 01 21:59:03 2010 +0200 @@ -0,0 +1,32 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +""" + Creole wiki markup parser + + See http://wikicreole.org/ for latest specs. + + Notes: + * No markup allowed in headings. + Creole 1.0 does not require us to support this. + * No markup allowed in table headings. + Creole 1.0 does not require us to support this. + * No (non-bracketed) generic url recognition: this is "mission impossible" + except if you want to risk lots of false positives. Only known protocols + are recognized. + * We do not allow ":" before "//" italic markup to avoid urls with + unrecognized schemes (like wtf://server/path) triggering italic rendering + for the rest of the paragraph. + + @copyright: 2007 MoinMoin:RadomirDopieralski (creole 0.5 implementation), + 2007 MoinMoin:ThomasWaldmann (updates) + @license: GNU GPL, see COPYING for details. + @license: BSD, see COPYING for details. +""" + +__version__ = '1.2' + +from creole.rules import Rules +from creole.parser import Parser +from creole.document import DocNode +from creole.html_emitter import HtmlEmitter +
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/creole/document.py Tue Jun 01 21:59:03 2010 +0200 @@ -0,0 +1,17 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +class DocNode(object): + """ + A node in the document. + """ + + def __init__(self, kind='', parent=None, content=None): + self.children = [] + self.parent = parent + self.kind = kind + self.content = content + if self.parent is not None: + self.parent.children.append(self) + +
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/creole/html_emitter.py Tue Jun 01 21:59:03 2010 +0200 @@ -0,0 +1,159 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + + +import re + +from creole.parser import Parser + + +class Rules(object): + # For the link targets: + proto = r'http|https|ftp|nntp|news|mailto|telnet|file|irc' + extern = r'(?P<extern_addr>(?P<extern_proto>%s):.*)' % proto + interwiki = r''' + (?P<inter_wiki> [A-Z][a-zA-Z]+ ) : + (?P<inter_page> .* ) + ''' + +class HtmlEmitter(object): + """ + Generate HTML output for the document + tree consisting of DocNodes. + """ + + addr_re = re.compile('|'.join([ + Rules.extern, + Rules.interwiki, + ]), re.X | re.U) # for addresses + + def __init__(self, root): + self.root = root + + def get_text(self, node): + """Try to emit whatever text is in the node.""" + + try: + return node.children[0].content or '' + except: + return node.content or '' + + def html_escape(self, text): + return text.replace('&', '&').replace('<', '<').replace('>', '>') + + def attr_escape(self, text): + return self.html_escape(text).replace('"', '"') + + # *_emit methods for emitting nodes of the document: + + def document_emit(self, node): + return self.emit_children(node) + + def text_emit(self, node): + return self.html_escape(node.content) + + def separator_emit(self, node): + return u'<hr>'; + + def paragraph_emit(self, node): + return u'<p>%s</p>\n' % self.emit_children(node) + + def bullet_list_emit(self, node): + return u'<ul>\n%s</ul>\n' % self.emit_children(node) + + def number_list_emit(self, node): + return u'<ol>\n%s</ol>\n' % self.emit_children(node) + + def list_item_emit(self, node): + return u'<li>%s</li>\n' % self.emit_children(node) + + def table_emit(self, node): + return u'<table>\n%s</table>\n' % self.emit_children(node) + + def table_row_emit(self, node): + return u'<tr>%s</tr>\n' % self.emit_children(node) + + def table_cell_emit(self, node): + return u'<td>%s</td>' % self.emit_children(node) + + def table_head_emit(self, node): + return u'<th>%s</th>' % self.emit_children(node) + + def emphasis_emit(self, node): + return u'<i>%s</i>' % self.emit_children(node) + + def strong_emit(self, node): + return u'<b>%s</b>' % self.emit_children(node) + + def header_emit(self, node): + return u'<h%d>%s</h%d>\n' % ( + node.level, self.html_escape(node.content), node.level) + + def code_emit(self, node): + return u'<tt>%s</tt>' % self.html_escape(node.content) + + def link_emit(self, node): + target = node.content + if node.children: + inside = self.emit_children(node) + else: + inside = self.html_escape(target) + m = self.addr_re.match(target) + if m: + if m.group('extern_addr'): + return u'<a href="%s">%s</a>' % ( + self.attr_escape(target), inside) + elif m.group('inter_wiki'): + raise NotImplementedError + return u'<a href="%s">%s</a>' % ( + self.attr_escape(target), inside) + + def image_emit(self, node): + target = node.content + text = self.get_text(node) + m = self.addr_re.match(target) + if m: + if m.group('extern_addr'): + return u'<img src="%s" alt="%s">' % ( + self.attr_escape(target), self.attr_escape(text)) + elif m.group('inter_wiki'): + raise NotImplementedError + return u'<img src="%s" alt="%s">' % ( + self.attr_escape(target), self.attr_escape(text)) + + def macro_emit(self, node): + raise NotImplementedError + + def break_emit(self, node): + return u"<br>" + + def preformatted_emit(self, node): + return u"<pre>%s</pre>" % self.html_escape(node.content) + + def default_emit(self, node): + """Fallback function for emitting unknown nodes.""" + + raise TypeError + + def emit_children(self, node): + """Emit all the children of a node.""" + + return u''.join([self.emit_node(child) for child in node.children]) + + def emit_node(self, node): + """Emit a single node.""" + + emit = getattr(self, '%s_emit' % node.kind, self.default_emit) + return emit(node) + + def emit(self): + """Emit the document represented by self.root DOM tree.""" + + return self.emit_node(self.root) + +if __name__=="__main__": + import sys + document = Parser(unicode(sys.stdin.read(), 'utf-8', 'ignore')).parse() + sys.stdout.write(HtmlEmitter(document).emit().encode('utf-8', 'ignore')) + +
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/creole/parser.py Tue Jun 01 21:59:03 2010 +0200 @@ -0,0 +1,270 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +import re +import sys + +from creole.rules import Rules +from creole.document import DocNode + + +class Parser(object): + """ + Parse the raw text and create a document object + that can be converted into output using Emitter. + + A separate instance should be created for parsing a new document. + The first parameter is the raw text to be parsed. An optional second + argument is the Rules object to use. You can customize the parsing + rules to enable optional features or extend the parser. + """ + + def __init__(self, raw, rules=None): + self.rules = rules or Rules() + self.raw = raw + self.root = DocNode('document', None) + self.cur = self.root # The most recent document node + self.text = None # The node to add inline characters to + + def _upto(self, node, kinds): + """ + Look up the tree to the first occurence + of one of the listed kinds of nodes or root. + Start at the node node. + """ + while node.parent is not None and not node.kind in kinds: + node = node.parent + return node + + # The _*_repl methods called for matches in regexps. Sometimes the + # same method needs several names, because of group names in regexps. + + def _url_repl(self, groups): + """Handle raw urls in text.""" + + if not groups.get('escaped_url'): + # this url is NOT escaped + target = groups.get('url_target', '') + node = DocNode('link', self.cur) + node.content = target + DocNode('text', node, node.content) + self.text = None + else: + # this url is escaped, we render it as text + if self.text is None: + self.text = DocNode('text', self.cur, u'') + self.text.content += groups.get('url_target') + _url_target_repl = _url_repl + _url_proto_repl = _url_repl + _escaped_url = _url_repl + + def _link_repl(self, groups): + """Handle all kinds of links.""" + + target = groups.get('link_target', '') + text = (groups.get('link_text', '') or '').strip() + parent = self.cur + self.cur = DocNode('link', self.cur) + self.cur.content = target + self.text = None + re.sub(self.rules.link_re, self._replace, text) + self.cur = parent + self.text = None + _link_target_repl = _link_repl + _link_text_repl = _link_repl + + def _wiki_repl(self, groups): + """Handle WikiWord links, if enabled.""" + + text = groups.get('wiki', '') + node = DocNode('link', self.cur) + node.content = text + DocNode('text', node, node.content) + self.text = None + + def _macro_repl(self, groups): + """Handles macros using the placeholder syntax.""" + + name = groups.get('macro_name', '') + text = (groups.get('macro_text', '') or '').strip() + node = DocNode('macro', self.cur, name) + node.args = groups.get('macro_args', '') or '' + DocNode('text', node, text or name) + self.text = None + _macro_name_repl = _macro_repl + _macro_args_repl = _macro_repl + _macro_text_repl = _macro_repl + + def _image_repl(self, groups): + """Handles images and attachemnts included in the page.""" + + target = groups.get('image_target', '').strip() + text = (groups.get('image_text', '') or '').strip() + node = DocNode("image", self.cur, target) + DocNode('text', node, text or node.content) + self.text = None + _image_target_repl = _image_repl + _image_text_repl = _image_repl + + def _separator_repl(self, groups): + self.cur = self._upto(self.cur, ('document', 'section', 'blockquote')) + DocNode('separator', self.cur) + + def _item_repl(self, groups): + bullet = groups.get('item_head', u'') + text = groups.get('item_text', u'') + if bullet[-1] == '#': + kind = 'number_list' + else: + kind = 'bullet_list' + level = len(bullet) + lst = self.cur + # Find a list of the same kind and level up the tree + while (lst and + not (lst.kind in ('number_list', 'bullet_list') and + lst.level == level) and + not lst.kind in ('document', 'section', 'blockquote')): + lst = lst.parent + if lst and lst.kind == kind: + self.cur = lst + else: + # Create a new level of list + self.cur = self._upto(self.cur, + ('list_item', 'document', 'section', 'blockquote')) + self.cur = DocNode(kind, self.cur) + self.cur.level = level + self.cur = DocNode('list_item', self.cur) + self.parse_inline(text) + self.text = None + _item_text_repl = _item_repl + _item_head_repl = _item_repl + + def _list_repl(self, groups): + text = groups.get('list', u'') + self.rules.item_re.sub(self._replace, text) + + def _head_repl(self, groups): + self.cur = self._upto(self.cur, ('document', 'section', 'blockquote')) + node = DocNode('header', self.cur, groups.get('head_text', '').strip()) + node.level = len(groups.get('head_head', ' ')) + _head_head_repl = _head_repl + _head_text_repl = _head_repl + + def _text_repl(self, groups): + text = groups.get('text', '') + if self.cur.kind in ('table', 'table_row', 'bullet_list', + 'number_list'): + self.cur = self._upto(self.cur, + ('document', 'section', 'blockquote')) + if self.cur.kind in ('document', 'section', 'blockquote'): + self.cur = DocNode('paragraph', self.cur) + else: + text = u' ' + text + self.parse_inline(text) + if groups.get('break') and self.cur.kind in ('paragraph', + 'emphasis', 'strong', 'code'): + DocNode('break', self.cur, '') + self.text = None + _break_repl = _text_repl + + def _table_repl(self, groups): + row = groups.get('table', '|').strip() + self.cur = self._upto(self.cur, ( + 'table', 'document', 'section', 'blockquote')) + if self.cur.kind != 'table': + self.cur = DocNode('table', self.cur) + tb = self.cur + tr = DocNode('table_row', tb) + + text = '' + for m in self.rules.cell_re.finditer(row): + cell = m.group('cell') + if cell: + self.cur = DocNode('table_cell', tr) + self.text = None + self.parse_inline(cell) + else: + cell = m.group('head') + self.cur = DocNode('table_head', tr) + self.text = DocNode('text', self.cur, u'') + self.text.content = cell.strip('=') + self.cur = tb + self.text = None + + def _pre_repl(self, groups): + self.cur = self._upto(self.cur, ('document', 'section', 'blockquote')) + kind = groups.get('pre_kind', None) + text = groups.get('pre_text', u'') + def remove_tilde(m): + return m.group('indent') + m.group('rest') + text = self.rules.pre_escape_re.sub(remove_tilde, text) + node = DocNode('preformatted', self.cur, text) + node.sect = kind or '' + self.text = None + _pre_text_repl = _pre_repl + _pre_head_repl = _pre_repl + _pre_kind_repl = _pre_repl + + def _line_repl(self, groups): + self.cur = self._upto(self.cur, ('document', 'section', 'blockquote')) + + def _code_repl(self, groups): + DocNode('code', self.cur, groups.get('code_text', u'').strip()) + self.text = None + _code_text_repl = _code_repl + _code_head_repl = _code_repl + + def _emph_repl(self, groups): + if self.cur.kind != 'emphasis': + self.cur = DocNode('emphasis', self.cur) + else: + self.cur = self._upto(self.cur, ('emphasis', )).parent + self.text = None + + def _strong_repl(self, groups): + if self.cur.kind != 'strong': + self.cur = DocNode('strong', self.cur) + else: + self.cur = self._upto(self.cur, ('strong', )).parent + self.text = None + + def _break_repl(self, groups): + DocNode('break', self.cur, None) + self.text = None + + def _escape_repl(self, groups): + if self.text is None: + self.text = DocNode('text', self.cur, u'') + self.text.content += groups.get('escaped_char', u'') + + def _char_repl(self, groups): + if self.text is None: + self.text = DocNode('text', self.cur, u'') + self.text.content += groups.get('char', u'') + + def _replace(self, match): + """Invoke appropriate _*_repl method. Called for every matched group.""" + + groups = match.groupdict() + for name, text in groups.iteritems(): + if text is not None: + replace = getattr(self, '_%s_repl' % name) + replace(groups) + return + + def parse_inline(self, raw): + """Recognize inline elements inside blocks.""" + + re.sub(self.rules.inline_re, self._replace, raw) + + def parse_block(self, raw): + """Recognize block elements.""" + + re.sub(self.rules.block_re, self._replace, raw) + + def parse(self): + """Parse the text given as self.raw and return DOM tree.""" + + self.parse_block(self.raw) + return self.root +
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/creole/rules.py Tue Jun 01 21:59:03 2010 +0200 @@ -0,0 +1,130 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +import re +import sys + + +class Rules(object): + """Hold all the rules for generating regular expressions.""" + + # For the inline elements: + proto = r'http|https|ftp|nntp|news|mailto|telnet|file|irc' + link = r'''(?P<link> + \[\[ + (?P<link_target>.+?) \s* + ([|] \s* (?P<link_text>.+?) \s*)? + ]] + )''' + image = r'''(?P<image> + {{ + (?P<image_target>.+?) \s* + ([|] \s* (?P<image_text>.+?) \s*)? + }} + )''' + macro = r'''(?P<macro> + << + (?P<macro_name> \w+) + (\( (?P<macro_args> .*?) \))? \s* + ([|] \s* (?P<macro_text> .+?) \s* )? + >> + )''' + code = r'(?P<code> {{{ (?P<code_text>.*?) }}} )' + emph = r'(?P<emph> (?<!:)// )' # there must be no : in front of the // + # avoids italic rendering in urls with + # unknown protocols + strong = r'(?P<strong> \*\* )' + linebreak = r'(?P<break> \\\\ )' + escape = r'(?P<escape> ~ (?P<escaped_char>\S) )' + char = r'(?P<char> . )' + + # For the block elements: + separator = r'(?P<separator> ^ \s* ---- \s* $ )' # horizontal line + line = r'(?P<line> ^ \s* $ )' # empty line that separates paragraphs + head = r'''(?P<head> + ^ \s* + (?P<head_head>=+) \s* + (?P<head_text> .*? ) \s* + (?P<head_tail>=*) \s* + $ + )''' + text = r'(?P<text> .+ )' + list = r'''(?P<list> + ^ [ \t]* ([*][^*\#]|[\#][^\#*]).* $ + ( \n[ \t]* [*\#]+.* $ )* + )''' # Matches the whole list, separate items are parsed later. The + # list *must* start with a single bullet. + item = r'''(?P<item> + ^ \s* + (?P<item_head> [\#*]+) \s* + (?P<item_text> .*?) + $ + )''' # Matches single list items + pre = r'''(?P<pre> + ^{{{ \s* $ + (\n)? + (?P<pre_text> + ([\#]!(?P<pre_kind>\w*?)(\s+.*)?$)? + (.|\n)+? + ) + (\n)? + ^}}} \s*$ + )''' + pre_escape = r' ^(?P<indent>\s*) ~ (?P<rest> \}\}\} \s*) $' + table = r'''(?P<table> + ^ \s* + [|].*? \s* + [|]? \s* + $ + )''' + + # For splitting table cells: + cell = r''' + \| \s* + ( + (?P<head> [=][^|]+ ) | + (?P<cell> ( %s | [^|])+ ) + ) \s* + ''' % '|'.join([link, macro, image, code]) + + def __init__(self, bloglike_lines=False, url_protocols=None, + wiki_words=False): + c = re.compile + # For pre escaping, in creole 1.0 done with ~: + self.pre_escape_re = c(self.pre_escape, re.M | re.X) + # for link descriptions + self.link_re = c('|'.join([self.image, self.linebreak, + self.char]), re.X | re.U) + # for list items + self.item_re = c(self.item, re.X | re.U | re.M) + # for table cells + self.cell_re = c(self.cell, re.X | re.U) + + # For block elements: + if bloglike_lines: + self.text = r'(?P<text> .+ ) (?P<break> (?<!\\)$\n(?!\s*$) )?' + self.block_re = c('|'.join([self.line, self.head, self.separator, + self.pre, self.list, self.table, + self.text]), re.X | re.U | re.M) + + # For inline elements: + if url_protocols is not None: + self.proto = '|'.join(re.escape(p) for p in url_protocols) + self.url = r'''(?P<url> + (^ | (?<=\s | [.,:;!?()/=])) + (?P<escaped_url>~)? + (?P<url_target> (?P<url_proto> %s ):\S+? ) + ($ | (?=\s | [,.:;!?()] (\s | $))))''' % self.proto + inline_elements = [self.link, self.url, self.macro, + self.code, self.image, self.strong, + self.emph, self.linebreak, + self.escape, self.char] + if wiki_words: + import unicodedata + up_case = u''.join(unichr(i) for i in xrange(sys.maxunicode) + if unicodedata.category(unichr(i))=='Lu') + self.wiki = ur'''(?P<wiki>[%s]\w+[%s]\w+)''' % (up_case, up_case) + inline_elements.insert(3, self.wiki) + self.inline_re = c('|'.join(inline_elements), re.X | re.U) + +
--- a/creole2html.py Sat Jan 16 23:11:50 2010 +0100 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,294 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -ur""" -WikiCreole to HTML converter -This program is an example of how the creole.py WikiCreole parser -can be used. - -@copyright: 2007 MoinMoin:RadomirDopieralski -@license: BSD, see COPYING for details. - -Test cases contributed by Jan Klopper (janklopper@underdark.nl), -modified by Radomir Dopieralski (MoinMoin:RadomirDopieralski). - ->>> import lxml.html.usedoctest ->>> import creole ->>> def parse(text): -... print HtmlEmitter(creole.Parser(text).parse()).emit() ->>> def wiki_parse(text): -... rules = creole.Rules(wiki_words=True) -... print HtmlEmitter(creole.Parser(text, rules).parse()).emit() - ->>> parse(u'test') -<p>test</p> - ->>> parse(u'test\ntest') -<p>test test</p> - ->>> HtmlEmitter(Parser(u'test\ntest').parse()).emit() -u'<p>test test</p>\n' - ->>> parse(u'test\n\ntest') -<p>test</p><p>test</p> - ->>> parse(u'test\\\\test') -<p>test<br>test</p> - ->>> parse(u'ÓÔÕÖØÙÚÛÜÝßàáâãäåæçèéêëìíîïñòóôõöøùúûüýÿŒœ%0A') -<p>ÓÔÕÖØÙÚÛÜÝßàáâãäåæçèéêëìíîïñòóôõöøùúûüýÿŒœ%0A</p> - ->>> parse(u'----') -<hr> - ->>> parse(u'==test==') -<h2>test</h2> - ->>> parse(u'== test') -<h2>test</h2> - ->>> parse(u'==test====') -<h2>test</h2> - ->>> parse(u'=====test') -<h5>test</h5> - ->>> parse(u'==test==\ntest\n===test===') -<h2>test</h2> -<p>test</p> -<h3>test</h3> - ->>> parse(u'test\n* test line one\n * test line two\ntest\n\ntest') -<p>test</p> -<ul> - <li>test line one</li> - <li>test line two test</li> -</ul> -<p>test</p> - ->>> parse(u'* test line one\n* test line two\n** Nested item') -<ul> - <li>test line one</li> - <li>test line two<ul> - <li>Nested item</li> - </ul></li> -</ul> - ->>> parse(u'* test line one\n* test line two\n# Nested item') -<ul> - <li>test line one</li> - <li>test line two<ol> - <li>Nested item</li> - </ol></li> -</ul> - ->>> parse(u'test //test test// test **test test** test') -<p>test <i>test test</i> test <b>test test</b> test</p> - ->>> parse(u'test //test **test// test** test') -<p>test <i>test <b>test<i> test<b> test</b></i></b></i></p> - ->>> parse(u'**test') -<p><b>test</b></p> - ->>> parse(u'|x|y|z|\n|a|b|c|\n|d|e|f|\ntest') -<table> - <tr><td>x</td><td>y</td><td>z</td></tr> - <tr><td>a</td><td>b</td><td>c</td></tr> - <tr><td>d</td><td>e</td><td>f</td></tr> -</table> -<p>test</p> - ->>> parse(u'|=x|y|=z=|\n|a|b|c|\n|d|e|f|') -<table> - <tr><th>x</th><td>y</td><th>z</th></tr> - <tr><td>a</td><td>b</td><td>c</td></tr> - <tr><td>d</td><td>e</td><td>f</td></tr> -</table> - ->>> parse(u'test http://example.com/test test') -<p>test <a href="http://example.com/test">http://example.com/test</a> test</p> - ->>> parse(u'http://example.com/,test, test') -<p><a href="http://example.com/,test">http://example.com/,test</a>, test</p> - ->>> parse(u'(http://example.com/test)') -<p>(<a href="http://example.com/test">http://example.com/test</a>)</p> - -XXX This might be considered a bug, but it's impossible to detect in general. ->>> parse(u'http://example.com/(test)') -<p><a href="http://example.com/(test">http://example.com/(test</a>)</p> - ->>> parse(u'http://example.com/test?test&test=1') -<p><a href="http://example.com/test?test&test=1">http://example.com/test?test&test=1</a></p> - ->>> parse(u'~http://example.com/test') -<p>http://example.com/test</p> - ->>> parse(u'http://example.com/~test') -<p><a href="http://example.com/~test">http://example.com/~test</a></p> - ->>> parse(u'[[test]] [[tset|test]]') -<p><a href="test">test</a> <a href="tset">test</a></p> - ->>> parse(u'[[http://example.com|test]]') -<p><a href="http://example.com">test</a></p> - ->>> wiki_parse(u'Lorem WikiWord iPsum sit ameT.') -<p>Lorem <a href="WikiWord">WikiWord</a> iPsum sit ameT.</p> - -""" - -import re -from creole import Parser, Rules - -class Rules: - # For the link targets: - proto = r'http|https|ftp|nntp|news|mailto|telnet|file|irc' - extern = r'(?P<extern_addr>(?P<extern_proto>%s):.*)' % proto - interwiki = r''' - (?P<inter_wiki> [A-Z][a-zA-Z]+ ) : - (?P<inter_page> .* ) - ''' - -class HtmlEmitter: - """ - Generate HTML output for the document - tree consisting of DocNodes. - """ - - addr_re = re.compile('|'.join([ - Rules.extern, - Rules.interwiki, - ]), re.X | re.U) # for addresses - - def __init__(self, root): - self.root = root - - def get_text(self, node): - """Try to emit whatever text is in the node.""" - - try: - return node.children[0].content or '' - except: - return node.content or '' - - def html_escape(self, text): - return text.replace('&', '&').replace('<', '<').replace('>', '>') - - def attr_escape(self, text): - return self.html_escape(text).replace('"', '"') - - # *_emit methods for emitting nodes of the document: - - def document_emit(self, node): - return self.emit_children(node) - - def text_emit(self, node): - return self.html_escape(node.content) - - def separator_emit(self, node): - return u'<hr>'; - - def paragraph_emit(self, node): - return u'<p>%s</p>\n' % self.emit_children(node) - - def bullet_list_emit(self, node): - return u'<ul>\n%s</ul>\n' % self.emit_children(node) - - def number_list_emit(self, node): - return u'<ol>\n%s</ol>\n' % self.emit_children(node) - - def list_item_emit(self, node): - return u'<li>%s</li>\n' % self.emit_children(node) - - def table_emit(self, node): - return u'<table>\n%s</table>\n' % self.emit_children(node) - - def table_row_emit(self, node): - return u'<tr>%s</tr>\n' % self.emit_children(node) - - def table_cell_emit(self, node): - return u'<td>%s</td>' % self.emit_children(node) - - def table_head_emit(self, node): - return u'<th>%s</th>' % self.emit_children(node) - - def emphasis_emit(self, node): - return u'<i>%s</i>' % self.emit_children(node) - - def strong_emit(self, node): - return u'<b>%s</b>' % self.emit_children(node) - - def header_emit(self, node): - return u'<h%d>%s</h%d>\n' % ( - node.level, self.html_escape(node.content), node.level) - - def code_emit(self, node): - return u'<tt>%s</tt>' % self.html_escape(node.content) - - def link_emit(self, node): - target = node.content - if node.children: - inside = self.emit_children(node) - else: - inside = self.html_escape(target) - m = self.addr_re.match(target) - if m: - if m.group('extern_addr'): - return u'<a href="%s">%s</a>' % ( - self.attr_escape(target), inside) - elif m.group('inter_wiki'): - raise NotImplementedError - return u'<a href="%s">%s</a>' % ( - self.attr_escape(target), inside) - - def image_emit(self, node): - target = node.content - text = self.get_text(node) - m = self.addr_re.match(target) - if m: - if m.group('extern_addr'): - return u'<img src="%s" alt="%s">' % ( - self.attr_escape(target), self.attr_escape(text)) - elif m.group('inter_wiki'): - raise NotImplementedError - return u'<img src="%s" alt="%s">' % ( - self.attr_escape(target), self.attr_escape(text)) - - def macro_emit(self, node): - raise NotImplementedError - - def break_emit(self, node): - return u"<br>" - - def preformatted_emit(self, node): - return u"<pre>%s</pre>" % self.html_escape(node.content) - - def default_emit(self, node): - """Fallback function for emitting unknown nodes.""" - - raise TypeError - - def emit_children(self, node): - """Emit all the children of a node.""" - - return u''.join([self.emit_node(child) for child in node.children]) - - def emit_node(self, node): - """Emit a single node.""" - - emit = getattr(self, '%s_emit' % node.kind, self.default_emit) - return emit(node) - - def emit(self): - """Emit the document represented by self.root DOM tree.""" - - return self.emit_node(self.root) - -if __name__=="__main__": - import sys - document = Parser(unicode(sys.stdin.read(), 'utf-8', 'ignore')).parse() - sys.stdout.write(HtmlEmitter(document).emit().encode('utf-8', 'ignore')) - -
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/examples/creole2html.py Tue Jun 01 21:59:03 2010 +0200 @@ -0,0 +1,11 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +import sys + +from creole import Parser, HtmlEmitter + +if __name__=="__main__": + document = Parser(unicode(sys.stdin.read(), 'utf-8', 'ignore')).parse() + sys.stdout.write(HtmlEmitter(document).emit().encode('utf-8', 'ignore')) +
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/examples/headings.py Tue Jun 01 21:59:03 2010 +0200 @@ -0,0 +1,94 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +import creole + +class HtmlEmitter(creole.HtmlEmitter): + """ + This class changes how we actually emit the HTML for some + of the nodes. You should also override the link generation + and macro support code here. + """ + + def header_emit(self, node): + """Instead of using node.content, we emit the children.""" + + return u'<h%d>%s</h%d>\n' % (node.level, self.emit_children(node), + node.level) + + +class Rules(creole.Rules): + """ + This class would be used to add or change any regular expressions. + We don't need it in this particular case though, so we just leave + it like the original class. + """ + +class Parser(creole.Parser): + r""" + This class is for overriding the behavior of our parser. We change the + rule that handles headings, so that instead of just putting the text + inside the heading, it parses it, looking for any markup. + + Here are some simple tests: + + >>> import lxml.html.usedoctest + >>> def parse(text): + ... print HtmlEmitter(Parser(text).parse()).emit() + + >>> parse(u'====test====') + <h4>test</h4> + + >>> parse(u'===[[test]]===') + <h3><a href="test">test</a></h3> + + >>> parse(u'==test //italic// **bold** test==') + <h2>test <i>italic</i> <b>bold</b> test</h2> + + >>> parse(u'==test==\ntest') + <h2>test</h2> + <p>test</p> + """ + + def _head_repl(self, groups): + # Note: this code can be shorter, I wrote it like this + # so that I can comment on every step. + # + # The shorter version would look like this: + # + # self.cur = creole.DocNode('header', self.cur) + # self.cur.level = len(groups.get('head_head', ' ')) + # self.parse_inline(groups.get('head_text', '').strip()) + # self.cur = self._upto(self.cur, ('document', 'section', 'blockquote')) + # self.text = None + + # Make sure we are not inside a paragraph, list, other heading, etc. + self.cur = self._upto(self.cur, ('document', 'section', 'blockquote')) + # Remember where we are. + parent = self.cur + # Create the heading node. + node = creole.DocNode('header', self.cur) + # Set its level. + node.level = len(groups.get('head_head', ' ')) + # Make it the curret node. + self.cur = node + # Parse the text inside the heading. + text = groups.get('head_text', '').strip() + self.parse_inline(text) + # Go back to where we started. + self.cur = parent + self.text = None + +if __name__=="__main__": +# import sys +# raw = unicode(sys.stdin.read(), 'utf-8', 'ignore') +# rules = Rules() +# document = Parser(raw, rules).parse() +# html = HtmlEmitter(document).emit() +# sys.stdout.write(html.encode('utf-8', 'ignore')) + +# Instead of the above code to convert text from command line, we run tests: + + import doctest + doctest.testmod() +
--- a/setup.py Sat Jan 16 23:11:50 2010 +0100 +++ b/setup.py Tue Jun 01 21:59:03 2010 +0200 @@ -17,11 +17,12 @@ keywords='wiki wikicreole creole markup text', py_modules=['creole'], data_files=[ - ('share/doc/creole.py/examples', ['creole2html.py']), - ('share/doc/creole.py/tests', ['test.py']), + ('share/doc/creole.py/examples', + ['exmaples/creole2html.py', 'exmaples/headings.py']), + ('share/doc/creole.py/tests', ['tests/test_html_emitter.py']), ('share/doc/creole.py', ['COPYING']), ], - scripts=['creole2html.py'], + scripts=['examples/creole2html.py'], platforms='any', classifiers=[ 'Topic :: Text Processing :: Markup',
--- a/test.py Sat Jan 16 23:11:50 2010 +0100 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,8 +0,0 @@ -#!/usr/bin/python -# -*- coding: utf-8 -*- - -import doctest -import creole2html - -if __name__ == "__main__": - doctest.testmod(creole2html)
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tests/test_html_emitter.py Tue Jun 01 21:59:03 2010 +0200 @@ -0,0 +1,13 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +import os +import sys +import doctest + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..')) + +import creole.html_emitter + +if __name__ == "__main__": + doctest.testmod(creole.html_emitter)