From ce65d98d57ea2b4332c291c0eed448f392201421 Mon Sep 17 00:00:00 2001 From: Brett Parker Date: Sun, 6 Jan 2008 22:39:04 +0000 Subject: [PATCH] * Serious reworking of HTML2Text to handle nested lists reasonably * Adding more unittests for the nested lists --- rss2maildir.py | 141 +++++++++++++++--- tests/expected/nestedlists-wellformed.txt | 7 + .../nestedorderedlists-wellformed.txt | 6 + tests/expected/unorderedlist-wellformed.txt | 1 - tests/html/nestedlists-wellformed.html | 16 ++ tests/html/nestedorderedlists-wellformed.html | 14 ++ tests/unittests/NestedListTests.py | 23 +++ tests/unittests/UnorderedListTests.py | 4 +- 8 files changed, 189 insertions(+), 23 deletions(-) create mode 100644 tests/expected/nestedlists-wellformed.txt create mode 100644 tests/expected/nestedorderedlists-wellformed.txt create mode 100644 tests/html/nestedlists-wellformed.html create mode 100644 tests/html/nestedorderedlists-wellformed.html create mode 100755 tests/unittests/NestedListTests.py diff --git a/rss2maildir.py b/rss2maildir.py index a51209c..c70c65b 100755 --- a/rss2maildir.py +++ b/rss2maildir.py @@ -92,6 +92,7 @@ class HTML2Text(HTMLParser): self.textwidth = textwidth self.opentags = [] self.indentlevel = 0 + self.listcount = [] HTMLParser.__init__(self) def handle_starttag(self, tag, attrs): @@ -105,11 +106,33 @@ class HTML2Text(HTMLParser): self.opentags.append(tag_name) self.opentags.pop() + if tag_name == u'ol': + self.handle_curdata() + self.listcount.append(1) + self.listlevel = len(self.listcount) - 1 + + if tag_name in self.liststarttags: + smallist = self.opentags[-3:] + smallist.reverse() + for prev_listtag in smallist: + if prev_listtag in [u'dl', u'ol']: + self.indentlevel = self.indentlevel + 4 + break + elif prev_listtag == u'ul': + self.indentlevel = self.indentlevel + 3 + break + if len(self.opentags) > 0: self.handle_curdata() - self.opentags.pop() + if tag_name not in self.cancontainflow: + self.opentags.pop() self.opentags.append(tag_name) else: + listcount = 0 + try: + listcount = self.listcount[-1] + except: + pass self.handle_curdata() self.opentags.append(tag_name) @@ -126,19 +149,31 @@ class HTML2Text(HTMLParser): if len(self.curdata) == 0: return + if len(self.curdata.strip()) == 0: + return + tag_thats_done = self.opentags[-1] if tag_thats_done in self.blockleveltags: newlinerequired = self.text != u'' if newlinerequired: - self.text = self.text + u'\n\n' + if newlinerequired \ + and len(self.text) > 2 \ + and self.text[-1] != u'\n' \ + and self.text[-2] != u'\n': + self.text = self.text + u'\n\n' if tag_thats_done in ["h1", "h2", "h3", "h4", "h5", "h6"]: underline = u'' underlinechar = u'=' headingtext = self.curdata.encode("utf-8").strip() - headingtext = u'\n'.join( \ - textwrap.wrap(headingtext, self.textwidth)) + seperator = u'\n' + u' '*self.indentlevel + headingtext = seperator.join( \ + textwrap.wrap( \ + headingtext, \ + self.textwidth - self.indentlevel \ + ) \ + ) if tag_thats_done == u'h2': underlinechar = u'-' @@ -146,37 +181,76 @@ class HTML2Text(HTMLParser): underlinechar = u'~' if u'\n' in headingtext: - underline = underlinechar * self.textwidth + underline = u' ' * self.indentlevel \ + + underlinechar * (self.textwidth - self.indentlevel) else: - underline = underlinechar * len(headingtext) + underline = u' ' * self.indentlevel \ + + underlinechar * len(headingtext) self.text = self.text \ + headingtext.encode("utf-8") + u'\n' \ + underline elif tag_thats_done == "p": paragraph = self.curdata.encode("utf-8").strip() + seperator = u'\n' + u' ' * self.indentlevel self.text = self.text \ - + u'\n'.join(textwrap.wrap(paragraph, self.textwidth)) + + u' ' * self.indentlevel \ + + seperator.join(textwrap.wrap(paragraph, self.textwidth - self.indentlevel)) elif tag_thats_done == "pre": self.text = self.text + self.curdata elif tag_thats_done == "blockquote": quote = self.curdata.encode("utf-8").strip() + seperator = u'\n' + u' ' * self.indentlevel + u'> ' self.text = self.text \ + u'> ' \ - + u'> '.join(textwrap.wrap(quote, self.textwidth - 2)) + + seperator.join( \ + textwrap.wrap( \ + quote, \ + self.textwidth - self.indentlevel - 2 \ + ) + ) elif tag_thats_done == "li": item = self.curdata.encode("utf-8").strip() if len(self.text) > 0 and self.text[-1] != u'\n': self.text = self.text + u'\n' + # work out if we're in an ol rather than a ul + latesttags = self.opentags[-4:] + latesttags.reverse() + isul = False + for thing in latesttags: + if thing == 'ul': + isul = True + break + elif thing == 'ol': + isul = False + break + + listindent = 3 + if not isul: + listindent = 4 + + listmarker = u' * ' + if not isul: + listmarker = u' %2d. ' %(self.listcount[-1]) + self.listcount[-1] = self.listcount[-1] + 1 + + seperator = u'\n' \ + + u' ' * self.indentlevel \ + + u' ' * listindent self.text = self.text \ - + u' * ' \ - + u'\n '.join( \ - textwrap.wrap(item, self.textwidth - 3)) + + u' ' * self.indentlevel \ + + listmarker \ + + seperator.join( \ + textwrap.wrap( \ + item, \ + self.textwidth - self.indentlevel - listindent \ + ) \ + ) self.curdata = u'' elif tag_thats_done == "dt": definition = self.curdata.encode("utf-8").strip() if len(self.text) > 0 and self.text[-1] != u'\n': self.text = self.text + u'\n\n' - elif len(self.text) > 0 and self.text[-2] != u'\n': + elif len(self.text) > 1 and self.text[-2] != u'\n': self.text = self.text + u'\n' definition = definition + "::" self.text = self.text \ @@ -185,13 +259,18 @@ class HTML2Text(HTMLParser): self.curdata = u'' elif tag_thats_done == "dd": definition = self.curdata.encode("utf-8").strip() - if len(self.text) > 0 and self.text[-1] != u'\n': - self.text = self.text + u'\n' - self.text = self.text \ - + ' ' \ - + '\n '.join( \ - textwrap.wrap(definition, self.textwidth - 4)) - self.curdata = u'' + if len(definition) > 0: + if len(self.text) > 0 and self.text[-1] != u'\n': + self.text = self.text + u'\n' + self.text = self.text \ + + ' ' \ + + '\n '.join( \ + textwrap.wrap( \ + definition, \ + self.textwidth - self.indentlevel - 4 \ + ) \ + ) + self.curdata = u'' elif tag_thats_done in self.liststarttags: pass else: @@ -215,6 +294,24 @@ class HTML2Text(HTMLParser): # err. weird. tagindex = 0 + tag = tag.lower() + + if tag in self.liststarttags: + if tag in [u'ol', u'dl', u'ul']: + # find if there was a previous list level + smalllist = self.opentags[:-1] + smalllist.reverse() + for prev_listtag in smalllist: + if prev_listtag in [u'ol', u'dl']: + self.indentlevel = self.indentlevel - 4 + break + elif prev_listtag == u'ul': + self.indentlevel = self.indentlevel - 3 + break + + if tag == u'ol': + self.listcount = self.listcount[:-1] + while tagindex < len(self.opentags) \ and tag in self.opentags[tagindex+1:]: try: @@ -226,7 +323,11 @@ class HTML2Text(HTMLParser): # Assuming the data was for the last opened tag first self.handle_curdata() # Now kill the list to be a slice before this tag was opened - self.opentags = self.opentags[:tagindex] + self.opentags = self.opentags[:tagindex + 1] + else: + self.handle_curdata() + if self.opentags[-1] == tag: + self.opentags.pop() def handle_data(self, data): self.curdata = self.curdata + unicode(data, "utf-8") diff --git a/tests/expected/nestedlists-wellformed.txt b/tests/expected/nestedlists-wellformed.txt new file mode 100644 index 0000000..8344bf7 --- /dev/null +++ b/tests/expected/nestedlists-wellformed.txt @@ -0,0 +1,7 @@ +When the clock strikes one:: + * bullet point 1 + * bullet point 2 + +When the clock strikes two:: + 1. first item + 2. second item diff --git a/tests/expected/nestedorderedlists-wellformed.txt b/tests/expected/nestedorderedlists-wellformed.txt new file mode 100644 index 0000000..8cb909f --- /dev/null +++ b/tests/expected/nestedorderedlists-wellformed.txt @@ -0,0 +1,6 @@ + 1. When the clock strikes one + 1. Weird things happen + 2. Really weird things + 2. And then we go on + 1. first item + 2. second item diff --git a/tests/expected/unorderedlist-wellformed.txt b/tests/expected/unorderedlist-wellformed.txt index c1edfe3..02f68ac 100644 --- a/tests/expected/unorderedlist-wellformed.txt +++ b/tests/expected/unorderedlist-wellformed.txt @@ -1,5 +1,4 @@ This is a well formed unordered list. - * The first item * The second item * The third item diff --git a/tests/html/nestedlists-wellformed.html b/tests/html/nestedlists-wellformed.html new file mode 100644 index 0000000..2efc06b --- /dev/null +++ b/tests/html/nestedlists-wellformed.html @@ -0,0 +1,16 @@ +
+
When the clock strikes one
+
+
    +
  • bullet point 1
    • +
  • bullet point 2
    • +
+
+
When the clock strikes two
+
+
    +
  1. first item
    2. +
  2. second item
    3. +
+
+
diff --git a/tests/html/nestedorderedlists-wellformed.html b/tests/html/nestedorderedlists-wellformed.html new file mode 100644 index 0000000..759c0d0 --- /dev/null +++ b/tests/html/nestedorderedlists-wellformed.html @@ -0,0 +1,14 @@ +
    +
  1. When the clock strikes one +
      +
    1. Weird things happen
      2. +
    2. Really weird things
      3. +
    +
    2. +
  2. And then we go on +
      +
    1. first item
      2. +
    2. second item
      3. +
    +
    3. + diff --git a/tests/unittests/NestedListTests.py b/tests/unittests/NestedListTests.py new file mode 100755 index 0000000..1e4ccf5 --- /dev/null +++ b/tests/unittests/NestedListTests.py @@ -0,0 +1,23 @@ +#!/usr/bin/python + +import unittest +import sys +import os + +import ParsingTests + +class NestedListTests(ParsingTests.ParsingTest): + def testWellFormedNestedLists(self): + return self.runParsingTest("nestedlists-wellformed") + + def testWellFormedNestedOrderedLists(self): + return self.runParsingTest("nestedorderedlists-wellformed") + +def suite(): + suite = unittest.TestSuite() + suite.addTest(NestedListTests("testWellFormedNestedLists")) + suite.addTest(NestedListTests("testWellFormedNestedOrderedLists")) + return suite + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unittests/UnorderedListTests.py b/tests/unittests/UnorderedListTests.py index a495461..026274a 100755 --- a/tests/unittests/UnorderedListTests.py +++ b/tests/unittests/UnorderedListTests.py @@ -8,10 +8,10 @@ import ParsingTests class UnorderedListTests(ParsingTests.ParsingTest): def testWellFormedList(self): - return self.runParsingTest("unordered-wellformed") + return self.runParsingTest("unorderedlist-wellformed") def testBadlyFormedList(self): - return self.runParsingTest("unordered-badlyformed") + return self.runParsingTest("unorderedlist-badlyformed") def suite(): suite = unittest.TestSuite() -- 2.39.5