* Serious reworking of HTML2Text to handle nested lists reasonably

* Adding more unittests for the nested lists

diff --git a/rss2maildir.py b/rss2maildir.py
index a51209ce0749e70fd5e9d40ff696d8cf6a2243e1..c70c65b24ec51b6c0334c40f7fc9bb07bbe60303 100755 (executable)
--- a/rss2maildir.py
+++ b/rss2maildir.py
@@ -92,6 +92,7 @@ class HTML2Text(HTMLParser):
         self.textwidth = textwidth
         self.opentags = []
         self.indentlevel = 0
         self.textwidth = textwidth
         self.opentags = []
         self.indentlevel = 0

+        self.listcount = []

         HTMLParser.__init__(self)
 
     def handle_starttag(self, tag, attrs):
         HTMLParser.__init__(self)
 
     def handle_starttag(self, tag, attrs):


@@ -105,11 +106,33 @@ class HTML2Text(HTMLParser):
                 self.opentags.append(tag_name)
                 self.opentags.pop()
 
                 self.opentags.append(tag_name)
                 self.opentags.pop()
 

+            if tag_name == u'ol':
+                self.handle_curdata()
+                self.listcount.append(1)
+                self.listlevel = len(self.listcount) - 1
+
+            if tag_name in self.liststarttags:
+                smallist = self.opentags[-3:]
+                smallist.reverse()
+                for prev_listtag in smallist:
+                    if prev_listtag in [u'dl', u'ol']:
+                        self.indentlevel = self.indentlevel + 4
+                        break
+                    elif prev_listtag == u'ul':
+                        self.indentlevel = self.indentlevel + 3
+                        break
+

             if len(self.opentags) > 0:
                 self.handle_curdata()
             if len(self.opentags) > 0:
                 self.handle_curdata()

-                self.opentags.pop()
+                if tag_name not in self.cancontainflow:
+                    self.opentags.pop()

             self.opentags.append(tag_name)
         else:
             self.opentags.append(tag_name)
         else:

+            listcount = 0
+            try:
+                listcount = self.listcount[-1]
+            except:
+                pass

             self.handle_curdata()
             self.opentags.append(tag_name)
 
             self.handle_curdata()
             self.opentags.append(tag_name)
 


@@ -126,19 +149,31 @@ class HTML2Text(HTMLParser):
         if len(self.curdata) == 0:
             return
 
         if len(self.curdata) == 0:
             return
 

+        if len(self.curdata.strip()) == 0:
+            return
+

         tag_thats_done = self.opentags[-1]
 
         if tag_thats_done in self.blockleveltags:
             newlinerequired = self.text != u''
             if newlinerequired:
         tag_thats_done = self.opentags[-1]
 
         if tag_thats_done in self.blockleveltags:
             newlinerequired = self.text != u''
             if newlinerequired:

-                self.text = self.text + u'\n\n'
+                if newlinerequired \
+                    and len(self.text) > 2 \
+                    and self.text[-1] != u'\n' \
+                    and self.text[-2] != u'\n':
+                    self.text = self.text + u'\n\n'

 
         if tag_thats_done in ["h1", "h2", "h3", "h4", "h5", "h6"]:
             underline = u''
             underlinechar = u'='
             headingtext = self.curdata.encode("utf-8").strip()
 
         if tag_thats_done in ["h1", "h2", "h3", "h4", "h5", "h6"]:
             underline = u''
             underlinechar = u'='
             headingtext = self.curdata.encode("utf-8").strip()

-            headingtext = u'\n'.join( \
-                textwrap.wrap(headingtext, self.textwidth))
+            seperator = u'\n' + u' '*self.indentlevel
+            headingtext = seperator.join( \
+                textwrap.wrap( \
+                    headingtext, \
+                    self.textwidth - self.indentlevel \
+                    ) \
+                )

 
             if tag_thats_done == u'h2':
                 underlinechar = u'-'
 
             if tag_thats_done == u'h2':
                 underlinechar = u'-'


@@ -146,37 +181,76 @@ class HTML2Text(HTMLParser):
                 underlinechar = u'~'
 
             if u'\n' in headingtext:
                 underlinechar = u'~'
 
             if u'\n' in headingtext:

-                underline = underlinechar * self.textwidth
+                underline = u' ' * self.indentlevel \
+                    + underlinechar * (self.textwidth - self.indentlevel)

             else:
             else:

-                underline = underlinechar * len(headingtext)
+                underline = u' ' * self.indentlevel \
+                    + underlinechar * len(headingtext)

             self.text = self.text \
                 + headingtext.encode("utf-8") + u'\n' \
                 + underline
         elif tag_thats_done == "p":
             paragraph = self.curdata.encode("utf-8").strip()
             self.text = self.text \
                 + headingtext.encode("utf-8") + u'\n' \
                 + underline
         elif tag_thats_done == "p":
             paragraph = self.curdata.encode("utf-8").strip()

+            seperator = u'\n' + u' ' * self.indentlevel

             self.text = self.text \
             self.text = self.text \

-                + u'\n'.join(textwrap.wrap(paragraph, self.textwidth))
+                + u' ' * self.indentlevel \
+                + seperator.join(textwrap.wrap(paragraph, self.textwidth - self.indentlevel))

         elif tag_thats_done == "pre":
             self.text = self.text + self.curdata
         elif tag_thats_done == "blockquote":
             quote = self.curdata.encode("utf-8").strip()
         elif tag_thats_done == "pre":
             self.text = self.text + self.curdata
         elif tag_thats_done == "blockquote":
             quote = self.curdata.encode("utf-8").strip()

+            seperator = u'\n' + u' ' * self.indentlevel + u'> '

             self.text = self.text \
                 + u'> ' \
             self.text = self.text \
                 + u'> ' \

-                + u'> '.join(textwrap.wrap(quote, self.textwidth - 2))
+                + seperator.join( \
+                    textwrap.wrap( \
+                        quote, \
+                        self.textwidth - self.indentlevel - 2 \
+                    )
+                )

         elif tag_thats_done == "li":
             item = self.curdata.encode("utf-8").strip()
             if len(self.text) > 0 and self.text[-1] != u'\n':
                 self.text = self.text + u'\n'
         elif tag_thats_done == "li":
             item = self.curdata.encode("utf-8").strip()
             if len(self.text) > 0 and self.text[-1] != u'\n':
                 self.text = self.text + u'\n'

+            # work out if we're in an ol rather than a ul
+            latesttags = self.opentags[-4:]
+            latesttags.reverse()
+            isul = False
+            for thing in latesttags:
+                if thing == 'ul':
+                    isul = True
+                    break
+                elif thing == 'ol':
+                    isul = False
+                    break
+
+            listindent = 3
+            if not isul:
+                listindent = 4
+
+            listmarker = u' * '
+            if not isul:
+                listmarker = u' %2d. ' %(self.listcount[-1])
+                self.listcount[-1] = self.listcount[-1] + 1
+
+            seperator = u'\n' \
+                + u' ' * self.indentlevel \
+                + u' ' * listindent

             self.text = self.text \
             self.text = self.text \

-                + u' * ' \
-                + u'\n   '.join( \
-                    textwrap.wrap(item, self.textwidth - 3))
+                + u' ' * self.indentlevel \
+                + listmarker \
+                + seperator.join( \
+                    textwrap.wrap( \
+                        item, \
+                        self.textwidth - self.indentlevel - listindent \
+                    ) \
+                )

             self.curdata = u''
         elif tag_thats_done == "dt":
             definition = self.curdata.encode("utf-8").strip()
             if len(self.text) > 0 and self.text[-1] != u'\n':
                 self.text = self.text + u'\n\n'
             self.curdata = u''
         elif tag_thats_done == "dt":
             definition = self.curdata.encode("utf-8").strip()
             if len(self.text) > 0 and self.text[-1] != u'\n':
                 self.text = self.text + u'\n\n'

-            elif len(self.text) > 0 and self.text[-2] != u'\n':
+            elif len(self.text) > 1 and self.text[-2] != u'\n':

                 self.text = self.text + u'\n'
             definition = definition + "::"
             self.text = self.text \
                 self.text = self.text + u'\n'
             definition = definition + "::"
             self.text = self.text \


@@ -185,13 +259,18 @@ class HTML2Text(HTMLParser):
             self.curdata = u''
         elif tag_thats_done == "dd":
             definition = self.curdata.encode("utf-8").strip()
             self.curdata = u''
         elif tag_thats_done == "dd":
             definition = self.curdata.encode("utf-8").strip()

-            if len(self.text) > 0 and self.text[-1] != u'\n':
-                self.text = self.text + u'\n'
-            self.text = self.text \
-                + '    ' \
-                + '\n    '.join( \
-                    textwrap.wrap(definition, self.textwidth - 4))
-            self.curdata = u''
+            if len(definition) > 0:
+                if len(self.text) > 0 and self.text[-1] != u'\n':
+                    self.text = self.text + u'\n'
+                self.text = self.text \
+                    + '    ' \
+                    + '\n    '.join( \
+                        textwrap.wrap( \
+                            definition, \
+                            self.textwidth - self.indentlevel - 4 \
+                            ) \
+                        )
+                self.curdata = u''

         elif tag_thats_done in self.liststarttags:
             pass
         else:
         elif tag_thats_done in self.liststarttags:
             pass
         else:


@@ -215,6 +294,24 @@ class HTML2Text(HTMLParser):
             # err. weird.
             tagindex = 0
 
             # err. weird.
             tagindex = 0
 

+        tag = tag.lower()
+
+        if tag in self.liststarttags:
+            if tag in [u'ol', u'dl', u'ul']:
+                # find if there was a previous list level
+                smalllist = self.opentags[:-1]
+                smalllist.reverse()
+                for prev_listtag in smalllist:
+                    if prev_listtag in [u'ol', u'dl']:
+                        self.indentlevel = self.indentlevel - 4
+                        break
+                    elif prev_listtag == u'ul':
+                        self.indentlevel = self.indentlevel - 3
+                        break
+
+        if tag == u'ol':
+            self.listcount = self.listcount[:-1]
+

         while tagindex < len(self.opentags) \
             and tag in self.opentags[tagindex+1:]:
             try:
         while tagindex < len(self.opentags) \
             and tag in self.opentags[tagindex+1:]:
             try:


@@ -226,7 +323,11 @@ class HTML2Text(HTMLParser):
             # Assuming the data was for the last opened tag first
             self.handle_curdata()
             # Now kill the list to be a slice before this tag was opened
             # Assuming the data was for the last opened tag first
             self.handle_curdata()
             # Now kill the list to be a slice before this tag was opened

-            self.opentags = self.opentags[:tagindex]
+            self.opentags = self.opentags[:tagindex + 1]
+        else:
+            self.handle_curdata()
+            if self.opentags[-1] == tag:
+                self.opentags.pop()

 
     def handle_data(self, data):
         self.curdata = self.curdata + unicode(data, "utf-8")
 
     def handle_data(self, data):
         self.curdata = self.curdata + unicode(data, "utf-8")

diff --git a/tests/expected/nestedlists-wellformed.txt b/tests/expected/nestedlists-wellformed.txt
new file mode 100644 (file)
index 0000000..8344bf7
--- /dev/null
+++ b/tests/expected/nestedlists-wellformed.txt
@@ -0,0 +1,7 @@
+When the clock strikes one::
+     * bullet point 1
+     * bullet point 2
+
+When the clock strikes two::
+      1. first item
+      2. second item

diff --git a/tests/expected/nestedorderedlists-wellformed.txt b/tests/expected/nestedorderedlists-wellformed.txt
new file mode 100644 (file)
index 0000000..8cb909f
--- /dev/null
+++ b/tests/expected/nestedorderedlists-wellformed.txt
@@ -0,0 +1,6 @@
+  1. When the clock strikes one
+      1. Weird things happen
+      2. Really weird things
+  2. And then we go on
+      1. first item
+      2. second item

diff --git a/tests/expected/unorderedlist-wellformed.txt b/tests/expected/unorderedlist-wellformed.txt
index c1edfe31a687bc214a40c3ed3ff9f9b7a02a2a4f..02f68acc012d1f432de49b06c606a14ac507b80b 100644 (file)
--- a/tests/expected/unorderedlist-wellformed.txt
+++ b/tests/expected/unorderedlist-wellformed.txt
@@ -1,5 +1,4 @@
 This is a well formed unordered list.
 This is a well formed unordered list.

-

  * The first item
  * The second item
  * The third item
  * The first item
  * The second item
  * The third item

diff --git a/tests/html/nestedlists-wellformed.html b/tests/html/nestedlists-wellformed.html
new file mode 100644 (file)
index 0000000..2efc06b
--- /dev/null
+++ b/tests/html/nestedlists-wellformed.html
@@ -0,0 +1,16 @@
+<dl>
+    <dt>When the clock strikes one</dt>
+    <dd>
+        <ul>
+            <li>bullet point 1</li>
+            <li>bullet point 2</li>
+        </ul>
+    </dd>
+    <dt>When the clock strikes two</dt>
+    <dd>
+        <ol>
+            <li>first item</li>
+            <li>second item</li>
+        </ol>
+    </dd>
+</dl>

diff --git a/tests/html/nestedorderedlists-wellformed.html b/tests/html/nestedorderedlists-wellformed.html
new file mode 100644 (file)
index 0000000..759c0d0
--- /dev/null
+++ b/tests/html/nestedorderedlists-wellformed.html
@@ -0,0 +1,14 @@
+<ol>
+    <li>When the clock strikes one
+        <ol>
+            <li>Weird things happen</li>
+            <li>Really weird things</li>
+        </ol>
+    </li>
+    <li>And then we go on
+        <ol>
+            <li>first item</li>
+            <li>second item</li>
+        </ol>
+    </li>
+</dl>

diff --git a/tests/unittests/NestedListTests.py b/tests/unittests/NestedListTests.py
new file mode 100755 (executable)
index 0000000..1e4ccf5
--- /dev/null
+++ b/tests/unittests/NestedListTests.py
@@ -0,0 +1,23 @@
+#!/usr/bin/python
+
+import unittest
+import sys
+import os
+
+import ParsingTests
+
+class NestedListTests(ParsingTests.ParsingTest):
+    def testWellFormedNestedLists(self):
+        return self.runParsingTest("nestedlists-wellformed")
+
+    def testWellFormedNestedOrderedLists(self):
+        return self.runParsingTest("nestedorderedlists-wellformed")
+
+def suite():
+    suite = unittest.TestSuite()
+    suite.addTest(NestedListTests("testWellFormedNestedLists"))
+    suite.addTest(NestedListTests("testWellFormedNestedOrderedLists"))
+    return suite
+
+if __name__ == "__main__":
+    unittest.main()

diff --git a/tests/unittests/UnorderedListTests.py b/tests/unittests/UnorderedListTests.py
index a49546165a18e1a32f6fb4e56e376f353423d6b0..026274a350eb249bb74733cccf960ac912a77620 100755 (executable)
--- a/tests/unittests/UnorderedListTests.py
+++ b/tests/unittests/UnorderedListTests.py
@@ -8,10 +8,10 @@ import ParsingTests
 
 class UnorderedListTests(ParsingTests.ParsingTest):
     def testWellFormedList(self):
 
 class UnorderedListTests(ParsingTests.ParsingTest):
     def testWellFormedList(self):

-        return self.runParsingTest("unordered-wellformed")
+        return self.runParsingTest("unorderedlist-wellformed")

 
     def testBadlyFormedList(self):
 
     def testBadlyFormedList(self):

-        return self.runParsingTest("unordered-badlyformed")
+        return self.runParsingTest("unorderedlist-badlyformed")

 
 def suite():
     suite = unittest.TestSuite()
 
 def suite():
     suite = unittest.TestSuite()