From f47e451a2ae7cde0b422266b10b131d00d398e87 Mon Sep 17 00:00:00 2001
From: Brett Parker <iDunno@sommitrealweird.co.uk>
Date: Sun, 6 Jan 2008 11:43:44 +0000
Subject: [PATCH] * Small improvements to the HTML2Text code * Reorganize
unittests for parsing to make it easier to add more tests later
---
rss2maildir.py | 27 +++++++++++++
tests/expected/definitionlist-badlyformed.txt | 6 +++
tests/expected/definitionlist-wellformed.txt | 4 +-
tests/html/definitionlist-badlyformed.html | 6 +++
tests/unittests/DefinitionListTests.py | 26 ++++---------
tests/unittests/ParagraphTests.py | 21 ++--------
tests/unittests/ParsingTests.py | 26 +++++++++++++
tests/unittests/UnorderedListTests.py | 38 ++-----------------
8 files changed, 81 insertions(+), 73 deletions(-)
create mode 100644 tests/expected/definitionlist-badlyformed.txt
create mode 100644 tests/html/definitionlist-badlyformed.html
create mode 100755 tests/unittests/ParsingTests.py
diff --git a/rss2maildir.py b/rss2maildir.py
index 739c1f3..a51209c 100755
--- a/rss2maildir.py
+++ b/rss2maildir.py
@@ -123,6 +123,9 @@ class HTML2Text(HTMLParser):
if len(self.opentags) == 0:
return
+ if len(self.curdata) == 0:
+ return
+
tag_thats_done = self.opentags[-1]
if tag_thats_done in self.blockleveltags:
@@ -169,6 +172,26 @@ class HTML2Text(HTMLParser):
+ u'\n '.join( \
textwrap.wrap(item, self.textwidth - 3))
self.curdata = u''
+ elif tag_thats_done == "dt":
+ definition = self.curdata.encode("utf-8").strip()
+ if len(self.text) > 0 and self.text[-1] != u'\n':
+ self.text = self.text + u'\n\n'
+ elif len(self.text) > 0 and self.text[-2] != u'\n':
+ self.text = self.text + u'\n'
+ definition = definition + "::"
+ self.text = self.text \
+ + '\n '.join(
+ textwrap.wrap(definition, self.textwidth - 1))
+ self.curdata = u''
+ elif tag_thats_done == "dd":
+ definition = self.curdata.encode("utf-8").strip()
+ if len(self.text) > 0 and self.text[-1] != u'\n':
+ self.text = self.text + u'\n'
+ self.text = self.text \
+ + ' ' \
+ + '\n '.join( \
+ textwrap.wrap(definition, self.textwidth - 4))
+ self.curdata = u''
elif tag_thats_done in self.liststarttags:
pass
else:
@@ -224,6 +247,10 @@ class HTML2Text(HTMLParser):
if len(self.text) == 0 or self.text[-1] != u'\n':
self.text = self.text + u'\n'
self.opentags = []
+ if len(self.text) > 0:
+ while len(self.text) > 1 and self.text[-1] == u'\n':
+ self.text = self.text[:-1]
+ self.text = self.text + u'\n'
return self.text
def open_url(method, url):
diff --git a/tests/expected/definitionlist-badlyformed.txt b/tests/expected/definitionlist-badlyformed.txt
new file mode 100644
index 0000000..3d9e83f
--- /dev/null
+++ b/tests/expected/definitionlist-badlyformed.txt
@@ -0,0 +1,6 @@
+An item::
+ It's definition
+
+Another item::
+ And it's got a much longer definition because we like to make sure
+ that we've got the test wrapping right don't we.
diff --git a/tests/expected/definitionlist-wellformed.txt b/tests/expected/definitionlist-wellformed.txt
index 98beb1b..3d9e83f 100644
--- a/tests/expected/definitionlist-wellformed.txt
+++ b/tests/expected/definitionlist-wellformed.txt
@@ -1,6 +1,6 @@
-An item
+An item::
It's definition
-Another item
+Another item::
And it's got a much longer definition because we like to make sure
that we've got the test wrapping right don't we.
diff --git a/tests/html/definitionlist-badlyformed.html b/tests/html/definitionlist-badlyformed.html
new file mode 100644
index 0000000..562bad4
--- /dev/null
+++ b/tests/html/definitionlist-badlyformed.html
@@ -0,0 +1,6 @@
+<dl>
+ <dt>An item
+ <dd>It's definition
+ <dt>Another item
+ <dd>And it's got a much longer definition because we like to make sure that we've got the test wrapping right don't we.
+</dl>
diff --git a/tests/unittests/DefinitionListTests.py b/tests/unittests/DefinitionListTests.py
index 9058973..a1abae0 100755
--- a/tests/unittests/DefinitionListTests.py
+++ b/tests/unittests/DefinitionListTests.py
@@ -1,33 +1,21 @@
#!/usr/bin/python
import unittest
-import sys
import os
-class DefinitionListTests(unittest.TestCase):
- def setUp(self):
- self.inputpath = os.path.sep.join(os.path.dirname(os.path.realpath(__file__)).split(os.path.sep)[0:-1])
+import ParsingTests
+class DefinitionListTests(ParsingTests.ParsingTest):
def testWellFormedDefinitionList(self):
- try:
- from rss2maildir import HTML2Text
- except:
- sys.path.append(os.path.sep.join(self.inputpath.split(os.path.sep)[0:-1]))
- try:
- from rss2maildir import HTML2Text
- except:
- self.assert_(False)
- input_path = os.path.sep.join(os.path.dirname(os.path.realpath(__file__)).split(os.path.sep)[0:-1])
- input = open(os.path.join(input_path, "html", "definitionlist-wellformed.html")).read()
- expectedoutput = open(os.path.join(input_path, "expected", "definitionlist-wellformed.txt")).read()
- parser = HTML2Text()
- parser.feed(input)
- output = parser.gettext()
- self.assertEqual(output, expectedoutput)
+ return self.runParsingTest("definitionlist-wellformed")
+
+ def testBadlyFormedDefinitionList(self):
+ return self.runParsingTest("definitionlist-badlyformed")
def suite():
suite = unittest.TestSuite()
suite.addTest(DefinitionListTests("testWellFormedDefinitionList"))
+ suite.addTest(DefinitionListTests("testBadlyFormedDefinitionList"))
return suite
if __name__ == "__main__":
diff --git a/tests/unittests/ParagraphTests.py b/tests/unittests/ParagraphTests.py
index 9111ecd..8b98a53 100755
--- a/tests/unittests/ParagraphTests.py
+++ b/tests/unittests/ParagraphTests.py
@@ -4,26 +4,11 @@ import unittest
import sys
import os
-class ParagraphTests(unittest.TestCase):
- def setUp(self):
- self.inputpath = os.path.sep.join(os.path.dirname(os.path.realpath(__file__)).split(os.path.sep)[0:-1])
+import ParsingTests
+class ParagraphTests(ParsingTests.ParsingTest):
def testWellFormedParagraphs(self):
- try:
- from rss2maildir import HTML2Text
- except:
- sys.path.append(os.path.sep.join(self.inputpath.split(os.path.sep)[0:-1]))
- try:
- from rss2maildir import HTML2Text
- except:
- self.assert_(False)
- input_path = os.path.sep.join(os.path.dirname(os.path.realpath(__file__)).split(os.path.sep)[0:-1])
- input = open(os.path.join(input_path, "html", "multiparagraph-wellformed.html")).read()
- expectedoutput = open(os.path.join(input_path, "expected", "multiparagraph-wellformed.txt")).read()
- parser = HTML2Text()
- parser.feed(input)
- output = parser.gettext()
- self.assertEqual(output, expectedoutput)
+ return self.runParsingTest("multiparagraph-wellformed")
def suite():
suite = unittest.TestSuite()
diff --git a/tests/unittests/ParsingTests.py b/tests/unittests/ParsingTests.py
new file mode 100755
index 0000000..20fc521
--- /dev/null
+++ b/tests/unittests/ParsingTests.py
@@ -0,0 +1,26 @@
+#!/usr/bin/python
+
+import unittest
+import sys
+import os
+
+class ParsingTest(unittest.TestCase):
+ def setUp(self):
+ self.inputpath = os.path.sep.join(os.path.dirname(os.path.realpath(__file__)).split(os.path.sep)[0:-1])
+
+ def runParsingTest(self, filename):
+ try:
+ from rss2maildir import HTML2Text
+ except:
+ sys.path.append(os.path.sep.join(self.inputpath.split(os.path.sep)[0:-1]))
+ try:
+ from rss2maildir import HTML2Text
+ except:
+ self.assert_(False)
+ input_path = os.path.sep.join(os.path.dirname(os.path.realpath(__file__)).split(os.path.sep)[0:-1])
+ input = open(os.path.join(input_path, "html", filename + ".html")).read()
+ expectedoutput = open(os.path.join(input_path, "expected", filename + ".txt")).read()
+ parser = HTML2Text()
+ parser.feed(input)
+ output = parser.gettext()
+ self.assertEqual(output, expectedoutput)
diff --git a/tests/unittests/UnorderedListTests.py b/tests/unittests/UnorderedListTests.py
index 2224e74..a495461 100755
--- a/tests/unittests/UnorderedListTests.py
+++ b/tests/unittests/UnorderedListTests.py
@@ -4,44 +4,14 @@ import unittest
import sys
import os
-class UnorderedListTests(unittest.TestCase):
- def setUp(self):
- self.inputpath = os.path.sep.join(os.path.dirname(os.path.realpath(__file__)).split(os.path.sep)[0:-1])
+import ParsingTests
+class UnorderedListTests(ParsingTests.ParsingTest):
def testWellFormedList(self):
- try:
- from rss2maildir import HTML2Text
- except:
- sys.path.append(os.path.sep.join(self.inputpath.split(os.path.sep)[0:-1]))
- try:
- from rss2maildir import HTML2Text
- except:
- self.assert_(False)
- input_path = os.path.sep.join(os.path.dirname(os.path.realpath(__file__)).split(os.path.sep)[0:-1])
- input = open(os.path.join(input_path, "html", "unorderedlist-wellformed.html")).read()
- expectedoutput = open(os.path.join(input_path, "expected", "unorderedlist-wellformed.txt")).read()
- parser = HTML2Text()
- parser.feed(input)
- output = parser.gettext()
- self.assertEqual(output, expectedoutput)
+ return self.runParsingTest("unordered-wellformed")
def testBadlyFormedList(self):
- try:
- from rss2maildir import HTML2Text
- except:
- sys.path.append(os.path.sep.join(self.inputpath.split(os.path.sep)[0:-1]))
- try:
- from rss2maildir import HTML2Text
- except:
- self.assert_(False)
-
- input_path = os.path.sep.join(os.path.dirname(os.path.realpath(__file__)).split(os.path.sep)[0:-1])
- input = open(os.path.join(input_path, "html", "unorderedlist-badlyformed.html")).read()
- expectedoutput = open(os.path.join(input_path, "expected", "unorderedlist-badlyformed.txt")).read()
- parser = HTML2Text()
- parser.feed(input)
- output = parser.gettext()
- self.assertEqual(output, expectedoutput)
+ return self.runParsingTest("unordered-badlyformed")
def suite():
suite = unittest.TestSuite()
--
2.39.5