From: Brett Parker <iDunno@sommitrealweird.co.uk>
Date: Sat, 1 Mar 2008 20:57:10 +0000 (+0000)
Subject: Normalise spaces where they should be.
X-Git-Url: https://git.sommitrealweird.co.uk/rss2maildir.git/commitdiff_plain/85df49e4c373d62bd5e9e6b45e1792a6d637ca25

Normalise spaces where they should be.
---

diff --git a/rss2maildir.py b/rss2maildir.py
index 2af32bc..ce6c342 100755
--- a/rss2maildir.py
+++ b/rss2maildir.py
@@ -235,7 +235,7 @@ class HTML2Text(HTMLParser):
         if tag_thats_done in ["h1", "h2", "h3", "h4", "h5", "h6"]:
             underline = u''
             underlinechar = u'='
-            headingtext = self.curdata
+            headingtext = " ".join(self.curdata.split())
             seperator = u'\n' + u' '*self.indentlevel
             headingtext = seperator.join( \
                 textwrap.wrap( \
@@ -260,7 +260,8 @@ class HTML2Text(HTMLParser):
                 + underline
         elif tag_thats_done in [u'p', u'div']:
             paragraph = unicode( \
-                self.curdata.strip().encode("utf-8"), "utf-8")
+                " ".join(self.curdata.strip().encode("utf-8").split()), \
+                "utf-8")
             seperator = u'\n' + u' ' * self.indentlevel
             self.text = self.text \
                 + u' ' * self.indentlevel \
@@ -269,10 +270,11 @@ class HTML2Text(HTMLParser):
                         paragraph, self.textwidth - self.indentlevel))
         elif tag_thats_done == "pre":
             self.text = self.text + unicode( \
-                self.curdata.encode("utf-8"), "utf-8")
+                " ".join(self.curdata.encode("utf-8").split()), "utf-8")
         elif tag_thats_done == u'blockquote':
             quote = unicode( \
-                self.curdata.encode("utf-8").strip(), "utf-8")
+                " ".join(self.curdata.encode("utf-8").strip().split()), \
+                "utf-8")
             seperator = u'\n' + u' ' * self.indentlevel + u'> '
             if len(self.text) > 0 and self.text[-1] != u'\n':
                 self.text = self.text + u'\n'
@@ -324,7 +326,9 @@ class HTML2Text(HTMLParser):
                 )
             self.curdata = u''
         elif tag_thats_done == u'dt':
-            definition = unicode(self.curdata.encode("utf-8").strip(), "utf-8")
+            definition = unicode(" ".join( \
+                    self.curdata.encode("utf-8").strip().split()), \
+                "utf-8")
             if len(self.text) > 0 and self.text[-1] != u'\n':
                 self.text = self.text + u'\n\n'
             elif len(self.text) > 1 and self.text[-2] != u'\n':
@@ -337,7 +341,9 @@ class HTML2Text(HTMLParser):
                         self.textwidth - self.indentlevel - 1))
             self.curdata = u''
         elif tag_thats_done == u'dd':
-            definition = unicode(self.curdata.encode("utf-8").strip(), "utf-8")
+            definition = unicode(" ".join( \
+                    self.curdata.encode("utf-8").strip().split()),
+                "utf-8")
             if len(definition) > 0:
                 if len(self.text) > 0 and self.text[-1] != u'\n':
                     self.text = self.text + u'\n'
diff --git a/tests/expected/non-normalised-spacing.txt b/tests/expected/non-normalised-spacing.txt
new file mode 100644
index 0000000..50d8fef
--- /dev/null
+++ b/tests/expected/non-normalised-spacing.txt
@@ -0,0 +1,4 @@
+This has some odd spacing
+=========================
+
+It's really great and hopefully shouldn't be too bad over all
diff --git a/tests/html/non-normalised-spacing.html b/tests/html/non-normalised-spacing.html
new file mode 100644
index 0000000..0c85190
--- /dev/null
+++ b/tests/html/non-normalised-spacing.html
@@ -0,0 +1,7 @@
+<h1>This  has some	odd    spacing</h1>
+<p>It's really  great    and hopefully  shouldn't
+be
+
+too bad over
+all
+</p>
diff --git a/tests/unittests/SpacingTests.py b/tests/unittests/SpacingTests.py
new file mode 100755
index 0000000..849e6ba
--- /dev/null
+++ b/tests/unittests/SpacingTests.py
@@ -0,0 +1,18 @@
+#!/usr/bin/python
+
+import unittest
+import os
+
+import ParsingTests
+
+class SpacingTests(ParsingTests.ParsingTest):
+    def testNormalisingSpacing(self):
+        return self.runParsingTest("non-normalised-spacing")
+
+def suite():
+    suite = unittest.TestSuite()
+    suite.addTest(SpacingTests("testNormalisingSpacing"))
+    return suite
+
+if __name__ == "__main__":
+    unittest.main()