* Handle unicode data more effectively.

[rss2maildir.git] / rss2maildir.py

diff --git a/rss2maildir.py b/rss2maildir.py
index c70c65b24ec51b6c0334c40f7fc9bb07bbe60303..8f7689314cc80f3548f33d8dde3acd18d202b59d 100755 (executable)
--- a/rss2maildir.py
+++ b/rss2maildir.py
@@ -106,13 +106,16 @@ class HTML2Text(HTMLParser):
                 self.opentags.append(tag_name)
                 self.opentags.pop()
 
                 self.opentags.append(tag_name)
                 self.opentags.pop()
 

+            if len(self.opentags) > 1 and self.opentags[-1] == u'li':
+                self.handle_curdata()
+

             if tag_name == u'ol':
                 self.handle_curdata()
                 self.listcount.append(1)
                 self.listlevel = len(self.listcount) - 1
 
             if tag_name in self.liststarttags:
             if tag_name == u'ol':
                 self.handle_curdata()
                 self.listcount.append(1)
                 self.listlevel = len(self.listcount) - 1
 
             if tag_name in self.liststarttags:

-                smallist = self.opentags[-3:]
+                smallist = self.opentags[-3:-1]

                 smallist.reverse()
                 for prev_listtag in smallist:
                     if prev_listtag in [u'dl', u'ol']:
                 smallist.reverse()
                 for prev_listtag in smallist:
                     if prev_listtag in [u'dl', u'ol']:


@@ -133,14 +136,24 @@ class HTML2Text(HTMLParser):
                 listcount = self.listcount[-1]
             except:
                 pass
                 listcount = self.listcount[-1]
             except:
                 pass

+
+            if tag_name == u'dd' and len(self.opentags) > 1 \
+                and self.opentags[-1] == u'dt':
+                self.handle_curdata()
+                self.opentags.pop()
+            elif tag_name == u'dt' and len(self.opentags) > 1 \
+                and self.opentags[-1] == u'dd':
+                self.handle_curdata()
+                self.opentags.pop()
+

             self.handle_curdata()
             self.opentags.append(tag_name)
 
     def handle_startendtag(self, tag, attrs):
         if tag.lower() == u'br':
             self.handle_curdata()
             self.opentags.append(tag_name)
 
     def handle_startendtag(self, tag, attrs):
         if tag.lower() == u'br':

-            self.tags.append(u'br')
+            self.opentags.append(u'br')

             self.handle_curdata() # just handle the data, don't do anything else
             self.handle_curdata() # just handle the data, don't do anything else

-            self.tags.pop()
+            self.opentags.pop()

 
     def handle_curdata(self):
         if len(self.opentags) == 0:
 
     def handle_curdata(self):
         if len(self.opentags) == 0:


@@ -166,7 +179,8 @@ class HTML2Text(HTMLParser):
         if tag_thats_done in ["h1", "h2", "h3", "h4", "h5", "h6"]:
             underline = u''
             underlinechar = u'='
         if tag_thats_done in ["h1", "h2", "h3", "h4", "h5", "h6"]:
             underline = u''
             underlinechar = u'='

-            headingtext = self.curdata.encode("utf-8").strip()
+            headingtext = unicode( \
+                self.curdata.encode("utf-8").strip(), "utf-8")

             seperator = u'\n' + u' '*self.indentlevel
             headingtext = seperator.join( \
                 textwrap.wrap( \
             seperator = u'\n' + u' '*self.indentlevel
             headingtext = seperator.join( \
                 textwrap.wrap( \


@@ -189,16 +203,21 @@ class HTML2Text(HTMLParser):
             self.text = self.text \
                 + headingtext.encode("utf-8") + u'\n' \
                 + underline
             self.text = self.text \
                 + headingtext.encode("utf-8") + u'\n' \
                 + underline

-        elif tag_thats_done == "p":
-            paragraph = self.curdata.encode("utf-8").strip()
+        elif tag_thats_done == u'p':
+            paragraph = unicode( \
+                self.curdata.strip().encode("utf-8"), "utf-8")

             seperator = u'\n' + u' ' * self.indentlevel
             self.text = self.text \
                 + u' ' * self.indentlevel \
             seperator = u'\n' + u' ' * self.indentlevel
             self.text = self.text \
                 + u' ' * self.indentlevel \

-                + seperator.join(textwrap.wrap(paragraph, self.textwidth - self.indentlevel))
+                + seperator.join( \
+                    textwrap.wrap( \
+                        paragraph, self.textwidth - self.indentlevel))

         elif tag_thats_done == "pre":
         elif tag_thats_done == "pre":

-            self.text = self.text + self.curdata
+            self.text = self.text + unicode( \
+                self.curdata.encode("utf-8"), "utf-8")

         elif tag_thats_done == "blockquote":
         elif tag_thats_done == "blockquote":

-            quote = self.curdata.encode("utf-8").strip()
+            quote = unicode( \
+                self.curdata.encode("utf-8").strip(), "utf-8")

             seperator = u'\n' + u' ' * self.indentlevel + u'> '
             self.text = self.text \
                 + u'> ' \
             seperator = u'\n' + u' ' * self.indentlevel + u'> '
             self.text = self.text \
                 + u'> ' \


@@ -209,13 +228,13 @@ class HTML2Text(HTMLParser):
                     )
                 )
         elif tag_thats_done == "li":
                     )
                 )
         elif tag_thats_done == "li":

-            item = self.curdata.encode("utf-8").strip()
+            item = unicode(self.curdata.encode("utf-8").strip(), "utf-8")

             if len(self.text) > 0 and self.text[-1] != u'\n':
                 self.text = self.text + u'\n'
             # work out if we're in an ol rather than a ul
             latesttags = self.opentags[-4:]
             latesttags.reverse()
             if len(self.text) > 0 and self.text[-1] != u'\n':
                 self.text = self.text + u'\n'
             # work out if we're in an ol rather than a ul
             latesttags = self.opentags[-4:]
             latesttags.reverse()

-            isul = False
+            isul = None

             for thing in latesttags:
                 if thing == 'ul':
                     isul = True
             for thing in latesttags:
                 if thing == 'ul':
                     isul = True


@@ -229,7 +248,7 @@ class HTML2Text(HTMLParser):
                 listindent = 4
 
             listmarker = u' * '
                 listindent = 4
 
             listmarker = u' * '

-            if not isul:
+            if isul == False:

                 listmarker = u' %2d. ' %(self.listcount[-1])
                 self.listcount[-1] = self.listcount[-1] + 1
 
                 listmarker = u' %2d. ' %(self.listcount[-1])
                 self.listcount[-1] = self.listcount[-1] + 1
 


@@ -246,25 +265,28 @@ class HTML2Text(HTMLParser):
                     ) \
                 )
             self.curdata = u''
                     ) \
                 )
             self.curdata = u''

-        elif tag_thats_done == "dt":
-            definition = self.curdata.encode("utf-8").strip()
+        elif tag_thats_done == u'dt':
+            definition = unicode(self.curdata.encode("utf-8").strip(), "utf-8")

             if len(self.text) > 0 and self.text[-1] != u'\n':
                 self.text = self.text + u'\n\n'
             elif len(self.text) > 1 and self.text[-2] != u'\n':
                 self.text = self.text + u'\n'
             if len(self.text) > 0 and self.text[-1] != u'\n':
                 self.text = self.text + u'\n\n'
             elif len(self.text) > 1 and self.text[-2] != u'\n':
                 self.text = self.text + u'\n'

-            definition = definition + "::"
+            definition = u' ' * self.indentlevel + definition + "::"
+            indentstring = u'\n' + u' ' * (self.indentlevel + 1)

             self.text = self.text \
             self.text = self.text \

-                + '\n '.join(
-                    textwrap.wrap(definition, self.textwidth - 1))
+                + indentstring.join(
+                    textwrap.wrap(definition, \
+                        self.textwidth - self.indentlevel - 1))

             self.curdata = u''
             self.curdata = u''

-        elif tag_thats_done == "dd":
-            definition = self.curdata.encode("utf-8").strip()
+        elif tag_thats_done == u'dd':
+            definition = unicode(self.curdata.encode("utf-8").strip(), "utf-8")

             if len(definition) > 0:
                 if len(self.text) > 0 and self.text[-1] != u'\n':
                     self.text = self.text + u'\n'
             if len(definition) > 0:
                 if len(self.text) > 0 and self.text[-1] != u'\n':
                     self.text = self.text + u'\n'

+                indentstring = u'\n' + u' ' * (self.indentlevel + 4)

                 self.text = self.text \
                 self.text = self.text \

-                    + '    ' \
-                    + '\n    '.join( \
+                    + u' ' * (self.indentlevel + 4) \
+                    + indentstring.join( \

                         textwrap.wrap( \
                             definition, \
                             self.textwidth - self.indentlevel - 4 \
                         textwrap.wrap( \
                             definition, \
                             self.textwidth - self.indentlevel - 4 \


@@ -280,7 +302,10 @@ class HTML2Text(HTMLParser):
                 self.text = self.text \
                     + u' ... ' \
                     + u'\n ... '.join( \
                 self.text = self.text \
                     + u' ... ' \
                     + u'\n ... '.join( \

-                        textwrap.wrap(self.curdata, self.textwidth - 5))
+                        textwrap.wrap( \
+                            unicode( \
+                                self.curdata.encode("utf-8").strip(), \
+                                "utf-8"), self.textwidth - 5))

             self.curdata = u''
 
         if tag_thats_done in self.blockleveltags:
             self.curdata = u''
 
         if tag_thats_done in self.blockleveltags:


@@ -298,6 +323,7 @@ class HTML2Text(HTMLParser):
 
         if tag in self.liststarttags:
             if tag in [u'ol', u'dl', u'ul']:
 
         if tag in self.liststarttags:
             if tag in [u'ol', u'dl', u'ul']:

+                self.handle_curdata()

                 # find if there was a previous list level
                 smalllist = self.opentags[:-1]
                 smalllist.reverse()
                 # find if there was a previous list level
                 smalllist = self.opentags[:-1]
                 smalllist.reverse()


@@ -557,7 +583,8 @@ def parse_and_deliver(maildir, url, statedir):
     if headers:
         data = []
         for header in headers:
     if headers:
         data = []
         for header in headers:

-            if header[0] in ["content-md5", "etag", "last-modified", "content-length"]:
+            if header[0] in \
+                ["content-md5", "etag", "last-modified", "content-length"]:

                 data.append((header[0], header[1]))
         if len(data) > 0:
             data = urllib.urlencode(data)
                 data.append((header[0], header[1]))
         if len(data) > 0:
             data = urllib.urlencode(data)