X-Git-Url: https://git.sommitrealweird.co.uk/rss2maildir.git/blobdiff_plain/f47e451a2ae7cde0b422266b10b131d00d398e87..84f14a811430b6a49cd2d9adac6d4bcaa34c3037:/rss2maildir.py

diff --git a/rss2maildir.py b/rss2maildir.py
index a51209c..f6078ff 100755
--- a/rss2maildir.py
+++ b/rss2maildir.py
@@ -92,6 +92,8 @@ class HTML2Text(HTMLParser):
         self.textwidth = textwidth
         self.opentags = []
         self.indentlevel = 0
+        self.listcount = []
+        self.urls = []
         HTMLParser.__init__(self)
 
     def handle_starttag(self, tag, attrs):
@@ -105,19 +107,61 @@ class HTML2Text(HTMLParser):
                 self.opentags.append(tag_name)
                 self.opentags.pop()
 
+            if len(self.opentags) > 1 and self.opentags[-1] == u'li':
+                self.handle_curdata()
+
+            if tag_name == u'ol':
+                self.handle_curdata()
+                self.listcount.append(1)
+                self.listlevel = len(self.listcount) - 1
+
+            if tag_name in self.liststarttags:
+                smallist = self.opentags[-3:-1]
+                smallist.reverse()
+                for prev_listtag in smallist:
+                    if prev_listtag in [u'dl', u'ol']:
+                        self.indentlevel = self.indentlevel + 4
+                        break
+                    elif prev_listtag == u'ul':
+                        self.indentlevel = self.indentlevel + 3
+                        break
+
             if len(self.opentags) > 0:
                 self.handle_curdata()
-                self.opentags.pop()
+                if tag_name not in self.cancontainflow:
+                    self.opentags.pop()
             self.opentags.append(tag_name)
         else:
+            listcount = 0
+            try:
+                listcount = self.listcount[-1]
+            except:
+                pass
+
+            if tag_name == u'dd' and len(self.opentags) > 1 \
+                and self.opentags[-1] == u'dt':
+                self.handle_curdata()
+                self.opentags.pop()
+            elif tag_name == u'dt' and len(self.opentags) > 1 \
+                and self.opentags[-1] == u'dd':
+                self.handle_curdata()
+                self.opentags.pop()
+            elif tag_name == u'a':
+                for attr in attrs:
+                    if attr[0].lower() == u'href':
+                        self.urls.append(attr[1])
+                self.curdata = self.curdata + u'`'
+                self.opentags.append(tag_name)
+                return
+
             self.handle_curdata()
             self.opentags.append(tag_name)
 
     def handle_startendtag(self, tag, attrs):
         if tag.lower() == u'br':
-            self.tags.append(u'br')
+            self.opentags.append(u'br')
             self.handle_curdata() # just handle the data, don't do anything else
-            self.tags.pop()
+            self.opentags.pop()
 
     def handle_curdata(self):
         if len(self.opentags) == 0:
@@ -126,19 +170,32 @@ class HTML2Text(HTMLParser):
         if len(self.curdata) == 0:
             return
 
+        if len(self.curdata.strip()) == 0:
+            return
+
         tag_thats_done = self.opentags[-1]
 
         if tag_thats_done in self.blockleveltags:
             newlinerequired = self.text != u''
             if newlinerequired:
-                self.text = self.text + u'\n\n'
+                if newlinerequired \
+                    and len(self.text) > 2 \
+                    and self.text[-1] != u'\n' \
+                    and self.text[-2] != u'\n':
+                    self.text = self.text + u'\n\n'
 
         if tag_thats_done in ["h1", "h2", "h3", "h4", "h5", "h6"]:
             underline = u''
             underlinechar = u'='
-            headingtext = self.curdata.encode("utf-8").strip()
-            headingtext = u'\n'.join( \
-                textwrap.wrap(headingtext, self.textwidth))
+            headingtext = unicode( \
+                self.curdata.encode("utf-8").strip(), "utf-8")
+            seperator = u'\n' + u' '*self.indentlevel
+            headingtext = seperator.join( \
+                textwrap.wrap( \
+                    headingtext, \
+                    self.textwidth - self.indentlevel \
+                    ) \
+                )
 
             if tag_thats_done == u'h2':
                 underlinechar = u'-'
@@ -146,52 +203,107 @@ class HTML2Text(HTMLParser):
                 underlinechar = u'~'
 
             if u'\n' in headingtext:
-                underline = underlinechar * self.textwidth
+                underline = u' ' * self.indentlevel \
+                    + underlinechar * (self.textwidth - self.indentlevel)
             else:
-                underline = underlinechar * len(headingtext)
+                underline = u' ' * self.indentlevel \
+                    + underlinechar * len(headingtext)
             self.text = self.text \
                 + headingtext.encode("utf-8") + u'\n' \
                 + underline
-        elif tag_thats_done == "p":
-            paragraph = self.curdata.encode("utf-8").strip()
+        elif tag_thats_done == u'p':
+            paragraph = unicode( \
+                self.curdata.strip().encode("utf-8"), "utf-8")
+            seperator = u'\n' + u' ' * self.indentlevel
             self.text = self.text \
-                + u'\n'.join(textwrap.wrap(paragraph, self.textwidth))
+                + u' ' * self.indentlevel \
+                + seperator.join( \
+                    textwrap.wrap( \
+                        paragraph, self.textwidth - self.indentlevel))
         elif tag_thats_done == "pre":
-            self.text = self.text + self.curdata
+            self.text = self.text + unicode( \
+                self.curdata.encode("utf-8"), "utf-8")
         elif tag_thats_done == "blockquote":
-            quote = self.curdata.encode("utf-8").strip()
+            quote = unicode( \
+                self.curdata.encode("utf-8").strip(), "utf-8")
+            seperator = u'\n' + u' ' * self.indentlevel + u'> '
             self.text = self.text \
                 + u'> ' \
-                + u'> '.join(textwrap.wrap(quote, self.textwidth - 2))
+                + seperator.join( \
+                    textwrap.wrap( \
+                        quote, \
+                        self.textwidth - self.indentlevel - 2 \
+                    )
+                )
         elif tag_thats_done == "li":
-            item = self.curdata.encode("utf-8").strip()
+            item = unicode(self.curdata.encode("utf-8").strip(), "utf-8")
             if len(self.text) > 0 and self.text[-1] != u'\n':
                 self.text = self.text + u'\n'
+            # work out if we're in an ol rather than a ul
+            latesttags = self.opentags[-4:]
+            latesttags.reverse()
+            isul = None
+            for thing in latesttags:
+                if thing == 'ul':
+                    isul = True
+                    break
+                elif thing == 'ol':
+                    isul = False
+                    break
+
+            listindent = 3
+            if not isul:
+                listindent = 4
+
+            listmarker = u' * '
+            if isul == False:
+                listmarker = u' %2d. ' %(self.listcount[-1])
+                self.listcount[-1] = self.listcount[-1] + 1
+
+            seperator = u'\n' \
+                + u' ' * self.indentlevel \
+                + u' ' * listindent
             self.text = self.text \
-                + u' * ' \
-                + u'\n   '.join( \
-                    textwrap.wrap(item, self.textwidth - 3))
+                + u' ' * self.indentlevel \
+                + listmarker \
+                + seperator.join( \
+                    textwrap.wrap( \
+                        item, \
+                        self.textwidth - self.indentlevel - listindent \
+                    ) \
+                )
             self.curdata = u''
-        elif tag_thats_done == "dt":
-            definition = self.curdata.encode("utf-8").strip()
+        elif tag_thats_done == u'dt':
+            definition = unicode(self.curdata.encode("utf-8").strip(), "utf-8")
             if len(self.text) > 0 and self.text[-1] != u'\n':
                 self.text = self.text + u'\n\n'
-            elif len(self.text) > 0 and self.text[-2] != u'\n':
-                self.text = self.text + u'\n'
-            definition = definition + "::"
-            self.text = self.text \
-                + '\n '.join(
-                    textwrap.wrap(definition, self.textwidth - 1))
-            self.curdata = u''
-        elif tag_thats_done == "dd":
-            definition = self.curdata.encode("utf-8").strip()
-            if len(self.text) > 0 and self.text[-1] != u'\n':
+            elif len(self.text) > 1 and self.text[-2] != u'\n':
                 self.text = self.text + u'\n'
+            definition = u' ' * self.indentlevel + definition + "::"
+            indentstring = u'\n' + u' ' * (self.indentlevel + 1)
             self.text = self.text \
-                + '    ' \
-                + '\n    '.join( \
-                    textwrap.wrap(definition, self.textwidth - 4))
+                + indentstring.join(
+                    textwrap.wrap(definition, \
+                        self.textwidth - self.indentlevel - 1))
             self.curdata = u''
+        elif tag_thats_done == u'dd':
+            definition = unicode(self.curdata.encode("utf-8").strip(), "utf-8")
+            if len(definition) > 0:
+                if len(self.text) > 0 and self.text[-1] != u'\n':
+                    self.text = self.text + u'\n'
+                indentstring = u'\n' + u' ' * (self.indentlevel + 4)
+                self.text = self.text \
+                    + u' ' * (self.indentlevel + 4) \
+                    + indentstring.join( \
+                        textwrap.wrap( \
+                            definition, \
+                            self.textwidth - self.indentlevel - 4 \
+                            ) \
+                        )
+                self.curdata = u''
+        elif tag_thats_done == u'a':
+            self.curdata = self.curdata + u'`__'
+            pass
         elif tag_thats_done in self.liststarttags:
             pass
         else:
@@ -201,7 +313,10 @@ class HTML2Text(HTMLParser):
                 self.text = self.text \
                     + u' ... ' \
                     + u'\n ... '.join( \
-                        textwrap.wrap(self.curdata, self.textwidth - 5))
+                        textwrap.wrap( \
+                            unicode( \
+                                self.curdata.encode("utf-8").strip(), \
+                                "utf-8"), self.textwidth - 5))
             self.curdata = u''
 
         if tag_thats_done in self.blockleveltags:
@@ -215,6 +330,25 @@ class HTML2Text(HTMLParser):
             # err. weird.
             tagindex = 0
 
+        tag = tag.lower()
+
+        if tag in self.liststarttags:
+            if tag in [u'ol', u'dl', u'ul']:
+                self.handle_curdata()
+                # find if there was a previous list level
+                smalllist = self.opentags[:-1]
+                smalllist.reverse()
+                for prev_listtag in smalllist:
+                    if prev_listtag in [u'ol', u'dl']:
+                        self.indentlevel = self.indentlevel - 4
+                        break
+                    elif prev_listtag == u'ul':
+                        self.indentlevel = self.indentlevel - 3
+                        break
+
+        if tag == u'ol':
+            self.listcount = self.listcount[:-1]
+
         while tagindex < len(self.opentags) \
             and tag in self.opentags[tagindex+1:]:
             try:
@@ -226,7 +360,11 @@ class HTML2Text(HTMLParser):
             # Assuming the data was for the last opened tag first
             self.handle_curdata()
             # Now kill the list to be a slice before this tag was opened
-            self.opentags = self.opentags[:tagindex]
+            self.opentags = self.opentags[:tagindex + 1]
+        else:
+            self.handle_curdata()
+            if self.opentags[-1] == tag:
+                self.opentags.pop()
 
     def handle_data(self, data):
         self.curdata = self.curdata + unicode(data, "utf-8")
@@ -251,6 +389,9 @@ class HTML2Text(HTMLParser):
             while len(self.text) > 1 and self.text[-1] == u'\n':
                 self.text = self.text[:-1]
             self.text = self.text + u'\n'
+        if len(self.urls) > 0:
+            self.text = self.text + u'\n__ ' + u'\n__ '.join(self.urls) + u'\n'
+            self.urls = []
         return self.text
 
 def open_url(method, url):
@@ -456,7 +597,8 @@ def parse_and_deliver(maildir, url, statedir):
     if headers:
         data = []
         for header in headers:
-            if header[0] in ["content-md5", "etag", "last-modified", "content-length"]:
+            if header[0] in \
+                ["content-md5", "etag", "last-modified", "content-length"]:
                 data.append((header[0], header[1]))
         if len(data) > 0:
             data = urllib.urlencode(data)