Rudimentary <a href="...">bleep</a> support.

[rss2maildir.git] / rss2maildir.py

diff --git a/rss2maildir.py b/rss2maildir.py
index 4421811152a18c37b587f9e46e7568e28fb47760..f6078ffc13fe99139551e91df8de2ab01d81944d 100755 (executable)
--- a/rss2maildir.py
+++ b/rss2maildir.py
@@ -93,6 +93,7 @@ class HTML2Text(HTMLParser):
         self.opentags = []
         self.indentlevel = 0
         self.listcount = []
         self.opentags = []
         self.indentlevel = 0
         self.listcount = []

+        self.urls = []

         HTMLParser.__init__(self)
 
     def handle_starttag(self, tag, attrs):
         HTMLParser.__init__(self)
 
     def handle_starttag(self, tag, attrs):


@@ -145,6 +146,13 @@ class HTML2Text(HTMLParser):
                 and self.opentags[-1] == u'dd':
                 self.handle_curdata()
                 self.opentags.pop()
                 and self.opentags[-1] == u'dd':
                 self.handle_curdata()
                 self.opentags.pop()

+            elif tag_name == u'a':
+                for attr in attrs:
+                    if attr[0].lower() == u'href':
+                        self.urls.append(attr[1])
+                self.curdata = self.curdata + u'`'
+                self.opentags.append(tag_name)
+                return

 
             self.handle_curdata()
             self.opentags.append(tag_name)
 
             self.handle_curdata()
             self.opentags.append(tag_name)


@@ -179,7 +187,8 @@ class HTML2Text(HTMLParser):
         if tag_thats_done in ["h1", "h2", "h3", "h4", "h5", "h6"]:
             underline = u''
             underlinechar = u'='
         if tag_thats_done in ["h1", "h2", "h3", "h4", "h5", "h6"]:
             underline = u''
             underlinechar = u'='

-            headingtext = self.curdata.encode("utf-8").strip()
+            headingtext = unicode( \
+                self.curdata.encode("utf-8").strip(), "utf-8")

             seperator = u'\n' + u' '*self.indentlevel
             headingtext = seperator.join( \
                 textwrap.wrap( \
             seperator = u'\n' + u' '*self.indentlevel
             headingtext = seperator.join( \
                 textwrap.wrap( \


@@ -203,15 +212,20 @@ class HTML2Text(HTMLParser):
                 + headingtext.encode("utf-8") + u'\n' \
                 + underline
         elif tag_thats_done == u'p':
                 + headingtext.encode("utf-8") + u'\n' \
                 + underline
         elif tag_thats_done == u'p':

-            paragraph = self.curdata.encode("utf-8").strip()
+            paragraph = unicode( \
+                self.curdata.strip().encode("utf-8"), "utf-8")

             seperator = u'\n' + u' ' * self.indentlevel
             self.text = self.text \
                 + u' ' * self.indentlevel \
             seperator = u'\n' + u' ' * self.indentlevel
             self.text = self.text \
                 + u' ' * self.indentlevel \

-                + seperator.join(textwrap.wrap(paragraph, self.textwidth - self.indentlevel))
+                + seperator.join( \
+                    textwrap.wrap( \
+                        paragraph, self.textwidth - self.indentlevel))

         elif tag_thats_done == "pre":
         elif tag_thats_done == "pre":

-            self.text = self.text + self.curdata
+            self.text = self.text + unicode( \
+                self.curdata.encode("utf-8"), "utf-8")

         elif tag_thats_done == "blockquote":
         elif tag_thats_done == "blockquote":

-            quote = self.curdata.encode("utf-8").strip()
+            quote = unicode( \
+                self.curdata.encode("utf-8").strip(), "utf-8")

             seperator = u'\n' + u' ' * self.indentlevel + u'> '
             self.text = self.text \
                 + u'> ' \
             seperator = u'\n' + u' ' * self.indentlevel + u'> '
             self.text = self.text \
                 + u'> ' \


@@ -222,13 +236,13 @@ class HTML2Text(HTMLParser):
                     )
                 )
         elif tag_thats_done == "li":
                     )
                 )
         elif tag_thats_done == "li":

-            item = self.curdata.encode("utf-8").strip()
+            item = unicode(self.curdata.encode("utf-8").strip(), "utf-8")

             if len(self.text) > 0 and self.text[-1] != u'\n':
                 self.text = self.text + u'\n'
             # work out if we're in an ol rather than a ul
             latesttags = self.opentags[-4:]
             latesttags.reverse()
             if len(self.text) > 0 and self.text[-1] != u'\n':
                 self.text = self.text + u'\n'
             # work out if we're in an ol rather than a ul
             latesttags = self.opentags[-4:]
             latesttags.reverse()

-            isul = False
+            isul = None

             for thing in latesttags:
                 if thing == 'ul':
                     isul = True
             for thing in latesttags:
                 if thing == 'ul':
                     isul = True


@@ -242,7 +256,7 @@ class HTML2Text(HTMLParser):
                 listindent = 4
 
             listmarker = u' * '
                 listindent = 4
 
             listmarker = u' * '

-            if not isul:
+            if isul == False:

                 listmarker = u' %2d. ' %(self.listcount[-1])
                 self.listcount[-1] = self.listcount[-1] + 1
 
                 listmarker = u' %2d. ' %(self.listcount[-1])
                 self.listcount[-1] = self.listcount[-1] + 1
 


@@ -260,7 +274,7 @@ class HTML2Text(HTMLParser):
                 )
             self.curdata = u''
         elif tag_thats_done == u'dt':
                 )
             self.curdata = u''
         elif tag_thats_done == u'dt':

-            definition = self.curdata.encode("utf-8").strip()
+            definition = unicode(self.curdata.encode("utf-8").strip(), "utf-8")

             if len(self.text) > 0 and self.text[-1] != u'\n':
                 self.text = self.text + u'\n\n'
             elif len(self.text) > 1 and self.text[-2] != u'\n':
             if len(self.text) > 0 and self.text[-1] != u'\n':
                 self.text = self.text + u'\n\n'
             elif len(self.text) > 1 and self.text[-2] != u'\n':


@@ -273,7 +287,7 @@ class HTML2Text(HTMLParser):
                         self.textwidth - self.indentlevel - 1))
             self.curdata = u''
         elif tag_thats_done == u'dd':
                         self.textwidth - self.indentlevel - 1))
             self.curdata = u''
         elif tag_thats_done == u'dd':

-            definition = self.curdata.encode("utf-8").strip()
+            definition = unicode(self.curdata.encode("utf-8").strip(), "utf-8")

             if len(definition) > 0:
                 if len(self.text) > 0 and self.text[-1] != u'\n':
                     self.text = self.text + u'\n'
             if len(definition) > 0:
                 if len(self.text) > 0 and self.text[-1] != u'\n':
                     self.text = self.text + u'\n'


@@ -287,6 +301,9 @@ class HTML2Text(HTMLParser):
                             ) \
                         )
                 self.curdata = u''
                             ) \
                         )
                 self.curdata = u''

+        elif tag_thats_done == u'a':
+            self.curdata = self.curdata + u'`__'
+            pass

         elif tag_thats_done in self.liststarttags:
             pass
         else:
         elif tag_thats_done in self.liststarttags:
             pass
         else:


@@ -296,7 +313,10 @@ class HTML2Text(HTMLParser):
                 self.text = self.text \
                     + u' ... ' \
                     + u'\n ... '.join( \
                 self.text = self.text \
                     + u' ... ' \
                     + u'\n ... '.join( \

-                        textwrap.wrap(self.curdata, self.textwidth - 5))
+                        textwrap.wrap( \
+                            unicode( \
+                                self.curdata.encode("utf-8").strip(), \
+                                "utf-8"), self.textwidth - 5))

             self.curdata = u''
 
         if tag_thats_done in self.blockleveltags:
             self.curdata = u''
 
         if tag_thats_done in self.blockleveltags:


@@ -369,6 +389,9 @@ class HTML2Text(HTMLParser):
             while len(self.text) > 1 and self.text[-1] == u'\n':
                 self.text = self.text[:-1]
             self.text = self.text + u'\n'
             while len(self.text) > 1 and self.text[-1] == u'\n':
                 self.text = self.text[:-1]
             self.text = self.text + u'\n'

+        if len(self.urls) > 0:
+            self.text = self.text + u'\n__ ' + u'\n__ '.join(self.urls) + u'\n'
+            self.urls = []

         return self.text
 
 def open_url(method, url):
         return self.text
 
 def open_url(method, url):


@@ -574,7 +597,8 @@ def parse_and_deliver(maildir, url, statedir):
     if headers:
         data = []
         for header in headers:
     if headers:
         data = []
         for header in headers:

-            if header[0] in ["content-md5", "etag", "last-modified", "content-length"]:
+            if header[0] in \
+                ["content-md5", "etag", "last-modified", "content-length"]:

                 data.append((header[0], header[1]))
         if len(data) > 0:
             data = urllib.urlencode(data)
                 data.append((header[0], header[1]))
         if len(data) > 0:
             data = urllib.urlencode(data)