Unicode handling of URLs fix

[rss2maildir.git] / rss2maildir.py

diff --git a/rss2maildir.py b/rss2maildir.py
index 4421811152a18c37b587f9e46e7568e28fb47760..2af32bc9105a49201e2bbba49d87b5e2cbcda1da 100755 (executable)
--- a/rss2maildir.py
+++ b/rss2maildir.py
@@ -48,42 +48,46 @@ from HTMLParser import HTMLParser
 
 class HTML2Text(HTMLParser):
     entities = {
 
 class HTML2Text(HTMLParser):
     entities = {

-        "amp": "&",
-        "lt": "<",
-        "gt": ">",
-        "pound": "£",
-        "copy": "©",
-        "apos": "'",
-        "quot": "\"",
-        "nbsp": " ",
+        u'amp': "&",
+        u'lt': "<",
+        u'gt': ">",
+        u'pound': "£",
+        u'copy': "©",
+        u'apos': "'",
+        u'quot': "\"",
+        u'nbsp': " ",

         }
 
     blockleveltags = [
         }
 
     blockleveltags = [

-        "h1",
-        "h2",
-        "h3",
-        "h4",
-        "h5",
-        "h6",
-        "pre",
-        "p",
-        "ul",
-        "ol",
-        "dl",
-        "br",
+        u'h1',
+        u'h2',
+        u'h3',
+        u'h4',
+        u'h5',
+        u'h6',
+        u'pre',
+        u'p',
+        u'ul',
+        u'ol',
+        u'dl',
+        u'li',
+        u'dt',
+        u'dd',
+        u'div',
+        #u'blockquote',

         ]
 
     liststarttags = [
         ]
 
     liststarttags = [

-        "ul",
-        "ol",
-        "dl",
+        u'ul',
+        u'ol',
+        u'dl',

         ]
 
     cancontainflow = [
         ]
 
     cancontainflow = [

-        "div",
-        "li",
-        "dd",
-        "blockquote",
+        u'div',
+        u'li',
+        u'dd',
+        u'blockquote',

     ]
 
     def __init__(self,textwidth=70):
     ]
 
     def __init__(self,textwidth=70):


@@ -92,7 +96,9 @@ class HTML2Text(HTMLParser):
         self.textwidth = textwidth
         self.opentags = []
         self.indentlevel = 0
         self.textwidth = textwidth
         self.opentags = []
         self.indentlevel = 0

+        self.ignorenodata = False

         self.listcount = []
         self.listcount = []

+        self.urls = []

         HTMLParser.__init__(self)
 
     def handle_starttag(self, tag, attrs):
         HTMLParser.__init__(self)
 
     def handle_starttag(self, tag, attrs):


@@ -101,11 +107,6 @@ class HTML2Text(HTMLParser):
             # handle starting a new block - unless we're in a block element
             # that can contain other blocks, we'll assume that we want to close
             # the container
             # handle starting a new block - unless we're in a block element
             # that can contain other blocks, we'll assume that we want to close
             # the container

-            if tag_name == u'br':
-                self.handle_curdata()
-                self.opentags.append(tag_name)
-                self.opentags.pop()
-

             if len(self.opentags) > 1 and self.opentags[-1] == u'li':
                 self.handle_curdata()
 
             if len(self.opentags) > 1 and self.opentags[-1] == u'li':
                 self.handle_curdata()
 


@@ -131,6 +132,8 @@ class HTML2Text(HTMLParser):
                     self.opentags.pop()
             self.opentags.append(tag_name)
         else:
                     self.opentags.pop()
             self.opentags.append(tag_name)
         else:

+            if tag_name == "span":
+                return

             listcount = 0
             try:
                 listcount = self.listcount[-1]
             listcount = 0
             try:
                 listcount = self.listcount[-1]


@@ -145,33 +148,86 @@ class HTML2Text(HTMLParser):
                 and self.opentags[-1] == u'dd':
                 self.handle_curdata()
                 self.opentags.pop()
                 and self.opentags[-1] == u'dd':
                 self.handle_curdata()
                 self.opentags.pop()

-
-            self.handle_curdata()
-            self.opentags.append(tag_name)
+            elif tag_name == u'a':
+                for attr in attrs:
+                    if attr[0].lower() == u'href':
+                        self.urls.append(attr[1].decode('utf-8'))
+                self.curdata = self.curdata + u'`'
+                self.opentags.append(tag_name)
+                return
+            elif tag_name == u'img':
+                self.handle_image(attrs)
+                return
+            elif tag_name == u'br':
+                self.handle_br()
+                return
+            else:
+                # we don't know the tag, so lets avoid handling it!
+                return 

 
     def handle_startendtag(self, tag, attrs):
         if tag.lower() == u'br':
 
     def handle_startendtag(self, tag, attrs):
         if tag.lower() == u'br':

+            self.handle_br()
+        elif tag.lower() == u'img':
+            self.handle_image(attrs)
+            return
+
+    def handle_br(self):
+            self.handle_curdata()

             self.opentags.append(u'br')
             self.opentags.append(u'br')

-            self.handle_curdata() # just handle the data, don't do anything else
+            self.handle_curdata()

             self.opentags.pop()
 
             self.opentags.pop()
 

+    def handle_image(self, attrs):
+        alt = u''
+        url = u''
+        for attr in attrs:
+            if attr[0] == 'alt':
+                alt = attr[1].decode('utf-8')
+            elif attr[0] == 'src':
+                url = attr[1].decode('utf-8')
+        if url:
+            self.curdata = self.curdata \
+                + u' [img:' \
+                + url
+            if alt:
+                self.curdata = self.curdata \
+                    + u'(' \
+                    + alt \
+                    + u')'
+            self.curdata = self.curdata \
+                + u']'
+

     def handle_curdata(self):
     def handle_curdata(self):

+

         if len(self.opentags) == 0:
             return
 
         if len(self.opentags) == 0:
             return
 

+        tag_thats_done = self.opentags[-1]
+

         if len(self.curdata) == 0:
             return
 
         if len(self.curdata) == 0:
             return
 

-        if len(self.curdata.strip()) == 0:
+        if tag_thats_done == u'br':
+            if len(self.text) == 0 or self.text[-1] != '\n':
+                self.text = self.text + '\n'
+                self.ignorenodata = True

             return
 
             return
 

-        tag_thats_done = self.opentags[-1]
+        if len(self.curdata.strip()) == 0:
+            return

 
         if tag_thats_done in self.blockleveltags:
             newlinerequired = self.text != u''
 
         if tag_thats_done in self.blockleveltags:
             newlinerequired = self.text != u''

+            if self.ignorenodata:
+                newlinerequired = False
+            self.ignorenodata = False

             if newlinerequired:
             if newlinerequired:

-                if newlinerequired \
-                    and len(self.text) > 2 \
+                if tag_thats_done in [u'dt', u'dd', u'li'] \
+                    and len(self.text) > 1 \
+                    and self.text[-1] != u'\n':
+                        self.text = self.text + u'\n'
+                elif len(self.text) > 2 \

                     and self.text[-1] != u'\n' \
                     and self.text[-2] != u'\n':
                     self.text = self.text + u'\n\n'
                     and self.text[-1] != u'\n' \
                     and self.text[-2] != u'\n':
                     self.text = self.text + u'\n\n'


@@ -179,7 +235,7 @@ class HTML2Text(HTMLParser):
         if tag_thats_done in ["h1", "h2", "h3", "h4", "h5", "h6"]:
             underline = u''
             underlinechar = u'='
         if tag_thats_done in ["h1", "h2", "h3", "h4", "h5", "h6"]:
             underline = u''
             underlinechar = u'='

-            headingtext = self.curdata.encode("utf-8").strip()
+            headingtext = self.curdata

             seperator = u'\n' + u' '*self.indentlevel
             headingtext = seperator.join( \
                 textwrap.wrap( \
             seperator = u'\n' + u' '*self.indentlevel
             headingtext = seperator.join( \
                 textwrap.wrap( \


@@ -200,19 +256,26 @@ class HTML2Text(HTMLParser):
                 underline = u' ' * self.indentlevel \
                     + underlinechar * len(headingtext)
             self.text = self.text \
                 underline = u' ' * self.indentlevel \
                     + underlinechar * len(headingtext)
             self.text = self.text \

-                + headingtext.encode("utf-8") + u'\n' \
+                + headingtext + u'\n' \

                 + underline
                 + underline

-        elif tag_thats_done == u'p':
-            paragraph = self.curdata.encode("utf-8").strip()
+        elif tag_thats_done in [u'p', u'div']:
+            paragraph = unicode( \
+                self.curdata.strip().encode("utf-8"), "utf-8")

             seperator = u'\n' + u' ' * self.indentlevel
             self.text = self.text \
                 + u' ' * self.indentlevel \
             seperator = u'\n' + u' ' * self.indentlevel
             self.text = self.text \
                 + u' ' * self.indentlevel \

-                + seperator.join(textwrap.wrap(paragraph, self.textwidth - self.indentlevel))
+                + seperator.join( \
+                    textwrap.wrap( \
+                        paragraph, self.textwidth - self.indentlevel))

         elif tag_thats_done == "pre":
         elif tag_thats_done == "pre":

-            self.text = self.text + self.curdata
-        elif tag_thats_done == "blockquote":
-            quote = self.curdata.encode("utf-8").strip()
+            self.text = self.text + unicode( \
+                self.curdata.encode("utf-8"), "utf-8")
+        elif tag_thats_done == u'blockquote':
+            quote = unicode( \
+                self.curdata.encode("utf-8").strip(), "utf-8")

             seperator = u'\n' + u' ' * self.indentlevel + u'> '
             seperator = u'\n' + u' ' * self.indentlevel + u'> '

+            if len(self.text) > 0 and self.text[-1] != u'\n':
+                self.text = self.text + u'\n'

             self.text = self.text \
                 + u'> ' \
                 + seperator.join( \
             self.text = self.text \
                 + u'> ' \
                 + seperator.join( \


@@ -221,14 +284,15 @@ class HTML2Text(HTMLParser):
                         self.textwidth - self.indentlevel - 2 \
                     )
                 )
                         self.textwidth - self.indentlevel - 2 \
                     )
                 )

+            self.curdata = u''

         elif tag_thats_done == "li":
         elif tag_thats_done == "li":

-            item = self.curdata.encode("utf-8").strip()
+            item = unicode(self.curdata.encode("utf-8").strip(), "utf-8")

             if len(self.text) > 0 and self.text[-1] != u'\n':
                 self.text = self.text + u'\n'
             # work out if we're in an ol rather than a ul
             latesttags = self.opentags[-4:]
             latesttags.reverse()
             if len(self.text) > 0 and self.text[-1] != u'\n':
                 self.text = self.text + u'\n'
             # work out if we're in an ol rather than a ul
             latesttags = self.opentags[-4:]
             latesttags.reverse()

-            isul = False
+            isul = None

             for thing in latesttags:
                 if thing == 'ul':
                     isul = True
             for thing in latesttags:
                 if thing == 'ul':
                     isul = True


@@ -242,7 +306,7 @@ class HTML2Text(HTMLParser):
                 listindent = 4
 
             listmarker = u' * '
                 listindent = 4
 
             listmarker = u' * '

-            if not isul:
+            if isul == False:

                 listmarker = u' %2d. ' %(self.listcount[-1])
                 self.listcount[-1] = self.listcount[-1] + 1
 
                 listmarker = u' %2d. ' %(self.listcount[-1])
                 self.listcount[-1] = self.listcount[-1] + 1
 


@@ -260,7 +324,7 @@ class HTML2Text(HTMLParser):
                 )
             self.curdata = u''
         elif tag_thats_done == u'dt':
                 )
             self.curdata = u''
         elif tag_thats_done == u'dt':

-            definition = self.curdata.encode("utf-8").strip()
+            definition = unicode(self.curdata.encode("utf-8").strip(), "utf-8")

             if len(self.text) > 0 and self.text[-1] != u'\n':
                 self.text = self.text + u'\n\n'
             elif len(self.text) > 1 and self.text[-2] != u'\n':
             if len(self.text) > 0 and self.text[-1] != u'\n':
                 self.text = self.text + u'\n\n'
             elif len(self.text) > 1 and self.text[-2] != u'\n':


@@ -273,7 +337,7 @@ class HTML2Text(HTMLParser):
                         self.textwidth - self.indentlevel - 1))
             self.curdata = u''
         elif tag_thats_done == u'dd':
                         self.textwidth - self.indentlevel - 1))
             self.curdata = u''
         elif tag_thats_done == u'dd':

-            definition = self.curdata.encode("utf-8").strip()
+            definition = unicode(self.curdata.encode("utf-8").strip(), "utf-8")

             if len(definition) > 0:
                 if len(self.text) > 0 and self.text[-1] != u'\n':
                     self.text = self.text + u'\n'
             if len(definition) > 0:
                 if len(self.text) > 0 and self.text[-1] != u'\n':
                     self.text = self.text + u'\n'


@@ -287,31 +351,31 @@ class HTML2Text(HTMLParser):
                             ) \
                         )
                 self.curdata = u''
                             ) \
                         )
                 self.curdata = u''

+        elif tag_thats_done == u'a':
+            self.curdata = self.curdata + u'`__'
+            pass

         elif tag_thats_done in self.liststarttags:
             pass
         elif tag_thats_done in self.liststarttags:
             pass

-        else:
-            # we've got no idea what this tag does, so we'll
-            # make an assumption that we're not going to know later
-            if len(self.curdata) > 0:
-                self.text = self.text \
-                    + u' ... ' \
-                    + u'\n ... '.join( \
-                        textwrap.wrap(self.curdata, self.textwidth - 5))
-            self.curdata = u''

 
         if tag_thats_done in self.blockleveltags:
             self.curdata = u''
 
 
         if tag_thats_done in self.blockleveltags:
             self.curdata = u''
 

+        self.ignorenodata = False
+

     def handle_endtag(self, tag):
     def handle_endtag(self, tag):

+        self.ignorenodata = False
+        if tag == "span":
+            return
+

         try:
             tagindex = self.opentags.index(tag)
         except:
         try:
             tagindex = self.opentags.index(tag)
         except:

-            # closing tag we know nothing about.
-            # err. weird.
-            tagindex = 0
-
+            return

         tag = tag.lower()
 
         tag = tag.lower()
 

+        if tag in [u'br', u'img']:
+            return
+

         if tag in self.liststarttags:
             if tag in [u'ol', u'dl', u'ul']:
                 self.handle_curdata()
         if tag in self.liststarttags:
             if tag in [u'ol', u'dl', u'ul']:
                 self.handle_curdata()


@@ -347,7 +411,9 @@ class HTML2Text(HTMLParser):
                 self.opentags.pop()
 
     def handle_data(self, data):
                 self.opentags.pop()
 
     def handle_data(self, data):

-        self.curdata = self.curdata + unicode(data, "utf-8")
+        if len(self.opentags) == 0:
+            self.opentags.append(u'p')
+        self.curdata = self.curdata + data.decode("utf-8")

 
     def handle_entityref(self, name):
         entity = name
 
     def handle_entityref(self, name):
         entity = name


@@ -369,6 +435,9 @@ class HTML2Text(HTMLParser):
             while len(self.text) > 1 and self.text[-1] == u'\n':
                 self.text = self.text[:-1]
             self.text = self.text + u'\n'
             while len(self.text) > 1 and self.text[-1] == u'\n':
                 self.text = self.text[:-1]
             self.text = self.text + u'\n'

+        if len(self.urls) > 0:
+            self.text = self.text + u'\n__ ' + u'\n__ '.join(self.urls) + u'\n'
+            self.urls = []

         return self.text
 
 def open_url(method, url):
         return self.text
 
 def open_url(method, url):


@@ -574,7 +643,8 @@ def parse_and_deliver(maildir, url, statedir):
     if headers:
         data = []
         for header in headers:
     if headers:
         data = []
         for header in headers:

-            if header[0] in ["content-md5", "etag", "last-modified", "content-length"]:
+            if header[0] in \
+                ["content-md5", "etag", "last-modified", "content-length"]:

                 data.append((header[0], header[1]))
         if len(data) > 0:
             data = urllib.urlencode(data)
                 data.append((header[0], header[1]))
         if len(data) > 0:
             data = urllib.urlencode(data)


@@ -653,11 +723,13 @@ if __name__ == "__main__":
     elif scp.has_option("general", "state_dir"):
         new_state_dir = scp.get("general", "state_dir")
         try:
     elif scp.has_option("general", "state_dir"):
         new_state_dir = scp.get("general", "state_dir")
         try:

-            mode = os.stat(state_dir)[stat.ST_MODE]
+            mode = os.stat(new_state_dir)[stat.ST_MODE]

             if not stat.S_ISDIR(mode):
                 sys.stderr.write( \
                     "State directory (%s) is not a directory\n" %(state_dir))
                 sys.exit(1)
             if not stat.S_ISDIR(mode):
                 sys.stderr.write( \
                     "State directory (%s) is not a directory\n" %(state_dir))
                 sys.exit(1)

+            else:
+                state_dir = new_state_dir

         except:
             # try to create it
             try:
         except:
             # try to create it
             try: