X-Git-Url: https://git.sommitrealweird.co.uk/rss2maildir.git/blobdiff_plain/629cdfb564c31d54236ba24cd6d891cba1c2308a..e7a1184b64e99278b75386776949a92f08033fab:/rss2maildir.py diff --git a/rss2maildir.py b/rss2maildir.py index 910b8a9..b0bc441 100755 --- a/rss2maildir.py +++ b/rss2maildir.py @@ -70,6 +70,9 @@ class HTML2Text(HTMLParser): u'ul', u'ol', u'dl', + u'li', + u'dt', + u'dd', u'div', #u'blockquote', ] @@ -96,6 +99,7 @@ class HTML2Text(HTMLParser): self.ignorenodata = False self.listcount = [] self.urls = [] + self.images = {} HTMLParser.__init__(self) def handle_starttag(self, tag, attrs): @@ -148,7 +152,7 @@ class HTML2Text(HTMLParser): elif tag_name == u'a': for attr in attrs: if attr[0].lower() == u'href': - self.urls.append(attr[1]) + self.urls.append(attr[1].decode('utf-8')) self.curdata = self.curdata + u'`' self.opentags.append(tag_name) return @@ -180,24 +184,35 @@ class HTML2Text(HTMLParser): url = u'' for attr in attrs: if attr[0] == 'alt': - alt = attr[1] + alt = attr[1].decode('utf-8') elif attr[0] == 'src': - url = attr[1] + url = attr[1].decode('utf-8') if url: - self.curdata = self.curdata \ - + u' [img:' \ - + unicode( \ - url.encode('utf-8'), \ - 'utf-8') if alt: - self.curdata = self.curdata \ - + u'(' \ - + unicode( \ - alt.encode('utf-8'), \ - 'utf-8') \ - + u')' - self.curdata = self.curdata \ - + u']' + if self.images.has_key(alt): + if self.images[alt]["url"] == url: + self.curdata = self.curdata \ + + u'|%s|' %(alt,) + else: + while self.images.has_key(alt): + alt = alt + "_" + self.images[alt]["url"] = url + self.curdata = self.curdata \ + + u'|%s|' %(alt,) + else: + self.images[alt] = {} + self.images[alt]["url"] = url + self.curdata = self.curdata \ + + u'|%s|' %(alt,) + else: + if self.images.has_key(url): + self.curdata = self.curdata \ + + u'|%s|' %(url,) + else: + self.images[url] = {} + self.images[url]["url"] =url + self.curdata = self.curdata \ + + u'|%s|' %(url,) def handle_curdata(self): @@ -223,17 +238,20 @@ class HTML2Text(HTMLParser): if self.ignorenodata: newlinerequired = False self.ignorenodata = False - if newlinerequired \ - and len(self.text) > 2 \ - and self.text[-1] != u'\n' \ - and self.text[-2] != u'\n': + if newlinerequired: + if tag_thats_done in [u'dt', u'dd', u'li'] \ + and len(self.text) > 1 \ + and self.text[-1] != u'\n': + self.text = self.text + u'\n' + elif len(self.text) > 2 \ + and self.text[-1] != u'\n' \ + and self.text[-2] != u'\n': self.text = self.text + u'\n\n' if tag_thats_done in ["h1", "h2", "h3", "h4", "h5", "h6"]: underline = u'' underlinechar = u'=' - headingtext = unicode( \ - self.curdata.encode("utf-8").strip(), "utf-8") + headingtext = " ".join(self.curdata.split()) seperator = u'\n' + u' '*self.indentlevel headingtext = seperator.join( \ textwrap.wrap( \ @@ -254,11 +272,12 @@ class HTML2Text(HTMLParser): underline = u' ' * self.indentlevel \ + underlinechar * len(headingtext) self.text = self.text \ - + headingtext.encode("utf-8") + u'\n' \ + + headingtext + u'\n' \ + underline elif tag_thats_done in [u'p', u'div']: paragraph = unicode( \ - self.curdata.strip().encode("utf-8"), "utf-8") + " ".join(self.curdata.strip().encode("utf-8").split()), \ + "utf-8") seperator = u'\n' + u' ' * self.indentlevel self.text = self.text \ + u' ' * self.indentlevel \ @@ -267,10 +286,11 @@ class HTML2Text(HTMLParser): paragraph, self.textwidth - self.indentlevel)) elif tag_thats_done == "pre": self.text = self.text + unicode( \ - self.curdata.encode("utf-8"), "utf-8") + " ".join(self.curdata.encode("utf-8").split()), "utf-8") elif tag_thats_done == u'blockquote': quote = unicode( \ - self.curdata.encode("utf-8").strip(), "utf-8") + " ".join(self.curdata.encode("utf-8").strip().split()), \ + "utf-8") seperator = u'\n' + u' ' * self.indentlevel + u'> ' if len(self.text) > 0 and self.text[-1] != u'\n': self.text = self.text + u'\n' @@ -322,7 +342,9 @@ class HTML2Text(HTMLParser): ) self.curdata = u'' elif tag_thats_done == u'dt': - definition = unicode(self.curdata.encode("utf-8").strip(), "utf-8") + definition = unicode(" ".join( \ + self.curdata.encode("utf-8").strip().split()), \ + "utf-8") if len(self.text) > 0 and self.text[-1] != u'\n': self.text = self.text + u'\n\n' elif len(self.text) > 1 and self.text[-2] != u'\n': @@ -335,7 +357,9 @@ class HTML2Text(HTMLParser): self.textwidth - self.indentlevel - 1)) self.curdata = u'' elif tag_thats_done == u'dd': - definition = unicode(self.curdata.encode("utf-8").strip(), "utf-8") + definition = unicode(" ".join( \ + self.curdata.encode("utf-8").strip().split()), + "utf-8") if len(definition) > 0: if len(self.text) > 0 and self.text[-1] != u'\n': self.text = self.text + u'\n' @@ -411,7 +435,7 @@ class HTML2Text(HTMLParser): def handle_data(self, data): if len(self.opentags) == 0: self.opentags.append(u'p') - self.curdata = self.curdata + unicode(data, "utf-8") + self.curdata = self.curdata + data.decode("utf-8") def handle_entityref(self, name): entity = name @@ -436,6 +460,12 @@ class HTML2Text(HTMLParser): if len(self.urls) > 0: self.text = self.text + u'\n__ ' + u'\n__ '.join(self.urls) + u'\n' self.urls = [] + if len(self.images.keys()) > 0: + self.text = self.text + u'\n.. ' \ + + u'.. '.join( \ + ["|%s| image:: %s" %(a, self.images[a]["url"]) \ + for a in self.images.keys()]) + u'\n' + self.images = {} return self.text def open_url(method, url): @@ -721,11 +751,13 @@ if __name__ == "__main__": elif scp.has_option("general", "state_dir"): new_state_dir = scp.get("general", "state_dir") try: - mode = os.stat(state_dir)[stat.ST_MODE] + mode = os.stat(new_state_dir)[stat.ST_MODE] if not stat.S_ISDIR(mode): sys.stderr.write( \ "State directory (%s) is not a directory\n" %(state_dir)) sys.exit(1) + else: + state_dir = new_state_dir except: # try to create it try: