X-Git-Url: https://git.sommitrealweird.co.uk/rss2maildir.git/blobdiff_plain/ed5781ac5244b22a6859fa21701acc462cace372..HEAD:/rss2maildir.py diff --git a/rss2maildir.py b/rss2maildir.py index 482aaaf..314a9c1 100755 --- a/rss2maildir.py +++ b/rss2maildir.py @@ -39,7 +39,13 @@ from optparse import OptionParser from ConfigParser import SafeConfigParser from base64 import b64encode -import md5 + +import chardet + +if sys.version_info[0] == 2 and sys.version_info[1] >= 6: + import hashlib as md5 +else: + import md5 import cgi import dbm @@ -271,7 +277,7 @@ class HTML2Text(HTMLParser): elif tag_name == u'a': for attr in attrs: if attr[0].lower() == u'href': - self.urls.append(attr[1].decode('utf-8')) + self.urls.append(attr[1]) self.curdata = self.curdata + u'`' self.opentags.append(tag_name) return @@ -303,9 +309,15 @@ class HTML2Text(HTMLParser): url = u'' for attr in attrs: if attr[0] == 'alt': - alt = attr[1].decode('utf-8') + if isinstance(attr[1], str): + alt = u'%s' %(attr[1]) + else: + alt = attr[1] elif attr[0] == 'src': - url = attr[1].decode('utf-8') + if isinstance(attr[1], str): + url = u'%s' %(attr[1]) + else: + url = attr[1] if url: if alt: if self.images.has_key(alt): @@ -556,7 +568,7 @@ class HTML2Text(HTMLParser): def handle_data(self, data): if len(self.opentags) == 0: self.opentags.append(u'p') - self.curdata = self.curdata + data.decode("utf-8") + self.curdata = "%s%s" %(self.curdata, data) def handle_charref(self, name): try: @@ -608,10 +620,17 @@ def open_url(method, url): (type, rest) = urllib.splittype(url) (host, path) = urllib.splithost(rest) (host, port) = urllib.splitport(host) - if port == None: + if type == "https": + if port == None: + port = 443 + elif port == None: port = 80 try: - conn = httplib.HTTPConnection("%s:%s" %(host, port)) + conn = None + if type == "http": + conn = httplib.HTTPConnection("%s:%s" %(host, port)) + else: + conn = httplib.HTTPSConnection("%s:%s" %(host, port)) conn.request(method, path) response = conn.getresponse() if response.status in [301, 302, 303, 307]: @@ -690,9 +709,18 @@ def parse_and_deliver(maildir, url, statedir): md5sum = md5.md5(content.encode("utf-8")).hexdigest() + # make sure content is unicode encoded + if not isinstance(content, unicode): + cd_res = chardet.detect(content) + chrset = cd_res['encoding'] + print "detected charset %s for item %s" %(chrset, item["link"]) + content = content.decode(chrset) + prevmessageid = None db_guid_key = None + if not item.has_key("link"): + item["link"] = u'#' + md5sum db_link_key = (url + u'|' + item["link"]).encode("utf-8") # check if there's a guid too - if that exists and we match the md5, @@ -730,8 +758,8 @@ def parse_and_deliver(maildir, url, statedir): ]) + "@" + socket.gethostname() + ">" msg.add_header("Message-ID", messageid) msg.set_unixfrom("\"%s\" " %(url)) - msg.add_header("From", "\"%s\" " %(author)) - msg.add_header("To", "\"%s\" " %(url)) + msg.add_header("From", "\"%s\" " %(author.encode("utf-8"))) + msg.add_header("To", "\"%s\" " %(url.encode("utf-8"))) if prevmessageid: msg.add_header("References", prevmessageid) createddate = datetime.datetime.now() \ @@ -743,12 +771,12 @@ def parse_and_deliver(maildir, url, statedir): pass msg.add_header("Date", createddate) msg.add_header("X-rss2maildir-rundate", datetime.datetime.now() \ - .strftime("%a, %e %b %Y %T -0000") + .strftime("%a, %e %b %Y %T -0000")) subj_gen = HTML2Text() title = item["title"] title = re.sub(u'<', u'<', title) title = re.sub(u'>', u'>', title) - subj_gen.feed(title.encode("utf-8")) + subj_gen.feed(title) msg.add_header("Subject", subj_gen.gettext()) msg.set_default_type("text/plain") @@ -759,7 +787,7 @@ def parse_and_deliver(maildir, url, statedir): item["link"] ) htmlpart = MIMEText(htmlcontent.encode("utf-8"), "html", "utf-8") textparser = HTML2Text() - textparser.feed(content.encode("utf-8")) + textparser.feed(content) textcontent = textparser.gettext() textcontent = "%s\n\nItem URL: %s" %( \ textcontent, \