X-Git-Url: https://git.sommitrealweird.co.uk/rss2maildir.git/blobdiff_plain/9e17dd6f0d7309794355fdc73fb796de422c76a7..5b6a0aea2a9275597048c8c83597080fbe345056:/rss2maildir.py diff --git a/rss2maildir.py b/rss2maildir.py index 6a319f4..6dad334 100755 --- a/rss2maildir.py +++ b/rss2maildir.py @@ -20,6 +20,7 @@ import sys import os import stat +import httplib import urllib import feedparser @@ -52,12 +53,12 @@ entities = { "pound": "£", "copy": "©", "apos": "'", - "quote": "\"", + "quot": "\"", "nbsp": " ", } class HTML2Text(HTMLParser): - + def __init__(self): self.inheadingone = False self.inheadingtwo = False @@ -88,23 +89,7 @@ class HTML2Text(HTMLParser): elif tag.lower() == "a": self.inlink = True elif tag.lower() == "br": - if self.inparagraph: - self.text = self.text \ - + u'\n'.join( \ - textwrap.wrap(self.currentparagraph, 70)) \ - + u'\n' - self.currentparagraph = "" - elif self.inblockquote: - self.text = self.text \ - + u'\n> ' \ - + u'\n> '.join( \ - [a.strip() \ - for a in textwrap.wrap(self.blockquote, 68) \ - ]) \ - + u'\n' - self.blockquote = u'' - else: - self.text = self.text + u'\n' + self.handle_br() elif tag.lower() == "blockquote": self.inblockquote = True self.text = self.text + u'\n' @@ -132,12 +117,16 @@ class HTML2Text(HTMLParser): else: self.text = self.text \ + u' * ' \ - + u'\n '.join([a.strip() for a in textwrap.wrap(self.item, 67)]) \ + + u'\n '.join([a.strip() for a in \ + textwrap.wrap(self.item, 67)]) \ + u'\n' self.item = u'' def handle_startendtag(self, tag, attrs): if tag.lower() == "br": + self.handle_br() + + def handle_br(self): if self.inparagraph: self.text = self.text \ + u'\n'.join( \ @@ -199,8 +188,10 @@ class HTML2Text(HTMLParser): self.text = self.text \ + u'\n> ' \ + u'\n> '.join( \ - [a.strip() for a in textwrap.wrap(self.blockquote, 68)] \ - ).encode("utf-8") \ + [a.strip() \ + for a in textwrap.wrap( \ + self.blockquote, 68)] \ + ) \ + u'\n' self.inblockquote = False self.blockquote = u'' @@ -248,11 +239,12 @@ class HTML2Text(HTMLParser): entity = "&" + name + ";" if self.inparagraph: - self.currentparagraph = self.currentparagraph + entity + self.currentparagraph = self.currentparagraph \ + + unicode(entity, "utf-8") elif self.inblockquote: - self.blockquote = self.blockquote + entity + self.blockquote = self.blockquote + unicode(entity, "utf-8") else: - self.text = self.text + entity + self.text = self.text + unicode(entity, "utf-8") def gettext(self): data = self.text @@ -260,8 +252,79 @@ class HTML2Text(HTMLParser): data = data + "\n".join(textwrap.wrap(self.currentparagraph, 70)) return data +def open_url(method, url): + redirectcount = 0 + while redirectcount < 3: + (type, rest) = urllib.splittype(url) + (host, path) = urllib.splithost(rest) + (host, port) = urllib.splitport(host) + if port == None: + port = 80 + try: + conn = httplib.HTTPConnection("%s:%s" %(host, port)) + conn.request(method, path) + response = conn.getresponse() + if response.status in [301, 302, 303, 307]: + headers = response.getheaders() + for header in headers: + if header[0] == "location": + url = header[1] + elif response.status == 200: + return response + except: + pass + redirectcount = redirectcount + 1 + return None + def parse_and_deliver(maildir, url, statedir): - fp = feedparser.parse(url) + feedhandle = None + headers = None + # first check if we know about this feed already + feeddb = dbm.open(os.path.join(statedir, "feeds"), "c") + if feeddb.has_key(url): + data = feeddb[url] + data = cgi.parse_qs(data) + response = open_url("HEAD", url) + headers = None + if response: + headers = response.getheaders() + ischanged = False + try: + for header in headers: + if header[0] == "content-length": + if header[1] != data["content-length"][0]: + ischanged = True + elif header[0] == "etag": + if header[1] != data["etag"][0]: + ischanged = True + elif header[0] == "last-modified": + if header[1] != data["last-modified"][0]: + ischanged = True + elif header[0] == "content-md5": + if header[1] != data["content-md5"][0]: + ischanged = True + except: + ischanged = True + if ischanged: + response = open_url("GET", url) + if response != None: + headers = response.getheaders() + feedhandle = response + else: + sys.stderr.write("Failed to fetch feed: %s\n" %(url)) + return + else: + return # don't need to do anything, nothings changed. + else: + response = open_url("GET", url) + if response != None: + headers = response.getheaders() + feedhandle = response + else: + sys.stderr.write("Failed to fetch feed: %s\n" %(url)) + return + + fp = feedparser.parse(feedhandle) db = dbm.open(os.path.join(statedir, "seen"), "c") for item in fp["items"]: # have we seen it before? @@ -274,9 +337,22 @@ def parse_and_deliver(maildir, url, statedir): md5sum = md5.md5(content.encode("utf-8")).hexdigest() + prevmessageid = None + + # check if there's a guid too - if that exists and we match the md5, + # return + if item.has_key("guid"): + if db.has_key(url + "|" + item["guid"]): + data = db[url + "|" + item["guid"]] + data = cgi.parse_qs(data) + if data["contentmd5"][0] == md5sum: + continue + if db.has_key(url + "|" + item["link"]): data = db[url + "|" + item["link"]] data = cgi.parse_qs(data) + if data.has_key("message-id"): + prevmessageid = data["message-id"][0] if data["contentmd5"][0] == md5sum: continue @@ -299,8 +375,15 @@ def parse_and_deliver(maildir, url, statedir): msg.set_unixfrom("\"%s\" " %(url)) msg.add_header("From", "\"%s\" " %(author)) msg.add_header("To", "\"%s\" " %(url)) - createddate = datetime.datetime(*item["updated_parsed"][0:6]) \ + if prevmessageid: + msg.add_header("References", prevmessageid) + createddate = datetime.datetime.now() \ .strftime("%a, %e %b %Y %T -0000") + try: + createddate = datetime.datetime(*item["updated_parsed"][0:6]) \ + .strftime("%a, %e %b %Y %T -0000") + except: + pass msg.add_header("Date", createddate) msg.add_header("Subject", item["title"]) msg.set_default_type("text/plain") @@ -333,14 +416,45 @@ def parse_and_deliver(maildir, url, statedir): os.unlink(fn) # now add to the database about the item - data = urllib.urlencode(( - ("message-id", messageid), \ - ("created", createddate), \ - ("contentmd5", md5sum) \ - )) - db[url + "|" + item["link"]] = data + if prevmessageid: + messageid = prevmessageid + " " + messageid + if item.has_key("guid") and item["guid"] != item["link"]: + data = urllib.urlencode(( \ + ("message-id", messageid), \ + ("created", createddate), \ + ("contentmd5", md5sum) \ + )) + db[url + "|" + item["guid"]] = data + try: + data = db[url + "|" + item["link"]] + data = cgi.parse_qs(data) + newdata = urllib.urlencode(( \ + ("message-id", messageid), \ + ("created", data["created"][0]), \ + ("contentmd5", data["contentmd5"][0]) \ + )) + db[url + "|" + item["link"]] = newdata + except: + db[url + "|" + item["link"]] = data + else: + data = urllib.urlencode(( \ + ("message-id", messageid), \ + ("created", createddate), \ + ("contentmd5", md5sum) \ + )) + db[url + "|" + item["link"]] = data + + if headers: + data = [] + for header in headers: + if header[0] in ["content-md5", "etag", "last-modified", "content-length"]: + data.append((header[0], header[1])) + if len(data) > 0: + data = urllib.urlencode(data) + feeddb[url] = data db.close() + feeddb.close() # first off, parse the command line arguments @@ -367,7 +481,8 @@ if options.conf != None: configfile = options.conf except: # should exit here as the specified file doesn't exist - sys.stderr.write("Config file %s does not exist. Exiting.\n" %(options.conf,)) + sys.stderr.write( \ + "Config file %s does not exist. Exiting.\n" %(options.conf,)) sys.exit(2) else: # check through the default locations @@ -396,7 +511,8 @@ if options.statedir != None: try: mode = os.stat(state_dir)[stat.ST_MODE] if not stat.S_ISDIR(mode): - sys.stderr.write("State directory (%s) is not a directory\n" %(state_dir)) + sys.stderr.write( \ + "State directory (%s) is not a directory\n" %(state_dir)) sys.exit(1) except: # try to make the directory @@ -410,7 +526,8 @@ elif scp.has_option("general", "state_dir"): try: mode = os.stat(state_dir)[stat.ST_MODE] if not stat.S_ISDIR(mode): - sys.stderr.write("State directory (%s) is not a directory\n" %(state_dir)) + sys.stderr.write( \ + "State directory (%s) is not a directory\n" %(state_dir)) sys.exit(1) except: # try to create it @@ -418,19 +535,22 @@ elif scp.has_option("general", "state_dir"): os.mkdir(new_state_dir) state_dir = new_state_dir except: - sys.stderr.write("Couldn't create state directory %s\n" %(new_state_dir)) + sys.stderr.write( \ + "Couldn't create state directory %s\n" %(new_state_dir)) sys.exit(1) else: try: mode = os.stat(state_dir)[stat.ST_MODE] if not stat.S_ISDIR(mode): - sys.stderr.write("State directory %s is not a directory\n" %(state_dir)) + sys.stderr.write( \ + "State directory %s is not a directory\n" %(state_dir)) sys.exit(1) except: try: os.mkdir(state_dir) except: - sys.stderr.write("State directory %s could not be created\n" %(state_dir)) + sys.stderr.write( \ + "State directory %s could not be created\n" %(state_dir)) sys.exit(1) if scp.has_option("general", "maildir_root"): @@ -439,7 +559,9 @@ if scp.has_option("general", "maildir_root"): try: mode = os.stat(maildir_root)[stat.ST_MODE] if not stat.S_ISDIR(mode): - sys.stderr.write("Maildir Root %s is not a directory\n" %(maildir_root)) + sys.stderr.write( \ + "Maildir Root %s is not a directory\n" \ + %(maildir_root)) sys.exit(1) except: try: @@ -500,7 +622,9 @@ for section in feeds: os.mkdir(os.path.join(maildir, "cur")) os.mkdir(os.path.join(maildir, "tmp")) except: - sys.stderr.write("Couldn't create required maildir directories for %s\n" %(section,)) + sys.stderr.write( \ + "Couldn't create required maildir directories for %s\n" \ + %(section,)) sys.exit(1) # right - we've got the directories, we've got the section, we know the