X-Git-Url: https://git.sommitrealweird.co.uk/rss2maildir.git/blobdiff_plain/9e17dd6f0d7309794355fdc73fb796de422c76a7..41170825eb09fcf3b3c69c9812073727ee8a2b4e:/rss2maildir.py diff --git a/rss2maildir.py b/rss2maildir.py index 6a319f4..25ddb3c 100755 --- a/rss2maildir.py +++ b/rss2maildir.py @@ -88,23 +88,7 @@ class HTML2Text(HTMLParser): elif tag.lower() == "a": self.inlink = True elif tag.lower() == "br": - if self.inparagraph: - self.text = self.text \ - + u'\n'.join( \ - textwrap.wrap(self.currentparagraph, 70)) \ - + u'\n' - self.currentparagraph = "" - elif self.inblockquote: - self.text = self.text \ - + u'\n> ' \ - + u'\n> '.join( \ - [a.strip() \ - for a in textwrap.wrap(self.blockquote, 68) \ - ]) \ - + u'\n' - self.blockquote = u'' - else: - self.text = self.text + u'\n' + self.handle_br() elif tag.lower() == "blockquote": self.inblockquote = True self.text = self.text + u'\n' @@ -132,12 +116,16 @@ class HTML2Text(HTMLParser): else: self.text = self.text \ + u' * ' \ - + u'\n '.join([a.strip() for a in textwrap.wrap(self.item, 67)]) \ + + u'\n '.join([a.strip() for a in \ + textwrap.wrap(self.item, 67)]) \ + u'\n' self.item = u'' def handle_startendtag(self, tag, attrs): if tag.lower() == "br": + self.handle_br() + + def handle_br(self): if self.inparagraph: self.text = self.text \ + u'\n'.join( \ @@ -199,8 +187,10 @@ class HTML2Text(HTMLParser): self.text = self.text \ + u'\n> ' \ + u'\n> '.join( \ - [a.strip() for a in textwrap.wrap(self.blockquote, 68)] \ - ).encode("utf-8") \ + [a.strip() \ + for a in textwrap.wrap( \ + self.blockquote, 68)] \ + ) \ + u'\n' self.inblockquote = False self.blockquote = u'' @@ -248,11 +238,12 @@ class HTML2Text(HTMLParser): entity = "&" + name + ";" if self.inparagraph: - self.currentparagraph = self.currentparagraph + entity + self.currentparagraph = self.currentparagraph \ + + unicode(entity, "utf-8") elif self.inblockquote: - self.blockquote = self.blockquote + entity + self.blockquote = self.blockquote + unicode(entity, "utf-8") else: - self.text = self.text + entity + self.text = self.text + unicode(entity, "utf-8") def gettext(self): data = self.text @@ -274,9 +265,13 @@ def parse_and_deliver(maildir, url, statedir): md5sum = md5.md5(content.encode("utf-8")).hexdigest() + prevmessageid = None + if db.has_key(url + "|" + item["link"]): data = db[url + "|" + item["link"]] data = cgi.parse_qs(data) + if data.has_key("message-id"): + prevmessageid = data["message-id"][0] if data["contentmd5"][0] == md5sum: continue @@ -299,6 +294,8 @@ def parse_and_deliver(maildir, url, statedir): msg.set_unixfrom("\"%s\" " %(url)) msg.add_header("From", "\"%s\" " %(author)) msg.add_header("To", "\"%s\" " %(url)) + if prevmessageid: + msg.add_header("References", prevmessageid) createddate = datetime.datetime(*item["updated_parsed"][0:6]) \ .strftime("%a, %e %b %Y %T -0000") msg.add_header("Date", createddate) @@ -333,6 +330,8 @@ def parse_and_deliver(maildir, url, statedir): os.unlink(fn) # now add to the database about the item + if prevmessageid: + messageid = prevmessageid + " " + messageid data = urllib.urlencode(( ("message-id", messageid), \ ("created", createddate), \ @@ -367,7 +366,8 @@ if options.conf != None: configfile = options.conf except: # should exit here as the specified file doesn't exist - sys.stderr.write("Config file %s does not exist. Exiting.\n" %(options.conf,)) + sys.stderr.write( \ + "Config file %s does not exist. Exiting.\n" %(options.conf,)) sys.exit(2) else: # check through the default locations @@ -396,7 +396,8 @@ if options.statedir != None: try: mode = os.stat(state_dir)[stat.ST_MODE] if not stat.S_ISDIR(mode): - sys.stderr.write("State directory (%s) is not a directory\n" %(state_dir)) + sys.stderr.write( \ + "State directory (%s) is not a directory\n" %(state_dir)) sys.exit(1) except: # try to make the directory @@ -410,7 +411,8 @@ elif scp.has_option("general", "state_dir"): try: mode = os.stat(state_dir)[stat.ST_MODE] if not stat.S_ISDIR(mode): - sys.stderr.write("State directory (%s) is not a directory\n" %(state_dir)) + sys.stderr.write( \ + "State directory (%s) is not a directory\n" %(state_dir)) sys.exit(1) except: # try to create it @@ -418,19 +420,22 @@ elif scp.has_option("general", "state_dir"): os.mkdir(new_state_dir) state_dir = new_state_dir except: - sys.stderr.write("Couldn't create state directory %s\n" %(new_state_dir)) + sys.stderr.write( \ + "Couldn't create state directory %s\n" %(new_state_dir)) sys.exit(1) else: try: mode = os.stat(state_dir)[stat.ST_MODE] if not stat.S_ISDIR(mode): - sys.stderr.write("State directory %s is not a directory\n" %(state_dir)) + sys.stderr.write( \ + "State directory %s is not a directory\n" %(state_dir)) sys.exit(1) except: try: os.mkdir(state_dir) except: - sys.stderr.write("State directory %s could not be created\n" %(state_dir)) + sys.stderr.write( \ + "State directory %s could not be created\n" %(state_dir)) sys.exit(1) if scp.has_option("general", "maildir_root"): @@ -439,7 +444,9 @@ if scp.has_option("general", "maildir_root"): try: mode = os.stat(maildir_root)[stat.ST_MODE] if not stat.S_ISDIR(mode): - sys.stderr.write("Maildir Root %s is not a directory\n" %(maildir_root)) + sys.stderr.write( \ + "Maildir Root %s is not a directory\n" \ + %(maildir_root)) sys.exit(1) except: try: @@ -500,7 +507,9 @@ for section in feeds: os.mkdir(os.path.join(maildir, "cur")) os.mkdir(os.path.join(maildir, "tmp")) except: - sys.stderr.write("Couldn't create required maildir directories for %s\n" %(section,)) + sys.stderr.write( \ + "Couldn't create required maildir directories for %s\n" \ + %(section,)) sys.exit(1) # right - we've got the directories, we've got the section, we know the