X-Git-Url: https://git.sommitrealweird.co.uk/rss2maildir.git/blobdiff_plain/ce65d98d57ea2b4332c291c0eed448f392201421..84f14a811430b6a49cd2d9adac6d4bcaa34c3037:/rss2maildir.py diff --git a/rss2maildir.py b/rss2maildir.py index c70c65b..f6078ff 100755 --- a/rss2maildir.py +++ b/rss2maildir.py @@ -93,6 +93,7 @@ class HTML2Text(HTMLParser): self.opentags = [] self.indentlevel = 0 self.listcount = [] + self.urls = [] HTMLParser.__init__(self) def handle_starttag(self, tag, attrs): @@ -106,13 +107,16 @@ class HTML2Text(HTMLParser): self.opentags.append(tag_name) self.opentags.pop() + if len(self.opentags) > 1 and self.opentags[-1] == u'li': + self.handle_curdata() + if tag_name == u'ol': self.handle_curdata() self.listcount.append(1) self.listlevel = len(self.listcount) - 1 if tag_name in self.liststarttags: - smallist = self.opentags[-3:] + smallist = self.opentags[-3:-1] smallist.reverse() for prev_listtag in smallist: if prev_listtag in [u'dl', u'ol']: @@ -133,14 +137,31 @@ class HTML2Text(HTMLParser): listcount = self.listcount[-1] except: pass + + if tag_name == u'dd' and len(self.opentags) > 1 \ + and self.opentags[-1] == u'dt': + self.handle_curdata() + self.opentags.pop() + elif tag_name == u'dt' and len(self.opentags) > 1 \ + and self.opentags[-1] == u'dd': + self.handle_curdata() + self.opentags.pop() + elif tag_name == u'a': + for attr in attrs: + if attr[0].lower() == u'href': + self.urls.append(attr[1]) + self.curdata = self.curdata + u'`' + self.opentags.append(tag_name) + return + self.handle_curdata() self.opentags.append(tag_name) def handle_startendtag(self, tag, attrs): if tag.lower() == u'br': - self.tags.append(u'br') + self.opentags.append(u'br') self.handle_curdata() # just handle the data, don't do anything else - self.tags.pop() + self.opentags.pop() def handle_curdata(self): if len(self.opentags) == 0: @@ -166,7 +187,8 @@ class HTML2Text(HTMLParser): if tag_thats_done in ["h1", "h2", "h3", "h4", "h5", "h6"]: underline = u'' underlinechar = u'=' - headingtext = self.curdata.encode("utf-8").strip() + headingtext = unicode( \ + self.curdata.encode("utf-8").strip(), "utf-8") seperator = u'\n' + u' '*self.indentlevel headingtext = seperator.join( \ textwrap.wrap( \ @@ -189,16 +211,21 @@ class HTML2Text(HTMLParser): self.text = self.text \ + headingtext.encode("utf-8") + u'\n' \ + underline - elif tag_thats_done == "p": - paragraph = self.curdata.encode("utf-8").strip() + elif tag_thats_done == u'p': + paragraph = unicode( \ + self.curdata.strip().encode("utf-8"), "utf-8") seperator = u'\n' + u' ' * self.indentlevel self.text = self.text \ + u' ' * self.indentlevel \ - + seperator.join(textwrap.wrap(paragraph, self.textwidth - self.indentlevel)) + + seperator.join( \ + textwrap.wrap( \ + paragraph, self.textwidth - self.indentlevel)) elif tag_thats_done == "pre": - self.text = self.text + self.curdata + self.text = self.text + unicode( \ + self.curdata.encode("utf-8"), "utf-8") elif tag_thats_done == "blockquote": - quote = self.curdata.encode("utf-8").strip() + quote = unicode( \ + self.curdata.encode("utf-8").strip(), "utf-8") seperator = u'\n' + u' ' * self.indentlevel + u'> ' self.text = self.text \ + u'> ' \ @@ -209,13 +236,13 @@ class HTML2Text(HTMLParser): ) ) elif tag_thats_done == "li": - item = self.curdata.encode("utf-8").strip() + item = unicode(self.curdata.encode("utf-8").strip(), "utf-8") if len(self.text) > 0 and self.text[-1] != u'\n': self.text = self.text + u'\n' # work out if we're in an ol rather than a ul latesttags = self.opentags[-4:] latesttags.reverse() - isul = False + isul = None for thing in latesttags: if thing == 'ul': isul = True @@ -229,7 +256,7 @@ class HTML2Text(HTMLParser): listindent = 4 listmarker = u' * ' - if not isul: + if isul == False: listmarker = u' %2d. ' %(self.listcount[-1]) self.listcount[-1] = self.listcount[-1] + 1 @@ -246,31 +273,37 @@ class HTML2Text(HTMLParser): ) \ ) self.curdata = u'' - elif tag_thats_done == "dt": - definition = self.curdata.encode("utf-8").strip() + elif tag_thats_done == u'dt': + definition = unicode(self.curdata.encode("utf-8").strip(), "utf-8") if len(self.text) > 0 and self.text[-1] != u'\n': self.text = self.text + u'\n\n' elif len(self.text) > 1 and self.text[-2] != u'\n': self.text = self.text + u'\n' - definition = definition + "::" + definition = u' ' * self.indentlevel + definition + "::" + indentstring = u'\n' + u' ' * (self.indentlevel + 1) self.text = self.text \ - + '\n '.join( - textwrap.wrap(definition, self.textwidth - 1)) + + indentstring.join( + textwrap.wrap(definition, \ + self.textwidth - self.indentlevel - 1)) self.curdata = u'' - elif tag_thats_done == "dd": - definition = self.curdata.encode("utf-8").strip() + elif tag_thats_done == u'dd': + definition = unicode(self.curdata.encode("utf-8").strip(), "utf-8") if len(definition) > 0: if len(self.text) > 0 and self.text[-1] != u'\n': self.text = self.text + u'\n' + indentstring = u'\n' + u' ' * (self.indentlevel + 4) self.text = self.text \ - + ' ' \ - + '\n '.join( \ + + u' ' * (self.indentlevel + 4) \ + + indentstring.join( \ textwrap.wrap( \ definition, \ self.textwidth - self.indentlevel - 4 \ ) \ ) self.curdata = u'' + elif tag_thats_done == u'a': + self.curdata = self.curdata + u'`__' + pass elif tag_thats_done in self.liststarttags: pass else: @@ -280,7 +313,10 @@ class HTML2Text(HTMLParser): self.text = self.text \ + u' ... ' \ + u'\n ... '.join( \ - textwrap.wrap(self.curdata, self.textwidth - 5)) + textwrap.wrap( \ + unicode( \ + self.curdata.encode("utf-8").strip(), \ + "utf-8"), self.textwidth - 5)) self.curdata = u'' if tag_thats_done in self.blockleveltags: @@ -298,6 +334,7 @@ class HTML2Text(HTMLParser): if tag in self.liststarttags: if tag in [u'ol', u'dl', u'ul']: + self.handle_curdata() # find if there was a previous list level smalllist = self.opentags[:-1] smalllist.reverse() @@ -352,6 +389,9 @@ class HTML2Text(HTMLParser): while len(self.text) > 1 and self.text[-1] == u'\n': self.text = self.text[:-1] self.text = self.text + u'\n' + if len(self.urls) > 0: + self.text = self.text + u'\n__ ' + u'\n__ '.join(self.urls) + u'\n' + self.urls = [] return self.text def open_url(method, url): @@ -557,7 +597,8 @@ def parse_and_deliver(maildir, url, statedir): if headers: data = [] for header in headers: - if header[0] in ["content-md5", "etag", "last-modified", "content-length"]: + if header[0] in \ + ["content-md5", "etag", "last-modified", "content-length"]: data.append((header[0], header[1])) if len(data) > 0: data = urllib.urlencode(data)