projects
/
rss2maildir.git
/ blobdiff
commit
grep
author
committer
pickaxe
?
search:
re
summary
|
shortlog
|
log
|
commit
|
commitdiff
|
tree
raw
|
inline
| side by side
Make sure that we feed the parser unicode data
[rss2maildir.git]
/
rss2maildir.py
diff --git
a/rss2maildir.py
b/rss2maildir.py
index a9abe4d1496cb676ffe4e3efe41d623431835dbe..e327bc3f046e6b345cef7f7a2bdfffd87ace643f 100755
(executable)
--- a/
rss2maildir.py
+++ b/
rss2maildir.py
@@
-40,6
+40,8
@@
from ConfigParser import SafeConfigParser
from base64 import b64encode
from base64 import b64encode
+import chardet
+
if sys.version_info[0] == 2 and sys.version_info[1] >= 6:
import hashlib as md5
else:
if sys.version_info[0] == 2 and sys.version_info[1] >= 6:
import hashlib as md5
else:
@@
-275,7
+277,7
@@
class HTML2Text(HTMLParser):
elif tag_name == u'a':
for attr in attrs:
if attr[0].lower() == u'href':
elif tag_name == u'a':
for attr in attrs:
if attr[0].lower() == u'href':
- self.urls.append(attr[1]
.decode('utf-8')
)
+ self.urls.append(attr[1])
self.curdata = self.curdata + u'`'
self.opentags.append(tag_name)
return
self.curdata = self.curdata + u'`'
self.opentags.append(tag_name)
return
@@
-308,12
+310,12
@@
class HTML2Text(HTMLParser):
for attr in attrs:
if attr[0] == 'alt':
if isinstance(attr[1], str):
for attr in attrs:
if attr[0] == 'alt':
if isinstance(attr[1], str):
- alt = u'%s' %(attr[1]
.decode("utf-8")
)
+ alt = u'%s' %(attr[1])
else:
alt = attr[1]
elif attr[0] == 'src':
if isinstance(attr[1], str):
else:
alt = attr[1]
elif attr[0] == 'src':
if isinstance(attr[1], str):
- url = u'%s' %(attr[1]
.decode("utf-8")
)
+ url = u'%s' %(attr[1])
else:
url = attr[1]
if url:
else:
url = attr[1]
if url:
@@
-566,7
+568,7
@@
class HTML2Text(HTMLParser):
def handle_data(self, data):
if len(self.opentags) == 0:
self.opentags.append(u'p')
def handle_data(self, data):
if len(self.opentags) == 0:
self.opentags.append(u'p')
- self.curdata =
self.curdata + data.decode("utf-8"
)
+ self.curdata =
"%s%s" %(self.curdata, data
)
def handle_charref(self, name):
try:
def handle_charref(self, name):
try:
@@
-707,6
+709,13
@@
def parse_and_deliver(maildir, url, statedir):
md5sum = md5.md5(content.encode("utf-8")).hexdigest()
md5sum = md5.md5(content.encode("utf-8")).hexdigest()
+ # make sure content is unicode encoded
+ if not isinstance(content, unicode):
+ cd_res = chardet.detect(content)
+ chrset = cd_res['encoding']
+ print "detected charset %s for item %s" %(chrset, item["link"])
+ content = content.decode(chrset)
+
prevmessageid = None
db_guid_key = None
prevmessageid = None
db_guid_key = None
@@
-765,7
+774,7
@@
def parse_and_deliver(maildir, url, statedir):
title = item["title"]
title = re.sub(u'<', u'<', title)
title = re.sub(u'>', u'>', title)
title = item["title"]
title = re.sub(u'<', u'<', title)
title = re.sub(u'>', u'>', title)
- subj_gen.feed(title
.encode("utf-8")
)
+ subj_gen.feed(title)
msg.add_header("Subject", subj_gen.gettext())
msg.set_default_type("text/plain")
msg.add_header("Subject", subj_gen.gettext())
msg.set_default_type("text/plain")
@@
-776,7
+785,7
@@
def parse_and_deliver(maildir, url, statedir):
item["link"] )
htmlpart = MIMEText(htmlcontent.encode("utf-8"), "html", "utf-8")
textparser = HTML2Text()
item["link"] )
htmlpart = MIMEText(htmlcontent.encode("utf-8"), "html", "utf-8")
textparser = HTML2Text()
- textparser.feed(content
.encode("utf-8")
)
+ textparser.feed(content)
textcontent = textparser.gettext()
textcontent = "%s\n\nItem URL: %s" %( \
textcontent, \
textcontent = textparser.gettext()
textcontent = "%s\n\nItem URL: %s" %( \
textcontent, \