projects
/
rss2maildir.git
/ commitdiff
commit
grep
author
committer
pickaxe
?
search:
re
summary
|
shortlog
|
log
|
commit
| commitdiff |
tree
raw
|
patch
|
inline
| side by side (from parent 1:
95250ed
)
* Handle unicode data more effectively.
author
Brett Parker
<iDunno@sommitrealweird.co.uk>
Thu, 10 Jan 2008 18:23:17 +0000
(18:23 +0000)
committer
Brett Parker
<iDunno@sommitrealweird.co.uk>
Thu, 10 Jan 2008 18:23:17 +0000
(18:23 +0000)
rss2maildir.py
patch
|
blob
|
history
diff --git
a/rss2maildir.py
b/rss2maildir.py
index 4421811152a18c37b587f9e46e7568e28fb47760..8f7689314cc80f3548f33d8dde3acd18d202b59d 100755
(executable)
--- a/
rss2maildir.py
+++ b/
rss2maildir.py
@@
-179,7
+179,8
@@
class HTML2Text(HTMLParser):
if tag_thats_done in ["h1", "h2", "h3", "h4", "h5", "h6"]:
underline = u''
underlinechar = u'='
if tag_thats_done in ["h1", "h2", "h3", "h4", "h5", "h6"]:
underline = u''
underlinechar = u'='
- headingtext = self.curdata.encode("utf-8").strip()
+ headingtext = unicode( \
+ self.curdata.encode("utf-8").strip(), "utf-8")
seperator = u'\n' + u' '*self.indentlevel
headingtext = seperator.join( \
textwrap.wrap( \
seperator = u'\n' + u' '*self.indentlevel
headingtext = seperator.join( \
textwrap.wrap( \
@@
-203,15
+204,20
@@
class HTML2Text(HTMLParser):
+ headingtext.encode("utf-8") + u'\n' \
+ underline
elif tag_thats_done == u'p':
+ headingtext.encode("utf-8") + u'\n' \
+ underline
elif tag_thats_done == u'p':
- paragraph = self.curdata.encode("utf-8").strip()
+ paragraph = unicode( \
+ self.curdata.strip().encode("utf-8"), "utf-8")
seperator = u'\n' + u' ' * self.indentlevel
self.text = self.text \
+ u' ' * self.indentlevel \
seperator = u'\n' + u' ' * self.indentlevel
self.text = self.text \
+ u' ' * self.indentlevel \
- + seperator.join(textwrap.wrap(paragraph, self.textwidth - self.indentlevel))
+ + seperator.join( \
+ textwrap.wrap( \
+ paragraph, self.textwidth - self.indentlevel))
elif tag_thats_done == "pre":
elif tag_thats_done == "pre":
- self.text = self.text + self.curdata
+ self.text = self.text + unicode( \
+ self.curdata.encode("utf-8"), "utf-8")
elif tag_thats_done == "blockquote":
elif tag_thats_done == "blockquote":
- quote = self.curdata.encode("utf-8").strip()
+ quote = unicode( \
+ self.curdata.encode("utf-8").strip(), "utf-8")
seperator = u'\n' + u' ' * self.indentlevel + u'> '
self.text = self.text \
+ u'> ' \
seperator = u'\n' + u' ' * self.indentlevel + u'> '
self.text = self.text \
+ u'> ' \
@@
-222,13
+228,13
@@
class HTML2Text(HTMLParser):
)
)
elif tag_thats_done == "li":
)
)
elif tag_thats_done == "li":
- item =
self.curdata.encode("utf-8").strip(
)
+ item =
unicode(self.curdata.encode("utf-8").strip(), "utf-8"
)
if len(self.text) > 0 and self.text[-1] != u'\n':
self.text = self.text + u'\n'
# work out if we're in an ol rather than a ul
latesttags = self.opentags[-4:]
latesttags.reverse()
if len(self.text) > 0 and self.text[-1] != u'\n':
self.text = self.text + u'\n'
# work out if we're in an ol rather than a ul
latesttags = self.opentags[-4:]
latesttags.reverse()
- isul =
Fals
e
+ isul =
Non
e
for thing in latesttags:
if thing == 'ul':
isul = True
for thing in latesttags:
if thing == 'ul':
isul = True
@@
-242,7
+248,7
@@
class HTML2Text(HTMLParser):
listindent = 4
listmarker = u' * '
listindent = 4
listmarker = u' * '
- if
not isul
:
+ if
isul == False
:
listmarker = u' %2d. ' %(self.listcount[-1])
self.listcount[-1] = self.listcount[-1] + 1
listmarker = u' %2d. ' %(self.listcount[-1])
self.listcount[-1] = self.listcount[-1] + 1
@@
-260,7
+266,7
@@
class HTML2Text(HTMLParser):
)
self.curdata = u''
elif tag_thats_done == u'dt':
)
self.curdata = u''
elif tag_thats_done == u'dt':
- definition =
self.curdata.encode("utf-8").strip(
)
+ definition =
unicode(self.curdata.encode("utf-8").strip(), "utf-8"
)
if len(self.text) > 0 and self.text[-1] != u'\n':
self.text = self.text + u'\n\n'
elif len(self.text) > 1 and self.text[-2] != u'\n':
if len(self.text) > 0 and self.text[-1] != u'\n':
self.text = self.text + u'\n\n'
elif len(self.text) > 1 and self.text[-2] != u'\n':
@@
-273,7
+279,7
@@
class HTML2Text(HTMLParser):
self.textwidth - self.indentlevel - 1))
self.curdata = u''
elif tag_thats_done == u'dd':
self.textwidth - self.indentlevel - 1))
self.curdata = u''
elif tag_thats_done == u'dd':
- definition =
self.curdata.encode("utf-8").strip(
)
+ definition =
unicode(self.curdata.encode("utf-8").strip(), "utf-8"
)
if len(definition) > 0:
if len(self.text) > 0 and self.text[-1] != u'\n':
self.text = self.text + u'\n'
if len(definition) > 0:
if len(self.text) > 0 and self.text[-1] != u'\n':
self.text = self.text + u'\n'
@@
-296,7
+302,10
@@
class HTML2Text(HTMLParser):
self.text = self.text \
+ u' ... ' \
+ u'\n ... '.join( \
self.text = self.text \
+ u' ... ' \
+ u'\n ... '.join( \
- textwrap.wrap(self.curdata, self.textwidth - 5))
+ textwrap.wrap( \
+ unicode( \
+ self.curdata.encode("utf-8").strip(), \
+ "utf-8"), self.textwidth - 5))
self.curdata = u''
if tag_thats_done in self.blockleveltags:
self.curdata = u''
if tag_thats_done in self.blockleveltags:
@@
-574,7
+583,8
@@
def parse_and_deliver(maildir, url, statedir):
if headers:
data = []
for header in headers:
if headers:
data = []
for header in headers:
- if header[0] in ["content-md5", "etag", "last-modified", "content-length"]:
+ if header[0] in \
+ ["content-md5", "etag", "last-modified", "content-length"]:
data.append((header[0], header[1]))
if len(data) > 0:
data = urllib.urlencode(data)
data.append((header[0], header[1]))
if len(data) > 0:
data = urllib.urlencode(data)