rss2maildir.py

   1 #!/usr/bin/python
   2 # coding=utf-8
   3
   4 # rss2maildir.py - RSS feeds to Maildir 1 email per item
   5 # Copyright (C) 2007  Brett Parker <iDunno@sommitrealweird.co.uk>
   6 #
   7 # This program is free software: you can redistribute it and/or modify
   8 # it under the terms of the GNU General Public License as published by
   9 # the Free Software Foundation, either version 3 of the License, or
  10 # (at your option) any later version.
  11 #
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU General Public License for more details.
  16 #
  17 # You should have received a copy of the GNU General Public License
  18 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  19
  20 import sys
  21 import os
  22 import stat
  23 import httplib
  24 import urllib
  25
  26 import feedparser
  27
  28 from email.MIMEMultipart import MIMEMultipart
  29 from email.MIMEText import MIMEText
  30
  31 import datetime
  32 import random
  33 import string
  34 import textwrap
  35
  36 import socket
  37
  38 from optparse import OptionParser
  39 from ConfigParser import SafeConfigParser
  40
  41 from base64 import b64encode
  42 import md5
  43
  44 import cgi
  45 import dbm
  46
  47 from HTMLParser import HTMLParser
  48
  49 entities = {
  50     "amp": "&",
  51     "lt": "<",
  52     "gt": ">",
  53     "pound": "£",
  54     "copy": "©",
  55     "apos": "'",
  56     "quot": "\"",
  57     "nbsp": " ",
  58     }
  59
  60 class HTML2Text(HTMLParser):
  61
  62     def __init__(self):
  63         self.inheadingone = False
  64         self.inheadingtwo = False
  65         self.inotherheading = False
  66         self.inparagraph = True
  67         self.inblockquote = False
  68         self.inlink = False
  69         self.text = u''
  70         self.currentparagraph = u''
  71         self.headingtext = u''
  72         self.blockquote = u''
  73         self.inpre = False
  74         self.inul = False
  75         self.initem = False
  76         self.item = u''
  77         HTMLParser.__init__(self)
  78
  79     def handle_starttag(self, tag, attrs):
  80         if tag.lower() == "h1":
  81             self.inheadingone = True
  82             self.inparagraph = False
  83         elif tag.lower() == "h2":
  84             self.inheadingtwo = True
  85             self.inparagraph = False
  86         elif tag.lower() in ["h3", "h4", "h5", "h6"]:
  87             self.inotherheading = True
  88             self.inparagraph = False
  89         elif tag.lower() == "a":
  90             self.inlink = True
  91         elif tag.lower() == "br":
  92             self.handle_br()
  93         elif tag.lower() == "blockquote":
  94             self.inblockquote = True
  95             self.text = self.text + u'\n'
  96         elif tag.lower() == "p":
  97             if self.text != "":
  98                 self.text = self.text + u'\n\n'
  99             if self.inparagraph:
 100                 self.text = self.text \
 101                     + u'\n'.join(textwrap.wrap(self.currentparagraph, 70))
 102             self.currentparagraph = u''
 103             self.inparagraph = True
 104         elif tag.lower() == "pre":
 105             self.text = self.text + "\n"
 106             self.inpre = True
 107             self.inparagraph = False
 108             self.inblockquote = False
 109         elif tag.lower() == "ul":
 110             self.item = u''
 111             self.inul = True
 112             self.text = self.text + "\n"
 113         elif tag.lower() == "li" and self.inul:
 114             if not self.initem:
 115                 self.initem = True
 116                 self.item = u''
 117             else:
 118                 self.text = self.text \
 119                     + u' * ' \
 120                     + u'\n   '.join([a.strip() for a in \
 121                         textwrap.wrap(self.item, 67)]) \
 122                     + u'\n'
 123                 self.item = u''
 124
 125     def handle_startendtag(self, tag, attrs):
 126         if tag.lower() == "br":
 127             self.handle_br()
 128
 129     def handle_br(self):
 130             if self.inparagraph:
 131                 self.text = self.text \
 132                 + u'\n'.join( \
 133                     [a \
 134                         for a in textwrap.wrap( \
 135                             self.currentparagraph, 70) \
 136                     ] \
 137                 ) \
 138                 + u'\n'
 139                 self.currentparagraph = u''
 140             elif self.inblockquote:
 141                 self.text = self.text \
 142                     + u'\n> ' \
 143                     + u'\n> '.join( \
 144                         [a \
 145                             for a in textwrap.wrap( \
 146                                 self.blockquote.encode("utf-8") \
 147                                 , 68) \
 148                         ] \
 149                     ) \
 150                     + u'\n'
 151                 self.blockquote = u''
 152             else:
 153                 self.text = self.text + "\n"
 154
 155     def handle_endtag(self, tag):
 156         if tag.lower() == "h1":
 157             self.inheadingone = False
 158             self.text = self.text \
 159                 + u'\n\n' \
 160                 + self.headingtext.encode("utf-8") \
 161                 + u'\n' \
 162                 + u'=' * len(self.headingtext.encode("utf-8").strip())
 163             self.headingtext = u''
 164         elif tag.lower() == "h2":
 165             self.inheadingtwo = False
 166             self.text = self.text \
 167                 + u'\n\n' \
 168                 + self.headingtext.encode("utf-8") \
 169                 + u'\n' \
 170                 + u'-' * len(self.headingtext.encode("utf-8").strip())
 171             self.headingtext = u''
 172         elif tag.lower() in ["h3", "h4", "h5", "h6"]:
 173             self.inotherheading = False
 174             self.text = self.text \
 175                 + u'\n\n' \
 176                 + self.headingtext.encode("utf-8") \
 177                 + u'\n' \
 178                 + u'~' * len(self.headingtext.encode("utf-8").strip())
 179             self.headingtext = u''
 180         elif tag.lower() == "p":
 181             self.text = self.text \
 182                 + u'\n'.join(textwrap.wrap( \
 183                     self.currentparagraph, 70) \
 184                 )
 185             self.inparagraph = False
 186             self.currentparagraph = u''
 187         elif tag.lower() == "blockquote":
 188             self.text = self.text \
 189                 + u'\n> ' \
 190                 + u'\n> '.join( \
 191                     [a.strip() \
 192                         for a in textwrap.wrap( \
 193                             self.blockquote, 68)] \
 194                     ) \
 195                 + u'\n'
 196             self.inblockquote = False
 197             self.blockquote = u''
 198         elif tag.lower() == "pre":
 199             self.inpre = False
 200         elif tag.lower() == "li":
 201             self.initem = False
 202             if self.item != "":
 203                 self.text = self.text \
 204                     + u' * ' \
 205                     + u'\n   '.join( \
 206                         [a.strip() for a in textwrap.wrap(self.item, 67)]) \
 207                     + u'\n'
 208             self.item = u''
 209         elif tag.lower() == "ul":
 210             self.inul = False
 211
 212     def handle_data(self, data):
 213         if self.inheadingone or self.inheadingtwo or self.inotherheading:
 214             self.headingtext = self.headingtext \
 215                 + unicode(data, "utf-8").strip() \
 216                 + u' '
 217         elif self.inblockquote:
 218             self.blockquote = self.blockquote \
 219                 + unicode(data, "utf-8").strip() \
 220                 + u' '
 221         elif self.inparagraph:
 222             self.currentparagraph = self.currentparagraph \
 223                 + unicode(data, "utf-8").strip() \
 224                 + u' '
 225         elif self.inul and self.initem:
 226             self.item = self.item + unicode(data, "utf-8")
 227         elif self.inpre:
 228             self.text = self.text + unicode(data, "utf-8")
 229         else:
 230             self.text = self.text + unicode(data, "utf-8").strip() + u' '
 231
 232     def handle_entityref(self, name):
 233         entity = name
 234         if entities.has_key(name.lower()):
 235             entity = entities[name.lower()]
 236         elif name[0] == "#":
 237             entity = unichr(int(name[1:]))
 238         else:
 239             entity = "&" + name + ";"
 240
 241         if self.inparagraph:
 242             self.currentparagraph = self.currentparagraph \
 243                 + unicode(entity, "utf-8")
 244         elif self.inblockquote:
 245             self.blockquote = self.blockquote + unicode(entity, "utf-8")
 246         else:
 247             self.text = self.text + unicode(entity, "utf-8")
 248
 249     def gettext(self):
 250         data = self.text
 251         if self.inparagraph:
 252             data = data + "\n".join(textwrap.wrap(self.currentparagraph, 70))
 253         return data
 254
 255 def open_url(method, url):
 256     redirectcount = 0
 257     while redirectcount < 3:
 258         (type, rest) = urllib.splittype(url)
 259         (host, path) = urllib.splithost(rest)
 260         (host, port) = urllib.splitport(host)
 261         if port == None:
 262             port = 80
 263         try:
 264             conn = httplib.HTTPConnection("%s:%s" %(host, port))
 265             conn.request(method, path)
 266             response = conn.getresponse()
 267             if response.status in [301, 302, 303, 307]:
 268                 headers = response.getheaders()
 269                 for header in headers:
 270                     if header[0] == "location":
 271                         url = header[1]
 272             elif response.status == 200:
 273                 return response
 274         except:
 275             pass
 276         redirectcount = redirectcount + 1
 277     return None
 278
 279 def parse_and_deliver(maildir, url, statedir):
 280     feedhandle = None
 281     headers = None
 282     # first check if we know about this feed already
 283     feeddb = dbm.open(os.path.join(statedir, "feeds"), "c")
 284     if feeddb.has_key(url):
 285         data = feeddb[url]
 286         data = cgi.parse_qs(data)
 287         response = open_url("HEAD", url)
 288         headers = None
 289         if response:
 290             headers = response.getheaders()
 291         ischanged = False
 292         try:
 293             for header in headers:
 294                 if header[0] == "content-length":
 295                     if header[1] != data["content-length"][0]:
 296                         ischanged = True
 297                 elif header[0] == "etag":
 298                     if header[1] != data["etag"][0]:
 299                         ischanged = True
 300                 elif header[0] == "last-modified":
 301                     if header[1] != data["last-modified"][0]:
 302                         ischanged = True
 303                 elif header[0] == "content-md5":
 304                     if header[1] != data["content-md5"][0]:
 305                         ischanged = True
 306         except:
 307             ischanged = True
 308         if ischanged:
 309             response = open_url("GET", url)
 310             if response != None:
 311                 headers = response.getheaders()
 312                 feedhandle = response
 313             else:
 314                 sys.stderr.write("Failed to fetch feed: %s\n" %(url))
 315                 return
 316         else:
 317             return # don't need to do anything, nothings changed.
 318     else:
 319         response = open_url("GET", url)
 320         if response != None:
 321             headers = response.getheaders()
 322             feedhandle = response
 323         else:
 324             sys.stderr.write("Failed to fetch feed: %s\n" %(url))
 325             return
 326
 327     fp = feedparser.parse(feedhandle)
 328     db = dbm.open(os.path.join(statedir, "seen"), "c")
 329     for item in fp["items"]:
 330         # have we seen it before?
 331         # need to work out what the content is first...
 332
 333         if item.has_key("content"):
 334             content = item["content"][0]["value"]
 335         else:
 336             content = item["summary"]
 337
 338         md5sum = md5.md5(content.encode("utf-8")).hexdigest()
 339
 340         prevmessageid = None
 341
 342         if db.has_key(url + "|" + item["link"]):
 343             data = db[url + "|" + item["link"]]
 344             data = cgi.parse_qs(data)
 345             if data.has_key("message-id"):
 346                 prevmessageid = data["message-id"][0]
 347             if data["contentmd5"][0] == md5sum:
 348                 continue
 349
 350         try:
 351             author = item["author"]
 352         except:
 353             author = url
 354
 355         # create a basic email message
 356         msg = MIMEMultipart("alternative")
 357         messageid = "<" \
 358             + datetime.datetime.now().strftime("%Y%m%d%H%M") \
 359             + "." \
 360             + "".join( \
 361                 [random.choice( \
 362                     string.ascii_letters + string.digits \
 363                     ) for a in range(0,6) \
 364                 ]) + "@" + socket.gethostname() + ">"
 365         msg.add_header("Message-ID", messageid)
 366         msg.set_unixfrom("\"%s\" <rss2maildir@localhost>" %(url))
 367         msg.add_header("From", "\"%s\" <rss2maildir@localhost>" %(author))
 368         msg.add_header("To", "\"%s\" <rss2maildir@localhost>" %(url))
 369         if prevmessageid:
 370             msg.add_header("References", prevmessageid)
 371         createddate = datetime.datetime.now() \
 372             .strftime("%a, %e %b %Y %T -0000")
 373         try:
 374             createddate = datetime.datetime(*item["updated_parsed"][0:6]) \
 375                 .strftime("%a, %e %b %Y %T -0000")
 376         except:
 377             pass
 378         msg.add_header("Date", createddate)
 379         msg.add_header("Subject", item["title"])
 380         msg.set_default_type("text/plain")
 381
 382         htmlpart = MIMEText(content.encode("utf-8"), "html", "utf-8")
 383         textparser = HTML2Text()
 384         textparser.feed(content.encode("utf-8"))
 385         textcontent = textparser.gettext()
 386         textpart = MIMEText(textcontent.encode("utf-8"), "plain", "utf-8")
 387         msg.attach(textpart)
 388         msg.attach(htmlpart)
 389
 390         # start by working out the filename we should be writting to, we do
 391         # this following the normal maildir style rules
 392         fname = str(os.getpid()) \
 393             + "." + socket.gethostname() \
 394             + "." + "".join( \
 395                 [random.choice( \
 396                     string.ascii_letters + string.digits \
 397                     ) for a in range(0,10) \
 398                 ]) + "." \
 399             + datetime.datetime.now().strftime('%s')
 400         fn = os.path.join(maildir, "tmp", fname)
 401         fh = open(fn, "w")
 402         fh.write(msg.as_string())
 403         fh.close()
 404         # now move it in to the new directory
 405         newfn = os.path.join(maildir, "new", fname)
 406         os.link(fn, newfn)
 407         os.unlink(fn)
 408
 409         # now add to the database about the item
 410         if prevmessageid:
 411             messageid = prevmessageid + " " + messageid
 412         data = urllib.urlencode((
 413             ("message-id", messageid), \
 414             ("created", createddate), \
 415             ("contentmd5", md5sum) \
 416             ))
 417         db[url + "|" + item["link"]] = data
 418
 419     if headers:
 420         data = []
 421         for header in headers:
 422             if header[0] in ["content-md5", "etag", "last-modified", "content-length"]:
 423                 data.append((header[0], header[1]))
 424         if len(data) > 0:
 425             data = urllib.urlencode(data)
 426             feeddb[url] = data
 427
 428     db.close()
 429     feeddb.close()
 430
 431 # first off, parse the command line arguments
 432
 433 oparser = OptionParser()
 434 oparser.add_option(
 435     "-c", "--conf", dest="conf",
 436     help="location of config file"
 437     )
 438 oparser.add_option(
 439     "-s", "--statedir", dest="statedir",
 440     help="location of directory to store state in"
 441     )
 442
 443 (options, args) = oparser.parse_args()
 444
 445 # check for the configfile
 446
 447 configfile = None
 448
 449 if options.conf != None:
 450     # does the file exist?
 451     try:
 452         os.stat(options.conf)
 453         configfile = options.conf
 454     except:
 455         # should exit here as the specified file doesn't exist
 456         sys.stderr.write( \
 457             "Config file %s does not exist. Exiting.\n" %(options.conf,))
 458         sys.exit(2)
 459 else:
 460     # check through the default locations
 461     try:
 462         os.stat("%s/.rss2maildir.conf" %(os.environ["HOME"],))
 463         configfile = "%s/.rss2maildir.conf" %(os.environ["HOME"],)
 464     except:
 465         try:
 466             os.stat("/etc/rss2maildir.conf")
 467             configfile = "/etc/rss2maildir.conf"
 468         except:
 469             sys.stderr.write("No config file found. Exiting.\n")
 470             sys.exit(2)
 471
 472 # Right - if we've got this far, we've got a config file, now for the hard
 473 # bits...
 474
 475 scp = SafeConfigParser()
 476 scp.read(configfile)
 477
 478 maildir_root = "RSSMaildir"
 479 state_dir = "state"
 480
 481 if options.statedir != None:
 482     state_dir = options.statedir
 483     try:
 484         mode = os.stat(state_dir)[stat.ST_MODE]
 485         if not stat.S_ISDIR(mode):
 486             sys.stderr.write( \
 487                 "State directory (%s) is not a directory\n" %(state_dir))
 488             sys.exit(1)
 489     except:
 490         # try to make the directory
 491         try:
 492             os.mkdir(state_dir)
 493         except:
 494             sys.stderr.write("Couldn't create statedir %s" %(state_dir))
 495             sys.exit(1)
 496 elif scp.has_option("general", "state_dir"):
 497     new_state_dir = scp.get("general", "state_dir")
 498     try:
 499         mode = os.stat(state_dir)[stat.ST_MODE]
 500         if not stat.S_ISDIR(mode):
 501             sys.stderr.write( \
 502                 "State directory (%s) is not a directory\n" %(state_dir))
 503             sys.exit(1)
 504     except:
 505         # try to create it
 506         try:
 507             os.mkdir(new_state_dir)
 508             state_dir = new_state_dir
 509         except:
 510             sys.stderr.write( \
 511                 "Couldn't create state directory %s\n" %(new_state_dir))
 512             sys.exit(1)
 513 else:
 514     try:
 515         mode = os.stat(state_dir)[stat.ST_MODE]
 516         if not stat.S_ISDIR(mode):
 517             sys.stderr.write( \
 518                 "State directory %s is not a directory\n" %(state_dir))
 519             sys.exit(1)
 520     except:
 521         try:
 522             os.mkdir(state_dir)
 523         except:
 524             sys.stderr.write( \
 525                 "State directory %s could not be created\n" %(state_dir))
 526             sys.exit(1)
 527
 528 if scp.has_option("general", "maildir_root"):
 529     maildir_root = scp.get("general", "maildir_root")
 530
 531 try:
 532     mode = os.stat(maildir_root)[stat.ST_MODE]
 533     if not stat.S_ISDIR(mode):
 534         sys.stderr.write( \
 535             "Maildir Root %s is not a directory\n" \
 536             %(maildir_root))
 537         sys.exit(1)
 538 except:
 539     try:
 540         os.mkdir(maildir_root)
 541     except:
 542         sys.stderr.write("Couldn't create Maildir Root %s\n" %(maildir_root))
 543         sys.exit(1)
 544
 545 feeds = scp.sections()
 546 try:
 547     feeds.remove("general")
 548 except:
 549     pass
 550
 551 for section in feeds:
 552     # check if the directory exists
 553     maildir = None
 554     try:
 555         maildir = scp.get(section, "maildir")
 556     except:
 557         maildir = section
 558
 559     maildir = urllib.urlencode(((section, maildir),)).split("=")[1]
 560     maildir = os.path.join(maildir_root, maildir)
 561
 562     try:
 563         exists = os.stat(maildir)
 564         if stat.S_ISDIR(exists[stat.ST_MODE]):
 565             # check if there's a new, cur and tmp directory
 566             try:
 567                 mode = os.stat(os.path.join(maildir, "cur"))[stat.ST_MODE]
 568             except:
 569                 os.mkdir(os.path.join(maildir, "cur"))
 570                 if not stat.S_ISDIR(mode):
 571                     sys.stderr.write("Broken maildir: %s\n" %(maildir))
 572             try:
 573                 mode = os.stat(os.path.join(maildir, "tmp"))[stat.ST_MODE]
 574             except:
 575                 os.mkdir(os.path.join(maildir, "tmp"))
 576                 if not stat.S_ISDIR(mode):
 577                     sys.stderr.write("Broken maildir: %s\n" %(maildir))
 578             try:
 579                 mode = os.stat(os.path.join(maildir, "new"))[stat.ST_MODE]
 580                 if not stat.S_ISDIR(mode):
 581                     sys.stderr.write("Broken maildir: %s\n" %(maildir))
 582             except:
 583                 os.mkdir(os.path.join(maildir, "new"))
 584         else:
 585             sys.stderr.write("Broken maildir: %s\n" %(maildir))
 586     except:
 587         try:
 588             os.mkdir(maildir)
 589         except:
 590             sys.stderr.write("Couldn't create root maildir %s\n" %(maildir))
 591             sys.exit(1)
 592         try:
 593             os.mkdir(os.path.join(maildir, "new"))
 594             os.mkdir(os.path.join(maildir, "cur"))
 595             os.mkdir(os.path.join(maildir, "tmp"))
 596         except:
 597             sys.stderr.write( \
 598                 "Couldn't create required maildir directories for %s\n" \
 599                 %(section,))
 600             sys.exit(1)
 601
 602     # right - we've got the directories, we've got the section, we know the
 603     # url... lets play!
 604
 605     parse_and_deliver(maildir, section, state_dir)