rss2maildir.py

   1 #!/usr/bin/python
   2 # coding=utf-8
   3
   4 # rss2maildir.py - RSS feeds to Maildir 1 email per item
   5 # Copyright (C) 2007  Brett Parker <iDunno@sommitrealweird.co.uk>
   6 #
   7 # This program is free software: you can redistribute it and/or modify
   8 # it under the terms of the GNU General Public License as published by
   9 # the Free Software Foundation, either version 3 of the License, or
  10 # (at your option) any later version.
  11 #
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU General Public License for more details.
  16 #
  17 # You should have received a copy of the GNU General Public License
  18 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  19
  20 import sys
  21 import os
  22 import stat
  23 import httplib
  24 import urllib
  25
  26 import feedparser
  27
  28 from email.MIMEMultipart import MIMEMultipart
  29 from email.MIMEText import MIMEText
  30
  31 import datetime
  32 import random
  33 import string
  34 import textwrap
  35
  36 import socket
  37
  38 from optparse import OptionParser
  39 from ConfigParser import SafeConfigParser
  40
  41 from base64 import b64encode
  42 import md5
  43
  44 import cgi
  45 import dbm
  46
  47 from HTMLParser import HTMLParser
  48
  49 class HTML2Text(HTMLParser):
  50     entities = {
  51         "amp": "&",
  52         "lt": "<",
  53         "gt": ">",
  54         "pound": "£",
  55         "copy": "©",
  56         "apos": "'",
  57         "quot": "\"",
  58         "nbsp": " ",
  59         }
  60
  61     blockleveltags = [
  62         "h1",
  63         "h2",
  64         "h3",
  65         "h4",
  66         "h5",
  67         "h6",
  68         "pre",
  69         "p",
  70         "ul",
  71         "ol",
  72         "dl",
  73         "br",
  74         ]
  75
  76     liststarttags = [
  77         "ul",
  78         "ol",
  79         "dl",
  80         ]
  81
  82     cancontainflow = [
  83         "div",
  84         "li",
  85         "dd",
  86         "blockquote",
  87     ]
  88
  89     def __init__(self,textwidth=70):
  90         self.text = u''
  91         self.curdata = u''
  92         self.textwidth = textwidth
  93         self.opentags = []
  94         self.indentlevel = 0
  95         self.listcount = []
  96         HTMLParser.__init__(self)
  97
  98     def handle_starttag(self, tag, attrs):
  99         tag_name = tag.lower()
 100         if tag_name in self.blockleveltags:
 101             # handle starting a new block - unless we're in a block element
 102             # that can contain other blocks, we'll assume that we want to close
 103             # the container
 104             if tag_name == u'br':
 105                 self.handle_curdata()
 106                 self.opentags.append(tag_name)
 107                 self.opentags.pop()
 108
 109             if len(self.opentags) > 1 and self.opentags[-1] == u'li':
 110                 self.handle_curdata()
 111
 112             if tag_name == u'ol':
 113                 self.handle_curdata()
 114                 self.listcount.append(1)
 115                 self.listlevel = len(self.listcount) - 1
 116
 117             if tag_name in self.liststarttags:
 118                 smallist = self.opentags[-3:-1]
 119                 smallist.reverse()
 120                 for prev_listtag in smallist:
 121                     if prev_listtag in [u'dl', u'ol']:
 122                         self.indentlevel = self.indentlevel + 4
 123                         break
 124                     elif prev_listtag == u'ul':
 125                         self.indentlevel = self.indentlevel + 3
 126                         break
 127
 128             if len(self.opentags) > 0:
 129                 self.handle_curdata()
 130                 if tag_name not in self.cancontainflow:
 131                     self.opentags.pop()
 132             self.opentags.append(tag_name)
 133         else:
 134             listcount = 0
 135             try:
 136                 listcount = self.listcount[-1]
 137             except:
 138                 pass
 139
 140             if tag_name == u'dd' and len(self.opentags) > 1 \
 141                 and self.opentags[-1] == u'dt':
 142                 self.handle_curdata()
 143                 self.opentags.pop()
 144             elif tag_name == u'dt' and len(self.opentags) > 1 \
 145                 and self.opentags[-1] == u'dd':
 146                 self.handle_curdata()
 147                 self.opentags.pop()
 148
 149             self.handle_curdata()
 150             self.opentags.append(tag_name)
 151
 152     def handle_startendtag(self, tag, attrs):
 153         if tag.lower() == u'br':
 154             self.opentags.append(u'br')
 155             self.handle_curdata() # just handle the data, don't do anything else
 156             self.opentags.pop()
 157
 158     def handle_curdata(self):
 159         if len(self.opentags) == 0:
 160             return
 161
 162         if len(self.curdata) == 0:
 163             return
 164
 165         if len(self.curdata.strip()) == 0:
 166             return
 167
 168         tag_thats_done = self.opentags[-1]
 169
 170         if tag_thats_done in self.blockleveltags:
 171             newlinerequired = self.text != u''
 172             if newlinerequired:
 173                 if newlinerequired \
 174                     and len(self.text) > 2 \
 175                     and self.text[-1] != u'\n' \
 176                     and self.text[-2] != u'\n':
 177                     self.text = self.text + u'\n\n'
 178
 179         if tag_thats_done in ["h1", "h2", "h3", "h4", "h5", "h6"]:
 180             underline = u''
 181             underlinechar = u'='
 182             headingtext = unicode( \
 183                 self.curdata.encode("utf-8").strip(), "utf-8")
 184             seperator = u'\n' + u' '*self.indentlevel
 185             headingtext = seperator.join( \
 186                 textwrap.wrap( \
 187                     headingtext, \
 188                     self.textwidth - self.indentlevel \
 189                     ) \
 190                 )
 191
 192             if tag_thats_done == u'h2':
 193                 underlinechar = u'-'
 194             elif tag_thats_done != u'h1':
 195                 underlinechar = u'~'
 196
 197             if u'\n' in headingtext:
 198                 underline = u' ' * self.indentlevel \
 199                     + underlinechar * (self.textwidth - self.indentlevel)
 200             else:
 201                 underline = u' ' * self.indentlevel \
 202                     + underlinechar * len(headingtext)
 203             self.text = self.text \
 204                 + headingtext.encode("utf-8") + u'\n' \
 205                 + underline
 206         elif tag_thats_done == u'p':
 207             paragraph = unicode( \
 208                 self.curdata.strip().encode("utf-8"), "utf-8")
 209             seperator = u'\n' + u' ' * self.indentlevel
 210             self.text = self.text \
 211                 + u' ' * self.indentlevel \
 212                 + seperator.join( \
 213                     textwrap.wrap( \
 214                         paragraph, self.textwidth - self.indentlevel))
 215         elif tag_thats_done == "pre":
 216             self.text = self.text + unicode( \
 217                 self.curdata.encode("utf-8"), "utf-8")
 218         elif tag_thats_done == "blockquote":
 219             quote = unicode( \
 220                 self.curdata.encode("utf-8").strip(), "utf-8")
 221             seperator = u'\n' + u' ' * self.indentlevel + u'> '
 222             self.text = self.text \
 223                 + u'> ' \
 224                 + seperator.join( \
 225                     textwrap.wrap( \
 226                         quote, \
 227                         self.textwidth - self.indentlevel - 2 \
 228                     )
 229                 )
 230         elif tag_thats_done == "li":
 231             item = unicode(self.curdata.encode("utf-8").strip(), "utf-8")
 232             if len(self.text) > 0 and self.text[-1] != u'\n':
 233                 self.text = self.text + u'\n'
 234             # work out if we're in an ol rather than a ul
 235             latesttags = self.opentags[-4:]
 236             latesttags.reverse()
 237             isul = None
 238             for thing in latesttags:
 239                 if thing == 'ul':
 240                     isul = True
 241                     break
 242                 elif thing == 'ol':
 243                     isul = False
 244                     break
 245
 246             listindent = 3
 247             if not isul:
 248                 listindent = 4
 249
 250             listmarker = u' * '
 251             if isul == False:
 252                 listmarker = u' %2d. ' %(self.listcount[-1])
 253                 self.listcount[-1] = self.listcount[-1] + 1
 254
 255             seperator = u'\n' \
 256                 + u' ' * self.indentlevel \
 257                 + u' ' * listindent
 258             self.text = self.text \
 259                 + u' ' * self.indentlevel \
 260                 + listmarker \
 261                 + seperator.join( \
 262                     textwrap.wrap( \
 263                         item, \
 264                         self.textwidth - self.indentlevel - listindent \
 265                     ) \
 266                 )
 267             self.curdata = u''
 268         elif tag_thats_done == u'dt':
 269             definition = unicode(self.curdata.encode("utf-8").strip(), "utf-8")
 270             if len(self.text) > 0 and self.text[-1] != u'\n':
 271                 self.text = self.text + u'\n\n'
 272             elif len(self.text) > 1 and self.text[-2] != u'\n':
 273                 self.text = self.text + u'\n'
 274             definition = u' ' * self.indentlevel + definition + "::"
 275             indentstring = u'\n' + u' ' * (self.indentlevel + 1)
 276             self.text = self.text \
 277                 + indentstring.join(
 278                     textwrap.wrap(definition, \
 279                         self.textwidth - self.indentlevel - 1))
 280             self.curdata = u''
 281         elif tag_thats_done == u'dd':
 282             definition = unicode(self.curdata.encode("utf-8").strip(), "utf-8")
 283             if len(definition) > 0:
 284                 if len(self.text) > 0 and self.text[-1] != u'\n':
 285                     self.text = self.text + u'\n'
 286                 indentstring = u'\n' + u' ' * (self.indentlevel + 4)
 287                 self.text = self.text \
 288                     + u' ' * (self.indentlevel + 4) \
 289                     + indentstring.join( \
 290                         textwrap.wrap( \
 291                             definition, \
 292                             self.textwidth - self.indentlevel - 4 \
 293                             ) \
 294                         )
 295                 self.curdata = u''
 296         elif tag_thats_done in self.liststarttags:
 297             pass
 298         else:
 299             # we've got no idea what this tag does, so we'll
 300             # make an assumption that we're not going to know later
 301             if len(self.curdata) > 0:
 302                 self.text = self.text \
 303                     + u' ... ' \
 304                     + u'\n ... '.join( \
 305                         textwrap.wrap( \
 306                             unicode( \
 307                                 self.curdata.encode("utf-8").strip(), \
 308                                 "utf-8"), self.textwidth - 5))
 309             self.curdata = u''
 310
 311         if tag_thats_done in self.blockleveltags:
 312             self.curdata = u''
 313
 314     def handle_endtag(self, tag):
 315         try:
 316             tagindex = self.opentags.index(tag)
 317         except:
 318             # closing tag we know nothing about.
 319             # err. weird.
 320             tagindex = 0
 321
 322         tag = tag.lower()
 323
 324         if tag in self.liststarttags:
 325             if tag in [u'ol', u'dl', u'ul']:
 326                 self.handle_curdata()
 327                 # find if there was a previous list level
 328                 smalllist = self.opentags[:-1]
 329                 smalllist.reverse()
 330                 for prev_listtag in smalllist:
 331                     if prev_listtag in [u'ol', u'dl']:
 332                         self.indentlevel = self.indentlevel - 4
 333                         break
 334                     elif prev_listtag == u'ul':
 335                         self.indentlevel = self.indentlevel - 3
 336                         break
 337
 338         if tag == u'ol':
 339             self.listcount = self.listcount[:-1]
 340
 341         while tagindex < len(self.opentags) \
 342             and tag in self.opentags[tagindex+1:]:
 343             try:
 344                 tagindex = self.opentags.index(tag, tagindex+1)
 345             except:
 346                 # well, we don't want to do that then
 347                 pass
 348         if tagindex != len(self.opentags) - 1:
 349             # Assuming the data was for the last opened tag first
 350             self.handle_curdata()
 351             # Now kill the list to be a slice before this tag was opened
 352             self.opentags = self.opentags[:tagindex + 1]
 353         else:
 354             self.handle_curdata()
 355             if self.opentags[-1] == tag:
 356                 self.opentags.pop()
 357
 358     def handle_data(self, data):
 359         self.curdata = self.curdata + unicode(data, "utf-8")
 360
 361     def handle_entityref(self, name):
 362         entity = name
 363         if HTML2Text.entities.has_key(name.lower()):
 364             entity = HTML2Text.entities[name.lower()]
 365         elif name[0] == "#":
 366             entity = unichr(int(name[1:]))
 367         else:
 368             entity = "&" + name + ";"
 369
 370         self.curdata = self.curdata + unicode(entity, "utf-8")
 371
 372     def gettext(self):
 373         self.handle_curdata()
 374         if len(self.text) == 0 or self.text[-1] != u'\n':
 375             self.text = self.text + u'\n'
 376         self.opentags = []
 377         if len(self.text) > 0:
 378             while len(self.text) > 1 and self.text[-1] == u'\n':
 379                 self.text = self.text[:-1]
 380             self.text = self.text + u'\n'
 381         return self.text
 382
 383 def open_url(method, url):
 384     redirectcount = 0
 385     while redirectcount < 3:
 386         (type, rest) = urllib.splittype(url)
 387         (host, path) = urllib.splithost(rest)
 388         (host, port) = urllib.splitport(host)
 389         if port == None:
 390             port = 80
 391         try:
 392             conn = httplib.HTTPConnection("%s:%s" %(host, port))
 393             conn.request(method, path)
 394             response = conn.getresponse()
 395             if response.status in [301, 302, 303, 307]:
 396                 headers = response.getheaders()
 397                 for header in headers:
 398                     if header[0] == "location":
 399                         url = header[1]
 400             elif response.status == 200:
 401                 return response
 402         except:
 403             pass
 404         redirectcount = redirectcount + 1
 405     return None
 406
 407 def parse_and_deliver(maildir, url, statedir):
 408     feedhandle = None
 409     headers = None
 410     # first check if we know about this feed already
 411     feeddb = dbm.open(os.path.join(statedir, "feeds"), "c")
 412     if feeddb.has_key(url):
 413         data = feeddb[url]
 414         data = cgi.parse_qs(data)
 415         response = open_url("HEAD", url)
 416         headers = None
 417         if response:
 418             headers = response.getheaders()
 419         ischanged = False
 420         try:
 421             for header in headers:
 422                 if header[0] == "content-length":
 423                     if header[1] != data["content-length"][0]:
 424                         ischanged = True
 425                 elif header[0] == "etag":
 426                     if header[1] != data["etag"][0]:
 427                         ischanged = True
 428                 elif header[0] == "last-modified":
 429                     if header[1] != data["last-modified"][0]:
 430                         ischanged = True
 431                 elif header[0] == "content-md5":
 432                     if header[1] != data["content-md5"][0]:
 433                         ischanged = True
 434         except:
 435             ischanged = True
 436         if ischanged:
 437             response = open_url("GET", url)
 438             if response != None:
 439                 headers = response.getheaders()
 440                 feedhandle = response
 441             else:
 442                 sys.stderr.write("Failed to fetch feed: %s\n" %(url))
 443                 return
 444         else:
 445             return # don't need to do anything, nothings changed.
 446     else:
 447         response = open_url("GET", url)
 448         if response != None:
 449             headers = response.getheaders()
 450             feedhandle = response
 451         else:
 452             sys.stderr.write("Failed to fetch feed: %s\n" %(url))
 453             return
 454
 455     fp = feedparser.parse(feedhandle)
 456     db = dbm.open(os.path.join(statedir, "seen"), "c")
 457     for item in fp["items"]:
 458         # have we seen it before?
 459         # need to work out what the content is first...
 460
 461         if item.has_key("content"):
 462             content = item["content"][0]["value"]
 463         else:
 464             content = item["summary"]
 465
 466         md5sum = md5.md5(content.encode("utf-8")).hexdigest()
 467
 468         prevmessageid = None
 469
 470         # check if there's a guid too - if that exists and we match the md5,
 471         # return
 472         if item.has_key("guid"):
 473             if db.has_key(url + "|" + item["guid"]):
 474                 data = db[url + "|" + item["guid"]]
 475                 data = cgi.parse_qs(data)
 476                 if data["contentmd5"][0] == md5sum:
 477                     continue
 478
 479         if db.has_key(url + "|" + item["link"]):
 480             data = db[url + "|" + item["link"]]
 481             data = cgi.parse_qs(data)
 482             if data.has_key("message-id"):
 483                 prevmessageid = data["message-id"][0]
 484             if data["contentmd5"][0] == md5sum:
 485                 continue
 486
 487         try:
 488             author = item["author"]
 489         except:
 490             author = url
 491
 492         # create a basic email message
 493         msg = MIMEMultipart("alternative")
 494         messageid = "<" \
 495             + datetime.datetime.now().strftime("%Y%m%d%H%M") \
 496             + "." \
 497             + "".join( \
 498                 [random.choice( \
 499                     string.ascii_letters + string.digits \
 500                     ) for a in range(0,6) \
 501                 ]) + "@" + socket.gethostname() + ">"
 502         msg.add_header("Message-ID", messageid)
 503         msg.set_unixfrom("\"%s\" <rss2maildir@localhost>" %(url))
 504         msg.add_header("From", "\"%s\" <rss2maildir@localhost>" %(author))
 505         msg.add_header("To", "\"%s\" <rss2maildir@localhost>" %(url))
 506         if prevmessageid:
 507             msg.add_header("References", prevmessageid)
 508         createddate = datetime.datetime.now() \
 509             .strftime("%a, %e %b %Y %T -0000")
 510         try:
 511             createddate = datetime.datetime(*item["updated_parsed"][0:6]) \
 512                 .strftime("%a, %e %b %Y %T -0000")
 513         except:
 514             pass
 515         msg.add_header("Date", createddate)
 516         msg.add_header("Subject", item["title"])
 517         msg.set_default_type("text/plain")
 518
 519         htmlcontent = content.encode("utf-8")
 520         htmlcontent = "%s\n\n<p>Item URL: <a href='%s'>%s</a></p>" %( \
 521             content, \
 522             item["link"], \
 523             item["link"] )
 524         htmlpart = MIMEText(htmlcontent.encode("utf-8"), "html", "utf-8")
 525         textparser = HTML2Text()
 526         textparser.feed(content.encode("utf-8"))
 527         textcontent = textparser.gettext()
 528         textcontent = "%s\n\nItem URL: %s" %( \
 529             textcontent, \
 530             item["link"] )
 531         textpart = MIMEText(textcontent.encode("utf-8"), "plain", "utf-8")
 532         msg.attach(textpart)
 533         msg.attach(htmlpart)
 534
 535         # start by working out the filename we should be writting to, we do
 536         # this following the normal maildir style rules
 537         fname = str(os.getpid()) \
 538             + "." + socket.gethostname() \
 539             + "." + "".join( \
 540                 [random.choice( \
 541                     string.ascii_letters + string.digits \
 542                     ) for a in range(0,10) \
 543                 ]) + "." \
 544             + datetime.datetime.now().strftime('%s')
 545         fn = os.path.join(maildir, "tmp", fname)
 546         fh = open(fn, "w")
 547         fh.write(msg.as_string())
 548         fh.close()
 549         # now move it in to the new directory
 550         newfn = os.path.join(maildir, "new", fname)
 551         os.link(fn, newfn)
 552         os.unlink(fn)
 553
 554         # now add to the database about the item
 555         if prevmessageid:
 556             messageid = prevmessageid + " " + messageid
 557         if item.has_key("guid") and item["guid"] != item["link"]:
 558             data = urllib.urlencode(( \
 559                 ("message-id", messageid), \
 560                 ("created", createddate), \
 561                 ("contentmd5", md5sum) \
 562                 ))
 563             db[url + "|" + item["guid"]] = data
 564             try:
 565                 data = db[url + "|" + item["link"]]
 566                 data = cgi.parse_qs(data)
 567                 newdata = urllib.urlencode(( \
 568                     ("message-id", messageid), \
 569                     ("created", data["created"][0]), \
 570                     ("contentmd5", data["contentmd5"][0]) \
 571                     ))
 572                 db[url + "|" + item["link"]] = newdata
 573             except:
 574                 db[url + "|" + item["link"]] = data
 575         else:
 576             data = urllib.urlencode(( \
 577                 ("message-id", messageid), \
 578                 ("created", createddate), \
 579                 ("contentmd5", md5sum) \
 580                 ))
 581             db[url + "|" + item["link"]] = data
 582
 583     if headers:
 584         data = []
 585         for header in headers:
 586             if header[0] in \
 587                 ["content-md5", "etag", "last-modified", "content-length"]:
 588                 data.append((header[0], header[1]))
 589         if len(data) > 0:
 590             data = urllib.urlencode(data)
 591             feeddb[url] = data
 592
 593     db.close()
 594     feeddb.close()
 595
 596 if __name__ == "__main__":
 597     # This only gets executed if we really called the program
 598     # first off, parse the command line arguments
 599
 600     oparser = OptionParser()
 601     oparser.add_option(
 602         "-c", "--conf", dest="conf",
 603         help="location of config file"
 604         )
 605     oparser.add_option(
 606         "-s", "--statedir", dest="statedir",
 607         help="location of directory to store state in"
 608         )
 609
 610     (options, args) = oparser.parse_args()
 611
 612     # check for the configfile
 613
 614     configfile = None
 615
 616     if options.conf != None:
 617         # does the file exist?
 618         try:
 619             os.stat(options.conf)
 620             configfile = options.conf
 621         except:
 622             # should exit here as the specified file doesn't exist
 623             sys.stderr.write( \
 624                 "Config file %s does not exist. Exiting.\n" %(options.conf,))
 625             sys.exit(2)
 626     else:
 627         # check through the default locations
 628         try:
 629             os.stat("%s/.rss2maildir.conf" %(os.environ["HOME"],))
 630             configfile = "%s/.rss2maildir.conf" %(os.environ["HOME"],)
 631         except:
 632             try:
 633                 os.stat("/etc/rss2maildir.conf")
 634                 configfile = "/etc/rss2maildir.conf"
 635             except:
 636                 sys.stderr.write("No config file found. Exiting.\n")
 637                 sys.exit(2)
 638
 639     # Right - if we've got this far, we've got a config file, now for the hard
 640     # bits...
 641
 642     scp = SafeConfigParser()
 643     scp.read(configfile)
 644
 645     maildir_root = "RSSMaildir"
 646     state_dir = "state"
 647
 648     if options.statedir != None:
 649         state_dir = options.statedir
 650         try:
 651             mode = os.stat(state_dir)[stat.ST_MODE]
 652             if not stat.S_ISDIR(mode):
 653                 sys.stderr.write( \
 654                     "State directory (%s) is not a directory\n" %(state_dir))
 655                 sys.exit(1)
 656         except:
 657             # try to make the directory
 658             try:
 659                 os.mkdir(state_dir)
 660             except:
 661                 sys.stderr.write("Couldn't create statedir %s" %(state_dir))
 662                 sys.exit(1)
 663     elif scp.has_option("general", "state_dir"):
 664         new_state_dir = scp.get("general", "state_dir")
 665         try:
 666             mode = os.stat(state_dir)[stat.ST_MODE]
 667             if not stat.S_ISDIR(mode):
 668                 sys.stderr.write( \
 669                     "State directory (%s) is not a directory\n" %(state_dir))
 670                 sys.exit(1)
 671         except:
 672             # try to create it
 673             try:
 674                 os.mkdir(new_state_dir)
 675                 state_dir = new_state_dir
 676             except:
 677                 sys.stderr.write( \
 678                     "Couldn't create state directory %s\n" %(new_state_dir))
 679                 sys.exit(1)
 680     else:
 681         try:
 682             mode = os.stat(state_dir)[stat.ST_MODE]
 683             if not stat.S_ISDIR(mode):
 684                 sys.stderr.write( \
 685                     "State directory %s is not a directory\n" %(state_dir))
 686                 sys.exit(1)
 687         except:
 688             try:
 689                 os.mkdir(state_dir)
 690             except:
 691                 sys.stderr.write( \
 692                     "State directory %s could not be created\n" %(state_dir))
 693                 sys.exit(1)
 694
 695     if scp.has_option("general", "maildir_root"):
 696         maildir_root = scp.get("general", "maildir_root")
 697
 698     try:
 699         mode = os.stat(maildir_root)[stat.ST_MODE]
 700         if not stat.S_ISDIR(mode):
 701             sys.stderr.write( \
 702                 "Maildir Root %s is not a directory\n" \
 703                 %(maildir_root))
 704             sys.exit(1)
 705     except:
 706         try:
 707             os.mkdir(maildir_root)
 708         except:
 709             sys.stderr.write("Couldn't create Maildir Root %s\n" \
 710                 %(maildir_root))
 711             sys.exit(1)
 712
 713     feeds = scp.sections()
 714     try:
 715         feeds.remove("general")
 716     except:
 717         pass
 718
 719     for section in feeds:
 720         # check if the directory exists
 721         maildir = None
 722         try:
 723             maildir = scp.get(section, "maildir")
 724         except:
 725             maildir = section
 726
 727         maildir = urllib.urlencode(((section, maildir),)).split("=")[1]
 728         maildir = os.path.join(maildir_root, maildir)
 729
 730         try:
 731             exists = os.stat(maildir)
 732             if stat.S_ISDIR(exists[stat.ST_MODE]):
 733                 # check if there's a new, cur and tmp directory
 734                 try:
 735                     mode = os.stat(os.path.join(maildir, "cur"))[stat.ST_MODE]
 736                 except:
 737                     os.mkdir(os.path.join(maildir, "cur"))
 738                     if not stat.S_ISDIR(mode):
 739                         sys.stderr.write("Broken maildir: %s\n" %(maildir))
 740                 try:
 741                     mode = os.stat(os.path.join(maildir, "tmp"))[stat.ST_MODE]
 742                 except:
 743                     os.mkdir(os.path.join(maildir, "tmp"))
 744                     if not stat.S_ISDIR(mode):
 745                         sys.stderr.write("Broken maildir: %s\n" %(maildir))
 746                 try:
 747                     mode = os.stat(os.path.join(maildir, "new"))[stat.ST_MODE]
 748                     if not stat.S_ISDIR(mode):
 749                         sys.stderr.write("Broken maildir: %s\n" %(maildir))
 750                 except:
 751                     os.mkdir(os.path.join(maildir, "new"))
 752             else:
 753                 sys.stderr.write("Broken maildir: %s\n" %(maildir))
 754         except:
 755             try:
 756                 os.mkdir(maildir)
 757             except:
 758                 sys.stderr.write("Couldn't create root maildir %s\n" \
 759                     %(maildir))
 760                 sys.exit(1)
 761             try:
 762                 os.mkdir(os.path.join(maildir, "new"))
 763                 os.mkdir(os.path.join(maildir, "cur"))
 764                 os.mkdir(os.path.join(maildir, "tmp"))
 765             except:
 766                 sys.stderr.write( \
 767                     "Couldn't create required maildir directories for %s\n" \
 768                     %(section,))
 769                 sys.exit(1)
 770
 771         # right - we've got the directories, we've got the section, we know the
 772         # url... lets play!
 773
 774         parse_and_deliver(maildir, section, state_dir)