rss2maildir.py

   1 #!/usr/bin/python
   2 # coding=utf-8
   3
   4 # rss2maildir.py - RSS feeds to Maildir 1 email per item
   5 # Copyright (C) 2007  Brett Parker <iDunno@sommitrealweird.co.uk>
   6 #
   7 # This program is free software: you can redistribute it and/or modify
   8 # it under the terms of the GNU General Public License as published by
   9 # the Free Software Foundation, either version 3 of the License, or
  10 # (at your option) any later version.
  11 #
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU General Public License for more details.
  16 #
  17 # You should have received a copy of the GNU General Public License
  18 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  19
  20 import sys
  21 import os
  22 import stat
  23 import httplib
  24 import urllib
  25
  26 import feedparser
  27
  28 from email.MIMEMultipart import MIMEMultipart
  29 from email.MIMEText import MIMEText
  30
  31 import datetime
  32 import random
  33 import string
  34 import textwrap
  35
  36 import socket
  37
  38 from optparse import OptionParser
  39 from ConfigParser import SafeConfigParser
  40
  41 from base64 import b64encode
  42 import md5
  43
  44 import cgi
  45 import dbm
  46
  47 from HTMLParser import HTMLParser
  48
  49 class HTML2Text(HTMLParser):
  50     entities = {
  51         "amp": "&",
  52         "lt": "<",
  53         "gt": ">",
  54         "pound": "£",
  55         "copy": "©",
  56         "apos": "'",
  57         "quot": "\"",
  58         "nbsp": " ",
  59         }
  60
  61     blockleveltags = [
  62         "h1",
  63         "h2",
  64         "h3",
  65         "h4",
  66         "h5",
  67         "h6",
  68         "pre",
  69         "p",
  70         "ul",
  71         "ol",
  72         "dl",
  73         "br",
  74         ]
  75
  76     liststarttags = [
  77         "ul",
  78         "ol",
  79         "dl",
  80         ]
  81
  82     cancontainflow = [
  83         "div",
  84         "li",
  85         "dd",
  86         "blockquote",
  87     ]
  88
  89     def __init__(self,textwidth=70):
  90         self.text = u''
  91         self.curdata = u''
  92         self.textwidth = textwidth
  93         self.opentags = []
  94         self.indentlevel = 0
  95         self.listcount = []
  96         self.urls = []
  97         HTMLParser.__init__(self)
  98
  99     def handle_starttag(self, tag, attrs):
 100         tag_name = tag.lower()
 101         if tag_name in self.blockleveltags:
 102             # handle starting a new block - unless we're in a block element
 103             # that can contain other blocks, we'll assume that we want to close
 104             # the container
 105             if tag_name == u'br':
 106                 self.handle_curdata()
 107                 self.opentags.append(tag_name)
 108                 self.opentags.pop()
 109
 110             if len(self.opentags) > 1 and self.opentags[-1] == u'li':
 111                 self.handle_curdata()
 112
 113             if tag_name == u'ol':
 114                 self.handle_curdata()
 115                 self.listcount.append(1)
 116                 self.listlevel = len(self.listcount) - 1
 117
 118             if tag_name in self.liststarttags:
 119                 smallist = self.opentags[-3:-1]
 120                 smallist.reverse()
 121                 for prev_listtag in smallist:
 122                     if prev_listtag in [u'dl', u'ol']:
 123                         self.indentlevel = self.indentlevel + 4
 124                         break
 125                     elif prev_listtag == u'ul':
 126                         self.indentlevel = self.indentlevel + 3
 127                         break
 128
 129             if len(self.opentags) > 0:
 130                 self.handle_curdata()
 131                 if tag_name not in self.cancontainflow:
 132                     self.opentags.pop()
 133             self.opentags.append(tag_name)
 134         else:
 135             listcount = 0
 136             try:
 137                 listcount = self.listcount[-1]
 138             except:
 139                 pass
 140
 141             if tag_name == u'dd' and len(self.opentags) > 1 \
 142                 and self.opentags[-1] == u'dt':
 143                 self.handle_curdata()
 144                 self.opentags.pop()
 145             elif tag_name == u'dt' and len(self.opentags) > 1 \
 146                 and self.opentags[-1] == u'dd':
 147                 self.handle_curdata()
 148                 self.opentags.pop()
 149             elif tag_name == u'a':
 150                 for attr in attrs:
 151                     if attr[0].lower() == u'href':
 152                         self.urls.append(attr[1])
 153                 self.curdata = self.curdata + u'`'
 154                 self.opentags.append(tag_name)
 155                 return
 156
 157             self.handle_curdata()
 158             self.opentags.append(tag_name)
 159
 160     def handle_startendtag(self, tag, attrs):
 161         if tag.lower() == u'br':
 162             self.opentags.append(u'br')
 163             self.handle_curdata() # just handle the data, don't do anything else
 164             self.opentags.pop()
 165
 166     def handle_curdata(self):
 167         if len(self.opentags) == 0:
 168             return
 169
 170         if len(self.curdata) == 0:
 171             return
 172
 173         if len(self.curdata.strip()) == 0:
 174             return
 175
 176         tag_thats_done = self.opentags[-1]
 177
 178         if tag_thats_done in self.blockleveltags:
 179             newlinerequired = self.text != u''
 180             if newlinerequired:
 181                 if newlinerequired \
 182                     and len(self.text) > 2 \
 183                     and self.text[-1] != u'\n' \
 184                     and self.text[-2] != u'\n':
 185                     self.text = self.text + u'\n\n'
 186
 187         if tag_thats_done in ["h1", "h2", "h3", "h4", "h5", "h6"]:
 188             underline = u''
 189             underlinechar = u'='
 190             headingtext = unicode( \
 191                 self.curdata.encode("utf-8").strip(), "utf-8")
 192             seperator = u'\n' + u' '*self.indentlevel
 193             headingtext = seperator.join( \
 194                 textwrap.wrap( \
 195                     headingtext, \
 196                     self.textwidth - self.indentlevel \
 197                     ) \
 198                 )
 199
 200             if tag_thats_done == u'h2':
 201                 underlinechar = u'-'
 202             elif tag_thats_done != u'h1':
 203                 underlinechar = u'~'
 204
 205             if u'\n' in headingtext:
 206                 underline = u' ' * self.indentlevel \
 207                     + underlinechar * (self.textwidth - self.indentlevel)
 208             else:
 209                 underline = u' ' * self.indentlevel \
 210                     + underlinechar * len(headingtext)
 211             self.text = self.text \
 212                 + headingtext.encode("utf-8") + u'\n' \
 213                 + underline
 214         elif tag_thats_done == u'p':
 215             paragraph = unicode( \
 216                 self.curdata.strip().encode("utf-8"), "utf-8")
 217             seperator = u'\n' + u' ' * self.indentlevel
 218             self.text = self.text \
 219                 + u' ' * self.indentlevel \
 220                 + seperator.join( \
 221                     textwrap.wrap( \
 222                         paragraph, self.textwidth - self.indentlevel))
 223         elif tag_thats_done == "pre":
 224             self.text = self.text + unicode( \
 225                 self.curdata.encode("utf-8"), "utf-8")
 226         elif tag_thats_done == "blockquote":
 227             quote = unicode( \
 228                 self.curdata.encode("utf-8").strip(), "utf-8")
 229             seperator = u'\n' + u' ' * self.indentlevel + u'> '
 230             self.text = self.text \
 231                 + u'> ' \
 232                 + seperator.join( \
 233                     textwrap.wrap( \
 234                         quote, \
 235                         self.textwidth - self.indentlevel - 2 \
 236                     )
 237                 )
 238         elif tag_thats_done == "li":
 239             item = unicode(self.curdata.encode("utf-8").strip(), "utf-8")
 240             if len(self.text) > 0 and self.text[-1] != u'\n':
 241                 self.text = self.text + u'\n'
 242             # work out if we're in an ol rather than a ul
 243             latesttags = self.opentags[-4:]
 244             latesttags.reverse()
 245             isul = None
 246             for thing in latesttags:
 247                 if thing == 'ul':
 248                     isul = True
 249                     break
 250                 elif thing == 'ol':
 251                     isul = False
 252                     break
 253
 254             listindent = 3
 255             if not isul:
 256                 listindent = 4
 257
 258             listmarker = u' * '
 259             if isul == False:
 260                 listmarker = u' %2d. ' %(self.listcount[-1])
 261                 self.listcount[-1] = self.listcount[-1] + 1
 262
 263             seperator = u'\n' \
 264                 + u' ' * self.indentlevel \
 265                 + u' ' * listindent
 266             self.text = self.text \
 267                 + u' ' * self.indentlevel \
 268                 + listmarker \
 269                 + seperator.join( \
 270                     textwrap.wrap( \
 271                         item, \
 272                         self.textwidth - self.indentlevel - listindent \
 273                     ) \
 274                 )
 275             self.curdata = u''
 276         elif tag_thats_done == u'dt':
 277             definition = unicode(self.curdata.encode("utf-8").strip(), "utf-8")
 278             if len(self.text) > 0 and self.text[-1] != u'\n':
 279                 self.text = self.text + u'\n\n'
 280             elif len(self.text) > 1 and self.text[-2] != u'\n':
 281                 self.text = self.text + u'\n'
 282             definition = u' ' * self.indentlevel + definition + "::"
 283             indentstring = u'\n' + u' ' * (self.indentlevel + 1)
 284             self.text = self.text \
 285                 + indentstring.join(
 286                     textwrap.wrap(definition, \
 287                         self.textwidth - self.indentlevel - 1))
 288             self.curdata = u''
 289         elif tag_thats_done == u'dd':
 290             definition = unicode(self.curdata.encode("utf-8").strip(), "utf-8")
 291             if len(definition) > 0:
 292                 if len(self.text) > 0 and self.text[-1] != u'\n':
 293                     self.text = self.text + u'\n'
 294                 indentstring = u'\n' + u' ' * (self.indentlevel + 4)
 295                 self.text = self.text \
 296                     + u' ' * (self.indentlevel + 4) \
 297                     + indentstring.join( \
 298                         textwrap.wrap( \
 299                             definition, \
 300                             self.textwidth - self.indentlevel - 4 \
 301                             ) \
 302                         )
 303                 self.curdata = u''
 304         elif tag_thats_done == u'a':
 305             self.curdata = self.curdata + u'`__'
 306             pass
 307         elif tag_thats_done in self.liststarttags:
 308             pass
 309         else:
 310             # we've got no idea what this tag does, so we'll
 311             # make an assumption that we're not going to know later
 312             if len(self.curdata) > 0:
 313                 self.text = self.text \
 314                     + u' ... ' \
 315                     + u'\n ... '.join( \
 316                         textwrap.wrap( \
 317                             unicode( \
 318                                 self.curdata.encode("utf-8").strip(), \
 319                                 "utf-8"), self.textwidth - 5))
 320             self.curdata = u''
 321
 322         if tag_thats_done in self.blockleveltags:
 323             self.curdata = u''
 324
 325     def handle_endtag(self, tag):
 326         try:
 327             tagindex = self.opentags.index(tag)
 328         except:
 329             # closing tag we know nothing about.
 330             # err. weird.
 331             tagindex = 0
 332
 333         tag = tag.lower()
 334
 335         if tag in self.liststarttags:
 336             if tag in [u'ol', u'dl', u'ul']:
 337                 self.handle_curdata()
 338                 # find if there was a previous list level
 339                 smalllist = self.opentags[:-1]
 340                 smalllist.reverse()
 341                 for prev_listtag in smalllist:
 342                     if prev_listtag in [u'ol', u'dl']:
 343                         self.indentlevel = self.indentlevel - 4
 344                         break
 345                     elif prev_listtag == u'ul':
 346                         self.indentlevel = self.indentlevel - 3
 347                         break
 348
 349         if tag == u'ol':
 350             self.listcount = self.listcount[:-1]
 351
 352         while tagindex < len(self.opentags) \
 353             and tag in self.opentags[tagindex+1:]:
 354             try:
 355                 tagindex = self.opentags.index(tag, tagindex+1)
 356             except:
 357                 # well, we don't want to do that then
 358                 pass
 359         if tagindex != len(self.opentags) - 1:
 360             # Assuming the data was for the last opened tag first
 361             self.handle_curdata()
 362             # Now kill the list to be a slice before this tag was opened
 363             self.opentags = self.opentags[:tagindex + 1]
 364         else:
 365             self.handle_curdata()
 366             if self.opentags[-1] == tag:
 367                 self.opentags.pop()
 368
 369     def handle_data(self, data):
 370         self.curdata = self.curdata + unicode(data, "utf-8")
 371
 372     def handle_entityref(self, name):
 373         entity = name
 374         if HTML2Text.entities.has_key(name.lower()):
 375             entity = HTML2Text.entities[name.lower()]
 376         elif name[0] == "#":
 377             entity = unichr(int(name[1:]))
 378         else:
 379             entity = "&" + name + ";"
 380
 381         self.curdata = self.curdata + unicode(entity, "utf-8")
 382
 383     def gettext(self):
 384         self.handle_curdata()
 385         if len(self.text) == 0 or self.text[-1] != u'\n':
 386             self.text = self.text + u'\n'
 387         self.opentags = []
 388         if len(self.text) > 0:
 389             while len(self.text) > 1 and self.text[-1] == u'\n':
 390                 self.text = self.text[:-1]
 391             self.text = self.text + u'\n'
 392         if len(self.urls) > 0:
 393             self.text = self.text + u'\n__ ' + u'\n__ '.join(self.urls) + u'\n'
 394             self.urls = []
 395         return self.text
 396
 397 def open_url(method, url):
 398     redirectcount = 0
 399     while redirectcount < 3:
 400         (type, rest) = urllib.splittype(url)
 401         (host, path) = urllib.splithost(rest)
 402         (host, port) = urllib.splitport(host)
 403         if port == None:
 404             port = 80
 405         try:
 406             conn = httplib.HTTPConnection("%s:%s" %(host, port))
 407             conn.request(method, path)
 408             response = conn.getresponse()
 409             if response.status in [301, 302, 303, 307]:
 410                 headers = response.getheaders()
 411                 for header in headers:
 412                     if header[0] == "location":
 413                         url = header[1]
 414             elif response.status == 200:
 415                 return response
 416         except:
 417             pass
 418         redirectcount = redirectcount + 1
 419     return None
 420
 421 def parse_and_deliver(maildir, url, statedir):
 422     feedhandle = None
 423     headers = None
 424     # first check if we know about this feed already
 425     feeddb = dbm.open(os.path.join(statedir, "feeds"), "c")
 426     if feeddb.has_key(url):
 427         data = feeddb[url]
 428         data = cgi.parse_qs(data)
 429         response = open_url("HEAD", url)
 430         headers = None
 431         if response:
 432             headers = response.getheaders()
 433         ischanged = False
 434         try:
 435             for header in headers:
 436                 if header[0] == "content-length":
 437                     if header[1] != data["content-length"][0]:
 438                         ischanged = True
 439                 elif header[0] == "etag":
 440                     if header[1] != data["etag"][0]:
 441                         ischanged = True
 442                 elif header[0] == "last-modified":
 443                     if header[1] != data["last-modified"][0]:
 444                         ischanged = True
 445                 elif header[0] == "content-md5":
 446                     if header[1] != data["content-md5"][0]:
 447                         ischanged = True
 448         except:
 449             ischanged = True
 450         if ischanged:
 451             response = open_url("GET", url)
 452             if response != None:
 453                 headers = response.getheaders()
 454                 feedhandle = response
 455             else:
 456                 sys.stderr.write("Failed to fetch feed: %s\n" %(url))
 457                 return
 458         else:
 459             return # don't need to do anything, nothings changed.
 460     else:
 461         response = open_url("GET", url)
 462         if response != None:
 463             headers = response.getheaders()
 464             feedhandle = response
 465         else:
 466             sys.stderr.write("Failed to fetch feed: %s\n" %(url))
 467             return
 468
 469     fp = feedparser.parse(feedhandle)
 470     db = dbm.open(os.path.join(statedir, "seen"), "c")
 471     for item in fp["items"]:
 472         # have we seen it before?
 473         # need to work out what the content is first...
 474
 475         if item.has_key("content"):
 476             content = item["content"][0]["value"]
 477         else:
 478             content = item["summary"]
 479
 480         md5sum = md5.md5(content.encode("utf-8")).hexdigest()
 481
 482         prevmessageid = None
 483
 484         # check if there's a guid too - if that exists and we match the md5,
 485         # return
 486         if item.has_key("guid"):
 487             if db.has_key(url + "|" + item["guid"]):
 488                 data = db[url + "|" + item["guid"]]
 489                 data = cgi.parse_qs(data)
 490                 if data["contentmd5"][0] == md5sum:
 491                     continue
 492
 493         if db.has_key(url + "|" + item["link"]):
 494             data = db[url + "|" + item["link"]]
 495             data = cgi.parse_qs(data)
 496             if data.has_key("message-id"):
 497                 prevmessageid = data["message-id"][0]
 498             if data["contentmd5"][0] == md5sum:
 499                 continue
 500
 501         try:
 502             author = item["author"]
 503         except:
 504             author = url
 505
 506         # create a basic email message
 507         msg = MIMEMultipart("alternative")
 508         messageid = "<" \
 509             + datetime.datetime.now().strftime("%Y%m%d%H%M") \
 510             + "." \
 511             + "".join( \
 512                 [random.choice( \
 513                     string.ascii_letters + string.digits \
 514                     ) for a in range(0,6) \
 515                 ]) + "@" + socket.gethostname() + ">"
 516         msg.add_header("Message-ID", messageid)
 517         msg.set_unixfrom("\"%s\" <rss2maildir@localhost>" %(url))
 518         msg.add_header("From", "\"%s\" <rss2maildir@localhost>" %(author))
 519         msg.add_header("To", "\"%s\" <rss2maildir@localhost>" %(url))
 520         if prevmessageid:
 521             msg.add_header("References", prevmessageid)
 522         createddate = datetime.datetime.now() \
 523             .strftime("%a, %e %b %Y %T -0000")
 524         try:
 525             createddate = datetime.datetime(*item["updated_parsed"][0:6]) \
 526                 .strftime("%a, %e %b %Y %T -0000")
 527         except:
 528             pass
 529         msg.add_header("Date", createddate)
 530         msg.add_header("Subject", item["title"])
 531         msg.set_default_type("text/plain")
 532
 533         htmlcontent = content.encode("utf-8")
 534         htmlcontent = "%s\n\n<p>Item URL: <a href='%s'>%s</a></p>" %( \
 535             content, \
 536             item["link"], \
 537             item["link"] )
 538         htmlpart = MIMEText(htmlcontent.encode("utf-8"), "html", "utf-8")
 539         textparser = HTML2Text()
 540         textparser.feed(content.encode("utf-8"))
 541         textcontent = textparser.gettext()
 542         textcontent = "%s\n\nItem URL: %s" %( \
 543             textcontent, \
 544             item["link"] )
 545         textpart = MIMEText(textcontent.encode("utf-8"), "plain", "utf-8")
 546         msg.attach(textpart)
 547         msg.attach(htmlpart)
 548
 549         # start by working out the filename we should be writting to, we do
 550         # this following the normal maildir style rules
 551         fname = str(os.getpid()) \
 552             + "." + socket.gethostname() \
 553             + "." + "".join( \
 554                 [random.choice( \
 555                     string.ascii_letters + string.digits \
 556                     ) for a in range(0,10) \
 557                 ]) + "." \
 558             + datetime.datetime.now().strftime('%s')
 559         fn = os.path.join(maildir, "tmp", fname)
 560         fh = open(fn, "w")
 561         fh.write(msg.as_string())
 562         fh.close()
 563         # now move it in to the new directory
 564         newfn = os.path.join(maildir, "new", fname)
 565         os.link(fn, newfn)
 566         os.unlink(fn)
 567
 568         # now add to the database about the item
 569         if prevmessageid:
 570             messageid = prevmessageid + " " + messageid
 571         if item.has_key("guid") and item["guid"] != item["link"]:
 572             data = urllib.urlencode(( \
 573                 ("message-id", messageid), \
 574                 ("created", createddate), \
 575                 ("contentmd5", md5sum) \
 576                 ))
 577             db[url + "|" + item["guid"]] = data
 578             try:
 579                 data = db[url + "|" + item["link"]]
 580                 data = cgi.parse_qs(data)
 581                 newdata = urllib.urlencode(( \
 582                     ("message-id", messageid), \
 583                     ("created", data["created"][0]), \
 584                     ("contentmd5", data["contentmd5"][0]) \
 585                     ))
 586                 db[url + "|" + item["link"]] = newdata
 587             except:
 588                 db[url + "|" + item["link"]] = data
 589         else:
 590             data = urllib.urlencode(( \
 591                 ("message-id", messageid), \
 592                 ("created", createddate), \
 593                 ("contentmd5", md5sum) \
 594                 ))
 595             db[url + "|" + item["link"]] = data
 596
 597     if headers:
 598         data = []
 599         for header in headers:
 600             if header[0] in \
 601                 ["content-md5", "etag", "last-modified", "content-length"]:
 602                 data.append((header[0], header[1]))
 603         if len(data) > 0:
 604             data = urllib.urlencode(data)
 605             feeddb[url] = data
 606
 607     db.close()
 608     feeddb.close()
 609
 610 if __name__ == "__main__":
 611     # This only gets executed if we really called the program
 612     # first off, parse the command line arguments
 613
 614     oparser = OptionParser()
 615     oparser.add_option(
 616         "-c", "--conf", dest="conf",
 617         help="location of config file"
 618         )
 619     oparser.add_option(
 620         "-s", "--statedir", dest="statedir",
 621         help="location of directory to store state in"
 622         )
 623
 624     (options, args) = oparser.parse_args()
 625
 626     # check for the configfile
 627
 628     configfile = None
 629
 630     if options.conf != None:
 631         # does the file exist?
 632         try:
 633             os.stat(options.conf)
 634             configfile = options.conf
 635         except:
 636             # should exit here as the specified file doesn't exist
 637             sys.stderr.write( \
 638                 "Config file %s does not exist. Exiting.\n" %(options.conf,))
 639             sys.exit(2)
 640     else:
 641         # check through the default locations
 642         try:
 643             os.stat("%s/.rss2maildir.conf" %(os.environ["HOME"],))
 644             configfile = "%s/.rss2maildir.conf" %(os.environ["HOME"],)
 645         except:
 646             try:
 647                 os.stat("/etc/rss2maildir.conf")
 648                 configfile = "/etc/rss2maildir.conf"
 649             except:
 650                 sys.stderr.write("No config file found. Exiting.\n")
 651                 sys.exit(2)
 652
 653     # Right - if we've got this far, we've got a config file, now for the hard
 654     # bits...
 655
 656     scp = SafeConfigParser()
 657     scp.read(configfile)
 658
 659     maildir_root = "RSSMaildir"
 660     state_dir = "state"
 661
 662     if options.statedir != None:
 663         state_dir = options.statedir
 664         try:
 665             mode = os.stat(state_dir)[stat.ST_MODE]
 666             if not stat.S_ISDIR(mode):
 667                 sys.stderr.write( \
 668                     "State directory (%s) is not a directory\n" %(state_dir))
 669                 sys.exit(1)
 670         except:
 671             # try to make the directory
 672             try:
 673                 os.mkdir(state_dir)
 674             except:
 675                 sys.stderr.write("Couldn't create statedir %s" %(state_dir))
 676                 sys.exit(1)
 677     elif scp.has_option("general", "state_dir"):
 678         new_state_dir = scp.get("general", "state_dir")
 679         try:
 680             mode = os.stat(state_dir)[stat.ST_MODE]
 681             if not stat.S_ISDIR(mode):
 682                 sys.stderr.write( \
 683                     "State directory (%s) is not a directory\n" %(state_dir))
 684                 sys.exit(1)
 685         except:
 686             # try to create it
 687             try:
 688                 os.mkdir(new_state_dir)
 689                 state_dir = new_state_dir
 690             except:
 691                 sys.stderr.write( \
 692                     "Couldn't create state directory %s\n" %(new_state_dir))
 693                 sys.exit(1)
 694     else:
 695         try:
 696             mode = os.stat(state_dir)[stat.ST_MODE]
 697             if not stat.S_ISDIR(mode):
 698                 sys.stderr.write( \
 699                     "State directory %s is not a directory\n" %(state_dir))
 700                 sys.exit(1)
 701         except:
 702             try:
 703                 os.mkdir(state_dir)
 704             except:
 705                 sys.stderr.write( \
 706                     "State directory %s could not be created\n" %(state_dir))
 707                 sys.exit(1)
 708
 709     if scp.has_option("general", "maildir_root"):
 710         maildir_root = scp.get("general", "maildir_root")
 711
 712     try:
 713         mode = os.stat(maildir_root)[stat.ST_MODE]
 714         if not stat.S_ISDIR(mode):
 715             sys.stderr.write( \
 716                 "Maildir Root %s is not a directory\n" \
 717                 %(maildir_root))
 718             sys.exit(1)
 719     except:
 720         try:
 721             os.mkdir(maildir_root)
 722         except:
 723             sys.stderr.write("Couldn't create Maildir Root %s\n" \
 724                 %(maildir_root))
 725             sys.exit(1)
 726
 727     feeds = scp.sections()
 728     try:
 729         feeds.remove("general")
 730     except:
 731         pass
 732
 733     for section in feeds:
 734         # check if the directory exists
 735         maildir = None
 736         try:
 737             maildir = scp.get(section, "maildir")
 738         except:
 739             maildir = section
 740
 741         maildir = urllib.urlencode(((section, maildir),)).split("=")[1]
 742         maildir = os.path.join(maildir_root, maildir)
 743
 744         try:
 745             exists = os.stat(maildir)
 746             if stat.S_ISDIR(exists[stat.ST_MODE]):
 747                 # check if there's a new, cur and tmp directory
 748                 try:
 749                     mode = os.stat(os.path.join(maildir, "cur"))[stat.ST_MODE]
 750                 except:
 751                     os.mkdir(os.path.join(maildir, "cur"))
 752                     if not stat.S_ISDIR(mode):
 753                         sys.stderr.write("Broken maildir: %s\n" %(maildir))
 754                 try:
 755                     mode = os.stat(os.path.join(maildir, "tmp"))[stat.ST_MODE]
 756                 except:
 757                     os.mkdir(os.path.join(maildir, "tmp"))
 758                     if not stat.S_ISDIR(mode):
 759                         sys.stderr.write("Broken maildir: %s\n" %(maildir))
 760                 try:
 761                     mode = os.stat(os.path.join(maildir, "new"))[stat.ST_MODE]
 762                     if not stat.S_ISDIR(mode):
 763                         sys.stderr.write("Broken maildir: %s\n" %(maildir))
 764                 except:
 765                     os.mkdir(os.path.join(maildir, "new"))
 766             else:
 767                 sys.stderr.write("Broken maildir: %s\n" %(maildir))
 768         except:
 769             try:
 770                 os.mkdir(maildir)
 771             except:
 772                 sys.stderr.write("Couldn't create root maildir %s\n" \
 773                     %(maildir))
 774                 sys.exit(1)
 775             try:
 776                 os.mkdir(os.path.join(maildir, "new"))
 777                 os.mkdir(os.path.join(maildir, "cur"))
 778                 os.mkdir(os.path.join(maildir, "tmp"))
 779             except:
 780                 sys.stderr.write( \
 781                     "Couldn't create required maildir directories for %s\n" \
 782                     %(section,))
 783                 sys.exit(1)
 784
 785         # right - we've got the directories, we've got the section, we know the
 786         # url... lets play!
 787
 788         parse_and_deliver(maildir, section, state_dir)