rss2maildir.py

   1 #!/usr/bin/python
   2 # coding=utf-8
   3
   4 # rss2maildir.py - RSS feeds to Maildir 1 email per item
   5 # Copyright (C) 2007  Brett Parker <iDunno@sommitrealweird.co.uk>
   6 #
   7 # This program is free software: you can redistribute it and/or modify
   8 # it under the terms of the GNU General Public License as published by
   9 # the Free Software Foundation, either version 3 of the License, or
  10 # (at your option) any later version.
  11 #
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU General Public License for more details.
  16 #
  17 # You should have received a copy of the GNU General Public License
  18 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  19
  20 import sys
  21 import os
  22 import stat
  23 import httplib
  24 import urllib
  25
  26 import feedparser
  27
  28 from email.MIMEMultipart import MIMEMultipart
  29 from email.MIMEText import MIMEText
  30
  31 import datetime
  32 import random
  33 import string
  34 import textwrap
  35
  36 import socket
  37
  38 from optparse import OptionParser
  39 from ConfigParser import SafeConfigParser
  40
  41 from base64 import b64encode
  42 import md5
  43
  44 import cgi
  45 import dbm
  46
  47 from HTMLParser import HTMLParser
  48
  49 class HTML2Text(HTMLParser):
  50     entities = {
  51         "amp": "&",
  52         "lt": "<",
  53         "gt": ">",
  54         "pound": "£",
  55         "copy": "©",
  56         "apos": "'",
  57         "quot": "\"",
  58         "nbsp": " ",
  59         }
  60
  61     blockleveltags = [
  62         "h1",
  63         "h2",
  64         "h3",
  65         "h4",
  66         "h5",
  67         "h6",
  68         "pre",
  69         "p",
  70         "ul",
  71         "ol",
  72         "dl",
  73         "br",
  74         ]
  75
  76     liststarttags = [
  77         "ul",
  78         "ol",
  79         "dl",
  80         ]
  81
  82     cancontainflow = [
  83         "div",
  84         "li",
  85         "dd",
  86         "blockquote",
  87     ]
  88
  89     def __init__(self,textwidth=70):
  90         self.text = u''
  91         self.curdata = u''
  92         self.textwidth = textwidth
  93         self.opentags = []
  94         self.indentlevel = 0
  95         self.listcount = []
  96         HTMLParser.__init__(self)
  97
  98     def handle_starttag(self, tag, attrs):
  99         tag_name = tag.lower()
 100         if tag_name in self.blockleveltags:
 101             # handle starting a new block - unless we're in a block element
 102             # that can contain other blocks, we'll assume that we want to close
 103             # the container
 104             if tag_name == u'br':
 105                 self.handle_curdata()
 106                 self.opentags.append(tag_name)
 107                 self.opentags.pop()
 108
 109             if len(self.opentags) > 1 and self.opentags[-1] == u'li':
 110                 self.handle_curdata()
 111
 112             if tag_name == u'ol':
 113                 self.handle_curdata()
 114                 self.listcount.append(1)
 115                 self.listlevel = len(self.listcount) - 1
 116
 117             if tag_name in self.liststarttags:
 118                 smallist = self.opentags[-3:-1]
 119                 smallist.reverse()
 120                 for prev_listtag in smallist:
 121                     if prev_listtag in [u'dl', u'ol']:
 122                         self.indentlevel = self.indentlevel + 4
 123                         break
 124                     elif prev_listtag == u'ul':
 125                         self.indentlevel = self.indentlevel + 3
 126                         break
 127
 128             if len(self.opentags) > 0:
 129                 self.handle_curdata()
 130                 if tag_name not in self.cancontainflow:
 131                     self.opentags.pop()
 132             self.opentags.append(tag_name)
 133         else:
 134             listcount = 0
 135             try:
 136                 listcount = self.listcount[-1]
 137             except:
 138                 pass
 139
 140             if tag_name == u'dd' and len(self.opentags) > 1 \
 141                 and self.opentags[-1] == u'dt':
 142                 self.handle_curdata()
 143                 self.opentags.pop()
 144             elif tag_name == u'dt' and len(self.opentags) > 1 \
 145                 and self.opentags[-1] == u'dd':
 146                 self.handle_curdata()
 147                 self.opentags.pop()
 148
 149             self.handle_curdata()
 150             self.opentags.append(tag_name)
 151
 152     def handle_startendtag(self, tag, attrs):
 153         if tag.lower() == u'br':
 154             self.opentags.append(u'br')
 155             self.handle_curdata() # just handle the data, don't do anything else
 156             self.opentags.pop()
 157
 158     def handle_curdata(self):
 159         if len(self.opentags) == 0:
 160             return
 161
 162         if len(self.curdata) == 0:
 163             return
 164
 165         if len(self.curdata.strip()) == 0:
 166             return
 167
 168         tag_thats_done = self.opentags[-1]
 169
 170         if tag_thats_done in self.blockleveltags:
 171             newlinerequired = self.text != u''
 172             if newlinerequired:
 173                 if newlinerequired \
 174                     and len(self.text) > 2 \
 175                     and self.text[-1] != u'\n' \
 176                     and self.text[-2] != u'\n':
 177                     self.text = self.text + u'\n\n'
 178
 179         if tag_thats_done in ["h1", "h2", "h3", "h4", "h5", "h6"]:
 180             underline = u''
 181             underlinechar = u'='
 182             headingtext = self.curdata.encode("utf-8").strip()
 183             seperator = u'\n' + u' '*self.indentlevel
 184             headingtext = seperator.join( \
 185                 textwrap.wrap( \
 186                     headingtext, \
 187                     self.textwidth - self.indentlevel \
 188                     ) \
 189                 )
 190
 191             if tag_thats_done == u'h2':
 192                 underlinechar = u'-'
 193             elif tag_thats_done != u'h1':
 194                 underlinechar = u'~'
 195
 196             if u'\n' in headingtext:
 197                 underline = u' ' * self.indentlevel \
 198                     + underlinechar * (self.textwidth - self.indentlevel)
 199             else:
 200                 underline = u' ' * self.indentlevel \
 201                     + underlinechar * len(headingtext)
 202             self.text = self.text \
 203                 + headingtext.encode("utf-8") + u'\n' \
 204                 + underline
 205         elif tag_thats_done == u'p':
 206             paragraph = self.curdata.encode("utf-8").strip()
 207             seperator = u'\n' + u' ' * self.indentlevel
 208             self.text = self.text \
 209                 + u' ' * self.indentlevel \
 210                 + seperator.join(textwrap.wrap(paragraph, self.textwidth - self.indentlevel))
 211         elif tag_thats_done == "pre":
 212             self.text = self.text + self.curdata
 213         elif tag_thats_done == "blockquote":
 214             quote = self.curdata.encode("utf-8").strip()
 215             seperator = u'\n' + u' ' * self.indentlevel + u'> '
 216             self.text = self.text \
 217                 + u'> ' \
 218                 + seperator.join( \
 219                     textwrap.wrap( \
 220                         quote, \
 221                         self.textwidth - self.indentlevel - 2 \
 222                     )
 223                 )
 224         elif tag_thats_done == "li":
 225             item = self.curdata.encode("utf-8").strip()
 226             if len(self.text) > 0 and self.text[-1] != u'\n':
 227                 self.text = self.text + u'\n'
 228             # work out if we're in an ol rather than a ul
 229             latesttags = self.opentags[-4:]
 230             latesttags.reverse()
 231             isul = False
 232             for thing in latesttags:
 233                 if thing == 'ul':
 234                     isul = True
 235                     break
 236                 elif thing == 'ol':
 237                     isul = False
 238                     break
 239
 240             listindent = 3
 241             if not isul:
 242                 listindent = 4
 243
 244             listmarker = u' * '
 245             if not isul:
 246                 listmarker = u' %2d. ' %(self.listcount[-1])
 247                 self.listcount[-1] = self.listcount[-1] + 1
 248
 249             seperator = u'\n' \
 250                 + u' ' * self.indentlevel \
 251                 + u' ' * listindent
 252             self.text = self.text \
 253                 + u' ' * self.indentlevel \
 254                 + listmarker \
 255                 + seperator.join( \
 256                     textwrap.wrap( \
 257                         item, \
 258                         self.textwidth - self.indentlevel - listindent \
 259                     ) \
 260                 )
 261             self.curdata = u''
 262         elif tag_thats_done == u'dt':
 263             definition = self.curdata.encode("utf-8").strip()
 264             if len(self.text) > 0 and self.text[-1] != u'\n':
 265                 self.text = self.text + u'\n\n'
 266             elif len(self.text) > 1 and self.text[-2] != u'\n':
 267                 self.text = self.text + u'\n'
 268             definition = u' ' * self.indentlevel + definition + "::"
 269             indentstring = u'\n' + u' ' * (self.indentlevel + 1)
 270             self.text = self.text \
 271                 + indentstring.join(
 272                     textwrap.wrap(definition, \
 273                         self.textwidth - self.indentlevel - 1))
 274             self.curdata = u''
 275         elif tag_thats_done == u'dd':
 276             definition = self.curdata.encode("utf-8").strip()
 277             if len(definition) > 0:
 278                 if len(self.text) > 0 and self.text[-1] != u'\n':
 279                     self.text = self.text + u'\n'
 280                 indentstring = u'\n' + u' ' * (self.indentlevel + 4)
 281                 self.text = self.text \
 282                     + u' ' * (self.indentlevel + 4) \
 283                     + indentstring.join( \
 284                         textwrap.wrap( \
 285                             definition, \
 286                             self.textwidth - self.indentlevel - 4 \
 287                             ) \
 288                         )
 289                 self.curdata = u''
 290         elif tag_thats_done in self.liststarttags:
 291             pass
 292         else:
 293             # we've got no idea what this tag does, so we'll
 294             # make an assumption that we're not going to know later
 295             if len(self.curdata) > 0:
 296                 self.text = self.text \
 297                     + u' ... ' \
 298                     + u'\n ... '.join( \
 299                         textwrap.wrap(self.curdata, self.textwidth - 5))
 300             self.curdata = u''
 301
 302         if tag_thats_done in self.blockleveltags:
 303             self.curdata = u''
 304
 305     def handle_endtag(self, tag):
 306         try:
 307             tagindex = self.opentags.index(tag)
 308         except:
 309             # closing tag we know nothing about.
 310             # err. weird.
 311             tagindex = 0
 312
 313         tag = tag.lower()
 314
 315         if tag in self.liststarttags:
 316             if tag in [u'ol', u'dl', u'ul']:
 317                 self.handle_curdata()
 318                 # find if there was a previous list level
 319                 smalllist = self.opentags[:-1]
 320                 smalllist.reverse()
 321                 for prev_listtag in smalllist:
 322                     if prev_listtag in [u'ol', u'dl']:
 323                         self.indentlevel = self.indentlevel - 4
 324                         break
 325                     elif prev_listtag == u'ul':
 326                         self.indentlevel = self.indentlevel - 3
 327                         break
 328
 329         if tag == u'ol':
 330             self.listcount = self.listcount[:-1]
 331
 332         while tagindex < len(self.opentags) \
 333             and tag in self.opentags[tagindex+1:]:
 334             try:
 335                 tagindex = self.opentags.index(tag, tagindex+1)
 336             except:
 337                 # well, we don't want to do that then
 338                 pass
 339         if tagindex != len(self.opentags) - 1:
 340             # Assuming the data was for the last opened tag first
 341             self.handle_curdata()
 342             # Now kill the list to be a slice before this tag was opened
 343             self.opentags = self.opentags[:tagindex + 1]
 344         else:
 345             self.handle_curdata()
 346             if self.opentags[-1] == tag:
 347                 self.opentags.pop()
 348
 349     def handle_data(self, data):
 350         self.curdata = self.curdata + unicode(data, "utf-8")
 351
 352     def handle_entityref(self, name):
 353         entity = name
 354         if HTML2Text.entities.has_key(name.lower()):
 355             entity = HTML2Text.entities[name.lower()]
 356         elif name[0] == "#":
 357             entity = unichr(int(name[1:]))
 358         else:
 359             entity = "&" + name + ";"
 360
 361         self.curdata = self.curdata + unicode(entity, "utf-8")
 362
 363     def gettext(self):
 364         self.handle_curdata()
 365         if len(self.text) == 0 or self.text[-1] != u'\n':
 366             self.text = self.text + u'\n'
 367         self.opentags = []
 368         if len(self.text) > 0:
 369             while len(self.text) > 1 and self.text[-1] == u'\n':
 370                 self.text = self.text[:-1]
 371             self.text = self.text + u'\n'
 372         return self.text
 373
 374 def open_url(method, url):
 375     redirectcount = 0
 376     while redirectcount < 3:
 377         (type, rest) = urllib.splittype(url)
 378         (host, path) = urllib.splithost(rest)
 379         (host, port) = urllib.splitport(host)
 380         if port == None:
 381             port = 80
 382         try:
 383             conn = httplib.HTTPConnection("%s:%s" %(host, port))
 384             conn.request(method, path)
 385             response = conn.getresponse()
 386             if response.status in [301, 302, 303, 307]:
 387                 headers = response.getheaders()
 388                 for header in headers:
 389                     if header[0] == "location":
 390                         url = header[1]
 391             elif response.status == 200:
 392                 return response
 393         except:
 394             pass
 395         redirectcount = redirectcount + 1
 396     return None
 397
 398 def parse_and_deliver(maildir, url, statedir):
 399     feedhandle = None
 400     headers = None
 401     # first check if we know about this feed already
 402     feeddb = dbm.open(os.path.join(statedir, "feeds"), "c")
 403     if feeddb.has_key(url):
 404         data = feeddb[url]
 405         data = cgi.parse_qs(data)
 406         response = open_url("HEAD", url)
 407         headers = None
 408         if response:
 409             headers = response.getheaders()
 410         ischanged = False
 411         try:
 412             for header in headers:
 413                 if header[0] == "content-length":
 414                     if header[1] != data["content-length"][0]:
 415                         ischanged = True
 416                 elif header[0] == "etag":
 417                     if header[1] != data["etag"][0]:
 418                         ischanged = True
 419                 elif header[0] == "last-modified":
 420                     if header[1] != data["last-modified"][0]:
 421                         ischanged = True
 422                 elif header[0] == "content-md5":
 423                     if header[1] != data["content-md5"][0]:
 424                         ischanged = True
 425         except:
 426             ischanged = True
 427         if ischanged:
 428             response = open_url("GET", url)
 429             if response != None:
 430                 headers = response.getheaders()
 431                 feedhandle = response
 432             else:
 433                 sys.stderr.write("Failed to fetch feed: %s\n" %(url))
 434                 return
 435         else:
 436             return # don't need to do anything, nothings changed.
 437     else:
 438         response = open_url("GET", url)
 439         if response != None:
 440             headers = response.getheaders()
 441             feedhandle = response
 442         else:
 443             sys.stderr.write("Failed to fetch feed: %s\n" %(url))
 444             return
 445
 446     fp = feedparser.parse(feedhandle)
 447     db = dbm.open(os.path.join(statedir, "seen"), "c")
 448     for item in fp["items"]:
 449         # have we seen it before?
 450         # need to work out what the content is first...
 451
 452         if item.has_key("content"):
 453             content = item["content"][0]["value"]
 454         else:
 455             content = item["summary"]
 456
 457         md5sum = md5.md5(content.encode("utf-8")).hexdigest()
 458
 459         prevmessageid = None
 460
 461         # check if there's a guid too - if that exists and we match the md5,
 462         # return
 463         if item.has_key("guid"):
 464             if db.has_key(url + "|" + item["guid"]):
 465                 data = db[url + "|" + item["guid"]]
 466                 data = cgi.parse_qs(data)
 467                 if data["contentmd5"][0] == md5sum:
 468                     continue
 469
 470         if db.has_key(url + "|" + item["link"]):
 471             data = db[url + "|" + item["link"]]
 472             data = cgi.parse_qs(data)
 473             if data.has_key("message-id"):
 474                 prevmessageid = data["message-id"][0]
 475             if data["contentmd5"][0] == md5sum:
 476                 continue
 477
 478         try:
 479             author = item["author"]
 480         except:
 481             author = url
 482
 483         # create a basic email message
 484         msg = MIMEMultipart("alternative")
 485         messageid = "<" \
 486             + datetime.datetime.now().strftime("%Y%m%d%H%M") \
 487             + "." \
 488             + "".join( \
 489                 [random.choice( \
 490                     string.ascii_letters + string.digits \
 491                     ) for a in range(0,6) \
 492                 ]) + "@" + socket.gethostname() + ">"
 493         msg.add_header("Message-ID", messageid)
 494         msg.set_unixfrom("\"%s\" <rss2maildir@localhost>" %(url))
 495         msg.add_header("From", "\"%s\" <rss2maildir@localhost>" %(author))
 496         msg.add_header("To", "\"%s\" <rss2maildir@localhost>" %(url))
 497         if prevmessageid:
 498             msg.add_header("References", prevmessageid)
 499         createddate = datetime.datetime.now() \
 500             .strftime("%a, %e %b %Y %T -0000")
 501         try:
 502             createddate = datetime.datetime(*item["updated_parsed"][0:6]) \
 503                 .strftime("%a, %e %b %Y %T -0000")
 504         except:
 505             pass
 506         msg.add_header("Date", createddate)
 507         msg.add_header("Subject", item["title"])
 508         msg.set_default_type("text/plain")
 509
 510         htmlcontent = content.encode("utf-8")
 511         htmlcontent = "%s\n\n<p>Item URL: <a href='%s'>%s</a></p>" %( \
 512             content, \
 513             item["link"], \
 514             item["link"] )
 515         htmlpart = MIMEText(htmlcontent.encode("utf-8"), "html", "utf-8")
 516         textparser = HTML2Text()
 517         textparser.feed(content.encode("utf-8"))
 518         textcontent = textparser.gettext()
 519         textcontent = "%s\n\nItem URL: %s" %( \
 520             textcontent, \
 521             item["link"] )
 522         textpart = MIMEText(textcontent.encode("utf-8"), "plain", "utf-8")
 523         msg.attach(textpart)
 524         msg.attach(htmlpart)
 525
 526         # start by working out the filename we should be writting to, we do
 527         # this following the normal maildir style rules
 528         fname = str(os.getpid()) \
 529             + "." + socket.gethostname() \
 530             + "." + "".join( \
 531                 [random.choice( \
 532                     string.ascii_letters + string.digits \
 533                     ) for a in range(0,10) \
 534                 ]) + "." \
 535             + datetime.datetime.now().strftime('%s')
 536         fn = os.path.join(maildir, "tmp", fname)
 537         fh = open(fn, "w")
 538         fh.write(msg.as_string())
 539         fh.close()
 540         # now move it in to the new directory
 541         newfn = os.path.join(maildir, "new", fname)
 542         os.link(fn, newfn)
 543         os.unlink(fn)
 544
 545         # now add to the database about the item
 546         if prevmessageid:
 547             messageid = prevmessageid + " " + messageid
 548         if item.has_key("guid") and item["guid"] != item["link"]:
 549             data = urllib.urlencode(( \
 550                 ("message-id", messageid), \
 551                 ("created", createddate), \
 552                 ("contentmd5", md5sum) \
 553                 ))
 554             db[url + "|" + item["guid"]] = data
 555             try:
 556                 data = db[url + "|" + item["link"]]
 557                 data = cgi.parse_qs(data)
 558                 newdata = urllib.urlencode(( \
 559                     ("message-id", messageid), \
 560                     ("created", data["created"][0]), \
 561                     ("contentmd5", data["contentmd5"][0]) \
 562                     ))
 563                 db[url + "|" + item["link"]] = newdata
 564             except:
 565                 db[url + "|" + item["link"]] = data
 566         else:
 567             data = urllib.urlencode(( \
 568                 ("message-id", messageid), \
 569                 ("created", createddate), \
 570                 ("contentmd5", md5sum) \
 571                 ))
 572             db[url + "|" + item["link"]] = data
 573
 574     if headers:
 575         data = []
 576         for header in headers:
 577             if header[0] in ["content-md5", "etag", "last-modified", "content-length"]:
 578                 data.append((header[0], header[1]))
 579         if len(data) > 0:
 580             data = urllib.urlencode(data)
 581             feeddb[url] = data
 582
 583     db.close()
 584     feeddb.close()
 585
 586 if __name__ == "__main__":
 587     # This only gets executed if we really called the program
 588     # first off, parse the command line arguments
 589
 590     oparser = OptionParser()
 591     oparser.add_option(
 592         "-c", "--conf", dest="conf",
 593         help="location of config file"
 594         )
 595     oparser.add_option(
 596         "-s", "--statedir", dest="statedir",
 597         help="location of directory to store state in"
 598         )
 599
 600     (options, args) = oparser.parse_args()
 601
 602     # check for the configfile
 603
 604     configfile = None
 605
 606     if options.conf != None:
 607         # does the file exist?
 608         try:
 609             os.stat(options.conf)
 610             configfile = options.conf
 611         except:
 612             # should exit here as the specified file doesn't exist
 613             sys.stderr.write( \
 614                 "Config file %s does not exist. Exiting.\n" %(options.conf,))
 615             sys.exit(2)
 616     else:
 617         # check through the default locations
 618         try:
 619             os.stat("%s/.rss2maildir.conf" %(os.environ["HOME"],))
 620             configfile = "%s/.rss2maildir.conf" %(os.environ["HOME"],)
 621         except:
 622             try:
 623                 os.stat("/etc/rss2maildir.conf")
 624                 configfile = "/etc/rss2maildir.conf"
 625             except:
 626                 sys.stderr.write("No config file found. Exiting.\n")
 627                 sys.exit(2)
 628
 629     # Right - if we've got this far, we've got a config file, now for the hard
 630     # bits...
 631
 632     scp = SafeConfigParser()
 633     scp.read(configfile)
 634
 635     maildir_root = "RSSMaildir"
 636     state_dir = "state"
 637
 638     if options.statedir != None:
 639         state_dir = options.statedir
 640         try:
 641             mode = os.stat(state_dir)[stat.ST_MODE]
 642             if not stat.S_ISDIR(mode):
 643                 sys.stderr.write( \
 644                     "State directory (%s) is not a directory\n" %(state_dir))
 645                 sys.exit(1)
 646         except:
 647             # try to make the directory
 648             try:
 649                 os.mkdir(state_dir)
 650             except:
 651                 sys.stderr.write("Couldn't create statedir %s" %(state_dir))
 652                 sys.exit(1)
 653     elif scp.has_option("general", "state_dir"):
 654         new_state_dir = scp.get("general", "state_dir")
 655         try:
 656             mode = os.stat(state_dir)[stat.ST_MODE]
 657             if not stat.S_ISDIR(mode):
 658                 sys.stderr.write( \
 659                     "State directory (%s) is not a directory\n" %(state_dir))
 660                 sys.exit(1)
 661         except:
 662             # try to create it
 663             try:
 664                 os.mkdir(new_state_dir)
 665                 state_dir = new_state_dir
 666             except:
 667                 sys.stderr.write( \
 668                     "Couldn't create state directory %s\n" %(new_state_dir))
 669                 sys.exit(1)
 670     else:
 671         try:
 672             mode = os.stat(state_dir)[stat.ST_MODE]
 673             if not stat.S_ISDIR(mode):
 674                 sys.stderr.write( \
 675                     "State directory %s is not a directory\n" %(state_dir))
 676                 sys.exit(1)
 677         except:
 678             try:
 679                 os.mkdir(state_dir)
 680             except:
 681                 sys.stderr.write( \
 682                     "State directory %s could not be created\n" %(state_dir))
 683                 sys.exit(1)
 684
 685     if scp.has_option("general", "maildir_root"):
 686         maildir_root = scp.get("general", "maildir_root")
 687
 688     try:
 689         mode = os.stat(maildir_root)[stat.ST_MODE]
 690         if not stat.S_ISDIR(mode):
 691             sys.stderr.write( \
 692                 "Maildir Root %s is not a directory\n" \
 693                 %(maildir_root))
 694             sys.exit(1)
 695     except:
 696         try:
 697             os.mkdir(maildir_root)
 698         except:
 699             sys.stderr.write("Couldn't create Maildir Root %s\n" \
 700                 %(maildir_root))
 701             sys.exit(1)
 702
 703     feeds = scp.sections()
 704     try:
 705         feeds.remove("general")
 706     except:
 707         pass
 708
 709     for section in feeds:
 710         # check if the directory exists
 711         maildir = None
 712         try:
 713             maildir = scp.get(section, "maildir")
 714         except:
 715             maildir = section
 716
 717         maildir = urllib.urlencode(((section, maildir),)).split("=")[1]
 718         maildir = os.path.join(maildir_root, maildir)
 719
 720         try:
 721             exists = os.stat(maildir)
 722             if stat.S_ISDIR(exists[stat.ST_MODE]):
 723                 # check if there's a new, cur and tmp directory
 724                 try:
 725                     mode = os.stat(os.path.join(maildir, "cur"))[stat.ST_MODE]
 726                 except:
 727                     os.mkdir(os.path.join(maildir, "cur"))
 728                     if not stat.S_ISDIR(mode):
 729                         sys.stderr.write("Broken maildir: %s\n" %(maildir))
 730                 try:
 731                     mode = os.stat(os.path.join(maildir, "tmp"))[stat.ST_MODE]
 732                 except:
 733                     os.mkdir(os.path.join(maildir, "tmp"))
 734                     if not stat.S_ISDIR(mode):
 735                         sys.stderr.write("Broken maildir: %s\n" %(maildir))
 736                 try:
 737                     mode = os.stat(os.path.join(maildir, "new"))[stat.ST_MODE]
 738                     if not stat.S_ISDIR(mode):
 739                         sys.stderr.write("Broken maildir: %s\n" %(maildir))
 740                 except:
 741                     os.mkdir(os.path.join(maildir, "new"))
 742             else:
 743                 sys.stderr.write("Broken maildir: %s\n" %(maildir))
 744         except:
 745             try:
 746                 os.mkdir(maildir)
 747             except:
 748                 sys.stderr.write("Couldn't create root maildir %s\n" \
 749                     %(maildir))
 750                 sys.exit(1)
 751             try:
 752                 os.mkdir(os.path.join(maildir, "new"))
 753                 os.mkdir(os.path.join(maildir, "cur"))
 754                 os.mkdir(os.path.join(maildir, "tmp"))
 755             except:
 756                 sys.stderr.write( \
 757                     "Couldn't create required maildir directories for %s\n" \
 758                     %(section,))
 759                 sys.exit(1)
 760
 761         # right - we've got the directories, we've got the section, we know the
 762         # url... lets play!
 763
 764         parse_and_deliver(maildir, section, state_dir)