rss2maildir.py

   1 #!/usr/bin/python
   2 # coding=utf-8
   3
   4 # rss2maildir.py - RSS feeds to Maildir 1 email per item
   5 # Copyright (C) 2007  Brett Parker <iDunno@sommitrealweird.co.uk>
   6 #
   7 # This program is free software: you can redistribute it and/or modify
   8 # it under the terms of the GNU General Public License as published by
   9 # the Free Software Foundation, either version 3 of the License, or
  10 # (at your option) any later version.
  11 #
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU General Public License for more details.
  16 #
  17 # You should have received a copy of the GNU General Public License
  18 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  19
  20 import sys
  21 import os
  22 import stat
  23 import httplib
  24 import urllib
  25
  26 import feedparser
  27
  28 from email.MIMEMultipart import MIMEMultipart
  29 from email.MIMEText import MIMEText
  30
  31 import datetime
  32 import random
  33 import string
  34 import textwrap
  35
  36 import socket
  37
  38 from optparse import OptionParser
  39 from ConfigParser import SafeConfigParser
  40
  41 from base64 import b64encode
  42 import md5
  43
  44 import cgi
  45 import dbm
  46
  47 from HTMLParser import HTMLParser
  48
  49 class HTML2Text(HTMLParser):
  50     entities = {
  51         u'amp': "&",
  52         u'lt': "<",
  53         u'gt': ">",
  54         u'pound': "£",
  55         u'copy': "©",
  56         u'apos': "'",
  57         u'quot': "\"",
  58         u'nbsp': " ",
  59         }
  60
  61     blockleveltags = [
  62         u'h1',
  63         u'h2',
  64         u'h3',
  65         u'h4',
  66         u'h5',
  67         u'h6',
  68         u'pre',
  69         u'p',
  70         u'ul',
  71         u'ol',
  72         u'dl',
  73         u'div',
  74         #u'blockquote',
  75         ]
  76
  77     liststarttags = [
  78         u'ul',
  79         u'ol',
  80         u'dl',
  81         ]
  82
  83     cancontainflow = [
  84         u'div',
  85         u'li',
  86         u'dd',
  87         u'blockquote',
  88     ]
  89
  90     def __init__(self,textwidth=70):
  91         self.text = u''
  92         self.curdata = u''
  93         self.textwidth = textwidth
  94         self.opentags = []
  95         self.indentlevel = 0
  96         self.ignorenodata = False
  97         self.listcount = []
  98         self.urls = []
  99         HTMLParser.__init__(self)
 100
 101     def handle_starttag(self, tag, attrs):
 102         tag_name = tag.lower()
 103         if tag_name in self.blockleveltags:
 104             # handle starting a new block - unless we're in a block element
 105             # that can contain other blocks, we'll assume that we want to close
 106             # the container
 107             if len(self.opentags) > 1 and self.opentags[-1] == u'li':
 108                 self.handle_curdata()
 109
 110             if tag_name == u'ol':
 111                 self.handle_curdata()
 112                 self.listcount.append(1)
 113                 self.listlevel = len(self.listcount) - 1
 114
 115             if tag_name in self.liststarttags:
 116                 smallist = self.opentags[-3:-1]
 117                 smallist.reverse()
 118                 for prev_listtag in smallist:
 119                     if prev_listtag in [u'dl', u'ol']:
 120                         self.indentlevel = self.indentlevel + 4
 121                         break
 122                     elif prev_listtag == u'ul':
 123                         self.indentlevel = self.indentlevel + 3
 124                         break
 125
 126             if len(self.opentags) > 0:
 127                 self.handle_curdata()
 128                 if tag_name not in self.cancontainflow:
 129                     self.opentags.pop()
 130             self.opentags.append(tag_name)
 131         else:
 132             if tag_name == "span":
 133                 return
 134             listcount = 0
 135             try:
 136                 listcount = self.listcount[-1]
 137             except:
 138                 pass
 139
 140             if tag_name == u'dd' and len(self.opentags) > 1 \
 141                 and self.opentags[-1] == u'dt':
 142                 self.handle_curdata()
 143                 self.opentags.pop()
 144             elif tag_name == u'dt' and len(self.opentags) > 1 \
 145                 and self.opentags[-1] == u'dd':
 146                 self.handle_curdata()
 147                 self.opentags.pop()
 148             elif tag_name == u'a':
 149                 for attr in attrs:
 150                     if attr[0].lower() == u'href':
 151                         self.urls.append(attr[1])
 152                 self.curdata = self.curdata + u'`'
 153                 self.opentags.append(tag_name)
 154                 return
 155             elif tag_name == u'img':
 156                 self.handle_image(attrs)
 157                 return
 158             elif tag_name == u'br':
 159                 self.handle_br()
 160                 return
 161             else:
 162                 # we don't know the tag, so lets avoid handling it!
 163                 return
 164
 165     def handle_startendtag(self, tag, attrs):
 166         if tag.lower() == u'br':
 167             self.handle_br()
 168         elif tag.lower() == u'img':
 169             self.handle_image(attrs)
 170             return
 171
 172     def handle_br(self):
 173             self.handle_curdata()
 174             self.opentags.append(u'br')
 175             self.handle_curdata()
 176             self.opentags.pop()
 177
 178     def handle_image(self, attrs):
 179         alt = u''
 180         url = u''
 181         for attr in attrs:
 182             if attr[0] == 'alt':
 183                 alt = attr[1].decode('utf-8')
 184             elif attr[0] == 'src':
 185                 url = attr[1].decode('utf-8')
 186         if url:
 187             self.curdata = self.curdata \
 188                 + u' [img:' \
 189                 + url
 190             if alt:
 191                 self.curdata = self.curdata \
 192                     + u'(' \
 193                     + alt \
 194                     + u')'
 195             self.curdata = self.curdata \
 196                 + u']'
 197
 198     def handle_curdata(self):
 199
 200         if len(self.opentags) == 0:
 201             return
 202
 203         tag_thats_done = self.opentags[-1]
 204
 205         if len(self.curdata) == 0:
 206             return
 207
 208         if tag_thats_done == u'br':
 209             if len(self.text) == 0 or self.text[-1] != '\n':
 210                 self.text = self.text + '\n'
 211                 self.ignorenodata = True
 212             return
 213
 214         if len(self.curdata.strip()) == 0:
 215             return
 216
 217         if tag_thats_done in self.blockleveltags:
 218             newlinerequired = self.text != u''
 219             if self.ignorenodata:
 220                 newlinerequired = False
 221             self.ignorenodata = False
 222             if newlinerequired \
 223                 and len(self.text) > 2 \
 224                 and self.text[-1] != u'\n' \
 225                 and self.text[-2] != u'\n':
 226                     self.text = self.text + u'\n\n'
 227
 228         if tag_thats_done in ["h1", "h2", "h3", "h4", "h5", "h6"]:
 229             underline = u''
 230             underlinechar = u'='
 231             headingtext = unicode( \
 232                 self.curdata.encode("utf-8").strip(), "utf-8")
 233             seperator = u'\n' + u' '*self.indentlevel
 234             headingtext = seperator.join( \
 235                 textwrap.wrap( \
 236                     headingtext, \
 237                     self.textwidth - self.indentlevel \
 238                     ) \
 239                 )
 240
 241             if tag_thats_done == u'h2':
 242                 underlinechar = u'-'
 243             elif tag_thats_done != u'h1':
 244                 underlinechar = u'~'
 245
 246             if u'\n' in headingtext:
 247                 underline = u' ' * self.indentlevel \
 248                     + underlinechar * (self.textwidth - self.indentlevel)
 249             else:
 250                 underline = u' ' * self.indentlevel \
 251                     + underlinechar * len(headingtext)
 252             self.text = self.text \
 253                 + headingtext.encode("utf-8") + u'\n' \
 254                 + underline
 255         elif tag_thats_done in [u'p', u'div']:
 256             paragraph = unicode( \
 257                 self.curdata.strip().encode("utf-8"), "utf-8")
 258             seperator = u'\n' + u' ' * self.indentlevel
 259             self.text = self.text \
 260                 + u' ' * self.indentlevel \
 261                 + seperator.join( \
 262                     textwrap.wrap( \
 263                         paragraph, self.textwidth - self.indentlevel))
 264         elif tag_thats_done == "pre":
 265             self.text = self.text + unicode( \
 266                 self.curdata.encode("utf-8"), "utf-8")
 267         elif tag_thats_done == u'blockquote':
 268             quote = unicode( \
 269                 self.curdata.encode("utf-8").strip(), "utf-8")
 270             seperator = u'\n' + u' ' * self.indentlevel + u'> '
 271             if len(self.text) > 0 and self.text[-1] != u'\n':
 272                 self.text = self.text + u'\n'
 273             self.text = self.text \
 274                 + u'> ' \
 275                 + seperator.join( \
 276                     textwrap.wrap( \
 277                         quote, \
 278                         self.textwidth - self.indentlevel - 2 \
 279                     )
 280                 )
 281             self.curdata = u''
 282         elif tag_thats_done == "li":
 283             item = unicode(self.curdata.encode("utf-8").strip(), "utf-8")
 284             if len(self.text) > 0 and self.text[-1] != u'\n':
 285                 self.text = self.text + u'\n'
 286             # work out if we're in an ol rather than a ul
 287             latesttags = self.opentags[-4:]
 288             latesttags.reverse()
 289             isul = None
 290             for thing in latesttags:
 291                 if thing == 'ul':
 292                     isul = True
 293                     break
 294                 elif thing == 'ol':
 295                     isul = False
 296                     break
 297
 298             listindent = 3
 299             if not isul:
 300                 listindent = 4
 301
 302             listmarker = u' * '
 303             if isul == False:
 304                 listmarker = u' %2d. ' %(self.listcount[-1])
 305                 self.listcount[-1] = self.listcount[-1] + 1
 306
 307             seperator = u'\n' \
 308                 + u' ' * self.indentlevel \
 309                 + u' ' * listindent
 310             self.text = self.text \
 311                 + u' ' * self.indentlevel \
 312                 + listmarker \
 313                 + seperator.join( \
 314                     textwrap.wrap( \
 315                         item, \
 316                         self.textwidth - self.indentlevel - listindent \
 317                     ) \
 318                 )
 319             self.curdata = u''
 320         elif tag_thats_done == u'dt':
 321             definition = unicode(self.curdata.encode("utf-8").strip(), "utf-8")
 322             if len(self.text) > 0 and self.text[-1] != u'\n':
 323                 self.text = self.text + u'\n\n'
 324             elif len(self.text) > 1 and self.text[-2] != u'\n':
 325                 self.text = self.text + u'\n'
 326             definition = u' ' * self.indentlevel + definition + "::"
 327             indentstring = u'\n' + u' ' * (self.indentlevel + 1)
 328             self.text = self.text \
 329                 + indentstring.join(
 330                     textwrap.wrap(definition, \
 331                         self.textwidth - self.indentlevel - 1))
 332             self.curdata = u''
 333         elif tag_thats_done == u'dd':
 334             definition = unicode(self.curdata.encode("utf-8").strip(), "utf-8")
 335             if len(definition) > 0:
 336                 if len(self.text) > 0 and self.text[-1] != u'\n':
 337                     self.text = self.text + u'\n'
 338                 indentstring = u'\n' + u' ' * (self.indentlevel + 4)
 339                 self.text = self.text \
 340                     + u' ' * (self.indentlevel + 4) \
 341                     + indentstring.join( \
 342                         textwrap.wrap( \
 343                             definition, \
 344                             self.textwidth - self.indentlevel - 4 \
 345                             ) \
 346                         )
 347                 self.curdata = u''
 348         elif tag_thats_done == u'a':
 349             self.curdata = self.curdata + u'`__'
 350             pass
 351         elif tag_thats_done in self.liststarttags:
 352             pass
 353
 354         if tag_thats_done in self.blockleveltags:
 355             self.curdata = u''
 356
 357         self.ignorenodata = False
 358
 359     def handle_endtag(self, tag):
 360         self.ignorenodata = False
 361         if tag == "span":
 362             return
 363
 364         try:
 365             tagindex = self.opentags.index(tag)
 366         except:
 367             return
 368         tag = tag.lower()
 369
 370         if tag in [u'br', u'img']:
 371             return
 372
 373         if tag in self.liststarttags:
 374             if tag in [u'ol', u'dl', u'ul']:
 375                 self.handle_curdata()
 376                 # find if there was a previous list level
 377                 smalllist = self.opentags[:-1]
 378                 smalllist.reverse()
 379                 for prev_listtag in smalllist:
 380                     if prev_listtag in [u'ol', u'dl']:
 381                         self.indentlevel = self.indentlevel - 4
 382                         break
 383                     elif prev_listtag == u'ul':
 384                         self.indentlevel = self.indentlevel - 3
 385                         break
 386
 387         if tag == u'ol':
 388             self.listcount = self.listcount[:-1]
 389
 390         while tagindex < len(self.opentags) \
 391             and tag in self.opentags[tagindex+1:]:
 392             try:
 393                 tagindex = self.opentags.index(tag, tagindex+1)
 394             except:
 395                 # well, we don't want to do that then
 396                 pass
 397         if tagindex != len(self.opentags) - 1:
 398             # Assuming the data was for the last opened tag first
 399             self.handle_curdata()
 400             # Now kill the list to be a slice before this tag was opened
 401             self.opentags = self.opentags[:tagindex + 1]
 402         else:
 403             self.handle_curdata()
 404             if self.opentags[-1] == tag:
 405                 self.opentags.pop()
 406
 407     def handle_data(self, data):
 408         if len(self.opentags) == 0:
 409             self.opentags.append(u'p')
 410         self.curdata = self.curdata + unicode(data, "utf-8")
 411
 412     def handle_entityref(self, name):
 413         entity = name
 414         if HTML2Text.entities.has_key(name.lower()):
 415             entity = HTML2Text.entities[name.lower()]
 416         elif name[0] == "#":
 417             entity = unichr(int(name[1:]))
 418         else:
 419             entity = "&" + name + ";"
 420
 421         self.curdata = self.curdata + unicode(entity, "utf-8")
 422
 423     def gettext(self):
 424         self.handle_curdata()
 425         if len(self.text) == 0 or self.text[-1] != u'\n':
 426             self.text = self.text + u'\n'
 427         self.opentags = []
 428         if len(self.text) > 0:
 429             while len(self.text) > 1 and self.text[-1] == u'\n':
 430                 self.text = self.text[:-1]
 431             self.text = self.text + u'\n'
 432         if len(self.urls) > 0:
 433             self.text = self.text + u'\n__ ' + u'\n__ '.join(self.urls) + u'\n'
 434             self.urls = []
 435         return self.text
 436
 437 def open_url(method, url):
 438     redirectcount = 0
 439     while redirectcount < 3:
 440         (type, rest) = urllib.splittype(url)
 441         (host, path) = urllib.splithost(rest)
 442         (host, port) = urllib.splitport(host)
 443         if port == None:
 444             port = 80
 445         try:
 446             conn = httplib.HTTPConnection("%s:%s" %(host, port))
 447             conn.request(method, path)
 448             response = conn.getresponse()
 449             if response.status in [301, 302, 303, 307]:
 450                 headers = response.getheaders()
 451                 for header in headers:
 452                     if header[0] == "location":
 453                         url = header[1]
 454             elif response.status == 200:
 455                 return response
 456         except:
 457             pass
 458         redirectcount = redirectcount + 1
 459     return None
 460
 461 def parse_and_deliver(maildir, url, statedir):
 462     feedhandle = None
 463     headers = None
 464     # first check if we know about this feed already
 465     feeddb = dbm.open(os.path.join(statedir, "feeds"), "c")
 466     if feeddb.has_key(url):
 467         data = feeddb[url]
 468         data = cgi.parse_qs(data)
 469         response = open_url("HEAD", url)
 470         headers = None
 471         if response:
 472             headers = response.getheaders()
 473         ischanged = False
 474         try:
 475             for header in headers:
 476                 if header[0] == "content-length":
 477                     if header[1] != data["content-length"][0]:
 478                         ischanged = True
 479                 elif header[0] == "etag":
 480                     if header[1] != data["etag"][0]:
 481                         ischanged = True
 482                 elif header[0] == "last-modified":
 483                     if header[1] != data["last-modified"][0]:
 484                         ischanged = True
 485                 elif header[0] == "content-md5":
 486                     if header[1] != data["content-md5"][0]:
 487                         ischanged = True
 488         except:
 489             ischanged = True
 490         if ischanged:
 491             response = open_url("GET", url)
 492             if response != None:
 493                 headers = response.getheaders()
 494                 feedhandle = response
 495             else:
 496                 sys.stderr.write("Failed to fetch feed: %s\n" %(url))
 497                 return
 498         else:
 499             return # don't need to do anything, nothings changed.
 500     else:
 501         response = open_url("GET", url)
 502         if response != None:
 503             headers = response.getheaders()
 504             feedhandle = response
 505         else:
 506             sys.stderr.write("Failed to fetch feed: %s\n" %(url))
 507             return
 508
 509     fp = feedparser.parse(feedhandle)
 510     db = dbm.open(os.path.join(statedir, "seen"), "c")
 511     for item in fp["items"]:
 512         # have we seen it before?
 513         # need to work out what the content is first...
 514
 515         if item.has_key("content"):
 516             content = item["content"][0]["value"]
 517         else:
 518             content = item["summary"]
 519
 520         md5sum = md5.md5(content.encode("utf-8")).hexdigest()
 521
 522         prevmessageid = None
 523
 524         # check if there's a guid too - if that exists and we match the md5,
 525         # return
 526         if item.has_key("guid"):
 527             if db.has_key(url + "|" + item["guid"]):
 528                 data = db[url + "|" + item["guid"]]
 529                 data = cgi.parse_qs(data)
 530                 if data["contentmd5"][0] == md5sum:
 531                     continue
 532
 533         if db.has_key(url + "|" + item["link"]):
 534             data = db[url + "|" + item["link"]]
 535             data = cgi.parse_qs(data)
 536             if data.has_key("message-id"):
 537                 prevmessageid = data["message-id"][0]
 538             if data["contentmd5"][0] == md5sum:
 539                 continue
 540
 541         try:
 542             author = item["author"]
 543         except:
 544             author = url
 545
 546         # create a basic email message
 547         msg = MIMEMultipart("alternative")
 548         messageid = "<" \
 549             + datetime.datetime.now().strftime("%Y%m%d%H%M") \
 550             + "." \
 551             + "".join( \
 552                 [random.choice( \
 553                     string.ascii_letters + string.digits \
 554                     ) for a in range(0,6) \
 555                 ]) + "@" + socket.gethostname() + ">"
 556         msg.add_header("Message-ID", messageid)
 557         msg.set_unixfrom("\"%s\" <rss2maildir@localhost>" %(url))
 558         msg.add_header("From", "\"%s\" <rss2maildir@localhost>" %(author))
 559         msg.add_header("To", "\"%s\" <rss2maildir@localhost>" %(url))
 560         if prevmessageid:
 561             msg.add_header("References", prevmessageid)
 562         createddate = datetime.datetime.now() \
 563             .strftime("%a, %e %b %Y %T -0000")
 564         try:
 565             createddate = datetime.datetime(*item["updated_parsed"][0:6]) \
 566                 .strftime("%a, %e %b %Y %T -0000")
 567         except:
 568             pass
 569         msg.add_header("Date", createddate)
 570         msg.add_header("Subject", item["title"])
 571         msg.set_default_type("text/plain")
 572
 573         htmlcontent = content.encode("utf-8")
 574         htmlcontent = "%s\n\n<p>Item URL: <a href='%s'>%s</a></p>" %( \
 575             content, \
 576             item["link"], \
 577             item["link"] )
 578         htmlpart = MIMEText(htmlcontent.encode("utf-8"), "html", "utf-8")
 579         textparser = HTML2Text()
 580         textparser.feed(content.encode("utf-8"))
 581         textcontent = textparser.gettext()
 582         textcontent = "%s\n\nItem URL: %s" %( \
 583             textcontent, \
 584             item["link"] )
 585         textpart = MIMEText(textcontent.encode("utf-8"), "plain", "utf-8")
 586         msg.attach(textpart)
 587         msg.attach(htmlpart)
 588
 589         # start by working out the filename we should be writting to, we do
 590         # this following the normal maildir style rules
 591         fname = str(os.getpid()) \
 592             + "." + socket.gethostname() \
 593             + "." + "".join( \
 594                 [random.choice( \
 595                     string.ascii_letters + string.digits \
 596                     ) for a in range(0,10) \
 597                 ]) + "." \
 598             + datetime.datetime.now().strftime('%s')
 599         fn = os.path.join(maildir, "tmp", fname)
 600         fh = open(fn, "w")
 601         fh.write(msg.as_string())
 602         fh.close()
 603         # now move it in to the new directory
 604         newfn = os.path.join(maildir, "new", fname)
 605         os.link(fn, newfn)
 606         os.unlink(fn)
 607
 608         # now add to the database about the item
 609         if prevmessageid:
 610             messageid = prevmessageid + " " + messageid
 611         if item.has_key("guid") and item["guid"] != item["link"]:
 612             data = urllib.urlencode(( \
 613                 ("message-id", messageid), \
 614                 ("created", createddate), \
 615                 ("contentmd5", md5sum) \
 616                 ))
 617             db[url + "|" + item["guid"]] = data
 618             try:
 619                 data = db[url + "|" + item["link"]]
 620                 data = cgi.parse_qs(data)
 621                 newdata = urllib.urlencode(( \
 622                     ("message-id", messageid), \
 623                     ("created", data["created"][0]), \
 624                     ("contentmd5", data["contentmd5"][0]) \
 625                     ))
 626                 db[url + "|" + item["link"]] = newdata
 627             except:
 628                 db[url + "|" + item["link"]] = data
 629         else:
 630             data = urllib.urlencode(( \
 631                 ("message-id", messageid), \
 632                 ("created", createddate), \
 633                 ("contentmd5", md5sum) \
 634                 ))
 635             db[url + "|" + item["link"]] = data
 636
 637     if headers:
 638         data = []
 639         for header in headers:
 640             if header[0] in \
 641                 ["content-md5", "etag", "last-modified", "content-length"]:
 642                 data.append((header[0], header[1]))
 643         if len(data) > 0:
 644             data = urllib.urlencode(data)
 645             feeddb[url] = data
 646
 647     db.close()
 648     feeddb.close()
 649
 650 if __name__ == "__main__":
 651     # This only gets executed if we really called the program
 652     # first off, parse the command line arguments
 653
 654     oparser = OptionParser()
 655     oparser.add_option(
 656         "-c", "--conf", dest="conf",
 657         help="location of config file"
 658         )
 659     oparser.add_option(
 660         "-s", "--statedir", dest="statedir",
 661         help="location of directory to store state in"
 662         )
 663
 664     (options, args) = oparser.parse_args()
 665
 666     # check for the configfile
 667
 668     configfile = None
 669
 670     if options.conf != None:
 671         # does the file exist?
 672         try:
 673             os.stat(options.conf)
 674             configfile = options.conf
 675         except:
 676             # should exit here as the specified file doesn't exist
 677             sys.stderr.write( \
 678                 "Config file %s does not exist. Exiting.\n" %(options.conf,))
 679             sys.exit(2)
 680     else:
 681         # check through the default locations
 682         try:
 683             os.stat("%s/.rss2maildir.conf" %(os.environ["HOME"],))
 684             configfile = "%s/.rss2maildir.conf" %(os.environ["HOME"],)
 685         except:
 686             try:
 687                 os.stat("/etc/rss2maildir.conf")
 688                 configfile = "/etc/rss2maildir.conf"
 689             except:
 690                 sys.stderr.write("No config file found. Exiting.\n")
 691                 sys.exit(2)
 692
 693     # Right - if we've got this far, we've got a config file, now for the hard
 694     # bits...
 695
 696     scp = SafeConfigParser()
 697     scp.read(configfile)
 698
 699     maildir_root = "RSSMaildir"
 700     state_dir = "state"
 701
 702     if options.statedir != None:
 703         state_dir = options.statedir
 704         try:
 705             mode = os.stat(state_dir)[stat.ST_MODE]
 706             if not stat.S_ISDIR(mode):
 707                 sys.stderr.write( \
 708                     "State directory (%s) is not a directory\n" %(state_dir))
 709                 sys.exit(1)
 710         except:
 711             # try to make the directory
 712             try:
 713                 os.mkdir(state_dir)
 714             except:
 715                 sys.stderr.write("Couldn't create statedir %s" %(state_dir))
 716                 sys.exit(1)
 717     elif scp.has_option("general", "state_dir"):
 718         new_state_dir = scp.get("general", "state_dir")
 719         try:
 720             mode = os.stat(state_dir)[stat.ST_MODE]
 721             if not stat.S_ISDIR(mode):
 722                 sys.stderr.write( \
 723                     "State directory (%s) is not a directory\n" %(state_dir))
 724                 sys.exit(1)
 725         except:
 726             # try to create it
 727             try:
 728                 os.mkdir(new_state_dir)
 729                 state_dir = new_state_dir
 730             except:
 731                 sys.stderr.write( \
 732                     "Couldn't create state directory %s\n" %(new_state_dir))
 733                 sys.exit(1)
 734     else:
 735         try:
 736             mode = os.stat(state_dir)[stat.ST_MODE]
 737             if not stat.S_ISDIR(mode):
 738                 sys.stderr.write( \
 739                     "State directory %s is not a directory\n" %(state_dir))
 740                 sys.exit(1)
 741         except:
 742             try:
 743                 os.mkdir(state_dir)
 744             except:
 745                 sys.stderr.write( \
 746                     "State directory %s could not be created\n" %(state_dir))
 747                 sys.exit(1)
 748
 749     if scp.has_option("general", "maildir_root"):
 750         maildir_root = scp.get("general", "maildir_root")
 751
 752     try:
 753         mode = os.stat(maildir_root)[stat.ST_MODE]
 754         if not stat.S_ISDIR(mode):
 755             sys.stderr.write( \
 756                 "Maildir Root %s is not a directory\n" \
 757                 %(maildir_root))
 758             sys.exit(1)
 759     except:
 760         try:
 761             os.mkdir(maildir_root)
 762         except:
 763             sys.stderr.write("Couldn't create Maildir Root %s\n" \
 764                 %(maildir_root))
 765             sys.exit(1)
 766
 767     feeds = scp.sections()
 768     try:
 769         feeds.remove("general")
 770     except:
 771         pass
 772
 773     for section in feeds:
 774         # check if the directory exists
 775         maildir = None
 776         try:
 777             maildir = scp.get(section, "maildir")
 778         except:
 779             maildir = section
 780
 781         maildir = urllib.urlencode(((section, maildir),)).split("=")[1]
 782         maildir = os.path.join(maildir_root, maildir)
 783
 784         try:
 785             exists = os.stat(maildir)
 786             if stat.S_ISDIR(exists[stat.ST_MODE]):
 787                 # check if there's a new, cur and tmp directory
 788                 try:
 789                     mode = os.stat(os.path.join(maildir, "cur"))[stat.ST_MODE]
 790                 except:
 791                     os.mkdir(os.path.join(maildir, "cur"))
 792                     if not stat.S_ISDIR(mode):
 793                         sys.stderr.write("Broken maildir: %s\n" %(maildir))
 794                 try:
 795                     mode = os.stat(os.path.join(maildir, "tmp"))[stat.ST_MODE]
 796                 except:
 797                     os.mkdir(os.path.join(maildir, "tmp"))
 798                     if not stat.S_ISDIR(mode):
 799                         sys.stderr.write("Broken maildir: %s\n" %(maildir))
 800                 try:
 801                     mode = os.stat(os.path.join(maildir, "new"))[stat.ST_MODE]
 802                     if not stat.S_ISDIR(mode):
 803                         sys.stderr.write("Broken maildir: %s\n" %(maildir))
 804                 except:
 805                     os.mkdir(os.path.join(maildir, "new"))
 806             else:
 807                 sys.stderr.write("Broken maildir: %s\n" %(maildir))
 808         except:
 809             try:
 810                 os.mkdir(maildir)
 811             except:
 812                 sys.stderr.write("Couldn't create root maildir %s\n" \
 813                     %(maildir))
 814                 sys.exit(1)
 815             try:
 816                 os.mkdir(os.path.join(maildir, "new"))
 817                 os.mkdir(os.path.join(maildir, "cur"))
 818                 os.mkdir(os.path.join(maildir, "tmp"))
 819             except:
 820                 sys.stderr.write( \
 821                     "Couldn't create required maildir directories for %s\n" \
 822                     %(section,))
 823                 sys.exit(1)
 824
 825         # right - we've got the directories, we've got the section, we know the
 826         # url... lets play!
 827
 828         parse_and_deliver(maildir, section, state_dir)