From: Jonathan McDowell Date: Fri, 15 Apr 2005 16:01:58 +0000 (+0000) Subject: Make onak-mail.pl queue requests. X-Git-Url: https://git.sommitrealweird.co.uk/onak.git/commitdiff_plain/80989f28059ef378b7473f65a5cf3afdecc722f3?ds=sidebyside Make onak-mail.pl queue requests. onak-mail.pl sequentially processes requests. Rather than having a lot of these processes hang around waiting for other requests to complete we now dump requests to a file and then the active copy will process all pending requests. --- diff --git a/onak-mail.pl.in b/onak-mail.pl.in index 7ad5960..263b727 100644 --- a/onak-mail.pl.in +++ b/onak-mail.pl.in @@ -3,12 +3,13 @@ # onak-mail.pl - Mail processing interface for onak, an OpenPGP Keyserver. # # Written by Jonathan McDowell -# Copyright 2002 Project Purple +# Copyright 2002-2005 Project Purple # Released under the GPL. # use strict; -use Fcntl ':flock'; +use Fcntl; +use IO::Handle; use IPC::Open3; my %config; @@ -39,6 +40,8 @@ sub readconfig { $config{'pks_bin_dir'} = $1; } elsif (/^db_dir (.*)/) { $config{'db_dir'} = $1; + } elsif (/^mail_dir (.*)/) { + $config{'mail_dir'} = $1; } elsif (/^syncsite (.*)/) { push @{$config{'syncsites'}}, $1; } @@ -56,146 +59,196 @@ sub readconfig { # difference between what we just added and what we had before (ie the least # data need to get from what we had to what we have). # -sub submitupdate { - my @data = @_; +sub submitupdate($) { + my $data = shift; my (@errors, @mergedata); - open(LOCKFILE, '>'.$config{'db_dir'}.'/onak-mail.lck'); - flock(LOCKFILE, LOCK_EX); - print LOCKFILE "$$"; - - open3(\*MERGEIN, \*MERGEOUT, \*MERGEERR, + my $pid = open3(\*MERGEIN, \*MERGEOUT, \*MERGEERR, $config{'pks_bin_dir'}."/onak", "-u", "add"); - print MERGEIN @data; + print MERGEIN @$data; close MERGEIN; @mergedata = ; close MERGEOUT; @errors = ; close MERGEERR; - - flock(LOCKFILE, LOCK_UN); - close(LOCKFILE); + waitpid $pid, 0; return @mergedata; } -my ($inheader, %seenby, $subject, $from, $replyto, @body, @syncmail); - -$inheader = 1; -$subject = ""; -&readconfig; -while (<>) { - if ($inheader) { - if (/^Subject:\s*(.*)\s*$/i) { - $subject = $1; - } elsif (/^X-KeyServer-Sent:\s*(.*)\s*$/i) { - $seenby{$1} = 1; - } elsif (/^From:\s*(.*)\s*$/i) { - $from = $1; - } elsif (/^Reply-To:\s*(.*)\s*$/i) { - $replyto = $1; - } elsif (/^$/) { - $inheader = 0; - } - } - if (!$inheader) { - push @body, $_; - } -} -if (! defined($replyto)) { - $replyto = $from; -} - -# HELP, ADD, INCREMENTAL, VERBOSE INDEX , INDEX , GET , -# LAST - -if ($subject =~ /^INCREMENTAL$/i) { - my $site; - my $count; - my $i; - my @newupdate = submitupdate(@body); - my @time; - - $count = 0; - foreach $i (@{$config{'syncsites'}}) { - if (! defined($seenby{$i})) { - $count++; +sub processmail($$$$$) { + my $subject = shift; + my $from = shift; + my $replyto = shift; + my $seenby = shift; + my $body = shift; + + # HELP, ADD, INCREMENTAL, VERBOSE INDEX , INDEX , + # GET , LAST + + if ($subject =~ /^INCREMENTAL$/i) { + my $site; + my $count; + my $i; + my @newupdate = submitupdate($body); + my @time; + + $count = 0; + foreach $i (@{$config{'syncsites'}}) { + if (! defined($seenby->{$i})) { + $count++; + } } - } - - open (LOG, ">>$config{'logfile'}"); - @time = localtime(time); - print LOG "["; - print LOG sprintf "%02d/%02d/%04d %02d:%02d:%02d", - $time[3], $time[4] + 1, $time[5] + 1900, - $time[2], $time[1], $time[0]; - print LOG "] onak-mail[$$]: Syncing with $count sites.\n"; - close LOG; - - if ((! defined($newupdate[0])) || $newupdate[0] eq '') { + open (LOG, ">>$config{'logfile'}"); + @time = localtime(time); print LOG "["; print LOG sprintf "%02d/%02d/%04d %02d:%02d:%02d", $time[3], $time[4] + 1, $time[5] + 1900, $time[2], $time[1], $time[0]; - print LOG "] onak-mail[$$]: Nothing to sync.\n"; + print LOG "] onak-mail[$$]: Syncing with $count sites.\n"; close LOG; - $count = 0; - } - - if ($count > 0) { - open(MAIL, "|$config{mta}"); - print MAIL "From: $config{adminemail}\n"; - print MAIL "To: "; - foreach $i (@{$config{'syncsites'}}) { - if (! defined($seenby{$i})) { - print MAIL "$i"; - $count--; - if ($count > 0) { - print MAIL ", "; + + if ((! defined($newupdate[0])) || $newupdate[0] eq '') { + open (LOG, ">>$config{'logfile'}"); + print LOG "["; + print LOG sprintf "%02d/%02d/%04d %02d:%02d:%02d", + $time[3], $time[4] + 1, $time[5] + 1900, + $time[2], $time[1], $time[0]; + print LOG "] onak-mail[$$]: Nothing to sync.\n"; + close LOG; + $count = 0; + } + + if ($count > 0) { + open(MAIL, "|$config{mta}"); + print MAIL "From: $config{adminemail}\n"; + print MAIL "To: "; + foreach $i (@{$config{'syncsites'}}) { + if (! defined($seenby->{$i})) { + print MAIL "$i"; + $count--; + if ($count > 0) { + print MAIL ", "; + } } } + print MAIL "\n"; + print MAIL "Subject: incremental\n"; + foreach $site (keys %$seenby) { + print MAIL "X-KeyServer-Sent: $site\n"; + } + print MAIL "X-KeyServer-Sent: $config{thissite}\n"; + print MAIL "Precedence: list\n"; + print MAIL "MIME-Version: 1.0\n"; + print MAIL "Content-Type: application/pgp-keys\n"; + print MAIL "\n"; + print MAIL @newupdate; + close MAIL; } - print MAIL "\n"; - print MAIL "Subject: incremental\n"; - foreach $site (keys %seenby) { - print MAIL "X-KeyServer-Sent: $site\n"; + } elsif ($subject =~ /^(VERBOSE )?INDEX (.*)$/i) { + my (@indexdata, $command); + + $command = "index"; + if (defined($1)) { + $command = "vindex"; } - print MAIL "X-KeyServer-Sent: $config{thissite}\n"; + + my $pid = open3(\*INDEXIN, \*INDEXOUT, \*INDEXERR, + $config{'pks_bin_dir'}."/onak", $command, "$2"); + close INDEXIN; + @indexdata = ; + close INDEXOUT; + close INDEXERR; + waitpid $pid, 0; + + open(MAIL, "|$config{mta}"); + print MAIL "From: $config{adminemail}\n"; + print MAIL "To: $replyto\n"; + print MAIL "Subject: Reply to INDEX $2\n"; print MAIL "Precedence: list\n"; print MAIL "MIME-Version: 1.0\n"; - print MAIL "Content-Type: application/pgp-keys\n"; + print MAIL "Content-Type: text/plain\n"; print MAIL "\n"; - print MAIL @newupdate; + print MAIL "Below follows the reply to your recent keyserver query:\n"; + print MAIL "\n"; + print MAIL @indexdata; close MAIL; } -} elsif ($subject =~ /^(VERBOSE )?INDEX (.*)$/i) { - my (@indexdata, $command); +} + +my ($inheader, %seenby, $subject, $from, $replyto, @body, @syncmail); + +&readconfig; + +# +# First dump the incoming mail to a file; this means that if we're receiving +# loads of updates we don't spawn lots of processes but instead leave the +# mails on disk to be dealt with sequentially. +# +my @time = localtime; +my $tmpfile = sprintf "%s/%04d%02d%02d-%02d%02d%02d-%d.onak", + $config{'mail_dir'}, + $time[5] + 1900, + $time[4], + $time[3], + $time[2], + $time[1], + $time[0], + $$; +open(MAILFILE, '>'.$tmpfile); +while (<>) { + print MAILFILE $_; +} +close(MAILFILE); - $command = "index"; - if (defined($1)) { - $command = "vindex"; +# +# Lock here to ensure that only one copy of us is processing the incoming +# mail queue at any point in time. +# +sysopen(LOCKFILE, $config{'db_dir'}.'/onak-mail.lck', + O_WRONLY|O_CREAT|O_EXCL) or exit; +print LOCKFILE "$$"; +close(LOCKFILE); + +my $file; +opendir(MAILDIR, $config{'mail_dir'}); +while ($file = readdir(MAILDIR)) { + next if $file !~ /\.onak$/; + + $inheader = 1; + $subject = $from = $replyto = ""; + undef %seenby; + @body = (); + + open(FILE, '<'.$config{'mail_dir'}.'/'.$file); + while () { + if ($inheader) { + if (/^Subject:\s*(.*)\s*$/i) { + $subject = $1; + } elsif (/^X-KeyServer-Sent:\s*(.*)\s*$/i) { + $seenby{$1} = 1; + } elsif (/^From:\s*(.*)\s*$/i) { + $from = $1; + } elsif (/^Reply-To:\s*(.*)\s*$/i) { + $replyto = $1; + } elsif (/^$/) { + $inheader = 0; + } + } + if (!$inheader) { + push @body, $_; + } + } + if (! defined($replyto)) { + $replyto = $from; } + close(FILE); + unlink $config{'mail_dir'}.'/'.$file; - open3(\*INDEXIN, \*INDEXOUT, \*INDEXERR, - $config{'pks_bin_dir'}."/onak", $command, "$2"); - close INDEXIN; - @indexdata = ; - close INDEXOUT; - close INDEXERR; - - open(MAIL, "|$config{mta}"); - print MAIL "From: $config{adminemail}\n"; - print MAIL "To: $replyto\n"; - print MAIL "Subject: Reply to INDEX $2\n"; - print MAIL "Precedence: list\n"; - print MAIL "MIME-Version: 1.0\n"; - print MAIL "Content-Type: text/plain\n"; - print MAIL "\n"; - print MAIL "Below follows the reply to your recent keyserver query:\n"; - print MAIL "\n"; - print MAIL @indexdata; - close MAIL; + processmail($subject, $from, $replyto, \%seenby, \@body); } +closedir(MAILDIR); +unlink $config{'db_dir'}.'/onak-mail.lck';