keydb_db4.c

   1 /*
   2  * keydb_db4.c - Routines to store and fetch keys in a DB4 database.
   3  *
   4  * Copyright 2002-2008 Jonathan McDowell <noodles@earth.li>
   5  */
   6
   7 #include <sys/types.h>
   8 #include <sys/stat.h>
   9 #include <sys/uio.h>
  10 #include <ctype.h>
  11 #include <errno.h>
  12 #include <fcntl.h>
  13 #include <stdio.h>
  14 #include <stdlib.h>
  15 #include <string.h>
  16 #include <unistd.h>
  17
  18 #include <db.h>
  19
  20 #include "charfuncs.h"
  21 #include "keyarray.h"
  22 #include "keydb.h"
  23 #include "keyid.h"
  24 #include "decodekey.h"
  25 #include "keystructs.h"
  26 #include "mem.h"
  27 #include "log.h"
  28 #include "onak-conf.h"
  29 #include "parsekey.h"
  30 #include "wordlist.h"
  31
  32 #define DB4_UPGRADE_FILE "db_upgrade.lck"
  33
  34 /**
  35  *      dbenv - our database environment.
  36  */
  37 static DB_ENV *dbenv = NULL;
  38
  39 /**
  40  *      numdb - The number of database files we have.
  41  */
  42 static int numdbs = 16;
  43
  44 /**
  45  *      dbconn - our connections to the key database files.
  46  */
  47 static DB **dbconns = NULL;
  48
  49 /**
  50  *      worddb - our connection to the word database.
  51  */
  52 static DB *worddb = NULL;
  53
  54 /**
  55  *      id32db - our connection to the 32bit ID database.
  56  */
  57 static DB *id32db = NULL;
  58
  59 /**
  60  *      skshashdb - our connection to the SKS hash database.
  61  */
  62 static DB *skshashdb = NULL;
  63
  64 /**
  65  *      txn - our current transaction id.
  66  */
  67 static DB_TXN *txn = NULL;
  68
  69 DB *keydb(uint64_t keyid)
  70 {
  71         uint64_t keytrun;
  72
  73         keytrun = keyid >> 8;
  74
  75         return(dbconns[keytrun % numdbs]);
  76 }
  77
  78 /**
  79  *      db4_errfunc - Direct DB errors to logfile
  80  *
  81  *      Basic function to take errors from the DB library and output them to
  82  *      the logfile rather than stderr.
  83  */
  84 #if (DB_VERSION_MAJOR == 4) && (DB_VERSION_MINOR < 3)
  85 static void db4_errfunc(const char *errpfx, const char *errmsg)
  86 #else
  87 static void db4_errfunc(const DB_ENV *edbenv, const char *errpfx,
  88                 const char *errmsg)
  89 #endif
  90 {
  91         if (errpfx) {
  92                 logthing(LOGTHING_DEBUG, "db4 error: %s:%s", errpfx, errmsg);
  93         } else {
  94                 logthing(LOGTHING_DEBUG, "db4 error: %s", errmsg);
  95         }
  96
  97         return;
  98 }
  99
 100 /**
 101  *      starttrans - Start a transaction.
 102  *
 103  *      Start a transaction. Intended to be used if we're about to perform many
 104  *      operations on the database to help speed it all up, or if we want
 105  *      something to only succeed if all relevant operations are successful.
 106  */
 107 static bool db4_starttrans(void)
 108 {
 109         int ret;
 110
 111         log_assert(dbenv != NULL);
 112         log_assert(txn == NULL);
 113
 114         ret = dbenv->txn_begin(dbenv,
 115                 NULL, /* No parent transaction */
 116                 &txn,
 117                 0);
 118         if (ret != 0) {
 119                 logthing(LOGTHING_CRITICAL,
 120                                 "Error starting transaction: %s",
 121                                 db_strerror(ret));
 122                 exit(1);
 123         }
 124
 125         return true;
 126 }
 127
 128 /**
 129  *      endtrans - End a transaction.
 130  *
 131  *      Ends a transaction.
 132  */
 133 static void db4_endtrans(void)
 134 {
 135         int ret;
 136
 137         log_assert(dbenv != NULL);
 138         log_assert(txn != NULL);
 139
 140         ret = txn->commit(txn,
 141                 0);
 142         if (ret != 0) {
 143                 logthing(LOGTHING_CRITICAL,
 144                                 "Error ending transaction: %s",
 145                                 db_strerror(ret));
 146                 exit(1);
 147         }
 148         txn = NULL;
 149
 150         return;
 151 }
 152
 153 /**
 154  *      cleanupdb - De-initialize the key database.
 155  *
 156  *      This function should be called upon program exit to allow the DB to
 157  *      cleanup after itself.
 158  */
 159 static void db4_cleanupdb(void)
 160 {
 161         int i = 0;
 162
 163         if (dbenv != NULL) {
 164                 dbenv->txn_checkpoint(dbenv, 0, 0, 0);
 165                 if (skshashdb != NULL) {
 166                         skshashdb->close(skshashdb, 0);
 167                         skshashdb = NULL;
 168                 }
 169                 if (id32db != NULL) {
 170                         id32db->close(id32db, 0);
 171                         id32db = NULL;
 172                 }
 173                 if (worddb != NULL) {
 174                         worddb->close(worddb, 0);
 175                         worddb = NULL;
 176                 }
 177                 for (i = 0; i < numdbs; i++) {
 178                         if (dbconns[i] != NULL) {
 179                                 dbconns[i]->close(dbconns[i], 0);
 180                                 dbconns[i] = NULL;
 181                         }
 182                 }
 183                 free(dbconns);
 184                 dbconns = NULL;
 185                 dbenv->close(dbenv, 0);
 186                 dbenv = NULL;
 187         }
 188 }
 189
 190 /**
 191  *      db4_upgradedb - Upgrade a DB4 database
 192  *
 193  *      Called if we discover we need to upgrade our DB4 database; ie if
 194  *      we're running with a newer version of db4 than the database was
 195  *      created with.
 196  */
 197 static int db4_upgradedb(int numdb)
 198 {
 199         DB *curdb = NULL;
 200         int ret;
 201         int i;
 202         char buf[1024];
 203         int lockfile_fd;
 204         struct stat statbuf;
 205
 206         snprintf(buf, sizeof(buf) - 1, "%s/%s", config.db_dir,
 207                         DB4_UPGRADE_FILE);
 208         lockfile_fd = open(buf, O_RDWR | O_CREAT | O_EXCL, 0600);
 209         if (lockfile_fd < 0) {
 210                 if (errno == EEXIST) {
 211                         while (stat(buf, &statbuf) == 0) ;
 212                         return 0;
 213                 } else {
 214                         logthing(LOGTHING_CRITICAL, "Couldn't open database "
 215                                 "update lock file: %s", strerror(errno));
 216                         return -1;
 217                 }
 218         }
 219         snprintf(buf, sizeof(buf) - 1, "%d", getpid());
 220         write(lockfile_fd, buf, strlen(buf));
 221         close(lockfile_fd);
 222
 223         logthing(LOGTHING_NOTICE, "Upgrading DB4 database");
 224         ret = db_env_create(&dbenv, 0);
 225         dbenv->set_errcall(dbenv, &db4_errfunc);
 226         dbenv->remove(dbenv, config.db_dir, 0);
 227         dbenv = NULL;
 228         for (i = 0; i < numdb; i++) {
 229                 ret = db_create(&curdb, NULL, 0);
 230                 if (ret == 0) {
 231                         snprintf(buf, sizeof(buf) - 1, "%s/keydb.%d.db",
 232                                 config.db_dir, i);
 233                         logthing(LOGTHING_DEBUG, "Upgrading %s", buf);
 234                         ret = curdb->upgrade(curdb, buf, 0);
 235                         curdb->close(curdb, 0);
 236                 } else {
 237                         logthing(LOGTHING_ERROR, "Error upgrading DB %s : %s",
 238                                 buf,
 239                                 db_strerror(ret));
 240                 }
 241         }
 242
 243         ret = db_create(&curdb, NULL, 0);
 244         if (ret == 0) {
 245                 snprintf(buf, sizeof(buf) - 1, "%s/worddb", config.db_dir);
 246                 logthing(LOGTHING_DEBUG, "Upgrading %s", buf);
 247                 ret = curdb->upgrade(curdb, buf, 0);
 248                 curdb->close(curdb, 0);
 249         } else {
 250                 logthing(LOGTHING_ERROR, "Error upgrading DB %s : %s",
 251                         buf,
 252                         db_strerror(ret));
 253         }
 254
 255         ret = db_create(&curdb, NULL, 0);
 256         if (ret == 0) {
 257                 snprintf(buf, sizeof(buf) - 1, "%s/id32db", config.db_dir);
 258                 logthing(LOGTHING_DEBUG, "Upgrading %s", buf);
 259                 ret = curdb->upgrade(curdb, buf, 0);
 260                 curdb->close(curdb, 0);
 261         } else {
 262                 logthing(LOGTHING_ERROR, "Error upgrading DB %s : %s",
 263                         buf,
 264                         db_strerror(ret));
 265         }
 266
 267         ret = db_create(&curdb, NULL, 0);
 268         if (ret == 0) {
 269                 snprintf(buf, sizeof(buf) - 1, "%s/skshashdb", config.db_dir);
 270                 logthing(LOGTHING_DEBUG, "Upgrading %s", buf);
 271                 ret = curdb->upgrade(curdb, buf, 0);
 272                 curdb->close(curdb, 0);
 273         } else {
 274                 logthing(LOGTHING_ERROR, "Error upgrading DB %s : %s",
 275                         buf,
 276                         db_strerror(ret));
 277         }
 278
 279         snprintf(buf, sizeof(buf) - 1, "%s/%s", config.db_dir,
 280                         DB4_UPGRADE_FILE);
 281         unlink(buf);
 282
 283         return ret;
 284 }
 285
 286 /**
 287  *      initdb - Initialize the key database.
 288  *
 289  *      This function should be called before any of the other functions in
 290  *      this file are called in order to allow the DB to be initialized ready
 291  *      for access.
 292  */
 293 static void db4_initdb(bool readonly)
 294 {
 295         char       buf[1024];
 296         FILE      *numdb = NULL;
 297         int        ret = 0;
 298         int        i = 0;
 299         u_int32_t  flags = 0;
 300         struct stat statbuf;
 301         int        maxlocks;
 302
 303         snprintf(buf, sizeof(buf) - 1, "%s/%s", config.db_dir,
 304                         DB4_UPGRADE_FILE);
 305         ret = stat(buf, &statbuf);
 306         while ((ret == 0) || (errno != ENOENT)) {
 307                 if (ret != 0) {
 308                         logthing(LOGTHING_CRITICAL, "Couldn't stat upgrade "
 309                                 "lock file: %s (%d)", strerror(errno), ret);
 310                         exit(1);
 311                 }
 312                 logthing(LOGTHING_DEBUG, "DB4 upgrade in progress; waiting.");
 313                 sleep(5);
 314                 ret = stat(buf, &statbuf);
 315         }
 316         ret = 0;
 317
 318         snprintf(buf, sizeof(buf) - 1, "%s/num_keydb", config.db_dir);
 319         numdb = fopen(buf, "r");
 320         if (numdb != NULL) {
 321                 if (fgets(buf, sizeof(buf), numdb) != NULL) {
 322                         numdbs = atoi(buf);
 323                 }
 324                 fclose(numdb);
 325         } else if (!readonly) {
 326                 logthing(LOGTHING_ERROR, "Couldn't open num_keydb: %s",
 327                                 strerror(errno));
 328                 numdb = fopen(buf, "w");
 329                 if (numdb != NULL) {
 330                         fprintf(numdb, "%d", numdbs);
 331                         fclose(numdb);
 332                 } else {
 333                         logthing(LOGTHING_ERROR,
 334                                 "Couldn't write num_keydb: %s",
 335                                 strerror(errno));
 336                 }
 337         }
 338
 339         dbconns = calloc(numdbs, sizeof (DB *));
 340         if (dbconns == NULL) {
 341                 logthing(LOGTHING_CRITICAL,
 342                                 "Couldn't allocate memory for dbconns");
 343                 ret = 1;
 344         }
 345
 346         if (ret == 0) {
 347                 ret = db_env_create(&dbenv, 0);
 348                 if (ret != 0) {
 349                         logthing(LOGTHING_CRITICAL,
 350                                 "db_env_create: %s", db_strerror(ret));
 351                 }
 352         }
 353
 354         /*
 355          * Up the number of locks we're allowed at once. We base this on
 356          * the maximum number of keys we're going to return.
 357          */
 358         maxlocks = config.maxkeys * 16;
 359         if (maxlocks < 1000) {
 360                 maxlocks = 1000;
 361         }
 362         dbenv->set_lk_max_locks(dbenv, maxlocks);
 363         dbenv->set_lk_max_objects(dbenv, maxlocks);
 364
 365         /*
 366          * Enable deadlock detection so that we don't block indefinitely on
 367          * anything. What we really want is simple 2 state locks, but I'm not
 368          * sure how to make the standard DB functions do that yet.
 369          */
 370         if (ret == 0) {
 371                 dbenv->set_errcall(dbenv, &db4_errfunc);
 372                 ret = dbenv->set_lk_detect(dbenv, DB_LOCK_DEFAULT);
 373                 if (ret != 0) {
 374                         logthing(LOGTHING_CRITICAL,
 375                                 "db_env_create: %s", db_strerror(ret));
 376                 }
 377         }
 378
 379         if (ret == 0) {
 380                 ret = dbenv->open(dbenv, config.db_dir,
 381                                 DB_INIT_LOG | DB_INIT_MPOOL | DB_INIT_LOCK |
 382                                 DB_INIT_TXN |
 383                                 DB_CREATE,
 384                                 0);
 385 #ifdef DB_VERSION_MISMATCH
 386                 if (ret == DB_VERSION_MISMATCH) {
 387                         dbenv->close(dbenv, 0);
 388                         dbenv = NULL;
 389                         ret = db4_upgradedb(numdbs);
 390                         if (ret == 0) {
 391                                 ret = db_env_create(&dbenv, 0);
 392                         }
 393                         if (ret == 0) {
 394                                 dbenv->set_errcall(dbenv, &db4_errfunc);
 395                                 dbenv->set_lk_detect(dbenv, DB_LOCK_DEFAULT);
 396                                 ret = dbenv->open(dbenv, config.db_dir,
 397                                         DB_INIT_LOG | DB_INIT_MPOOL |
 398                                         DB_INIT_LOCK | DB_INIT_TXN |
 399                                         DB_CREATE | DB_RECOVER,
 400                                         0);
 401
 402                                 if (ret == 0) {
 403                                         dbenv->txn_checkpoint(dbenv,
 404                                                         0,
 405                                                         0,
 406                                                         DB_FORCE);
 407                                 }
 408                         }
 409                 }
 410 #endif
 411                 if (ret != 0) {
 412                         logthing(LOGTHING_CRITICAL,
 413                                         "Error opening db environment: %s (%s)",
 414                                         config.db_dir,
 415                                         db_strerror(ret));
 416                         dbenv->close(dbenv, 0);
 417                         dbenv = NULL;
 418                 }
 419         }
 420
 421         if (ret == 0) {
 422                 db4_starttrans();
 423
 424                 for (i = 0; !ret && i < numdbs; i++) {
 425                         ret = db_create(&dbconns[i], dbenv, 0);
 426                         if (ret != 0) {
 427                                 logthing(LOGTHING_CRITICAL,
 428                                         "db_create: %s", db_strerror(ret));
 429                         }
 430
 431                         if (ret == 0) {
 432                                 snprintf(buf, 1023, "keydb.%d.db", i);
 433                                 flags = DB_CREATE;
 434                                 if (readonly) {
 435                                         flags = DB_RDONLY;
 436                                 }
 437                                 ret = dbconns[i]->open(dbconns[i],
 438                                                 txn,
 439                                                 buf,
 440                                                 "keydb",
 441                                                 DB_HASH,
 442                                                 flags,
 443                                                 0664);
 444                                 if (ret != 0) {
 445                                         logthing(LOGTHING_CRITICAL,
 446                                                 "Error opening key database:"
 447                                                 " %s (%s)",
 448                                                 buf,
 449                                                 db_strerror(ret));
 450                                 }
 451                         }
 452                 }
 453
 454         }
 455
 456         if (ret == 0) {
 457                 ret = db_create(&worddb, dbenv, 0);
 458                 if (ret != 0) {
 459                         logthing(LOGTHING_CRITICAL, "db_create: %s",
 460                                         db_strerror(ret));
 461                 }
 462         }
 463
 464         if (ret == 0) {
 465                 ret = worddb->set_flags(worddb, DB_DUP);
 466         }
 467
 468         if (ret == 0) {
 469                 ret = worddb->open(worddb, txn, "worddb", "worddb", DB_BTREE,
 470                                 flags,
 471                                 0664);
 472                 if (ret != 0) {
 473                         logthing(LOGTHING_CRITICAL,
 474                                         "Error opening word database: %s (%s)",
 475                                         "worddb",
 476                                         db_strerror(ret));
 477                 }
 478         }
 479
 480         if (ret == 0) {
 481                 ret = db_create(&id32db, dbenv, 0);
 482                 if (ret != 0) {
 483                         logthing(LOGTHING_CRITICAL, "db_create: %s",
 484                                         db_strerror(ret));
 485                 }
 486         }
 487
 488         if (ret == 0) {
 489                 ret = id32db->set_flags(id32db, DB_DUP);
 490         }
 491
 492         if (ret == 0) {
 493                 ret = id32db->open(id32db, txn, "id32db", "id32db", DB_HASH,
 494                                 flags,
 495                                 0664);
 496                 if (ret != 0) {
 497                         logthing(LOGTHING_CRITICAL,
 498                                         "Error opening id32 database: %s (%s)",
 499                                         "id32db",
 500                                         db_strerror(ret));
 501                 }
 502         }
 503
 504         if (ret == 0) {
 505                 ret = db_create(&skshashdb, dbenv, 0);
 506                 if (ret != 0) {
 507                         logthing(LOGTHING_CRITICAL, "db_create: %s",
 508                                         db_strerror(ret));
 509                 }
 510         }
 511
 512         if (ret == 0) {
 513                 ret = skshashdb->open(skshashdb, txn, "skshashdb",
 514                                 "skshashdb", DB_HASH,
 515                                 flags,
 516                                 0664);
 517                 if (ret != 0) {
 518                         logthing(LOGTHING_CRITICAL,
 519                                 "Error opening skshash database: %s (%s)",
 520                                 "skshashdb",
 521                                 db_strerror(ret));
 522                 }
 523         }
 524
 525         if (txn != NULL) {
 526                 db4_endtrans();
 527         }
 528
 529         if (ret != 0) {
 530                 db4_cleanupdb();
 531                 logthing(LOGTHING_CRITICAL,
 532                                 "Error opening database; exiting");
 533                 exit(EXIT_FAILURE);
 534         }
 535
 536         return;
 537 }
 538
 539 /**
 540  *      getfullkeyid - Maps a 32bit key id to a 64bit one.
 541  *      @keyid: The 32bit keyid.
 542  *
 543  *      This function maps a 32bit key id to the full 64bit one. It returns the
 544  *      full keyid. If the key isn't found a keyid of 0 is returned.
 545  */
 546 static uint64_t db4_getfullkeyid(uint64_t keyid)
 547 {
 548         DBT       key, data;
 549         DBC      *cursor = NULL;
 550         uint32_t  shortkeyid = 0;
 551         int       ret = 0;
 552
 553         if (keyid < 0x100000000LL) {
 554                 ret = id32db->cursor(id32db,
 555                                 txn,
 556                                 &cursor,
 557                                 0);   /* flags */
 558
 559                 shortkeyid = keyid & 0xFFFFFFFF;
 560
 561                 memset(&key, 0, sizeof(key));
 562                 memset(&data, 0, sizeof(data));
 563                 key.data = &shortkeyid;
 564                 key.size = sizeof(shortkeyid);
 565                 data.flags = DB_DBT_MALLOC;
 566
 567                 ret = cursor->c_get(cursor,
 568                         &key,
 569                         &data,
 570                         DB_SET);
 571
 572                 if (ret == 0) {
 573                         keyid = *(uint64_t *) data.data;
 574
 575                         if (data.data != NULL) {
 576                                 free(data.data);
 577                                 data.data = NULL;
 578                         }
 579                 }
 580
 581                 ret = cursor->c_close(cursor);
 582                 cursor = NULL;
 583         }
 584
 585         return keyid;
 586 }
 587
 588 /**
 589  *      fetch_key - Given a keyid fetch the key from storage.
 590  *      @keyid: The keyid to fetch.
 591  *      @publickey: A pointer to a structure to return the key in.
 592  *      @intrans: If we're already in a transaction.
 593  *
 594  *      We use the hex representation of the keyid as the filename to fetch the
 595  *      key from. The key is stored in the file as a binary OpenPGP stream of
 596  *      packets, so we can just use read_openpgp_stream() to read the packets
 597  *      in and then parse_keys() to parse the packets into a publickey
 598  *      structure.
 599  */
 600 static int db4_fetch_key(uint64_t keyid, struct openpgp_publickey **publickey,
 601                 bool intrans)
 602 {
 603         struct openpgp_packet_list *packets = NULL;
 604         DBT key, data;
 605         int ret = 0;
 606         int numkeys = 0;
 607         struct buffer_ctx fetchbuf;
 608
 609         if (keyid < 0x100000000LL) {
 610                 keyid = db4_getfullkeyid(keyid);
 611         }
 612
 613         memset(&key, 0, sizeof(key));
 614         memset(&data, 0, sizeof(data));
 615
 616         data.size = 0;
 617         data.data = NULL;
 618
 619         key.size = sizeof(keyid);
 620         key.data = &keyid;
 621
 622         if (!intrans) {
 623                 db4_starttrans();
 624         }
 625
 626         ret = keydb(keyid)->get(keydb(keyid),
 627                         txn,
 628                         &key,
 629                         &data,
 630                         0); /* flags*/
 631
 632         if (ret == 0) {
 633                 fetchbuf.buffer = data.data;
 634                 fetchbuf.offset = 0;
 635                 fetchbuf.size = data.size;
 636                 read_openpgp_stream(buffer_fetchchar, &fetchbuf,
 637                                 &packets, 0);
 638                 parse_keys(packets, publickey);
 639                 free_packet_list(packets);
 640                 packets = NULL;
 641                 numkeys++;
 642         } else if (ret != DB_NOTFOUND) {
 643                 logthing(LOGTHING_ERROR,
 644                                 "Problem retrieving key: %s",
 645                                 db_strerror(ret));
 646         }
 647
 648         if (!intrans) {
 649                 db4_endtrans();
 650         }
 651
 652         return (numkeys);
 653 }
 654
 655 int worddb_cmp(const void *d1, const void *d2)
 656 {
 657         return memcmp(d1, d2, 12);
 658 }
 659
 660 /**
 661  *      fetch_key_text - Trys to find the keys that contain the supplied text.
 662  *      @search: The text to search for.
 663  *      @publickey: A pointer to a structure to return the key in.
 664  *
 665  *      This function searches for the supplied text and returns the keys that
 666  *      contain it.
 667  */
 668 static int db4_fetch_key_text(const char *search,
 669                 struct openpgp_publickey **publickey)
 670 {
 671         DBC *cursor = NULL;
 672         DBT key, data;
 673         int ret;
 674         uint64_t keyid;
 675         int i;
 676         int numkeys;
 677         char *searchtext = NULL;
 678         struct ll *wordlist = NULL;
 679         struct ll *curword = NULL;
 680         struct keyarray keylist = { NULL, 0, 0 };
 681         struct keyarray newkeylist = { NULL, 0, 0 };
 682         int firstpass = 1;
 683
 684         numkeys = 0;
 685         searchtext = strdup(search);
 686         wordlist = makewordlist(wordlist, searchtext);
 687
 688         for (curword = wordlist; curword != NULL; curword = curword->next) {
 689                 db4_starttrans();
 690
 691                 ret = worddb->cursor(worddb,
 692                                 txn,
 693                                 &cursor,
 694                                 0);   /* flags */
 695
 696                 memset(&key, 0, sizeof(key));
 697                 memset(&data, 0, sizeof(data));
 698                 key.data = curword->object;
 699                 key.size = strlen(curword->object);
 700                 data.flags = DB_DBT_MALLOC;
 701                 ret = cursor->c_get(cursor,
 702                                 &key,
 703                                 &data,
 704                                 DB_SET);
 705                 while (ret == 0 && strncmp(key.data, curword->object,
 706                                         key.size) == 0 &&
 707                                 ((char *) curword->object)[key.size] == 0) {
 708                         keyid = 0;
 709                         for (i = 4; i < 12; i++) {
 710                                 keyid <<= 8;
 711                                 keyid += ((unsigned char *)
 712                                                 data.data)[i];
 713                         }
 714
 715                         /*
 716                          * Only add the keys containing this word if this is
 717                          * our first pass (ie we have no existing key list),
 718                          * or the key contained a previous word.
 719                          */
 720                         if (firstpass || array_find(&keylist, keyid)) {
 721                                 array_add(&newkeylist, keyid);
 722                         }
 723
 724                         free(data.data);
 725                         data.data = NULL;
 726
 727                         ret = cursor->c_get(cursor,
 728                                         &key,
 729                                         &data,
 730                                         DB_NEXT);
 731                 }
 732                 array_free(&keylist);
 733                 keylist = newkeylist;
 734                 newkeylist.keys = NULL;
 735                 newkeylist.count = newkeylist.size = 0;
 736                 if (data.data != NULL) {
 737                         free(data.data);
 738                         data.data = NULL;
 739                 }
 740                 ret = cursor->c_close(cursor);
 741                 cursor = NULL;
 742                 firstpass = 0;
 743                 db4_endtrans();
 744         }
 745         llfree(wordlist, NULL);
 746         wordlist = NULL;
 747
 748         if (keylist.count > config.maxkeys) {
 749                 keylist.count = config.maxkeys;
 750         }
 751
 752         db4_starttrans();
 753         for (i = 0; i < keylist.count; i++) {
 754                 numkeys += db4_fetch_key(keylist.keys[i],
 755                         publickey,
 756                         true);
 757         }
 758         array_free(&keylist);
 759         free(searchtext);
 760         searchtext = NULL;
 761
 762         db4_endtrans();
 763
 764         return (numkeys);
 765 }
 766
 767 static int db4_fetch_key_skshash(const struct skshash *hash,
 768                 struct openpgp_publickey **publickey)
 769 {
 770         DBT       key, data;
 771         DBC      *cursor = NULL;
 772         uint64_t  keyid = 0;
 773         int       ret = 0;
 774
 775         ret = skshashdb->cursor(skshashdb,
 776                         txn,
 777                         &cursor,
 778                         0);   /* flags */
 779
 780         memset(&key, 0, sizeof(key));
 781         memset(&data, 0, sizeof(data));
 782         key.data = (void *) hash->hash;
 783         key.size = sizeof(hash->hash);
 784         data.flags = DB_DBT_MALLOC;
 785
 786         ret = cursor->c_get(cursor,
 787                 &key,
 788                 &data,
 789                 DB_SET);
 790
 791         if (ret == 0) {
 792                 keyid = *(uint64_t *) data.data;
 793
 794                 if (data.data != NULL) {
 795                         free(data.data);
 796                         data.data = NULL;
 797                 }
 798         }
 799
 800         ret = cursor->c_close(cursor);
 801         cursor = NULL;
 802
 803         return db4_fetch_key(keyid, publickey, false);
 804 }
 805
 806 /**
 807  *      delete_key - Given a keyid delete the key from storage.
 808  *      @keyid: The keyid to delete.
 809  *      @intrans: If we're already in a transaction.
 810  *
 811  *      This function deletes a public key from whatever storage mechanism we
 812  *      are using. Returns 0 if the key existed.
 813  */
 814 static int db4_delete_key(uint64_t keyid, bool intrans)
 815 {
 816         struct openpgp_publickey *publickey = NULL;
 817         DBT key, data;
 818         DBC *cursor = NULL;
 819         uint32_t   shortkeyid = 0;
 820         uint64_t  *subkeyids = NULL;
 821         int ret = 0;
 822         int i;
 823         char **uids = NULL;
 824         char *primary = NULL;
 825         unsigned char worddb_data[12];
 826         struct ll *wordlist = NULL;
 827         struct ll *curword  = NULL;
 828         bool deadlock = false;
 829         struct skshash hash;
 830
 831         if (!intrans) {
 832                 db4_starttrans();
 833         }
 834
 835         db4_fetch_key(keyid, &publickey, true);
 836
 837         /*
 838          * Walk through the uids removing the words from the worddb.
 839          */
 840         if (publickey != NULL) {
 841                 uids = keyuids(publickey, &primary);
 842         }
 843         if (uids != NULL) {
 844                 for (i = 0; ret == 0 && uids[i] != NULL; i++) {
 845                         wordlist = makewordlist(wordlist, uids[i]);
 846                 }
 847
 848                 ret = worddb->cursor(worddb,
 849                         txn,
 850                         &cursor,
 851                         0);   /* flags */
 852
 853                 for (curword = wordlist; curword != NULL && !deadlock;
 854                                 curword = curword->next) {
 855                         memset(&key, 0, sizeof(key));
 856                         memset(&data, 0, sizeof(data));
 857                         key.data = curword->object;
 858                         key.size = strlen(key.data);
 859                         data.data = worddb_data;
 860                         data.size = sizeof(worddb_data);
 861
 862                         /*
 863                          * Our data is the key creation time followed by the
 864                          * key id.
 865                          */
 866                         worddb_data[ 0] = publickey->publickey->data[1];
 867                         worddb_data[ 1] = publickey->publickey->data[2];
 868                         worddb_data[ 2] = publickey->publickey->data[3];
 869                         worddb_data[ 3] = publickey->publickey->data[4];
 870                         worddb_data[ 4] = (keyid >> 56) & 0xFF;
 871                         worddb_data[ 5] = (keyid >> 48) & 0xFF;
 872                         worddb_data[ 6] = (keyid >> 40) & 0xFF;
 873                         worddb_data[ 7] = (keyid >> 32) & 0xFF;
 874                         worddb_data[ 8] = (keyid >> 24) & 0xFF;
 875                         worddb_data[ 9] = (keyid >> 16) & 0xFF;
 876                         worddb_data[10] = (keyid >>  8) & 0xFF;
 877                         worddb_data[11] = keyid & 0xFF;
 878
 879                         ret = cursor->c_get(cursor,
 880                                 &key,
 881                                 &data,
 882                                 DB_GET_BOTH);
 883
 884                         if (ret == 0) {
 885                                 ret = cursor->c_del(cursor, 0);
 886                                 if (ret != 0) {
 887                                         logthing(LOGTHING_ERROR,
 888                                                 "Problem deleting word: %s",
 889                                                 db_strerror(ret));
 890                                 }
 891                         }
 892
 893                         if (ret != 0) {
 894                                 logthing(LOGTHING_ERROR,
 895                                         "Problem deleting word: %s",
 896                                         db_strerror(ret));
 897                                 if (ret == DB_LOCK_DEADLOCK) {
 898                                         deadlock = true;
 899                                 }
 900                         }
 901                 }
 902                 ret = cursor->c_close(cursor);
 903                 cursor = NULL;
 904
 905                 /*
 906                  * Free our UID and word lists.
 907                  */
 908                 llfree(wordlist, NULL);
 909                 for (i = 0; uids[i] != NULL; i++) {
 910                         free(uids[i]);
 911                         uids[i] = NULL;
 912                 }
 913                 free(uids);
 914                 uids = NULL;
 915                 free_publickey(publickey);
 916                 publickey = NULL;
 917         }
 918
 919         if (!deadlock) {
 920                 ret = id32db->cursor(id32db,
 921                         txn,
 922                         &cursor,
 923                         0);   /* flags */
 924
 925                 shortkeyid = keyid & 0xFFFFFFFF;
 926
 927                 memset(&key, 0, sizeof(key));
 928                 memset(&data, 0, sizeof(data));
 929                 key.data = &shortkeyid;
 930                 key.size = sizeof(shortkeyid);
 931                 data.data = &keyid;
 932                 data.size = sizeof(keyid);
 933
 934                 ret = cursor->c_get(cursor,
 935                         &key,
 936                         &data,
 937                         DB_GET_BOTH);
 938
 939                 if (ret == 0) {
 940                         ret = cursor->c_del(cursor, 0);
 941                         if (ret != 0) {
 942                                 logthing(LOGTHING_ERROR,
 943                                         "Problem deleting short keyid: %s",
 944                                         db_strerror(ret));
 945                         }
 946                 }
 947
 948                 if (ret != 0) {
 949                         logthing(LOGTHING_ERROR,
 950                                 "Problem deleting short keyid: %s",
 951                                 db_strerror(ret));
 952                         if (ret == DB_LOCK_DEADLOCK) {
 953                                 deadlock = true;
 954                         }
 955                 }
 956
 957                 subkeyids = keysubkeys(publickey);
 958                 i = 0;
 959                 while (subkeyids != NULL && subkeyids[i] != 0) {
 960                         shortkeyid = subkeyids[i++] & 0xFFFFFFFF;
 961
 962                         memset(&key, 0, sizeof(key));
 963                         memset(&data, 0, sizeof(data));
 964                         key.data = &shortkeyid;
 965                         key.size = sizeof(shortkeyid);
 966                         data.data = &keyid;
 967                         data.size = sizeof(keyid);
 968
 969                         ret = cursor->c_get(cursor,
 970                                 &key,
 971                                 &data,
 972                                 DB_GET_BOTH);
 973
 974                         if (ret == 0) {
 975                                 ret = cursor->c_del(cursor, 0);
 976                                 if (ret != 0) {
 977                                         logthing(LOGTHING_ERROR,
 978                                                 "Problem deleting short"
 979                                                 " keyid: %s",
 980                                                 db_strerror(ret));
 981                                 }
 982                         }
 983
 984                         if (ret != 0) {
 985                                 logthing(LOGTHING_ERROR,
 986                                         "Problem deleting short keyid: %s",
 987                                         db_strerror(ret));
 988                                 if (ret == DB_LOCK_DEADLOCK) {
 989                                         deadlock = true;
 990                                 }
 991                         }
 992                 }
 993                 if (subkeyids != NULL) {
 994                         free(subkeyids);
 995                         subkeyids = NULL;
 996                 }
 997                 ret = cursor->c_close(cursor);
 998                 cursor = NULL;
 999
1000         }
1001
1002         if (!deadlock) {
1003                 get_skshash(publickey, &hash);
1004
1005                 memset(&key, 0, sizeof(key));
1006                 memset(&data, 0, sizeof(data));
1007                 key.data = hash.hash;
1008                 key.size = sizeof(hash.hash);
1009                 data.data = &keyid;
1010                 data.size = sizeof(keyid);
1011
1012                 ret = cursor->c_get(cursor,
1013                         &key,
1014                         &data,
1015                         DB_GET_BOTH);
1016
1017                 if (ret == 0) {
1018                         ret = cursor->c_del(cursor, 0);
1019                 }
1020
1021                 if (ret != 0) {
1022                         logthing(LOGTHING_ERROR,
1023                                 "Problem deleting skshash: %s",
1024                                 db_strerror(ret));
1025                         if (ret == DB_LOCK_DEADLOCK) {
1026                                 deadlock = true;
1027                         }
1028                 }
1029
1030                 ret = cursor->c_close(cursor);
1031                 cursor = NULL;
1032         }
1033
1034         if (!deadlock) {
1035                 key.data = &keyid;
1036                 key.size = sizeof(keyid);
1037
1038                 keydb(keyid)->del(keydb(keyid),
1039                                 txn,
1040                                 &key,
1041                                 0); /* flags */
1042         }
1043
1044         if (!intrans) {
1045                 db4_endtrans();
1046         }
1047
1048         return deadlock ? (-1) : (ret == DB_NOTFOUND);
1049 }
1050
1051 /**
1052  *      store_key - Takes a key and stores it.
1053  *      @publickey: A pointer to the public key to store.
1054  *      @intrans: If we're already in a transaction.
1055  *      @update: If true the key exists and should be updated.
1056  *
1057  *      Again we just use the hex representation of the keyid as the filename
1058  *      to store the key to. We flatten the public key to a list of OpenPGP
1059  *      packets and then use write_openpgp_stream() to write the stream out to
1060  *      the file. If update is true then we delete the old key first, otherwise
1061  *      we trust that it doesn't exist.
1062  */
1063 static int db4_store_key(struct openpgp_publickey *publickey, bool intrans,
1064                 bool update)
1065 {
1066         struct     openpgp_packet_list *packets = NULL;
1067         struct     openpgp_packet_list *list_end = NULL;
1068         struct     openpgp_publickey *next = NULL;
1069         int        ret = 0;
1070         int        i = 0;
1071         struct     buffer_ctx storebuf;
1072         DBT        key;
1073         DBT        data;
1074         uint64_t   keyid = 0;
1075         uint32_t   shortkeyid = 0;
1076         uint64_t  *subkeyids = NULL;
1077         char     **uids = NULL;
1078         char      *primary = NULL;
1079         unsigned char worddb_data[12];
1080         struct ll *wordlist = NULL;
1081         struct ll *curword  = NULL;
1082         bool       deadlock = false;
1083         struct skshash hash;
1084
1085         keyid = get_keyid(publickey);
1086
1087         if (!intrans) {
1088                 db4_starttrans();
1089         }
1090
1091         /*
1092          * Delete the key if we already have it.
1093          *
1094          * TODO: Can we optimize this perhaps? Possibly when other data is
1095          * involved as well? I suspect this is easiest and doesn't make a lot
1096          * of difference though - the largest chunk of data is the keydata and
1097          * it definitely needs updated.
1098          */
1099         if (update) {
1100                 deadlock = (db4_delete_key(keyid, true) == -1);
1101         }
1102
1103         /*
1104          * Convert the key to a flat set of binary data.
1105          */
1106         if (!deadlock) {
1107                 next = publickey->next;
1108                 publickey->next = NULL;
1109                 flatten_publickey(publickey, &packets, &list_end);
1110                 publickey->next = next;
1111
1112                 storebuf.offset = 0;
1113                 storebuf.size = 8192;
1114                 storebuf.buffer = malloc(8192);
1115
1116                 write_openpgp_stream(buffer_putchar, &storebuf, packets);
1117
1118                 /*
1119                  * Now we have the key data store it in the DB; the keyid is
1120                  * the key.
1121                  */
1122                 memset(&key, 0, sizeof(key));
1123                 memset(&data, 0, sizeof(data));
1124                 key.data = &keyid;
1125                 key.size = sizeof(keyid);
1126                 data.size = storebuf.offset;
1127                 data.data = storebuf.buffer;
1128
1129                 ret = keydb(keyid)->put(keydb(keyid),
1130                                 txn,
1131                                 &key,
1132                                 &data,
1133                                 0); /* flags*/
1134                 if (ret != 0) {
1135                         logthing(LOGTHING_ERROR,
1136                                         "Problem storing key: %s",
1137                                         db_strerror(ret));
1138                         if (ret == DB_LOCK_DEADLOCK) {
1139                                 deadlock = true;
1140                         }
1141                 }
1142
1143                 free(storebuf.buffer);
1144                 storebuf.buffer = NULL;
1145                 storebuf.size = 0;
1146                 storebuf.offset = 0;
1147
1148                 free_packet_list(packets);
1149                 packets = NULL;
1150         }
1151
1152         /*
1153          * Walk through our uids storing the words into the db with the keyid.
1154          */
1155         if (!deadlock) {
1156                 uids = keyuids(publickey, &primary);
1157         }
1158         if (uids != NULL) {
1159                 for (i = 0; ret == 0 && uids[i] != NULL; i++) {
1160                         wordlist = makewordlist(wordlist, uids[i]);
1161                 }
1162
1163                 for (curword = wordlist; curword != NULL && !deadlock;
1164                                 curword = curword->next) {
1165                         memset(&key, 0, sizeof(key));
1166                         memset(&data, 0, sizeof(data));
1167                         key.data = curword->object;
1168                         key.size = strlen(key.data);
1169                         data.data = worddb_data;
1170                         data.size = sizeof(worddb_data);
1171
1172                         /*
1173                          * Our data is the key creation time followed by the
1174                          * key id.
1175                          */
1176                         worddb_data[ 0] = publickey->publickey->data[1];
1177                         worddb_data[ 1] = publickey->publickey->data[2];
1178                         worddb_data[ 2] = publickey->publickey->data[3];
1179                         worddb_data[ 3] = publickey->publickey->data[4];
1180                         worddb_data[ 4] = (keyid >> 56) & 0xFF;
1181                         worddb_data[ 5] = (keyid >> 48) & 0xFF;
1182                         worddb_data[ 6] = (keyid >> 40) & 0xFF;
1183                         worddb_data[ 7] = (keyid >> 32) & 0xFF;
1184                         worddb_data[ 8] = (keyid >> 24) & 0xFF;
1185                         worddb_data[ 9] = (keyid >> 16) & 0xFF;
1186                         worddb_data[10] = (keyid >>  8) & 0xFF;
1187                         worddb_data[11] = keyid & 0xFF;
1188                         ret = worddb->put(worddb,
1189                                 txn,
1190                                 &key,
1191                                 &data,
1192                                 0);
1193                         if (ret != 0) {
1194                                 logthing(LOGTHING_ERROR,
1195                                         "Problem storing word: %s",
1196                                         db_strerror(ret));
1197                                 if (ret == DB_LOCK_DEADLOCK) {
1198                                         deadlock = true;
1199                                 }
1200                         }
1201                 }
1202
1203                 /*
1204                  * Free our UID and word lists.
1205                  */
1206                 llfree(wordlist, NULL);
1207                 for (i = 0; uids[i] != NULL; i++) {
1208                         free(uids[i]);
1209                         uids[i] = NULL;
1210                 }
1211                 free(uids);
1212                 uids = NULL;
1213         }
1214
1215         /*
1216          * Write the truncated 32 bit keyid so we can lookup the full id for
1217          * queries.
1218          */
1219         if (!deadlock) {
1220                 shortkeyid = keyid & 0xFFFFFFFF;
1221
1222                 memset(&key, 0, sizeof(key));
1223                 memset(&data, 0, sizeof(data));
1224                 key.data = &shortkeyid;
1225                 key.size = sizeof(shortkeyid);
1226                 data.data = &keyid;
1227                 data.size = sizeof(keyid);
1228
1229                 ret = id32db->put(id32db,
1230                         txn,
1231                         &key,
1232                         &data,
1233                         0);
1234                 if (ret != 0) {
1235                         logthing(LOGTHING_ERROR,
1236                                 "Problem storing short keyid: %s",
1237                                 db_strerror(ret));
1238                         if (ret == DB_LOCK_DEADLOCK) {
1239                                 deadlock = true;
1240                         }
1241                 }
1242         }
1243
1244         if (!deadlock) {
1245                 subkeyids = keysubkeys(publickey);
1246                 i = 0;
1247                 while (subkeyids != NULL && subkeyids[i] != 0) {
1248                         shortkeyid = subkeyids[i++] & 0xFFFFFFFF;
1249
1250                         memset(&key, 0, sizeof(key));
1251                         memset(&data, 0, sizeof(data));
1252                         key.data = &shortkeyid;
1253                         key.size = sizeof(shortkeyid);
1254                         data.data = &keyid;
1255                         data.size = sizeof(keyid);
1256
1257                         ret = id32db->put(id32db,
1258                                 txn,
1259                                 &key,
1260                                 &data,
1261                                 0);
1262                         if (ret != 0) {
1263                                 logthing(LOGTHING_ERROR,
1264                                         "Problem storing short keyid: %s",
1265                                         db_strerror(ret));
1266                                 if (ret == DB_LOCK_DEADLOCK) {
1267                                         deadlock = true;
1268                                 }
1269                         }
1270                 }
1271                 if (subkeyids != NULL) {
1272                         free(subkeyids);
1273                         subkeyids = NULL;
1274                 }
1275         }
1276
1277         if (!deadlock) {
1278                 get_skshash(publickey, &hash);
1279                 memset(&key, 0, sizeof(key));
1280                 memset(&data, 0, sizeof(data));
1281                 key.data = hash.hash;
1282                 key.size = sizeof(hash.hash);
1283                 data.data = &keyid;
1284                 data.size = sizeof(keyid);
1285
1286                 ret = skshashdb->put(skshashdb,
1287                         txn,
1288                         &key,
1289                         &data,
1290                         0);
1291                 if (ret != 0) {
1292                         logthing(LOGTHING_ERROR,
1293                                 "Problem storing SKS hash: %s",
1294                                 db_strerror(ret));
1295                         if (ret == DB_LOCK_DEADLOCK) {
1296                                 deadlock = true;
1297                         }
1298                 }
1299         }
1300
1301         if (!intrans) {
1302                 db4_endtrans();
1303         }
1304
1305         return deadlock ? -1 : 0 ;
1306 }
1307
1308 /**
1309  *      iterate_keys - call a function once for each key in the db.
1310  *      @iterfunc: The function to call.
1311  *      @ctx: A context pointer
1312  *
1313  *      Calls iterfunc once for each key in the database. ctx is passed
1314  *      unaltered to iterfunc. This function is intended to aid database dumps
1315  *      and statistic calculations.
1316  *
1317  *      Returns the number of keys we iterated over.
1318  */
1319 static int db4_iterate_keys(void (*iterfunc)(void *ctx,
1320                 struct openpgp_publickey *key), void *ctx)
1321 {
1322         DBT                         dbkey, data;
1323         DBC                        *cursor = NULL;
1324         int                         ret = 0;
1325         int                         i = 0;
1326         int                         numkeys = 0;
1327         struct buffer_ctx           fetchbuf;
1328         struct openpgp_packet_list *packets = NULL;
1329         struct openpgp_publickey   *key = NULL;
1330
1331         for (i = 0; i < numdbs; i++) {
1332                 ret = dbconns[i]->cursor(dbconns[i],
1333                         NULL,
1334                         &cursor,
1335                         0);   /* flags */
1336
1337                 memset(&dbkey, 0, sizeof(dbkey));
1338                 memset(&data, 0, sizeof(data));
1339                 ret = cursor->c_get(cursor, &dbkey, &data, DB_NEXT);
1340                 while (ret == 0) {
1341                         fetchbuf.buffer = data.data;
1342                         fetchbuf.offset = 0;
1343                         fetchbuf.size = data.size;
1344                         read_openpgp_stream(buffer_fetchchar, &fetchbuf,
1345                                 &packets, 0);
1346                         parse_keys(packets, &key);
1347
1348                         iterfunc(ctx, key);
1349
1350                         free_publickey(key);
1351                         key = NULL;
1352                         free_packet_list(packets);
1353                         packets = NULL;
1354
1355                         memset(&dbkey, 0, sizeof(dbkey));
1356                         memset(&data, 0, sizeof(data));
1357                         ret = cursor->c_get(cursor, &dbkey, &data,
1358                                         DB_NEXT);
1359                         numkeys++;
1360                 }
1361                 if (ret != DB_NOTFOUND) {
1362                         logthing(LOGTHING_ERROR,
1363                                 "Problem reading key: %s",
1364                                 db_strerror(ret));
1365                 }
1366
1367                 ret = cursor->c_close(cursor);
1368                 cursor = NULL;
1369         }
1370
1371         return numkeys;
1372 }
1373
1374 /*
1375  * Include the basic keydb routines.
1376  */
1377 #define NEED_GETKEYSIGS 1
1378 #define NEED_KEYID2UID 1
1379 #define NEED_UPDATEKEYS 1
1380 #include "keydb.c"
1381
1382 struct dbfuncs keydb_db4_funcs = {
1383         .initdb                 = db4_initdb,
1384         .cleanupdb              = db4_cleanupdb,
1385         .starttrans             = db4_starttrans,
1386         .endtrans               = db4_endtrans,
1387         .fetch_key              = db4_fetch_key,
1388         .fetch_key_text         = db4_fetch_key_text,
1389         .fetch_key_skshash      = db4_fetch_key_skshash,
1390         .store_key              = db4_store_key,
1391         .update_keys            = generic_update_keys,
1392         .delete_key             = db4_delete_key,
1393         .getkeysigs             = generic_getkeysigs,
1394         .cached_getkeysigs      = generic_cached_getkeysigs,
1395         .keyid2uid              = generic_keyid2uid,
1396         .getfullkeyid           = db4_getfullkeyid,
1397         .iterate_keys           = db4_iterate_keys,
1398 };