X-Git-Url: https://git.sommitrealweird.co.uk/onak.git/blobdiff_plain/b1cb08b256587f6c5d2c484438d942e58583ba88..c80e4acc7e15d50bb9398cfb77d4efd7ea2b6710:/keydb_db3.c?ds=inline diff --git a/keydb_db3.c b/keydb_db3.c index 6177ac7..7cc3b1e 100644 --- a/keydb_db3.c +++ b/keydb_db3.c @@ -6,6 +6,7 @@ * Copyright 2002 Project Purple */ +#include #include #include #include @@ -24,13 +25,25 @@ #include "decodekey.h" #include "keystructs.h" #include "mem.h" +#include "log.h" #include "onak-conf.h" #include "parsekey.h" +#include "wordlist.h" /** - * dbconn - our connection to the key database. + * dbenv - our database environment. */ -static DB *dbconn = NULL; +static DB_ENV *dbenv = NULL; + +/** + * numdb - The number of database files we have. + */ +static int numdbs = 16; + +/** + * dbconn - our connections to the key database files. + */ +static DB **dbconns = NULL; /** * worddb - our connection to the word database. @@ -38,53 +51,22 @@ static DB *dbconn = NULL; static DB *worddb = NULL; /** - * makewordlist - Takes a string and splits it into a set of unique words. - * @wordlist: The current word list. - * @words: The string to split and add. - * - * We take words and split it on non alpha numeric characters. These get - * added to the word list if they're not already present. If the wordlist - * is NULL then we start a new list, otherwise it's search for already - * added words. Note that words is modified in the process of scanning. - * - * Returns the new word list. + * id32db - our connection to the 32bit ID database. */ -struct ll *makewordlist(struct ll *wordlist, char *word) +static DB *id32db = NULL; + +/** + * txn - our current transaction id. + */ +static DB_TXN *txn = NULL; + +DB *keydb(uint64_t keyid) { - char *start = NULL; - char *end = NULL; + uint64_t keytrun; - /* - * Walk through the words string, spliting on non alphanumerics and - * then checking if the word already exists in the list. If not then - * we add it. - */ - end = word; - while (end != NULL && *end != 0) { - start = end; - while (*start != 0 && !isalnum(*start)) { - start++; - } - end = start; - while (*end != 0 && isalnum(*end)) { - *end = tolower(*end); - end++; - } - if (end - start > 1) { - if (*end != 0) { - *end = 0; - end++; - } - - if (llfind(wordlist, start, - strcmp) == NULL) { - wordlist = lladd(wordlist, - start); - } - } - } + keytrun = keyid >> 8; - return wordlist; + return(dbconns[keytrun % numdbs]); } /** @@ -94,43 +76,134 @@ struct ll *makewordlist(struct ll *wordlist, char *word) * this file are called in order to allow the DB to be initialized ready * for access. */ -void initdb(void) +void initdb(bool readonly) { - char buf[1024]; - int ret = 0; + char buf[1024]; + FILE *numdb = NULL; + int ret = 0; + int i = 0; + u_int32_t flags = 0; - strcpy(buf, config.db_dir); - strcat(buf, "/keydb.db"); - - ret = db_create(&dbconn, NULL, 0); + snprintf(buf, sizeof(buf) - 1, "%s/num_keydb", config.db_dir); + numdb = fopen(buf, "r"); + if (numdb != NULL) { + if (fgets(buf, sizeof(buf), numdb) != NULL) { + numdbs = atoi(buf); + } + fclose(numdb); + } else if (!readonly) { + logthing(LOGTHING_ERROR, "Couldn't open num_keydb: %s", + strerror(errno)); + numdb = fopen(buf, "w"); + if (numdb != NULL) { + fprintf(numdb, "%d", numdbs); + fclose(numdb); + } else { + logthing(LOGTHING_ERROR, + "Couldn't write num_keydb: %s", + strerror(errno)); + } + } + + dbconns = malloc(sizeof (DB *) * numdbs); + if (dbconns == NULL) { + logthing(LOGTHING_CRITICAL, + "Couldn't allocate memory for dbconns"); + exit(1); + } + + ret = db_env_create(&dbenv, 0); + if (ret != 0) { + logthing(LOGTHING_CRITICAL, + "db_env_create: %s", db_strerror(ret)); + exit(1); + } + + /* + * Enable deadlock detection so that we don't block indefinitely on + * anything. What we really want is simple 2 state locks, but I'm not + * sure how to make the standard DB functions do that yet. + */ + ret = dbenv->set_lk_detect(dbenv, DB_LOCK_DEFAULT); if (ret != 0) { - fprintf(stderr, "db_create: %s\n", db_strerror(ret)); + logthing(LOGTHING_CRITICAL, + "db_env_create: %s", db_strerror(ret)); exit(1); } - ret = dbconn->open(dbconn, buf, NULL, DB_HASH, + ret = dbenv->open(dbenv, config.db_dir, + DB_INIT_LOG | DB_INIT_MPOOL | DB_INIT_LOCK | + DB_INIT_TXN | DB_CREATE, - 0664); + 0); if (ret != 0) { - dbconn->err(dbconn, ret, "%s", buf); + logthing(LOGTHING_CRITICAL, + "Error opening db environment: %s (%s)", + config.db_dir, + db_strerror(ret)); exit(1); } - strcpy(buf, config.db_dir); - strcat(buf, "/worddb"); - - ret = db_create(&worddb, NULL, 0); + for (i = 0; i < numdbs; i++) { + ret = db_create(&dbconns[i], dbenv, 0); + if (ret != 0) { + logthing(LOGTHING_CRITICAL, + "db_create: %s", db_strerror(ret)); + exit(1); + } + + snprintf(buf, 1023, "keydb.%d.db", i); + flags = DB_CREATE; + if (readonly) { + flags = DB_RDONLY; + } + ret = dbconns[i]->open(dbconns[i], buf, + NULL, + DB_HASH, + flags, + 0664); + if (ret != 0) { + logthing(LOGTHING_CRITICAL, + "Error opening key database: %s (%s)", + buf, + db_strerror(ret)); + exit(1); + } + } + + ret = db_create(&worddb, dbenv, 0); if (ret != 0) { - fprintf(stderr, "db_create: %s\n", db_strerror(ret)); + logthing(LOGTHING_CRITICAL, "db_create: %s", db_strerror(ret)); exit(1); } ret = worddb->set_flags(worddb, DB_DUP); - ret = worddb->open(worddb, buf, NULL, DB_BTREE, - DB_CREATE, + ret = worddb->open(worddb, "worddb", NULL, DB_BTREE, + flags, + 0664); + if (ret != 0) { + logthing(LOGTHING_CRITICAL, + "Error opening word database: %s (%s)", + "worddb", + db_strerror(ret)); + exit(1); + } + + ret = db_create(&id32db, dbenv, 0); + if (ret != 0) { + logthing(LOGTHING_CRITICAL, "db_create: %s", db_strerror(ret)); + exit(1); + } + ret = id32db->set_flags(id32db, DB_DUP); + + ret = id32db->open(id32db, "id32db", NULL, DB_HASH, + flags, 0664); if (ret != 0) { - worddb->err(worddb, ret, "%s", buf); + logthing(LOGTHING_CRITICAL, + "Error opening id32 database: %s (%s)", + "id32db", + db_strerror(ret)); exit(1); } @@ -145,10 +218,19 @@ void initdb(void) */ void cleanupdb(void) { + int i = 0; + + txn_checkpoint(dbenv, 0, 0, 0); + id32db->close(id32db, 0); + id32db = NULL; worddb->close(worddb, 0); worddb = NULL; - dbconn->close(dbconn, 0); - dbconn = NULL; + for (i = 0; i < numdbs; i++) { + dbconns[i]->close(dbconns[i], 0); + dbconns[i] = NULL; + } + dbenv->close(dbenv, 0); + dbenv = NULL; } /** @@ -160,6 +242,22 @@ void cleanupdb(void) */ bool starttrans(void) { + int ret; + + assert(dbenv != NULL); + assert(txn == NULL); + + ret = txn_begin(dbenv, + NULL, /* No parent transaction */ + &txn, + 0); + if (ret != 0) { + logthing(LOGTHING_CRITICAL, + "Error starting transaction: %s", + db_strerror(ret)); + exit(1); + } + return true; } @@ -170,6 +268,21 @@ bool starttrans(void) */ void endtrans(void) { + int ret; + + assert(dbenv != NULL); + assert(txn != NULL); + + ret = txn_commit(txn, + 0); + if (ret != 0) { + logthing(LOGTHING_CRITICAL, + "Error ending transaction: %s", + db_strerror(ret)); + exit(1); + } + txn = NULL; + return; } @@ -194,6 +307,10 @@ int fetch_key(uint64_t keyid, struct openpgp_publickey **publickey, int numkeys = 0; struct buffer_ctx fetchbuf; + if (keyid < 0x100000000LL) { + keyid = getfullkeyid(keyid); + } + memset(&key, 0, sizeof(key)); memset(&data, 0, sizeof(data)); @@ -202,10 +319,13 @@ int fetch_key(uint64_t keyid, struct openpgp_publickey **publickey, key.size = sizeof(keyid); key.data = &keyid; - keyid &= 0xFFFFFFFF; - ret = dbconn->get(dbconn, - NULL, /* txn id */ + if (!intrans) { + starttrans(); + } + + ret = keydb(keyid)->get(keydb(keyid), + txn, &key, &data, 0); /* flags*/ @@ -215,19 +335,25 @@ int fetch_key(uint64_t keyid, struct openpgp_publickey **publickey, fetchbuf.offset = 0; fetchbuf.size = data.size; read_openpgp_stream(buffer_fetchchar, &fetchbuf, - &packets); + &packets, 0); parse_keys(packets, publickey); free_packet_list(packets); packets = NULL; numkeys++; } else if (ret != DB_NOTFOUND) { - dbconn->err(dbconn, ret, "Problem retrieving key"); + logthing(LOGTHING_ERROR, + "Problem retrieving key: %s", + db_strerror(ret)); + } + + if (!intrans) { + endtrans(); } return (numkeys); } -int worddb_cmp(const char *d1, const char *d2) +int worddb_cmp(const void *d1, const void *d2) { return memcmp(d1, d2, 12); } @@ -258,9 +384,10 @@ int fetch_key_text(const char *search, struct openpgp_publickey **publickey) searchtext = strdup(search); wordlist = makewordlist(wordlist, searchtext); + starttrans(); ret = worddb->cursor(worddb, - NULL, /* txn */ + txn, &cursor, 0); /* flags */ @@ -322,7 +449,7 @@ int fetch_key_text(const char *search, struct openpgp_publickey **publickey) numkeys += fetch_key(keyid, publickey, - false); + true); } llfree(keylist, free); keylist = NULL; @@ -331,6 +458,8 @@ int fetch_key_text(const char *search, struct openpgp_publickey **publickey) ret = cursor->c_close(cursor); cursor = NULL; + + endtrans(); return (numkeys); } @@ -358,14 +487,21 @@ int store_key(struct openpgp_publickey *publickey, bool intrans, bool update) DBT key; DBT data; uint64_t keyid = 0; + uint32_t shortkeyid = 0; + uint64_t *subkeyids = NULL; char **uids = NULL; char *primary = NULL; unsigned char worddb_data[12]; struct ll *wordlist = NULL; struct ll *curword = NULL; + bool deadlock = false; keyid = get_keyid(publickey); + if (!intrans) { + starttrans(); + } + /* * Delete the key if we already have it. * @@ -375,61 +511,70 @@ int store_key(struct openpgp_publickey *publickey, bool intrans, bool update) * it definitely needs updated. */ if (update) { - delete_key(keyid, true); + deadlock = (delete_key(keyid, true) == -1); } /* * Convert the key to a flat set of binary data. */ - next = publickey->next; - publickey->next = NULL; - flatten_publickey(publickey, &packets, &list_end); - publickey->next = next; - - storebuf.offset = 0; - storebuf.size = 8192; - storebuf.buffer = malloc(8192); + if (!deadlock) { + next = publickey->next; + publickey->next = NULL; + flatten_publickey(publickey, &packets, &list_end); + publickey->next = next; + + storebuf.offset = 0; + storebuf.size = 8192; + storebuf.buffer = malloc(8192); - write_openpgp_stream(buffer_putchar, &storebuf, packets); + write_openpgp_stream(buffer_putchar, &storebuf, packets); - /* - * Now we have the key data store it in the DB; the keyid is the key. - */ - memset(&key, 0, sizeof(key)); - memset(&data, 0, sizeof(data)); - key.data = &keyid; - key.size = sizeof(keyid); - keyid &= 0xFFFFFFFF; - data.size = storebuf.offset; - data.data = storebuf.buffer; - - ret = dbconn->put(dbconn, - NULL, /* txn id */ - &key, - &data, - 0); /* flags*/ - if (ret != 0) { - dbconn->err(dbconn, ret, "Problem storing key"); - } + /* + * Now we have the key data store it in the DB; the keyid is + * the key. + */ + memset(&key, 0, sizeof(key)); + memset(&data, 0, sizeof(data)); + key.data = &keyid; + key.size = sizeof(keyid); + data.size = storebuf.offset; + data.data = storebuf.buffer; - free(storebuf.buffer); - storebuf.buffer = NULL; - storebuf.size = 0; - storebuf.offset = 0; + ret = keydb(keyid)->put(keydb(keyid), + txn, + &key, + &data, + 0); /* flags*/ + if (ret != 0) { + logthing(LOGTHING_ERROR, + "Problem storing key: %s", + db_strerror(ret)); + if (ret == DB_LOCK_DEADLOCK) { + deadlock = true; + } + } - free_packet_list(packets); - packets = NULL; + free(storebuf.buffer); + storebuf.buffer = NULL; + storebuf.size = 0; + storebuf.offset = 0; + + free_packet_list(packets); + packets = NULL; + } /* * Walk through our uids storing the words into the db with the keyid. */ - uids = keyuids(publickey, &primary); + if (!deadlock) { + uids = keyuids(publickey, &primary); + } if (uids != NULL) { for (i = 0; ret == 0 && uids[i] != NULL; i++) { wordlist = makewordlist(wordlist, uids[i]); } - for (curword = wordlist; curword != NULL; + for (curword = wordlist; curword != NULL && !deadlock; curword = curword->next) { memset(&key, 0, sizeof(key)); memset(&data, 0, sizeof(data)); @@ -455,13 +600,17 @@ int store_key(struct openpgp_publickey *publickey, bool intrans, bool update) worddb_data[10] = (keyid >> 8) & 0xFF; worddb_data[11] = keyid & 0xFF; ret = worddb->put(worddb, - 0, + txn, &key, &data, 0); if (ret != 0) { - worddb->err(worddb, ret, - "Problem storing key"); + logthing(LOGTHING_ERROR, + "Problem storing word: %s", + db_strerror(ret)); + if (ret == DB_LOCK_DEADLOCK) { + deadlock = true; + } } } @@ -477,7 +626,73 @@ int store_key(struct openpgp_publickey *publickey, bool intrans, bool update) uids = NULL; } - return 0; + if (!intrans) { + endtrans(); + } + + /* + * Write the truncated 32 bit keyid so we can lookup the full id for + * queries. + */ + if (!deadlock) { + shortkeyid = keyid & 0xFFFFFFFF; + + memset(&key, 0, sizeof(key)); + memset(&data, 0, sizeof(data)); + key.data = &shortkeyid; + key.size = sizeof(shortkeyid); + data.data = &keyid; + data.size = sizeof(keyid); + + ret = id32db->put(id32db, + txn, + &key, + &data, + 0); + if (ret != 0) { + logthing(LOGTHING_ERROR, + "Problem storing short keyid: %s", + db_strerror(ret)); + if (ret == DB_LOCK_DEADLOCK) { + deadlock = true; + } + } + } + + if (!deadlock) { + subkeyids = keysubkeys(publickey); + i = 0; + while (subkeyids != NULL && subkeyids[i] != 0) { + shortkeyid = subkeyids[i++] & 0xFFFFFFFF; + + memset(&key, 0, sizeof(key)); + memset(&data, 0, sizeof(data)); + key.data = &shortkeyid; + key.size = sizeof(shortkeyid); + data.data = &keyid; + data.size = sizeof(keyid); + + ret = id32db->put(id32db, + txn, + &key, + &data, + 0); + if (ret != 0) { + logthing(LOGTHING_ERROR, + "Problem storing short keyid: %s", + db_strerror(ret)); + if (ret == DB_LOCK_DEADLOCK) { + deadlock = true; + } + } + } + if (subkeyids != NULL) { + free(subkeyids); + subkeyids = NULL; + } + } + + return deadlock ? -1 : 0 ; } /** @@ -493,6 +708,8 @@ int delete_key(uint64_t keyid, bool intrans) struct openpgp_publickey *publickey = NULL; DBT key, data; DBC *cursor = NULL; + uint32_t shortkeyid = 0; + uint64_t *subkeyids = NULL; int ret = 0; int i; char **uids = NULL; @@ -500,10 +717,13 @@ int delete_key(uint64_t keyid, bool intrans) unsigned char worddb_data[12]; struct ll *wordlist = NULL; struct ll *curword = NULL; + bool deadlock = false; - keyid &= 0xFFFFFFFF; + if (!intrans) { + starttrans(); + } - fetch_key(keyid, &publickey, intrans); + fetch_key(keyid, &publickey, true); /* * Walk through the uids removing the words from the worddb. @@ -517,11 +737,11 @@ int delete_key(uint64_t keyid, bool intrans) } ret = worddb->cursor(worddb, - NULL, /* txn */ + txn, &cursor, 0); /* flags */ - for (curword = wordlist; curword != NULL; + for (curword = wordlist; curword != NULL && !deadlock; curword = curword->next) { memset(&key, 0, sizeof(key)); memset(&data, 0, sizeof(data)); @@ -551,13 +771,23 @@ int delete_key(uint64_t keyid, bool intrans) &key, &data, DB_GET_BOTH); + if (ret == 0) { ret = cursor->c_del(cursor, 0); + if (ret != 0) { + logthing(LOGTHING_ERROR, + "Problem deleting word: %s", + db_strerror(ret)); + } } if (ret != 0) { - worddb->err(worddb, ret, - "Problem deleting word."); + logthing(LOGTHING_ERROR, + "Problem deleting word: %s", + db_strerror(ret)); + if (ret == DB_LOCK_DEADLOCK) { + deadlock = true; + } } } ret = cursor->c_close(cursor); @@ -577,21 +807,217 @@ int delete_key(uint64_t keyid, bool intrans) publickey = NULL; } - key.data = &keyid; - key.size = sizeof(keyid); + if (!deadlock) { + ret = id32db->cursor(id32db, + txn, + &cursor, + 0); /* flags */ + + shortkeyid = keyid & 0xFFFFFFFF; + + memset(&key, 0, sizeof(key)); + memset(&data, 0, sizeof(data)); + key.data = &shortkeyid; + key.size = sizeof(shortkeyid); + data.data = &keyid; + data.size = sizeof(keyid); + + ret = cursor->c_get(cursor, + &key, + &data, + DB_GET_BOTH); + + if (ret == 0) { + ret = cursor->c_del(cursor, 0); + if (ret != 0) { + logthing(LOGTHING_ERROR, + "Problem deleting short keyid: %s", + db_strerror(ret)); + } + } + + if (ret != 0) { + logthing(LOGTHING_ERROR, + "Problem deleting short keyid: %s", + db_strerror(ret)); + if (ret == DB_LOCK_DEADLOCK) { + deadlock = true; + } + } + + subkeyids = keysubkeys(publickey); + i = 0; + while (subkeyids != NULL && subkeyids[i] != 0) { + shortkeyid = subkeyids[i++] & 0xFFFFFFFF; + + memset(&key, 0, sizeof(key)); + memset(&data, 0, sizeof(data)); + key.data = &shortkeyid; + key.size = sizeof(shortkeyid); + data.data = &keyid; + data.size = sizeof(keyid); + + ret = cursor->c_get(cursor, + &key, + &data, + DB_GET_BOTH); + + if (ret == 0) { + ret = cursor->c_del(cursor, 0); + if (ret != 0) { + logthing(LOGTHING_ERROR, + "Problem deleting short" + " keyid: %s", + db_strerror(ret)); + } + } + + if (ret != 0) { + logthing(LOGTHING_ERROR, + "Problem deleting short keyid: %s", + db_strerror(ret)); + if (ret == DB_LOCK_DEADLOCK) { + deadlock = true; + } + } + } + if (subkeyids != NULL) { + free(subkeyids); + subkeyids = NULL; + } + + ret = cursor->c_close(cursor); + cursor = NULL; + } - dbconn->del(dbconn, - NULL, /* txn id */ + if (!deadlock) { + key.data = &keyid; + key.size = sizeof(keyid); + + keydb(keyid)->del(keydb(keyid), + txn, + &key, + 0); /* flags */ + } + + if (!intrans) { + endtrans(); + } + + return deadlock ? (-1) : (ret == DB_NOTFOUND); +} + +/** + * dumpdb - dump the key database + * @filenamebase: The base filename to use for the dump. + * + * Dumps the database into one or more files, which contain pure OpenPGP + * that can be reimported into onak or gpg. filenamebase provides a base + * file name for the dump; several files may be created, all of which will + * begin with this string and then have a unique number and a .pgp + * extension. + */ +int dumpdb(char *filenamebase) +{ + DBT key, data; + DBC *cursor = NULL; + int ret = 0; + int fd = -1; + int i = 0; + char filename[1024]; + + filename[1023] = 0; + for (i = 0; i < numdbs; i++) { + ret = dbconns[i]->cursor(dbconns[i], + NULL, + &cursor, + 0); /* flags */ + + snprintf(filename, 1023, "%s.%d.pgp", filenamebase, i); + fd = open(filename, O_CREAT | O_WRONLY | O_TRUNC, 0640); + if (fd == -1) { + logthing(LOGTHING_ERROR, + "Error opening keydump file (%s): %s", + filename, + strerror(errno)); + } else { + memset(&key, 0, sizeof(key)); + memset(&data, 0, sizeof(data)); + ret = cursor->c_get(cursor, &key, &data, DB_NEXT); + while (ret == 0) { + write(fd, data.data, data.size); + memset(&key, 0, sizeof(key)); + memset(&data, 0, sizeof(data)); + ret = cursor->c_get(cursor, &key, &data, + DB_NEXT); + } + if (ret != DB_NOTFOUND) { + logthing(LOGTHING_ERROR, + "Problem reading key: %s", + db_strerror(ret)); + } + close(fd); + } + + ret = cursor->c_close(cursor); + cursor = NULL; + } + + return 0; +} + +/** + * getfullkeyid - Maps a 32bit key id to a 64bit one. + * @keyid: The 32bit keyid. + * + * This function maps a 32bit key id to the full 64bit one. It returns the + * full keyid. If the key isn't found a keyid of 0 is returned. + */ +uint64_t getfullkeyid(uint64_t keyid) +{ + DBT key, data; + DBC *cursor = NULL; + uint32_t shortkeyid = 0; + int ret = 0; + + if (keyid < 0x100000000LL) { + ret = id32db->cursor(id32db, + txn, + &cursor, + 0); /* flags */ + + shortkeyid = keyid & 0xFFFFFFFF; + + memset(&key, 0, sizeof(key)); + memset(&data, 0, sizeof(data)); + key.data = &shortkeyid; + key.size = sizeof(shortkeyid); + data.flags = DB_DBT_MALLOC; + + ret = cursor->c_get(cursor, &key, - 0); /* flags */ + &data, + DB_SET); + + if (ret == 0) { + keyid = *(uint64_t *) data.data; + + if (data.data != NULL) { + free(data.data); + data.data = NULL; + } + } - return (ret == DB_NOTFOUND); + ret = cursor->c_close(cursor); + cursor = NULL; + } + + return keyid; } /* * Include the basic keydb routines. */ -#define NEED_GETFULLKEYID 1 #define NEED_GETKEYSIGS 1 #define NEED_KEYID2UID 1 #include "keydb.c"