X-Git-Url: https://git.sommitrealweird.co.uk/onak.git/blobdiff_plain/76c5dd1081bc9db3bf446480457b52d4ae7b3891..e94ac8da0e87d66927f746e6053fdafbee85cd16:/keydb_db3.c

diff --git a/keydb_db3.c b/keydb_db3.c
index de24f75..7cc3b1e 100644
--- a/keydb_db3.c
+++ b/keydb_db3.c
@@ -25,8 +25,10 @@
 #include "decodekey.h"
 #include "keystructs.h"
 #include "mem.h"
+#include "log.h"
 #include "onak-conf.h"
 #include "parsekey.h"
+#include "wordlist.h"
 
 /**
  *	dbenv - our database environment.
@@ -34,9 +36,14 @@
 static DB_ENV *dbenv = NULL;
 
 /**
- *	dbconn - our connection to the key database.
+ *	numdb - The number of database files we have.
  */
-static DB *dbconn = NULL;
+static int numdbs = 16;
+
+/**
+ *	dbconn - our connections to the key database files.
+ */
+static DB **dbconns = NULL;
 
 /**
  *	worddb - our connection to the word database.
@@ -44,58 +51,22 @@ static DB *dbconn = NULL;
 static DB *worddb = NULL;
 
 /**
- *	txn - our current transaction id.
+ *	id32db - our connection to the 32bit ID database.
  */
-static DB_TXN *txn = NULL;
+static DB *id32db = NULL;
 
 /**
- *	makewordlist - Takes a string and splits it into a set of unique words.
- *	@wordlist: The current word list.
- *	@words: The string to split and add.
- *
- *	We take words and split it on non alpha numeric characters. These get
- *	added to the word list if they're not already present. If the wordlist
- *	is NULL then we start a new list, otherwise it's search for already
- *	added words. Note that words is modified in the process of scanning.
- *
- *	Returns the new word list.
+ *	txn - our current transaction id.
  */
-struct ll *makewordlist(struct ll *wordlist, char *word)
+static DB_TXN *txn = NULL;
+
+DB *keydb(uint64_t keyid)
 {
-	char *start = NULL;
-	char *end = NULL;
+	uint64_t keytrun;
 
-	/*
-	 * Walk through the words string, spliting on non alphanumerics and
-	 * then checking if the word already exists in the list. If not then
-	 * we add it.
-	 */
-	end = word;
-	while (end != NULL && *end != 0) {
-		start = end;
-		while (*start != 0 && !isalnum(*start)) {
-			start++;
-		}
-		end = start;
-		while (*end != 0 && isalnum(*end)) {
-			*end = tolower(*end);
-			end++;
-		}
-		if (end - start > 1) {
-			if (*end != 0) {
-				*end = 0;
-				end++;
-			}
-			
-			if (llfind(wordlist, start,
-					strcmp) == NULL) {
-				wordlist = lladd(wordlist,
-						start);
-			}
-		}
-	}
+	keytrun = keyid >> 8;
 
-	return wordlist;
+	return(dbconns[keytrun % numdbs]);
 }
 
 /**
@@ -105,26 +76,60 @@ struct ll *makewordlist(struct ll *wordlist, char *word)
  *	this file are called in order to allow the DB to be initialized ready
  *	for access.
  */
-void initdb(void)
+void initdb(bool readonly)
 {
-	int ret = 0;
+	char       buf[1024];
+	FILE      *numdb = NULL;
+	int        ret = 0;
+	int        i = 0;
+	u_int32_t  flags = 0;
+
+	snprintf(buf, sizeof(buf) - 1, "%s/num_keydb", config.db_dir);
+	numdb = fopen(buf, "r");
+	if (numdb != NULL) {
+		if (fgets(buf, sizeof(buf), numdb) != NULL) {
+			numdbs = atoi(buf);
+		}
+		fclose(numdb);
+	} else if (!readonly) {
+		logthing(LOGTHING_ERROR, "Couldn't open num_keydb: %s",
+				strerror(errno));
+		numdb = fopen(buf, "w");
+		if (numdb != NULL) {
+			fprintf(numdb, "%d", numdbs);
+			fclose(numdb);
+		} else {
+			logthing(LOGTHING_ERROR,
+				"Couldn't write num_keydb: %s",
+				strerror(errno));
+		}
+	}
+
+	dbconns = malloc(sizeof (DB *) * numdbs);
+	if (dbconns == NULL) {
+		logthing(LOGTHING_CRITICAL,
+				"Couldn't allocate memory for dbconns");
+		exit(1);
+	}
 
 	ret = db_env_create(&dbenv, 0);
 	if (ret != 0) {
-		fprintf(stderr, "db_env_create: %s\n", db_strerror(ret));
+		logthing(LOGTHING_CRITICAL,
+			"db_env_create: %s", db_strerror(ret));
 		exit(1);
 	}
 
 	/*
-	 * This is a bit of a kludge. Either we run a separate process for
-	 * deadlock detection or we do this every time we run. What we really
-	 * want to do is specify that our locks are exclusive locks when we
-	 * start to do an update.
+	 * Enable deadlock detection so that we don't block indefinitely on
+	 * anything. What we really want is simple 2 state locks, but I'm not
+	 * sure how to make the standard DB functions do that yet.
 	 */
-	ret = lock_detect(dbenv,
-			0, /* flags */
-			DB_LOCK_RANDOM,
-			NULL); /* If non null int* for number broken */
+	ret = dbenv->set_lk_detect(dbenv, DB_LOCK_DEFAULT);
+	if (ret != 0) {
+		logthing(LOGTHING_CRITICAL,
+			"db_env_create: %s", db_strerror(ret));
+		exit(1);
+	}
 
 	ret = dbenv->open(dbenv, config.db_dir,
 			DB_INIT_LOG | DB_INIT_MPOOL | DB_INIT_LOCK |
@@ -132,38 +137,73 @@ void initdb(void)
 			DB_CREATE,
 			0);
 	if (ret != 0) {
-		dbenv->err(dbenv, ret, "%s", config.db_dir);
+		logthing(LOGTHING_CRITICAL,
+				"Error opening db environment: %s (%s)",
+				config.db_dir,
+				db_strerror(ret));
 		exit(1);
 	}
 
-	ret = db_create(&dbconn, dbenv, 0);
-	if (ret != 0) {
-		fprintf(stderr, "db_create: %s\n", db_strerror(ret));
-		exit(1);
-	}
+	for (i = 0; i < numdbs; i++) {
+		ret = db_create(&dbconns[i], dbenv, 0);
+		if (ret != 0) {
+			logthing(LOGTHING_CRITICAL,
+				"db_create: %s", db_strerror(ret));
+			exit(1);
+		}
 
-	ret = dbconn->open(dbconn, "keydb.db", 
+		snprintf(buf, 1023, "keydb.%d.db", i);
+		flags = DB_CREATE;
+		if (readonly) {
+			flags = DB_RDONLY;
+		}
+		ret = dbconns[i]->open(dbconns[i], buf,
 			NULL,
 			DB_HASH,
-			DB_CREATE,
+			flags,
 			0664);
-	if (ret != 0) {
-		dbconn->err(dbconn, ret, "keydb.db");
-		exit(1);
+		if (ret != 0) {
+			logthing(LOGTHING_CRITICAL,
+				"Error opening key database: %s (%s)",
+				buf,
+				db_strerror(ret));
+			exit(1);
+		}
 	}
 
 	ret = db_create(&worddb, dbenv, 0);
 	if (ret != 0) {
-		fprintf(stderr, "db_create: %s\n", db_strerror(ret));
+		logthing(LOGTHING_CRITICAL, "db_create: %s", db_strerror(ret));
 		exit(1);
 	}
 	ret = worddb->set_flags(worddb, DB_DUP);
 
 	ret = worddb->open(worddb, "worddb", NULL, DB_BTREE,
-			DB_CREATE,
+			flags,
 			0664);
 	if (ret != 0) {
-		worddb->err(worddb, ret, "worddb");
+		logthing(LOGTHING_CRITICAL,
+				"Error opening word database: %s (%s)",
+				"worddb",
+				db_strerror(ret));
+		exit(1);
+	}
+
+	ret = db_create(&id32db, dbenv, 0);
+	if (ret != 0) {
+		logthing(LOGTHING_CRITICAL, "db_create: %s", db_strerror(ret));
+		exit(1);
+	}
+	ret = id32db->set_flags(id32db, DB_DUP);
+
+	ret = id32db->open(id32db, "id32db", NULL, DB_HASH,
+			flags,
+			0664);
+	if (ret != 0) {
+		logthing(LOGTHING_CRITICAL,
+				"Error opening id32 database: %s (%s)",
+				"id32db",
+				db_strerror(ret));
 		exit(1);
 	}
 	
@@ -178,10 +218,17 @@ void initdb(void)
  */
 void cleanupdb(void)
 {
+	int i = 0;
+
+	txn_checkpoint(dbenv, 0, 0, 0);
+	id32db->close(id32db, 0);
+	id32db = NULL;
 	worddb->close(worddb, 0);
 	worddb = NULL;
-	dbconn->close(dbconn, 0);
-	dbconn = NULL;
+	for (i = 0; i < numdbs; i++) {
+		dbconns[i]->close(dbconns[i], 0);
+		dbconns[i] = NULL;
+	}
 	dbenv->close(dbenv, 0);
 	dbenv = NULL;
 }
@@ -205,7 +252,9 @@ bool starttrans(void)
 		&txn,
 		0);
 	if (ret != 0) {
-		dbenv->err(dbenv, ret, "starttrans():");
+		logthing(LOGTHING_CRITICAL,
+				"Error starting transaction: %s",
+				db_strerror(ret));
 		exit(1);
 	}
 
@@ -227,7 +276,9 @@ void endtrans(void)
 	ret = txn_commit(txn,
 		0);
 	if (ret != 0) {
-		dbenv->err(dbenv, ret, "endtrans():");
+		logthing(LOGTHING_CRITICAL,
+				"Error ending transaction: %s",
+				db_strerror(ret));
 		exit(1);
 	}
 	txn = NULL;
@@ -256,6 +307,10 @@ int fetch_key(uint64_t keyid, struct openpgp_publickey **publickey,
 	int numkeys = 0;
 	struct buffer_ctx fetchbuf;
 
+	if (keyid < 0x100000000LL) {
+		keyid = getfullkeyid(keyid);
+	}
+
 	memset(&key, 0, sizeof(key));
 	memset(&data, 0, sizeof(data));
 
@@ -264,13 +319,12 @@ int fetch_key(uint64_t keyid, struct openpgp_publickey **publickey,
 
 	key.size = sizeof(keyid);
 	key.data = &keyid;
-	keyid &= 0xFFFFFFFF;
 
 	if (!intrans) {
 		starttrans();
 	}
 
-	ret = dbconn->get(dbconn,
+	ret = keydb(keyid)->get(keydb(keyid),
 			txn,
 			&key,
 			&data,
@@ -281,13 +335,15 @@ int fetch_key(uint64_t keyid, struct openpgp_publickey **publickey,
 		fetchbuf.offset = 0;
 		fetchbuf.size = data.size;
 		read_openpgp_stream(buffer_fetchchar, &fetchbuf,
-				&packets);
+				&packets, 0);
 		parse_keys(packets, publickey);
 		free_packet_list(packets);
 		packets = NULL;
 		numkeys++;
 	} else if (ret != DB_NOTFOUND) {
-		dbconn->err(dbconn, ret, "Problem retrieving key");
+		logthing(LOGTHING_ERROR,
+				"Problem retrieving key: %s",
+				db_strerror(ret));
 	}
 
 	if (!intrans) {
@@ -297,7 +353,7 @@ int fetch_key(uint64_t keyid, struct openpgp_publickey **publickey,
 	return (numkeys);
 }
 
-int worddb_cmp(const char *d1, const char *d2)
+int worddb_cmp(const void *d1, const void *d2)
 {
 	return memcmp(d1, d2, 12);
 }
@@ -431,6 +487,8 @@ int store_key(struct openpgp_publickey *publickey, bool intrans, bool update)
 	DBT        key;
 	DBT        data;
 	uint64_t   keyid = 0;
+	uint32_t   shortkeyid = 0;
+	uint64_t  *subkeyids = NULL;
 	char     **uids = NULL;
 	char      *primary = NULL;
 	unsigned char worddb_data[12];
@@ -479,17 +537,18 @@ int store_key(struct openpgp_publickey *publickey, bool intrans, bool update)
 		memset(&data, 0, sizeof(data));
 		key.data = &keyid;
 		key.size = sizeof(keyid);
-		keyid &= 0xFFFFFFFF;
 		data.size = storebuf.offset;
 		data.data = storebuf.buffer;
 
-		ret = dbconn->put(dbconn,
+		ret = keydb(keyid)->put(keydb(keyid),
 				txn,
 				&key,
 				&data,
 				0); /* flags*/
 		if (ret != 0) {
-			dbconn->err(dbconn, ret, "Problem storing key");
+			logthing(LOGTHING_ERROR,
+					"Problem storing key: %s",
+					db_strerror(ret));
 			if (ret == DB_LOCK_DEADLOCK) {
 				deadlock = true;
 			}
@@ -546,8 +605,9 @@ int store_key(struct openpgp_publickey *publickey, bool intrans, bool update)
 				&data,
 				0);
 			if (ret != 0) {
-				worddb->err(worddb, ret,
-					"Problem storing key");
+				logthing(LOGTHING_ERROR,
+					"Problem storing word: %s",
+					db_strerror(ret));
 				if (ret == DB_LOCK_DEADLOCK) {
 					deadlock = true;
 				}
@@ -570,6 +630,68 @@ int store_key(struct openpgp_publickey *publickey, bool intrans, bool update)
 		endtrans();
 	}
 
+	/*
+	 * Write the truncated 32 bit keyid so we can lookup the full id for
+	 * queries.
+	 */
+	if (!deadlock) {
+		shortkeyid = keyid & 0xFFFFFFFF;
+
+		memset(&key, 0, sizeof(key));
+		memset(&data, 0, sizeof(data));
+		key.data = &shortkeyid;
+		key.size = sizeof(shortkeyid);
+		data.data = &keyid;
+		data.size = sizeof(keyid);
+
+		ret = id32db->put(id32db,
+			txn,
+			&key,
+			&data,
+			0);
+		if (ret != 0) {
+			logthing(LOGTHING_ERROR,
+				"Problem storing short keyid: %s",
+				db_strerror(ret));
+			if (ret == DB_LOCK_DEADLOCK) {
+				deadlock = true;
+			}
+		}
+	}
+
+	if (!deadlock) {
+		subkeyids = keysubkeys(publickey);
+		i = 0;
+		while (subkeyids != NULL && subkeyids[i] != 0) {
+			shortkeyid = subkeyids[i++] & 0xFFFFFFFF;
+
+			memset(&key, 0, sizeof(key));
+			memset(&data, 0, sizeof(data));
+			key.data = &shortkeyid;
+			key.size = sizeof(shortkeyid);
+			data.data = &keyid;
+			data.size = sizeof(keyid);
+
+			ret = id32db->put(id32db,
+				txn,
+				&key,
+				&data,
+				0);
+			if (ret != 0) {
+				logthing(LOGTHING_ERROR,
+					"Problem storing short keyid: %s",
+					db_strerror(ret));
+				if (ret == DB_LOCK_DEADLOCK) {
+					deadlock = true;
+				}
+			}
+		}
+		if (subkeyids != NULL) {
+			free(subkeyids);
+			subkeyids = NULL;
+		}
+	}
+
 	return deadlock ? -1 : 0 ;
 }
 
@@ -586,6 +708,8 @@ int delete_key(uint64_t keyid, bool intrans)
 	struct openpgp_publickey *publickey = NULL;
 	DBT key, data;
 	DBC *cursor = NULL;
+	uint32_t   shortkeyid = 0;
+	uint64_t  *subkeyids = NULL;
 	int ret = 0;
 	int i;
 	char **uids = NULL;
@@ -595,8 +719,6 @@ int delete_key(uint64_t keyid, bool intrans)
 	struct ll *curword  = NULL;
 	bool deadlock = false;
 
-	keyid &= 0xFFFFFFFF;
-
 	if (!intrans) {
 		starttrans();
 	}
@@ -653,14 +775,16 @@ int delete_key(uint64_t keyid, bool intrans)
 			if (ret == 0) {
 				ret = cursor->c_del(cursor, 0);
 				if (ret != 0) {
-					worddb->err(worddb, ret,
-						"Problem deleting word.");
+					logthing(LOGTHING_ERROR,
+						"Problem deleting word: %s",
+						db_strerror(ret));
 				}
 			}
 
 			if (ret != 0) {
-				worddb->err(worddb, ret,
-					"Problem deleting word.");
+				logthing(LOGTHING_ERROR,
+					"Problem deleting word: %s",
+					db_strerror(ret));
 				if (ret == DB_LOCK_DEADLOCK) {
 					deadlock = true;
 				}
@@ -683,11 +807,94 @@ int delete_key(uint64_t keyid, bool intrans)
 		publickey = NULL;
 	}
 
+	if (!deadlock) {
+		ret = id32db->cursor(id32db,
+			txn,
+			&cursor,
+			0);   /* flags */
+
+		shortkeyid = keyid & 0xFFFFFFFF;
+
+		memset(&key, 0, sizeof(key));
+		memset(&data, 0, sizeof(data));
+		key.data = &shortkeyid;
+		key.size = sizeof(shortkeyid);
+		data.data = &keyid;
+		data.size = sizeof(keyid);
+
+		ret = cursor->c_get(cursor,
+			&key,
+			&data,
+			DB_GET_BOTH);
+
+		if (ret == 0) {
+			ret = cursor->c_del(cursor, 0);
+			if (ret != 0) {
+				logthing(LOGTHING_ERROR,
+					"Problem deleting short keyid: %s",
+					db_strerror(ret));
+			}
+		}
+
+		if (ret != 0) {
+			logthing(LOGTHING_ERROR,
+				"Problem deleting short keyid: %s",
+				db_strerror(ret));
+			if (ret == DB_LOCK_DEADLOCK) {
+				deadlock = true;
+			}
+		}
+
+		subkeyids = keysubkeys(publickey);
+		i = 0;
+		while (subkeyids != NULL && subkeyids[i] != 0) {
+			shortkeyid = subkeyids[i++] & 0xFFFFFFFF;
+
+			memset(&key, 0, sizeof(key));
+			memset(&data, 0, sizeof(data));
+			key.data = &shortkeyid;
+			key.size = sizeof(shortkeyid);
+			data.data = &keyid;
+			data.size = sizeof(keyid);
+
+			ret = cursor->c_get(cursor,
+				&key,
+				&data,
+				DB_GET_BOTH);
+
+			if (ret == 0) {
+				ret = cursor->c_del(cursor, 0);
+				if (ret != 0) {
+					logthing(LOGTHING_ERROR,
+						"Problem deleting short"
+						" keyid: %s",
+						db_strerror(ret));
+				}
+			}
+
+			if (ret != 0) {
+				logthing(LOGTHING_ERROR,
+					"Problem deleting short keyid: %s",
+					db_strerror(ret));
+				if (ret == DB_LOCK_DEADLOCK) {
+					deadlock = true;
+				}
+			}
+		}
+		if (subkeyids != NULL) {
+			free(subkeyids);
+			subkeyids = NULL;
+		}
+
+		ret = cursor->c_close(cursor);
+		cursor = NULL;
+	}
+
 	if (!deadlock) {
 		key.data = &keyid;
 		key.size = sizeof(keyid);
 
-		dbconn->del(dbconn,
+		keydb(keyid)->del(keydb(keyid),
 				txn,
 				&key,
 				0); /* flags */
@@ -712,44 +919,105 @@ int delete_key(uint64_t keyid, bool intrans)
  */
 int dumpdb(char *filenamebase)
 {
-	DBT key, data;
-	DBC *cursor = NULL;
-	int ret = 0;
-	int fd = -1;
+	DBT   key, data;
+	DBC  *cursor = NULL;
+	int   ret = 0;
+	int   fd = -1;
+	int   i = 0;
+	char  filename[1024];
+
+	filename[1023] = 0;
+	for (i = 0; i < numdbs; i++) {
+		ret = dbconns[i]->cursor(dbconns[i],
+			NULL,
+			&cursor,
+			0);   /* flags */
 
-	starttrans();
+		snprintf(filename, 1023, "%s.%d.pgp", filenamebase, i);
+		fd = open(filename, O_CREAT | O_WRONLY | O_TRUNC, 0640);
+		if (fd == -1) {
+			logthing(LOGTHING_ERROR,
+				"Error opening keydump file (%s): %s",
+				filename,
+				strerror(errno));
+		} else {
+			memset(&key, 0, sizeof(key));
+			memset(&data, 0, sizeof(data));
+			ret = cursor->c_get(cursor, &key, &data, DB_NEXT);
+			while (ret == 0) {
+				write(fd, data.data, data.size);
+				memset(&key, 0, sizeof(key));
+				memset(&data, 0, sizeof(data));
+				ret = cursor->c_get(cursor, &key, &data,
+						DB_NEXT);
+			}
+			if (ret != DB_NOTFOUND) {
+				logthing(LOGTHING_ERROR,
+					"Problem reading key: %s",
+					db_strerror(ret));
+			}
+			close(fd);
+		}
+
+		ret = cursor->c_close(cursor);
+		cursor = NULL;
+	}
 	
-	ret = dbconn->cursor(dbconn,
-		txn,
-		&cursor,
-		0);   /* flags */
+	return 0;
+}
+
+/**
+ *	getfullkeyid - Maps a 32bit key id to a 64bit one.
+ *	@keyid: The 32bit keyid.
+ *
+ *	This function maps a 32bit key id to the full 64bit one. It returns the
+ *	full keyid. If the key isn't found a keyid of 0 is returned.
+ */
+uint64_t getfullkeyid(uint64_t keyid)
+{
+	DBT       key, data;
+	DBC      *cursor = NULL;
+	uint32_t  shortkeyid = 0;
+	int       ret = 0;
+
+	if (keyid < 0x100000000LL) {
+		ret = id32db->cursor(id32db,
+				txn,
+				&cursor,
+				0);   /* flags */
+
+		shortkeyid = keyid & 0xFFFFFFFF;
 
-	fd = open(filenamebase, O_CREAT | O_WRONLY | O_TRUNC, 0640);
-	memset(&key, 0, sizeof(key));
-	memset(&data, 0, sizeof(data));
-	ret = cursor->c_get(cursor, &key, &data, DB_NEXT);
-	while (ret == 0) {
-		write(fd, data.data, data.size);
 		memset(&key, 0, sizeof(key));
 		memset(&data, 0, sizeof(data));
-		ret = cursor->c_get(cursor, &key, &data, DB_NEXT);
-	}
-	dbconn->err(dbconn, ret, "Problem reading key");
+		key.data = &shortkeyid;
+		key.size = sizeof(shortkeyid);
+		data.flags = DB_DBT_MALLOC;
 
-	close(fd);
+		ret = cursor->c_get(cursor,
+			&key,
+			&data,
+			DB_SET);
 
-	ret = cursor->c_close(cursor);
-	cursor = NULL;
-	
-	endtrans();
+		if (ret == 0) {
+			keyid = *(uint64_t *) data.data;
+
+			if (data.data != NULL) {
+				free(data.data);
+				data.data = NULL;
+			}
+		}
+
+		ret = cursor->c_close(cursor);
+		cursor = NULL;
+	}
 	
-	return 0;
+	return keyid;
 }
 
 /*
  * Include the basic keydb routines.
  */
-#define NEED_GETFULLKEYID 1
 #define NEED_GETKEYSIGS 1
 #define NEED_KEYID2UID 1
 #include "keydb.c"