From 5bde3efabfec675b323501cae5e3a99784afcc31 Mon Sep 17 00:00:00 2001 From: Andrew Colin Kissa Date: Sun, 14 Aug 2016 13:45:08 +0100 Subject: [PATCH] LMDB: introduce as Experimental. Bug 1856 --- doc/doc-txt/NewStuff | 2 + doc/doc-txt/experimental-spec.txt | 44 +++++++ src/scripts/MakeLinks | 2 +- src/scripts/lookups-Makefile | 5 + src/src/EDITME | 7 ++ src/src/config.h.defaults | 1 + src/src/drtables.c | 7 ++ src/src/exim.c | 6 + src/src/lookups/Makefile | 2 + src/src/lookups/lmdb.c | 160 ++++++++++++++++++++++++++ test/aux-fixed/2800.lmdb-mkdb-dump.py | 15 +++ test/aux-fixed/2800.mdb | Bin 0 -> 12288 bytes test/aux-fixed/2800.mdb.src | 3 + test/confs/2800 | 9 ++ test/scripts/2800-lmdb/2800 | 11 ++ test/scripts/2800-lmdb/REQUIRES | 1 + test/stdout/2800 | 9 ++ 17 files changed, 283 insertions(+), 1 deletion(-) create mode 100644 src/src/lookups/lmdb.c create mode 100644 test/aux-fixed/2800.lmdb-mkdb-dump.py create mode 100644 test/aux-fixed/2800.mdb create mode 100644 test/aux-fixed/2800.mdb.src create mode 100644 test/confs/2800 create mode 100644 test/scripts/2800-lmdb/2800 create mode 100644 test/scripts/2800-lmdb/REQUIRES create mode 100644 test/stdout/2800 diff --git a/doc/doc-txt/NewStuff b/doc/doc-txt/NewStuff index 719bedf31..cf1cf6d56 100644 --- a/doc/doc-txt/NewStuff +++ b/doc/doc-txt/NewStuff @@ -33,6 +33,8 @@ Version 4.88 chunking_advertise_hosts, and smtp transport option hosts_try_chunking for control. + 8. LMDB lookup support, as Experimental. Patch supplied by Andrew Colin Kissa. + Version 4.87 ------------ diff --git a/doc/doc-txt/experimental-spec.txt b/doc/doc-txt/experimental-spec.txt index 993b5b05c..2e72def27 100644 --- a/doc/doc-txt/experimental-spec.txt +++ b/doc/doc-txt/experimental-spec.txt @@ -967,6 +967,50 @@ Rationale: Note that non-RFC-documented field names and data types are used. +LMDB Lookup support +------------------- +LMDB is an ultra-fast, ultra-compact, crash-proof key-value embedded data store. +It is modeled loosely on the BerkeleyDB API. You shoul read about the feature +set as well as operation modes at https://symas.com/products/lightning-memory-mapped-database/ + +LMDB single key lookup support is provided by linking to the LMDB C library. +The current implementation does not support writing to the LMDB database. + +Visit https://github.com/LMDB/lmdb to download the library or find it in your +operating systems package repository. + +If building from source, this description assumes that headers will be in +/usr/local/include, and that the libraries are in /usr/local/lib. + +1. In order to build exim with LMDB lookup support add or uncomment + +EXPERIMENTAL_LMDB=yes + +to your Local/Makefile. (Re-)build/install exim. exim -d should show +Experimental_LMDB in the line "Support for:". + +EXPERIMENTAL_LMDB=yes +LDFLAGS += -llmdb +# CFLAGS += -I/usr/local/include +# LDFLAGS += -L/usr/local/lib + +The first line sets the feature to include the correct code, and +the second line says to link the LMDB libraries into the +exim binary. The commented out lines should be uncommented if you +built LMDB from source and installed in the default location. +Adjust the paths if you installed them elsewhere, but you do not +need to uncomment them if an rpm (or you) installed them in the +package controlled locations (/usr/include and /usr/lib). + +2. Create your LMDB files, you can use the mdb_load utility which is +part of the LMDB distribution our your favourite language bindings. + +3. Add the single key lookups to your exim.conf file, example lookups +are below. + +${lookup{$sender_address_domain}lmdb{/var/lib/baruwa/data/db/relaydomains.mdb}{$value}} +${lookup{$sender_address_domain}lmdb{/var/lib/baruwa/data/db/relaydomains.mdb}{$value}fail} +${lookup{$sender_address_domain}lmdb{/var/lib/baruwa/data/db/relaydomains.mdb}} -------------------------------------------------------------- diff --git a/src/scripts/MakeLinks b/src/scripts/MakeLinks index 7a5649ef8..86f748c15 100755 --- a/src/scripts/MakeLinks +++ b/src/scripts/MakeLinks @@ -31,7 +31,7 @@ mkdir lookups cd lookups # Makefile is generated for f in README cdb.c dbmdb.c dnsdb.c dsearch.c ibase.c ldap.h ldap.c \ - lsearch.c mysql.c redis.c nis.c nisplus.c oracle.c passwd.c \ + lmdb.c lsearch.c mysql.c redis.c nis.c nisplus.c oracle.c passwd.c \ pgsql.c spf.c sqlite.c testdb.c whoson.c \ lf_functions.h lf_check_file.c lf_quote.c lf_sqlperform.c do diff --git a/src/scripts/lookups-Makefile b/src/scripts/lookups-Makefile index 4e69a9a77..db2d184a7 100755 --- a/src/scripts/lookups-Makefile +++ b/src/scripts/lookups-Makefile @@ -177,6 +177,11 @@ fi OBJ="${OBJ} spf.o" +if want_experimental LMDB +then + OBJ="${OBJ} lmdb.o" +fi + echo "MODS = $MODS" echo "OBJ = $OBJ" diff --git a/src/src/EDITME b/src/src/EDITME index ac8c642bc..10e5ccacb 100644 --- a/src/src/EDITME +++ b/src/src/EDITME @@ -488,6 +488,13 @@ EXIM_MONITOR=eximon.bin # Uncomment the following to include extra information in fail DSN message (bounces) # EXPERIMENTAL_DSN_INFO=yes +# Uncomment the following to add LMDB lookup support +# You need to have LMDB installed on your system (https://github.com/LMDB/lmdb) +# Depending on where it is installed you may have to edit the CFLAGS and LDFLAGS lines. +# EXPERIMENTAL_LMDB=yes +# CFLAGS += -I/usr/local/include +# LDFLAGS += -llmdb + ############################################################################### # THESE ARE THINGS YOU MIGHT WANT TO SPECIFY # ############################################################################### diff --git a/src/src/config.h.defaults b/src/src/config.h.defaults index 9d5f4c402..fe5878aaf 100644 --- a/src/src/config.h.defaults +++ b/src/src/config.h.defaults @@ -177,6 +177,7 @@ it's a default value. */ #define EXPERIMENTAL_DCC #define EXPERIMENTAL_DSN_INFO #define EXPERIMENTAL_DMARC +#define EXPERIMENTAL_LMDB #define EXPERIMENTAL_SPF #define EXPERIMENTAL_SRS diff --git a/src/src/drtables.c b/src/src/drtables.c index c83012944..c807d86c7 100644 --- a/src/src/drtables.c +++ b/src/src/drtables.c @@ -500,6 +500,9 @@ extern lookup_module_info pgsql_lookup_module_info; #if defined(LOOKUP_REDIS) && LOOKUP_REDIS!=2 extern lookup_module_info redis_lookup_module_info; #endif +#if defined(EXPERIMENTAL_LMDB) +extern lookup_module_info lmdb_lookup_module_info; +#endif #if defined(EXPERIMENTAL_SPF) extern lookup_module_info spf_lookup_module_info; #endif @@ -585,6 +588,10 @@ init_lookup_list(void) addlookupmodule(NULL, &redis_lookup_module_info); #endif +#ifdef EXPERIMENTAL_LMDB + addlookupmodule(NULL, &lmdb_lookup_module_info); +#endif + #ifdef EXPERIMENTAL_SPF addlookupmodule(NULL, &spf_lookup_module_info); #endif diff --git a/src/src/exim.c b/src/src/exim.c index 14e0b9d67..1ad76dea2 100644 --- a/src/src/exim.c +++ b/src/src/exim.c @@ -842,6 +842,9 @@ fprintf(f, "Support for:"); #ifdef SUPPORT_SOCKS fprintf(f, " SOCKS"); #endif +#ifdef EXPERIMENTAL_LMDB + fprintf(f, " Experimental_LMDB"); +#endif #ifdef EXPERIMENTAL_SPF fprintf(f, " Experimental_SPF"); #endif @@ -887,6 +890,9 @@ fprintf(f, "Lookups (built-in):"); #if defined(LOOKUP_LDAP) && LOOKUP_LDAP!=2 fprintf(f, " ldap ldapdn ldapm"); #endif +#ifdef EXPERIMENTAL_LMDB + fprintf(f, " lmdb"); +#endif #if defined(LOOKUP_MYSQL) && LOOKUP_MYSQL!=2 fprintf(f, " mysql"); #endif diff --git a/src/src/lookups/Makefile b/src/src/lookups/Makefile index 6ba0cb169..7c9700690 100644 --- a/src/src/lookups/Makefile +++ b/src/src/lookups/Makefile @@ -34,6 +34,7 @@ dnsdb.o: $(PHDRS) dnsdb.c dsearch.o: $(PHDRS) dsearch.c ibase.o: $(PHDRS) ibase.c ldap.o: $(PHDRS) ldap.c +lmdb.o: $(PHDRS) lmdb.c lsearch.o: $(PHDRS) lsearch.c mysql.o: $(PHDRS) mysql.c nis.o: $(PHDRS) nis.c @@ -53,6 +54,7 @@ dnsdb.so: $(PHDRS) dnsdb.c dsearch.so: $(PHDRS) dsearch.c ibase.so: $(PHDRS) ibase.c ldap.so: $(PHDRS) ldap.c +lmdb.so: $(PHDRS) lmdb.c lsearch.so: $(PHDRS) lsearch.c mysql.so: $(PHDRS) mysql.c nis.so: $(PHDRS) nis.c diff --git a/src/src/lookups/lmdb.c b/src/src/lookups/lmdb.c new file mode 100644 index 000000000..8b0ffd2dd --- /dev/null +++ b/src/src/lookups/lmdb.c @@ -0,0 +1,160 @@ +/************************************************* +* Exim - an Internet mail transport agent * +*************************************************/ + +/* Copyright (c) University of Cambridge 2016 */ +/* See the file NOTICE for conditions of use and distribution. */ + +#include "../exim.h" + +#ifdef EXPERIMENTAL_LMDB + +#include + +typedef struct lmdbstrct +{ +MDB_txn *txn; +MDB_dbi db_dbi; +} Lmdbstrct; + + +/************************************************* +* Open entry point * +*************************************************/ + +static void * +lmdb_open(uschar * filename, uschar ** errmsg) +{ +MDB_env * db_env = NULL; +Lmdbstrct * lmdb_p; +int ret, save_errno; +const uschar * errstr; + +lmdb_p = store_get(sizeof(Lmdbstrct)); +lmdb_p->txn = NULL; + +if ((ret = mdb_env_create(&db_env))) + { + errstr = US"create environment"; + goto bad; + } + +if ((ret = mdb_env_open(db_env, CS filename, MDB_NOSUBDIR|MDB_RDONLY, 0660))) + { + errstr = US"open environment"; + goto bad; + } + +if ((ret = mdb_txn_begin(db_env, NULL, MDB_RDONLY, &lmdb_p->txn))) + { + errstr = US"start transaction"; + goto bad; + } + +if ((ret = mdb_open(lmdb_p->txn, NULL, 0, &lmdb_p->db_dbi))) + { + errstr = US"open database"; + goto bad; + } + +return lmdb_p; + +bad: + save_errno = errno; + if (lmdb_p->txn) mdb_txn_abort(lmdb_p->txn); + if (db_env) mdb_env_close(db_env); + *errmsg = string_sprintf("LMDB: Unable to %s: %s", errstr, mdb_strerror(ret)); + errno = save_errno; + return NULL; +} + + +/************************************************* +* Find entry point * +*************************************************/ + +static int +lmdb_find(void * handle, uschar * filename, + const uschar * keystring, int length, uschar ** result, uschar ** errmsg, + uint * do_cache) +{ +int ret; +MDB_val dbkey, data; +Lmdbstrct * lmdb_p = handle; + +dbkey.mv_data = CS keystring; +dbkey.mv_size = length; + +DEBUG(D_lookup) debug_printf("LMDB: lookup key: %s\n", (char *)keystring); + +if ((ret = mdb_get(lmdb_p->txn, lmdb_p->db_dbi, &dbkey, &data)) == 0) + { + *result = string_copyn(US data.mv_data, data.mv_size); + DEBUG(D_lookup) debug_printf("LMDB: lookup result: %s\n", *result); + return OK; + } +else if (ret == MDB_NOTFOUND) + { + *errmsg = US"LMDB: lookup, no data found"; + DEBUG(D_lookup) debug_printf("%s\n", *errmsg); + return FAIL; + } +else + { + *errmsg = string_sprintf("LMDB: lookup error: %s", mdb_strerror(ret)); + DEBUG(D_lookup) debug_printf("%s\n", *errmsg); + return DEFER; + } +} + + +/************************************************* +* Close entry point * +*************************************************/ + +static void +lmdb_close(void * handle) +{ +Lmdbstrct * lmdb_p = handle; +MDB_env * db_env = mdb_txn_env(lmdb_p->txn); +mdb_txn_abort(lmdb_p->txn); +mdb_env_close(db_env); +} + + +/************************************************* +* Version reporting entry point * +*************************************************/ + +#include "../version.h" + +void +lmdb_version_report(FILE * f) +{ +fprintf(f, "Library version: LMDB: Compile: %d.%d.%d\n", + MDB_VERSION_MAJOR, MDB_VERSION_MINOR, MDB_VERSION_PATCH); +#ifdef DYNLOOKUP +fprintf(f, " Exim version %s\n", EXIM_VERSION_STR); +#endif +} + +static lookup_info lmdb_lookup_info = { + US"lmdb", /* lookup name */ + lookup_absfile, /* query-style lookup */ + lmdb_open, /* open function */ + NULL, /* no check function */ + lmdb_find, /* find function */ + lmdb_close, /* close function */ + NULL, /* tidy function */ + NULL, /* quoting function */ + lmdb_version_report /* version reporting */ +}; + +#ifdef DYNLOOKUP +# define lmdb_lookup_module_info _lookup_module_info +#endif /* DYNLOOKUP */ + +static lookup_info *_lookup_list[] = { &lmdb_lookup_info }; +lookup_module_info lmdb_lookup_module_info = { LOOKUP_MODULE_INFO_MAGIC, _lookup_list, 1 }; + +#endif /* EXPERIMENTAL_LMDB */ diff --git a/test/aux-fixed/2800.lmdb-mkdb-dump.py b/test/aux-fixed/2800.lmdb-mkdb-dump.py new file mode 100644 index 000000000..3de6ba13b --- /dev/null +++ b/test/aux-fixed/2800.lmdb-mkdb-dump.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python +import os +import lmdb + +if os.path.exists('2800.mdb'): + os.unlink('2800.mdb') + +env = lmdb.open('2800.mdb', subdir=False); +with env.begin(write=True) as txn: + txn.put('first', 'data for first') + txn.put('second', 'A=1 B=2') + txn.put('third', 'A1:B2:C3') + cursor = txn.cursor() + for key, value in cursor: + print key, "=>", value diff --git a/test/aux-fixed/2800.mdb b/test/aux-fixed/2800.mdb new file mode 100644 index 0000000000000000000000000000000000000000..2002ee1b5f99b26703f97d928155370a88f7201c GIT binary patch literal 12288 zcmeI%F%E)25J1sUgHnl=HCF&T5)#3a5RphD32-2fL1jFFN3gQ@2pV@GAO#IEvG`lC zv&#U(>n1DG3{wU7>+>>cw$yyqtCfhk)#1?fRga;Z?!6}e`OZdw~?-RRu?|)&HcQw|HtzPAbkTQ7X&wiT7rNeVBMVj9T#Ca! cJH>(T*rD&rQtcX&q data for first +> data for first +> failure value +> Failed: "lookup" failed and "fail" requested +> A=1 B=2 +> 1 +> 2 +> Failed: "extract" failed and "fail" requested +> -- 2.30.2