LMDB: introduce as Experimental. Bug 1856

diff --git a/doc/doc-txt/NewStuff b/doc/doc-txt/NewStuff
index 719bedf3179f9e937de9e57eabfa86e77b847d98..cf1cf6d56e9f9dd7b304f026cd518c4e353c459b 100644 (file)
--- a/doc/doc-txt/NewStuff
+++ b/doc/doc-txt/NewStuff
@@ -33,6 +33,8 @@ Version 4.88
     chunking_advertise_hosts, and smtp transport option hosts_try_chunking
     for control.
 
+ 8. LMDB lookup support, as Experimental. Patch supplied by Andrew Colin Kissa.
+
 
 Version 4.87
 ------------

diff --git a/doc/doc-txt/experimental-spec.txt b/doc/doc-txt/experimental-spec.txt
index 993b5b05cfdd260d30608e045b565ada6c70fc2e..2e72def272bf92d384ad3fb6f5d7f201853a9736 100644 (file)
--- a/doc/doc-txt/experimental-spec.txt
+++ b/doc/doc-txt/experimental-spec.txt
@@ -967,6 +967,50 @@ Rationale:
 Note that non-RFC-documented field names and data types are used.
 
 
+LMDB Lookup support
+-------------------
+LMDB is an ultra-fast, ultra-compact, crash-proof key-value embedded data store.
+It is modeled loosely on the BerkeleyDB API. You shoul read about the feature
+set as well as operation modes at https://symas.com/products/lightning-memory-mapped-database/
+
+LMDB single key lookup support is provided by linking to the LMDB C library.
+The current implementation does not support writing to the LMDB database.
+
+Visit https://github.com/LMDB/lmdb to download the library or find it in your
+operating systems package repository.
+
+If building from source, this description assumes that headers will be in
+/usr/local/include, and that the libraries are in /usr/local/lib.
+
+1. In order to build exim with LMDB lookup support add or uncomment
+
+EXPERIMENTAL_LMDB=yes
+
+to your Local/Makefile. (Re-)build/install exim. exim -d should show
+Experimental_LMDB in the line "Support for:".
+
+EXPERIMENTAL_LMDB=yes
+LDFLAGS += -llmdb
+# CFLAGS += -I/usr/local/include
+# LDFLAGS += -L/usr/local/lib
+
+The first line sets the feature to include the correct code, and
+the second line says to link the LMDB libraries into the
+exim binary.  The commented out lines should be uncommented if you
+built LMDB from source and installed in the default location.
+Adjust the paths if you installed them elsewhere, but you do not
+need to uncomment them if an rpm (or you) installed them in the
+package controlled locations (/usr/include and /usr/lib).
+
+2. Create your LMDB files, you can use the mdb_load utility which is
+part of the LMDB distribution our your favourite language bindings.
+
+3. Add the single key lookups to your exim.conf file, example lookups
+are below.
+
+${lookup{$sender_address_domain}lmdb{/var/lib/baruwa/data/db/relaydomains.mdb}{$value}}
+${lookup{$sender_address_domain}lmdb{/var/lib/baruwa/data/db/relaydomains.mdb}{$value}fail}
+${lookup{$sender_address_domain}lmdb{/var/lib/baruwa/data/db/relaydomains.mdb}}
 
 
 --------------------------------------------------------------

diff --git a/src/scripts/MakeLinks b/src/scripts/MakeLinks
index 7a5649ef8af17e8771d0425170a3100f3f6ea6ac..86f748c1524023f8aa0404781dd1fe8bc60ea090 100755 (executable)
--- a/src/scripts/MakeLinks
+++ b/src/scripts/MakeLinks
@@ -31,7 +31,7 @@ mkdir lookups
 cd lookups
 # Makefile is generated
 for f in README cdb.c dbmdb.c dnsdb.c dsearch.c ibase.c ldap.h ldap.c \
-  lsearch.c mysql.c redis.c nis.c nisplus.c oracle.c passwd.c \
+  lmdb.c lsearch.c mysql.c redis.c nis.c nisplus.c oracle.c passwd.c \
   pgsql.c spf.c sqlite.c testdb.c whoson.c \
   lf_functions.h lf_check_file.c lf_quote.c lf_sqlperform.c
 do

diff --git a/src/scripts/lookups-Makefile b/src/scripts/lookups-Makefile
index 4e69a9a775afa014465ab92e70b7b9c778f82870..db2d184a731a6995e89ee0c281893a41f5c8da6f 100755 (executable)
--- a/src/scripts/lookups-Makefile
+++ b/src/scripts/lookups-Makefile
@@ -177,6 +177,11 @@ fi
 
 OBJ="${OBJ} spf.o"
 
+if want_experimental LMDB
+then
+  OBJ="${OBJ} lmdb.o"
+fi
+
 echo "MODS = $MODS"
 echo "OBJ = $OBJ"
 

diff --git a/src/src/EDITME b/src/src/EDITME
index ac8c642bcf02800445ef2dc534755335208d9eb0..10e5ccacb80cab7ba7bb65f3d51fc7ef25bef40a 100644 (file)
--- a/src/src/EDITME
+++ b/src/src/EDITME
@@ -488,6 +488,13 @@ EXIM_MONITOR=eximon.bin
 # Uncomment the following to include extra information in fail DSN message (bounces)
 # EXPERIMENTAL_DSN_INFO=yes
 
+# Uncomment the following to add LMDB lookup support
+# You need to have LMDB installed on your system (https://github.com/LMDB/lmdb)
+# Depending on where it is installed you may have to edit the CFLAGS and LDFLAGS lines.
+# EXPERIMENTAL_LMDB=yes
+# CFLAGS += -I/usr/local/include
+# LDFLAGS += -llmdb
+
 ###############################################################################
 #                 THESE ARE THINGS YOU MIGHT WANT TO SPECIFY                  #
 ###############################################################################

diff --git a/src/src/config.h.defaults b/src/src/config.h.defaults
index 9d5f4c402eec5ac4dbf25e0536c7c8a53b588374..fe5878aafe9cf27ce19bb3dc6b7ad2527f71a7c5 100644 (file)
--- a/src/src/config.h.defaults
+++ b/src/src/config.h.defaults
@@ -177,6 +177,7 @@ it's a default value. */
 #define EXPERIMENTAL_DCC
 #define EXPERIMENTAL_DSN_INFO
 #define EXPERIMENTAL_DMARC
+#define EXPERIMENTAL_LMDB
 #define EXPERIMENTAL_SPF
 #define EXPERIMENTAL_SRS
 

diff --git a/src/src/drtables.c b/src/src/drtables.c
index c83012944398c7aca717526dae5579fc5333d2df..c807d86c7ebb64b45793fe54615847bbd6537fdd 100644 (file)
--- a/src/src/drtables.c
+++ b/src/src/drtables.c
@@ -500,6 +500,9 @@ extern lookup_module_info pgsql_lookup_module_info;
 #if defined(LOOKUP_REDIS) && LOOKUP_REDIS!=2
 extern lookup_module_info redis_lookup_module_info;
 #endif
+#if defined(EXPERIMENTAL_LMDB)
+extern lookup_module_info lmdb_lookup_module_info;
+#endif
 #if defined(EXPERIMENTAL_SPF)
 extern lookup_module_info spf_lookup_module_info;
 #endif
@@ -585,6 +588,10 @@ init_lookup_list(void)
   addlookupmodule(NULL, &redis_lookup_module_info);
 #endif
 
+#ifdef EXPERIMENTAL_LMDB
+  addlookupmodule(NULL, &lmdb_lookup_module_info);
+#endif
+
 #ifdef EXPERIMENTAL_SPF
   addlookupmodule(NULL, &spf_lookup_module_info);
 #endif

diff --git a/src/src/exim.c b/src/src/exim.c
index 14e0b9d67d76a49f2cda24ed9c0a6975b7998ef3..1ad76dea236dba748ef2f210c3230f1f05df0d80 100644 (file)
--- a/src/src/exim.c
+++ b/src/src/exim.c
@@ -842,6 +842,9 @@ fprintf(f, "Support for:");
 #ifdef SUPPORT_SOCKS
   fprintf(f, " SOCKS");
 #endif
+#ifdef EXPERIMENTAL_LMDB
+  fprintf(f, " Experimental_LMDB");
+#endif
 #ifdef EXPERIMENTAL_SPF
   fprintf(f, " Experimental_SPF");
 #endif
@@ -887,6 +890,9 @@ fprintf(f, "Lookups (built-in):");
 #if defined(LOOKUP_LDAP) && LOOKUP_LDAP!=2
   fprintf(f, " ldap ldapdn ldapm");
 #endif
+#ifdef EXPERIMENTAL_LMDB
+  fprintf(f, " lmdb");
+#endif
 #if defined(LOOKUP_MYSQL) && LOOKUP_MYSQL!=2
   fprintf(f, " mysql");
 #endif

diff --git a/src/src/lookups/Makefile b/src/src/lookups/Makefile
index 6ba0cb1691663d6e6665b93acbc242515baa3888..7c9700690580e3e878ecbb34235a2db41e739ebf 100644 (file)
--- a/src/src/lookups/Makefile
+++ b/src/src/lookups/Makefile
@@ -34,6 +34,7 @@ dnsdb.o:         $(PHDRS) dnsdb.c
 dsearch.o:       $(PHDRS) dsearch.c
 ibase.o:         $(PHDRS) ibase.c
 ldap.o:          $(PHDRS) ldap.c
+lmdb.o:          $(PHDRS) lmdb.c
 lsearch.o:       $(PHDRS) lsearch.c
 mysql.o:         $(PHDRS) mysql.c
 nis.o:           $(PHDRS) nis.c
@@ -53,6 +54,7 @@ dnsdb.so:         $(PHDRS) dnsdb.c
 dsearch.so:       $(PHDRS) dsearch.c
 ibase.so:         $(PHDRS) ibase.c
 ldap.so:          $(PHDRS) ldap.c
+lmdb.so:          $(PHDRS) lmdb.c
 lsearch.so:       $(PHDRS) lsearch.c
 mysql.so:         $(PHDRS) mysql.c
 nis.so:           $(PHDRS) nis.c

diff --git a/src/src/lookups/lmdb.c b/src/src/lookups/lmdb.c
new file mode 100644 (file)
index 0000000..8b0ffd2
--- /dev/null
+++ b/src/src/lookups/lmdb.c
@@ -0,0 +1,160 @@
+/*************************************************
+*     Exim - an Internet mail transport agent    *
+*************************************************/
+
+/* Copyright (c) University of Cambridge 2016 */
+/* See the file NOTICE for conditions of use and distribution. */
+
+#include "../exim.h"
+
+#ifdef EXPERIMENTAL_LMDB
+
+#include <lmdb.h>
+
+typedef struct lmdbstrct
+{
+MDB_txn *txn;
+MDB_dbi db_dbi;
+} Lmdbstrct;
+
+
+/*************************************************
+*              Open entry point                  *
+*************************************************/
+
+static void *
+lmdb_open(uschar * filename, uschar ** errmsg)
+{
+MDB_env * db_env = NULL;
+Lmdbstrct * lmdb_p;
+int ret, save_errno;
+const uschar * errstr;
+
+lmdb_p = store_get(sizeof(Lmdbstrct));
+lmdb_p->txn = NULL;
+
+if ((ret = mdb_env_create(&db_env)))
+  {
+  errstr = US"create environment";
+  goto bad;
+  }
+
+if ((ret = mdb_env_open(db_env, CS filename, MDB_NOSUBDIR|MDB_RDONLY, 0660)))
+  {
+  errstr = US"open environment";
+  goto bad;
+  }
+
+if ((ret = mdb_txn_begin(db_env, NULL, MDB_RDONLY, &lmdb_p->txn)))
+  {
+  errstr = US"start transaction";
+  goto bad;
+  }
+
+if ((ret = mdb_open(lmdb_p->txn, NULL, 0, &lmdb_p->db_dbi)))
+  {
+  errstr = US"open database";
+  goto bad;
+  }
+
+return lmdb_p;
+
+bad:
+  save_errno = errno;
+  if (lmdb_p->txn) mdb_txn_abort(lmdb_p->txn);
+  if (db_env) mdb_env_close(db_env);
+  *errmsg = string_sprintf("LMDB: Unable to %s: %s", errstr,  mdb_strerror(ret));
+  errno = save_errno;
+  return NULL;
+}
+
+
+/*************************************************
+*              Find entry point                  *
+*************************************************/
+
+static int
+lmdb_find(void * handle, uschar * filename,
+    const uschar * keystring, int length, uschar ** result, uschar ** errmsg,
+    uint * do_cache)
+{
+int ret;
+MDB_val dbkey, data;
+Lmdbstrct * lmdb_p = handle;
+
+dbkey.mv_data = CS keystring;
+dbkey.mv_size = length;
+
+DEBUG(D_lookup) debug_printf("LMDB: lookup key: %s\n", (char *)keystring);
+
+if ((ret = mdb_get(lmdb_p->txn, lmdb_p->db_dbi, &dbkey, &data)) == 0)
+  {
+  *result = string_copyn(US data.mv_data, data.mv_size);
+  DEBUG(D_lookup) debug_printf("LMDB: lookup result: %s\n", *result);
+  return OK;
+  }
+else if (ret == MDB_NOTFOUND)
+  {
+  *errmsg = US"LMDB: lookup, no data found";
+  DEBUG(D_lookup) debug_printf("%s\n", *errmsg);
+  return FAIL;
+  }
+else
+  {
+  *errmsg = string_sprintf("LMDB: lookup error: %s", mdb_strerror(ret));
+  DEBUG(D_lookup) debug_printf("%s\n", *errmsg);
+  return DEFER;
+  }
+}
+
+
+/*************************************************
+*              Close entry point                 *
+*************************************************/
+
+static void
+lmdb_close(void * handle)
+{
+Lmdbstrct * lmdb_p = handle;
+MDB_env * db_env = mdb_txn_env(lmdb_p->txn);
+mdb_txn_abort(lmdb_p->txn);
+mdb_env_close(db_env);
+}
+
+
+/*************************************************
+*         Version reporting entry point          *
+*************************************************/
+
+#include "../version.h"
+
+void
+lmdb_version_report(FILE * f)
+{
+fprintf(f, "Library version: LMDB: Compile: %d.%d.%d\n",
+    MDB_VERSION_MAJOR, MDB_VERSION_MINOR, MDB_VERSION_PATCH);
+#ifdef DYNLOOKUP
+fprintf(f, "                        Exim version %s\n", EXIM_VERSION_STR);
+#endif
+}
+
+static lookup_info lmdb_lookup_info = {
+  US"lmdb",                     /* lookup name */
+  lookup_absfile,               /* query-style lookup */
+  lmdb_open,                    /* open function */
+  NULL,                         /* no check function */
+  lmdb_find,                    /* find function */
+  lmdb_close,                   /* close function */
+  NULL,                         /* tidy function */
+  NULL,                         /* quoting function */
+  lmdb_version_report           /* version reporting */
+};
+
+#ifdef DYNLOOKUP
+# define lmdb_lookup_module_info _lookup_module_info
+#endif /* DYNLOOKUP */
+
+static lookup_info *_lookup_list[] = { &lmdb_lookup_info };
+lookup_module_info lmdb_lookup_module_info = { LOOKUP_MODULE_INFO_MAGIC, _lookup_list, 1 };
+
+#endif /* EXPERIMENTAL_LMDB */

diff --git a/test/aux-fixed/2800.lmdb-mkdb-dump.py b/test/aux-fixed/2800.lmdb-mkdb-dump.py
new file mode 100644 (file)
index 0000000..3de6ba1
--- /dev/null
+++ b/test/aux-fixed/2800.lmdb-mkdb-dump.py
@@ -0,0 +1,15 @@
+#!/usr/bin/env python
+import os
+import lmdb
+
+if os.path.exists('2800.mdb'):
+    os.unlink('2800.mdb')
+
+env = lmdb.open('2800.mdb', subdir=False);
+with env.begin(write=True) as txn:
+   txn.put('first', 'data for first')
+   txn.put('second', 'A=1 B=2')
+   txn.put('third', 'A1:B2:C3')
+   cursor = txn.cursor()
+   for key, value in cursor:
+       print key, "=>", value

diff --git a/test/aux-fixed/2800.mdb b/test/aux-fixed/2800.mdb
new file mode 100644 (file)
index 0000000..2002ee1
Binary files /dev/null and b/test/aux-fixed/2800.mdb differ

diff --git a/test/aux-fixed/2800.mdb.src b/test/aux-fixed/2800.mdb.src
new file mode 100644 (file)
index 0000000..9579527
--- /dev/null
+++ b/test/aux-fixed/2800.mdb.src
@@ -0,0 +1,3 @@
+first: data for first
+second: A=1 B=2
+third: A1:B2:C3

diff --git a/test/confs/2800 b/test/confs/2800
new file mode 100644 (file)
index 0000000..607fcf5
--- /dev/null
+++ b/test/confs/2800
@@ -0,0 +1,9 @@
+# Exim test configuration 2800
+
+.include DIR/aux-var/std_conf_prefix
+
+primary_hostname = myhost.test.ex
+
+# ----- Main settings -----
+
+# End

diff --git a/test/scripts/2800-lmdb/2800 b/test/scripts/2800-lmdb/2800
new file mode 100644 (file)
index 0000000..a074ae6
--- /dev/null
+++ b/test/scripts/2800-lmdb/2800
@@ -0,0 +1,11 @@
+# lmdb lookup
+exim -be
+${lookup{first}lmdb{DIR/aux-fixed/TESTNUM.mdb}}
+${lookup{first}lmdb{DIR/aux-fixed/TESTNUM.mdb}{$value}fail}
+${lookup{fail}lmdb{DIR/aux-fixed/TESTNUM.mdb}{$value}{failure value}}
+${lookup{fail}lmdb{DIR/aux-fixed/TESTNUM.mdb}{$value}fail}
+${lookup{second}lmdb{DIR/aux-fixed/TESTNUM.mdb}{$value}fail}
+${extract{A}{${lookup{second}lmdb{DIR/aux-fixed/TESTNUM.mdb}{$value}fail}}}
+${extract{B}{${lookup{second}lmdb{DIR/aux-fixed/TESTNUM.mdb}{$value}fail}}}
+${extract{C}{${lookup{second}lmdb{DIR/aux-fixed/TESTNUM.mdb}{$value}fail}}{$value}fail}
+****

diff --git a/test/scripts/2800-lmdb/REQUIRES b/test/scripts/2800-lmdb/REQUIRES
new file mode 100644 (file)
index 0000000..c160c46
--- /dev/null
+++ b/test/scripts/2800-lmdb/REQUIRES
@@ -0,0 +1 @@
+lookup lmdb

diff --git a/test/stdout/2800 b/test/stdout/2800
new file mode 100644 (file)
index 0000000..30e14b0
--- /dev/null
+++ b/test/stdout/2800
@@ -0,0 +1,9 @@
+> data for first
+> data for first
+> failure value
+> Failed: "lookup" failed and "fail" requested
+> A=1 B=2
+> 1
+> 2
+> Failed: "extract" failed and "fail" requested
+>