From 0a34949459c8ec5f79599a458704b7b11cdbb248 Mon Sep 17 00:00:00 2001 From: Phil Pennock Date: Sun, 16 Jan 2011 02:15:53 -0500 Subject: [PATCH] Bug 139: portability fixes and documentation. Document the dynamic lookup module capability in spec.xfpt. Include a ChangeLog item. Avoid the GNU-specific "export" make(1) directive. Build the lookups Makefile using the existing framework. Build with BSD Make once more. The src/lookups/Makefile that is used at build time now has the dynamic content come from scripts/lookups-Makefile. Add CFLAGS_DYNAMIC support, which can be set in Local/Makefile. Provide defaults for Linux & FreeBSD. Ensure that build fails early if a dynamic module is requested but CFLAGS_DYNAMIC is not defined. --- doc/doc-docbook/spec.xfpt | 37 ++++++++ doc/doc-txt/ChangeLog | 8 ++ src/Makefile | 6 +- src/OS/Makefile-Base | 1 + src/OS/Makefile-FreeBSD | 3 + src/OS/Makefile-Linux | 1 + src/scripts/Configure-Makefile | 21 +++-- src/scripts/MakeLinks | 2 +- src/scripts/lookups-Makefile | 85 ++++++++++++++++++ src/src/EDITME | 5 ++ src/src/lookups/Makefile | 160 ++------------------------------- 11 files changed, 167 insertions(+), 162 deletions(-) create mode 100755 src/scripts/lookups-Makefile diff --git a/doc/doc-docbook/spec.xfpt b/doc/doc-docbook/spec.xfpt index 66dfab7cd..15b3a2b89 100644 --- a/doc/doc-docbook/spec.xfpt +++ b/doc/doc-docbook/spec.xfpt @@ -1946,6 +1946,36 @@ support has not been tested for some time. +.section "Dynamically loaded lookup module support" "SECTdynamicmodules" +.cindex "lookup modules" +.cindex "dynamic modules" +.cindex ".so building" +On some platforms, Exim supports not compiling all lookup types directly into +the main binary, instead putting some into external modules which can be loaded +on demand. +This permits packagers to build Exim with support for lookups with extensive +library dependencies without requiring all users to install all of those +dependencies. +Most, but not all, lookup types can be built this way. + +Set &`LOOKUP_MODULE_DIR`& to the directory into which the modules will be +installed; Exim will only load modules from that directory, as a security +measure. You will need to set &`CFLAGS_DYNAMIC`& if not already defined +for your OS; see &_OS/Makefile-Linux_& for an example. +Some other requirements for adjusting &`EXTRALIBS`& may also be necessary, +see &_src/EDITME_& for details. + +Then, for each module to be loaded dynamically, define the relevant +&`LOOKUP_`&<&'lookup_type'&> flags to have the value "2" instead of "yes". +For example, this will build in lsearch but load sqlite and mysql support +on demand: +.code +LOOKUP_LSEARCH=yes +LOOKUP_SQLITE=2 +LOOKUP_MYSQL=2 +.endd + + .section "The building process" "SECID29" .cindex "build directory" Once &_Local/Makefile_& (and &_Local/eximon.conf_&, if required) have been @@ -34208,6 +34238,13 @@ arbitrary program's being run as exim, not as root. +.section "Dynamic module directory" "SECTdynmoddir" +Any dynamically loadable modules must be installed into the directory +defined in &`LOOKUP_MODULE_DIR`& in &_Local/Makefile_& for Exim to permit +loading it. + + + .section "Use of sprintf()" "SECID279" .cindex "&[sprintf()]&" A large number of occurrences of &"sprintf"& in the code are actually calls to diff --git a/doc/doc-txt/ChangeLog b/doc/doc-txt/ChangeLog index ef82a0cc9..e27496b75 100644 --- a/doc/doc-txt/ChangeLog +++ b/doc/doc-txt/ChangeLog @@ -14,6 +14,14 @@ TF/02 Log LMTP confirmation messages in the same way as SMTP, TF/03 Include the error message when we fail to unlink a spool file. +DW/01 Bugzilla 139: Support dynamically loaded lookups as modules. + With thanks to Steve Haslam, Johannes Berg & Serge Demonchaux + for maintaining out-of-tree patches for some time. + +PP/01 Bugzilla 139: Documentation and portability issues. + Avoid GNU Makefile-isms, let Exim continue to build on BSD. + Handle per-OS dynamic-module compilation flags. + Exim version 4.73 ----------------- diff --git a/src/Makefile b/src/Makefile index eb9df5015..8ad750d74 100644 --- a/src/Makefile +++ b/src/Makefile @@ -54,14 +54,16 @@ build-directory: configure: build-directory @cd build-$(buildname); \ - build=$(build) $(SHELL) ../scripts/Configure-Makefile + build=$(build) $(SHELL) ../scripts/Configure-Makefile; \ + $(SHELL) ../scripts/lookups-Makefile # The "makefile" target forces a rebuild of the makefile (as opposed to # "configure", which doesn't force it). makefile: build-directory @cd build-$(buildname); $(RM_COMMAND) -f Makefile; \ - build=$(build) $(SHELL) ../scripts/Configure-Makefile + build=$(build) $(SHELL) ../scripts/Configure-Makefile; \ + $(SHELL) ../scripts/lookups-Makefile # The installation commands are kept in a separate script, which expects # to be run from inside the build directory. diff --git a/src/OS/Makefile-Base b/src/OS/Makefile-Base index 5793869e2..29a6ad371 100644 --- a/src/OS/Makefile-Base +++ b/src/OS/Makefile-Base @@ -641,6 +641,7 @@ $(MONBIN): $(HDRS) buildlookups lookups/lookups.a: config.h @cd lookups; $(MAKE) SHELL=$(SHELL) AR="$(AR)" $(MFLAGS) CC="$(CC)" CFLAGS="$(CFLAGS)" \ + CFLAGS_DYNAMIC="$(CFLAGS_DYNAMIC)" \ FE="$(FE)" RANLIB="$(RANLIB)" RM_COMMAND="$(RM_COMMAND)" HDRS="$(PHDRS)" \ INCLUDE="$(INCLUDE) $(IPV6_INCLUDE) $(TLS_INCLUDE) $(LOOKUP_INCLUDE)"; \ echo " " diff --git a/src/OS/Makefile-FreeBSD b/src/OS/Makefile-FreeBSD index c0f75ee78..3a2697bba 100644 --- a/src/OS/Makefile-FreeBSD +++ b/src/OS/Makefile-FreeBSD @@ -15,6 +15,9 @@ HAVE_SA_LEN=YES # crypt() is in a separate library LIBS=-lcrypt -lm -lutil +# Dynamicly loaded modules need to be built with -fPIC +CFLAGS_DYNAMIC=-shared -rdynamic -fPIC + # FreeBSD always ships with Berkeley DB USE_DB=yes diff --git a/src/OS/Makefile-Linux b/src/OS/Makefile-Linux index cc8dce728..4fe2436a6 100644 --- a/src/OS/Makefile-Linux +++ b/src/OS/Makefile-Linux @@ -11,6 +11,7 @@ CHGRP_COMMAND=look_for_it CHMOD_COMMAND=look_for_it CFLAGS=-O -D_FILE_OFFSET_BITS=64 -D_LARGEFILE_SOURCE +CFLAGS_DYNAMIC=-shared -rdynamic DBMLIB = -ldb USE_DB = yes diff --git a/src/scripts/Configure-Makefile b/src/scripts/Configure-Makefile index 1b2ea1e28..abef50017 100755 --- a/src/scripts/Configure-Makefile +++ b/src/scripts/Configure-Makefile @@ -78,10 +78,13 @@ mf=Makefile mft=$mf-t mftt=$mf-tt +look_mf=lookups/Makefile.predynamic +look_mft=${look_mf}-t + # Ensure the temporary does not exist and start the new one by setting # the OSTYPE and ARCHTYPE variables. -rm -f $mft $mftt +rm -f $mft $mftt $look_mf-t (echo "OSTYPE=$ostype"; echo "ARCHTYPE=$archtype"; echo "") > $mft || exit 1 # Now concatenate the files to the temporary file. Copy the files using sed to @@ -107,7 +110,13 @@ do if test -r ../$f echo "# End of $f" echo "" fi -done | sed 's/^LOOKUP_/export LOOKUP_/' >> $mft || exit 1 +done >> $mft || exit 1 + +# make the lookups Makefile with the definitions + +## prepend stuff here; eg: grep LOOKUP_ $mft > $look_mft +## cat ../src/lookups/Makefile >> $look_mft +cp ../src/lookups/Makefile $look_mft # See if there is a definition of EXIM_PERL in what we have built so far. # If so, run Perl to find the default values for PERL_CC, PERL_CCOPTS, @@ -158,18 +167,20 @@ cat ../OS/Makefile-Base >> $mft || exit 1 # If the new makefile is the same as the existing one, say so, and just # update the timestamp. Otherwise remove the old and install the new. -if [ -s $mf ] && cmp -s $mft $mf +if [ -s $mf ] && cmp -s $mft $mf && [ -s $look_mf ] && cmp -s $look_mft $look_mf then echo ">>> rebuilt $mf unchanged" echo " " touch $mf || exit rm -f $mft -elif rm -f $mf +elif rm -f $mf $look_mf mv $mft $mf -then echo ">>> New $mf installed" + mv $look_mft $look_mf +then echo ">>> New $mf & $look_mf installed" echo '>>> Use "make makefile" if you need to force rebuilding of the makefile' echo " " else echo " " echo "*** Failed to install $mf - see $mft" + echo " (or $look_mft)" echo " " exit 1; fi diff --git a/src/scripts/MakeLinks b/src/scripts/MakeLinks index 591813971..c021ace0e 100755 --- a/src/scripts/MakeLinks +++ b/src/scripts/MakeLinks @@ -31,7 +31,7 @@ echo ">>> Creating links to source files..." mkdir lookups cd lookups ln -s ../../src/lookups/README README -ln -s ../../src/lookups/Makefile Makefile +# Makefile is generated ln -s ../../src/lookups/cdb.c cdb.c ln -s ../../src/lookups/dbmdb.c dbmdb.c ln -s ../../src/lookups/dnsdb.c dnsdb.c diff --git a/src/scripts/lookups-Makefile b/src/scripts/lookups-Makefile new file mode 100755 index 000000000..7069cfb69 --- /dev/null +++ b/src/scripts/lookups-Makefile @@ -0,0 +1,85 @@ +#! /bin/sh + +# We turn the configure-built build-$foo/lookups/Makefile.predynamic into Makefile + +input=lookups/Makefile.predynamic +target=lookups/Makefile +defs_source=Makefile +tag_marker='MAGIC-TAG-MODS-OBJ-RULES-GO-HERE' + +tab=' ' +if grep -q "^LOOKUP.*=[ $tab]*2" "$defs_source" +then + # we have work to do +else + echo "No dynamic module loading support" + cp "$input" "$target" + exit 0 +fi + +if grep -q "^CFLAGS_DYNAMIC[ $tab]*=" "$defs_source" +then + # we have a definition, we're good to go +else + echo >&2 "Missing CFLAGS_DYNAMIC inhibits building dynamic module lookup" + exit 1 +fi + +tmp="$target.t" + +want_dynamic() { + local dyn_name="$1" + grep -q "^LOOKUP_${dyn_name}[ $tab]*=[ $tab]*2" "$defs_source" +} + +want_at_all() { + local want_name="$1" + grep -q "^LOOKUP_${want_name}[ $tab]*=[ $tab]*." "$defs_source" +} + +emit_module_rule() { + local lookup_name="$1" + local mod_name + if [ "${lookup_name%:*}" = "$lookup_name" ] + then + mod_name=$(echo $lookup_name | tr A-Z a-z) + else + mod_name="${lookup_name#*:}" + lookup_name="${lookup_name%:*}" + fi + + if want_dynamic "$lookup_name" + then + echo "MODS += ${mod_name}.so" + echo "LOOKUP_${mod_name}_INCLUDE = \$(LOOKUP_${lookup_name}_INCLUDE)" + echo "LOOKUP_${mod_name}_LIBS = \$(LOOKUP_${lookup_name}_LIBS)" + elif want_at_all "$lookup_name" + then + echo "OBJ += ${mod_name}.o" + fi +} + +exec 5>&1 +exec > "$tmp" + +sed -n "1,/$tag_marker/p" < "$input" + +for name_mod in \ + CDB DBM:dbmdb DNSDB DSEARCH IBASE LSEARCH MYSQL NIS NISPLUS ORACLE \ + PASSWD PGSQL SQLITE TESTDB WHOSON +do + emit_module_rule $name_mod +done + +if want_at_all LDAP +then + echo "OBJ += ldap.o" +fi + +sed -n "/$tag_marker/,\$p" < "$input" + +exec >&5 +mv "$tmp" "$target" + + +# vim: set ft=sh sw=2 : diff --git a/src/src/EDITME b/src/src/EDITME index be15e4ec9..74c507d3a 100644 --- a/src/src/EDITME +++ b/src/src/EDITME @@ -252,6 +252,11 @@ TRANSPORT_SMTP=yes # See below for dynamic lookup modules. # LOOKUP_MODULE_DIR=/usr/lib/exim/lookups/ +# To build a module dynamically, you'll need to define CFLAGS_DYNAMIC for +# your platform. Eg: +# CFLAGS_DYNAMIC=-shared -rdynamic +# CFLAGS_DYNAMIC=-shared -rdynamic -fPIC + #------------------------------------------------------------------------------ # These settings determine which file and database lookup methods are included # in the binary. See the manual chapter entitled "File and database lookups" diff --git a/src/src/lookups/Makefile b/src/src/lookups/Makefile index 76e56da61..623f24f96 100644 --- a/src/src/lookups/Makefile +++ b/src/src/lookups/Makefile @@ -1,4 +1,8 @@ # $Cambridge: exim/src/src/lookups/Makefile,v 1.9 2009/06/10 07:34:05 tom Exp $ +# +# nb: at build time, the version of this file used will have had some +# extra variable definitions and prepended to it and module build rules +# interpolated below. # Make file for building all the available lookups. # This is called from the main make file, after cd'ing @@ -10,159 +14,7 @@ OBJ=spf.o MODS= -ifeq ($(LOOKUP_CDB),2) -MODS += cdb.so -LOOKUP_cdb_INCLUDE = $(LOOKUP_CDB_INCLUDE) -LOOKUP_cdb_LIBS = $(LOOKUP_CDB_LIBS) -else -ifneq ($(LOOKUP_CDB),) -OBJ += cdb.o -endif -endif - -ifeq ($(LOOKUP_DBM),2) -MODS += dbmdb.so -LOOKUP_dbmdb_INCLUDE = $(LOOKUP_DBM_INCLUDE) -LOOKUP_dbmdb_LIBS = $(LOOKUP_DBM_LIBS) -else -ifneq ($(LOOKUP_DBM),) -OBJ += dbmdb.o -endif -endif - -ifeq ($(LOOKUP_DNSDB),2) -MODS += dnsdb.so -LOOKUP_dnsdb_INCLUDE = $(LOOKUP_DNSDB_INCLUDE) -LOOKUP_dnsdb_LIBS = $(LOOKUP_DNSDB_LIBS) -else -ifneq ($(LOOKUP_DNSDB),) -OBJ += dnsdb.o -endif -endif - -ifeq ($(LOOKUP_DSEARCH),2) -MODS += dsearch.so -LOOKUP_dsearch_INCLUDE = $(LOOKUP_DSEARCH_INCLUDE) -LOOKUP_dsearch_LIBS = $(LOOKUP_DSEARCH_LIBS) -else -ifneq ($(LOOKUP_DSEARCH),) -OBJ += dsearch.o -endif -endif - -ifeq ($(LOOKUP_IBASE),2) -MODS += ibase.so -LOOKUP_ibase_INCLUDE = $(LOOKUP_IBASE_INCLUDE) -LOOKUP_ibase_LIBS = $(LOOKUP_IBASE_LIBS) -else -ifneq ($(LOOKUP_IBASE),) -OBJ += ibase.o -endif -endif - -ifneq ($(LOOKUP_LDAP),) -OBJ += ldap.o -endif - -ifeq ($(LOOKUP_LSEARCH),2) -MODS += lsearch.so -LOOKUP_lsearch_INCLUDE = $(LOOKUP_LSEARCH_INCLUDE) -LOOKUP_lsearch_LIBS = $(LOOKUP_LSEARCH_LIBS) -else -ifneq ($(LOOKUP_LSEARCH),) -OBJ += lsearch.o -endif -endif - -ifeq ($(LOOKUP_MYSQL),2) -MODS += mysql.so -LOOKUP_mysql_INCLUDE = $(LOOKUP_MYSQL_INCLUDE) -LOOKUP_mysql_LIBS = $(LOOKUP_MYSQL_LIBS) -else -ifneq ($(LOOKUP_MYSQL),) -OBJ += mysql.o -endif -endif - -ifeq ($(LOOKUP_NIS),2) -MODS += nis.so -LOOKUP_nis_INCLUDE = $(LOOKUP_NIS_INCLUDE) -LOOKUP_nis_LIBS = $(LOOKUP_NIS_LIBS) -else -ifneq ($(LOOKUP_NIS),) -OBJ += nis.o -endif -endif - -ifeq ($(LOOKUP_NISPLUS),2) -MODS += nisplus.so -LOOKUP_nisplus_INCLUDE = $(LOOKUP_NISPLUS_INCLUDE) -LOOKUP_nisplus_LIBS = $(LOOKUP_NISPLUS_LIBS) -else -ifneq ($(LOOKUP_NISPLUS),) -OBJ += nisplus.o -endif -endif - -ifeq ($(LOOKUP_ORACLE),2) -MODS += oracle.so -LOOKUP_oracle_INCLUDE = $(LOOKUP_ORACLE_INCLUDE) -LOOKUP_oracle_LIBS = $(LOOKUP_ORACLE_LIBS) -else -ifneq ($(LOOKUP_ORACLE),) -OBJ += oracle.o -endif -endif - -ifeq ($(LOOKUP_PASSWD),2) -MODS += passwd.so -LOOKUP_passwd_INCLUDE = $(LOOKUP_PASSWD_INCLUDE) -LOOKUP_passwd_LIBS = $(LOOKUP_PASSWD_LIBS) -else -ifneq ($(LOOKUP_PASSWD),) -OBJ += passwd.o -endif -endif - -ifeq ($(LOOKUP_PGSQL),2) -MODS += pgsql.so -LOOKUP_pgsql_INCLUDE = $(LOOKUP_PGSQL_INCLUDE) -LOOKUP_pgsql_LIBS = $(LOOKUP_PGSQL_LIBS) -else -ifneq ($(LOOKUP_PGSQL),) -OBJ += pgsql.o -endif -endif - -ifeq ($(LOOKUP_SQLITE),2) -MODS += sqlite.so -LOOKUP_sqlite_INCLUDE = $(LOOKUP_SQLITE_INCLUDE) -LOOKUP_sqlite_LIBS = $(LOOKUP_SQLITE_LIBS) -else -ifneq ($(LOOKUP_SQLITE),) -OBJ += sqlite.o -endif -endif - -ifeq ($(LOOKUP_TESTDB),2) -MODS += testdb.so -LOOKUP_testdb_INCLUDE = $(LOOKUP_TESTDB_INCLUDE) -LOOKUP_testdb_LIBS = $(LOOKUP_TESTDB_LIBS) -else -ifneq ($(LOOKUP_TESTDB),) -OBJ += testdb.o -endif -endif - -ifeq ($(LOOKUP_WHOSON),2) -MODS += whoson.so -LOOKUP_whoson_INCLUDE = $(LOOKUP_WHOSON_INCLUDE) -LOOKUP_whoson_LIBS = $(LOOKUP_WHOSON_LIBS) -else -ifneq ($(LOOKUP_WHOSON),) -OBJ += whoson.o -endif -endif +# MAGIC-TAG-MODS-OBJ-RULES-GO-HERE all: lookups.a lf_quote.o lf_check_file.o lf_sqlperform.o $(MODS) @@ -178,7 +30,7 @@ lookups.a: $(OBJ) $(FE)$(CC) -c $(CFLAGS) $(INCLUDE) $*.c .c.so:; @echo "$(CC) -shared $*.c" - $(FE)$(CC) $(LOOKUP_$*_INCLUDE) $(LOOKUP_$*_LIBS) -DDYNLOOKUP -shared -rdynamic $(CFLAGS) $(INCLUDE) $(DLFLAGS) $*.c -o $@ + $(FE)$(CC) $(LOOKUP_$*_INCLUDE) $(LOOKUP_$*_LIBS) -DDYNLOOKUP $(CFLAGS_DYNAMIC) $(CFLAGS) $(INCLUDE) $(DLFLAGS) $*.c -o $@ lf_check_file.o: $(HDRS) lf_check_file.c lf_functions.h lf_quote.o: $(HDRS) lf_quote.c lf_functions.h -- 2.30.2