Merge branch 'master' into transp_logging_1031

Conflicts:
	doc/doc-txt/experimental-spec.txt
	src/src/EDITME
	src/src/deliver.c
	src/src/exim.c

diff --git a/doc/doc-txt/experimental-spec.txt b/doc/doc-txt/experimental-spec.txt
index 7fd2bd8ecf015d1383b2bb8362f4dfd76f24688c..565c01851a5bc80acbc663117b70c03cf8bfc1c1 100644 (file)
--- a/doc/doc-txt/experimental-spec.txt
+++ b/doc/doc-txt/experimental-spec.txt
@@ -844,6 +844,85 @@ b. Configure, somewhere before the DATA ACL, the control option to
 
 
 
+DBL (Database Logging)
+--------------------------------------------------------------
+
+This feature allows to write exim internal log information
+(not available otherwise) into a database. 
+Initially implemented is logging of details about successfully
+completed remote deliveries, which are needed for reputation
+systems, and deferrals caused by a host error.
+
+In order to use DBL, you must set
+
+EXPERIMENTAL_DBL=yes
+
+in your Local/Makefile
+
+and define the database queries in the runtime config file, to
+be executed at end of delivery.
+
+Additionally, there are 8 more variables, available at end of
+delivery:
+
+dbl_delivery_ip             IP of host, which has accepted delivery
+dbl_delivery_port           Port of remote host which has accepted delivery
+dbl_delivery_fqdn           FQDN of host, which has accepted delivery
+dbl_delivery_local_part     local part of address being delivered
+dbl_delivery_domain         domain part of address being delivered
+dbl_delivery_confirmation   SMTP confirmation message
+
+In case of a deferral caused by a host-error:
+dbl_defer_errno             Error number
+dbl_defer_errstr            Error string possibly containing more details
+
+
+To log successful deliveries, set the following option in the main
+option part of runtime config.
+
+dbl_delivery_query
+
+An example might look like:
+
+dbl_delivery_query = \
+${lookup pgsql {SELECT * FROM record_Delivery( \
+    '${quote_pgsql:$sender_address_domain}',\
+    '${quote_pgsql:${lc:$sender_address_local_part}}', \
+    '${quote_pgsql:$dbl_delivery_domain}', \
+    '${quote_pgsql:${lc:$dbl_delivery_local_part}}', \
+    '${quote_pgsql:$dbl_delivery_ip}', \
+    '${quote_pgsql:${lc:$dbl_delivery_fqdn}}', \
+    '${quote_pgsql:$message_exim_id}')}}
+
+
+In order to log host deferrals, add the following option to an SMTP
+transport:
+
+dbl_host_defer_query
+
+This is a private option of the SMTP transport. It is intended to
+log failures of remote hosts. It is executed only when exim has
+attempted to deliver a message to a remote host and failed due to
+an error which doesn't seem to be related to the individual
+message, sender, or recipient address.
+See section 45.2 of the exim documentation for more details on how
+this is determined.
+
+Example:
+
+dbl_host_defer_query = \
+${lookup mysql {insert into delivlog set \
+    msgid = '${quote_mysql:$message_exim_id}', \
+    senderlp = '${quote_mysql:${lc:$sender_address_local_part}}', \
+    senderdom = '${quote_mysql:$sender_address_domain}', \
+    delivlp = '${quote_mysql:${lc:$dbl_delivery_local_part}}', \
+    delivdom = '${quote_mysql:$dbl_delivery_domain}', \
+    delivip = '${quote_mysql:$dbl_delivery_ip}', \
+    delivport = '${quote_mysql:$dbl_delivery_port}', \
+    delivfqdn = '${quote_mysql:$dbl_delivery_fqdn}', \
+    deliverrno = '${quote_mysql:$dbl_defer_errno}', \
+    deliverrstr = '${quote_mysql:$dbl_defer_errstr}' \
+    }}
 
 --------------------------------------------------------------
 End of file

diff --git a/src/src/EDITME b/src/src/EDITME
index e29c1eb25e11ffe47f1af6750af24998d588e4d5..637256c7ddf518b330ce0cf2ab812c86c1f97852 100644 (file)
--- a/src/src/EDITME
+++ b/src/src/EDITME
@@ -469,6 +469,10 @@ EXIM_MONITOR=eximon.bin
 # Uncomment the following line to add Per-Recipient-Data-Response support.
 # EXPERIMENTAL_PRDR=yes
 
+# This feature allows to write log information about a completed
+#  delivery into a database
+# EXPERIMENTAL_DBL=yes
+
 
 ###############################################################################
 #                 THESE ARE THINGS YOU MIGHT WANT TO SPECIFY                  #

diff --git a/src/src/config.h.defaults b/src/src/config.h.defaults
index 1594f6858d95978a1deab0fdcb70af000b07dd5f..bcbc70b389d9fc8d9a6aa4fee876bc29de8a5f13 100644 (file)
--- a/src/src/config.h.defaults
+++ b/src/src/config.h.defaults
@@ -170,6 +170,7 @@ it's a default value. */
 #define EXPERIMENTAL_PRDR
 #define EXPERIMENTAL_SPF
 #define EXPERIMENTAL_SRS
+#define EXPERIMENTAL_DBL
 
 /* For developers */
 #define WANT_DEEPER_PRINTF_CHECKS

diff --git a/src/src/deliver.c b/src/src/deliver.c
index 23e63d553d188426538ad0e826a0b72d090812db..7ff25f22aa33231a1f7351bb0dc218e509001a84 100644 (file)
--- a/src/src/deliver.c
+++ b/src/src/deliver.c
@@ -713,6 +713,9 @@ if ((log_extra_selector & LX_sender_on_delivery) != 0  ||  msg)
 if(addr->p.srs_sender)
   s = string_append(s, &size, &ptr, 3, US" SRS=<", addr->p.srs_sender, US">");
 #endif
+#ifdef EXPERIMENTAL_DBL
+  dbl_delivery_ip = NULL;      /* presume no successful remote delivery */
+#endif
 
 /* You might think that the return path must always be set for a successful
 delivery; indeed, I did for some time, until this statement crashed. The case
@@ -827,6 +830,14 @@ store we used to build the line after writing it. */
 
 s[ptr] = 0;
 log_write(0, flags, "%s", s);
+#ifdef EXPERIMENTAL_DBL
+DEBUG(D_deliver)
+  {
+  debug_printf("  DBL(Delivery): dbl_delivery_query=|%s| dbl_delivery_IP=%s\n", dbl_delivery_query, dbl_delivery_ip);
+  }
+if (dbl_delivery_ip != NULL && dbl_delivery_query != NULL)
+  expand_string(dbl_delivery_query);
+#endif
 store_reset(reset_point);
 return;
 }

diff --git a/src/src/exim.c b/src/src/exim.c
index a27e391d1b1e40cf98e8706a6c41090c2e845090..1e4d6d64611fde419c9045503c34bed52576bafe 100644 (file)
--- a/src/src/exim.c
+++ b/src/src/exim.c
@@ -825,6 +825,9 @@ fprintf(f, "Support for:");
 #ifdef EXPERIMENTAL_PRDR
   fprintf(f, " Experimental_PRDR");
 #endif
+#ifdef EXPERIMENTAL_DBL
+  fprintf(f, " EXPERIMENTAL_DBL");
+#endif
 fprintf(f, "\n");
 
 fprintf(f, "Lookups (built-in):");

diff --git a/src/src/expand.c b/src/src/expand.c
index 1da2225637850b5ae8eda30854c96d81858c6499..d808d1bf3a90636930123f86586c917b28d0ce06 100644 (file)
--- a/src/src/expand.c
+++ b/src/src/expand.c
@@ -431,6 +431,16 @@ static var_entry var_table[] = {
   { "compile_date",        vtype_stringptr,   &version_date },
   { "compile_number",      vtype_stringptr,   &version_cnumber },
   { "csa_status",          vtype_stringptr,   &csa_status },
+#ifdef EXPERIMENTAL_DBL
+  { "dbl_defer_errno",     vtype_int,         &dbl_defer_errno },
+  { "dbl_defer_errstr",    vtype_stringptr,   &dbl_defer_errstr },
+  { "dbl_delivery_confirmation", vtype_stringptr,   &dbl_delivery_confirmation },
+  { "dbl_delivery_domain", vtype_stringptr,   &dbl_delivery_domain },
+  { "dbl_delivery_fqdn",   vtype_stringptr,   &dbl_delivery_fqdn },
+  { "dbl_delivery_ip",     vtype_stringptr,   &dbl_delivery_ip },
+  { "dbl_delivery_local_part",vtype_stringptr,&dbl_delivery_local_part },
+  { "dbl_delivery_port",   vtype_int,         &dbl_delivery_port },
+#endif
 #ifdef EXPERIMENTAL_DCC
   { "dcc_header",          vtype_stringptr,   &dcc_header },
   { "dcc_result",          vtype_stringptr,   &dcc_result },

diff --git a/src/src/globals.c b/src/src/globals.c
index 74b6edb016132ddd00d90e2e4d040e7800fa579e..e9f62666527a129e27794228a9d1198bcc6c061f 100644 (file)
--- a/src/src/globals.c
+++ b/src/src/globals.c
@@ -483,6 +483,18 @@ uschar *daemon_smtp_port       = US"smtp";
 int     daemon_startup_retries = 9;
 int     daemon_startup_sleep   = 30;
 
+#ifdef EXPERIMENTAL_DBL
+int     dbl_defer_errno        = 0;
+uschar *dbl_defer_errstr       = NULL;
+uschar *dbl_delivery_query     = NULL;
+uschar *dbl_delivery_ip        = NULL;
+int     dbl_delivery_port      = 0;
+uschar *dbl_delivery_fqdn      = NULL;
+uschar *dbl_delivery_local_part= NULL;
+uschar *dbl_delivery_domain    = NULL;
+uschar *dbl_delivery_confirmation = NULL;
+#endif
+
 #ifdef EXPERIMENTAL_DCC
 BOOL    dcc_direct_add_header  = FALSE;
 uschar *dcc_header             = NULL;

diff --git a/src/src/globals.h b/src/src/globals.h
index db436c06d18285dd6787ff9120f189ee88ea6d7b..4f22832969e09f669bc77e24373200b2fad3378d 100644 (file)
--- a/src/src/globals.h
+++ b/src/src/globals.h
@@ -283,6 +283,18 @@ extern uschar *dccifd_address;         /* address of the dccifd daemon */
 extern uschar *dccifd_options;         /* options for the dccifd daemon */
 #endif
 
+#ifdef EXPERIMENTAL_DBL
+extern int     dbl_defer_errno;        /* error number set when a remote delivery is deferred with a host error */
+extern uschar *dbl_defer_errstr;       /* error string set when a remote delivery is deferred with a host error */
+extern uschar *dbl_delivery_query;     /* query string to log delivery info in DB */
+extern uschar *dbl_delivery_ip;        /* IP of host, which has accepted delivery */
+extern int     dbl_delivery_port;       /* port of host, which has accepted delivery */
+extern uschar *dbl_delivery_fqdn;      /* FQDN of host, which has accepted delivery */
+extern uschar *dbl_delivery_local_part;/* local part of address being delivered */
+extern uschar *dbl_delivery_domain;    /* domain part of address being delivered */
+extern uschar *dbl_delivery_confirmation; /* SMTP confirmation message */
+#endif
+
 extern BOOL    debug_daemon;           /* Debug the daemon process only */
 extern int     debug_fd;               /* The fd for debug_file */
 extern FILE   *debug_file;             /* Where to write debugging info */

diff --git a/src/src/readconf.c b/src/src/readconf.c
index 7f42bb7a97179814cdbbe02713459c52b721557d..d0608d7dee8d94f0f44f306b8f4c95c977dcea96 100644 (file)
--- a/src/src/readconf.c
+++ b/src/src/readconf.c
@@ -195,6 +195,9 @@ static optionlist optionlist_config[] = {
   { "daemon_smtp_ports",        opt_stringptr,   &daemon_smtp_port },
   { "daemon_startup_retries",   opt_int,         &daemon_startup_retries },
   { "daemon_startup_sleep",     opt_time,        &daemon_startup_sleep },
+#ifdef EXPERIMENTAL_DBL
+  { "dbl_delivery_query",       opt_stringptr,   &dbl_delivery_query },
+#endif
 #ifdef EXPERIMENTAL_DCC
   { "dcc_direct_add_header",    opt_bool,        &dcc_direct_add_header },
   { "dccifd_address",           opt_stringptr,   &dccifd_address },

diff --git a/src/src/transports/smtp.c b/src/src/transports/smtp.c
index 25cc5490a8273ef394849da4b8435327b4894728..04bee7fab3d26ccdbfdcb2319d065aca81351379 100644 (file)
--- a/src/src/transports/smtp.c
+++ b/src/src/transports/smtp.c
@@ -35,6 +35,10 @@ optionlist smtp_transport_options[] = {
       (void *)offsetof(transport_instance, connection_max_messages) },
   { "data_timeout",         opt_time,
       (void *)offsetof(smtp_transport_options_block, data_timeout) },
+#ifdef EXPERIMENTAL_DBL
+  { "dbl_host_defer_query",      opt_stringptr,
+         (void *)offsetof(smtp_transport_options_block, dbl_host_defer_query) },
+#endif
   { "delay_after_cutoff", opt_bool,
       (void *)offsetof(smtp_transport_options_block, delay_after_cutoff) },
 #ifndef DISABLE_DKIM
@@ -233,6 +237,9 @@ smtp_transport_options_block smtp_transport_option_defaults = {
   NULL,                /* dkim_sign_headers */
   NULL                 /* dkim_strict */
 #endif
+#ifdef EXPERIMENTAL_DBL
+ ,NULL                 /* dbl_host_defer_query */
+#endif
 };
 
 
@@ -577,6 +584,52 @@ else
 
 
 
+#ifdef EXPERIMENTAL_DBL
+/*************************************************
+*          Write error message to database log   *
+*************************************************/
+
+/* This writes to the database log
+
+Arguments:
+  dbl_host_defer_query  dbl_host_defer_query from the transport options block
+  addr                  the address item containing error information
+  host                  the current host
+
+Returns:   nothing
+*/
+
+static void
+dbl_write_defer_log(uschar *dbl_host_defer_query, address_item *addr, host_item *host)
+{
+if (dbl_host_defer_query == NULL)
+       return;
+
+dbl_delivery_ip = string_copy(host->address);
+dbl_delivery_port = (host->port == PORT_NONE)? 25 : host->port;
+dbl_delivery_fqdn = string_copy(host->name);
+dbl_delivery_local_part = string_copy(addr->local_part);
+dbl_delivery_domain = string_copy(addr->domain);
+dbl_defer_errno = addr->basic_errno;
+
+dbl_defer_errstr = NULL;
+if (addr->message != NULL)
+  if (addr->basic_errno > 0)
+    dbl_defer_errstr = string_sprintf("%s: %s", addr->message, strerror(addr->basic_errno));
+  else
+       dbl_defer_errstr = string_copy(addr->message);
+else if (addr->basic_errno > 0)
+  dbl_defer_errstr = string_copy(strerror(addr->basic_errno));
+
+DEBUG(D_transport) {
+       debug_printf("  DBL(host defer): dbl_host_defer_query=|%s| dbl_delivery_IP=%s\n", dbl_host_defer_query, dbl_delivery_ip);
+}
+expand_string(dbl_host_defer_query);
+}
+#endif
+
+
+
 /*************************************************
 *           Synchronize SMTP responses           *
 *************************************************/
@@ -1912,7 +1965,12 @@ if (!ok) ok = TRUE; else
 
     /* Set up confirmation if needed - applies only to SMTP */
 
-    if ((log_extra_selector & LX_smtp_confirmation) != 0 && !lmtp)
+    if (
+               #ifndef EXPERIMENTAL_DBL
+                       (log_extra_selector & LX_smtp_confirmation) != 0 &&
+               #endif
+                       !lmtp
+        )
       {
       uschar *s = string_printing(buffer);
       conf = (s == buffer)? (uschar *)string_copy(s) : s;
@@ -3051,6 +3109,11 @@ for (cutoff_retry = 0; expired &&
                          first_addr->basic_errno != ERRNO_TLSFAILURE)
         write_logs(first_addr, host);
 
+      #ifdef EXPERIMENTAL_DBL
+      if (rc == DEFER)
+        dbl_write_defer_log(ob->dbl_host_defer_query, first_addr, host);
+      #endif
+
       /* If STARTTLS was accepted, but there was a failure in setting up the
       TLS session (usually a certificate screwup), and the host is not in
       hosts_require_tls, and tls_tempfail_tryclear is true, try again, with
@@ -3073,6 +3136,10 @@ for (cutoff_retry = 0; expired &&
           expanded_hosts != NULL, &message_defer, TRUE);
         if (rc == DEFER && first_addr->basic_errno != ERRNO_AUTHFAIL)
           write_logs(first_addr, host);
+        #ifdef EXPERIMENTAL_DBL
+        if (rc == DEFER)
+          dbl_write_defer_log(ob->dbl_host_defer_query, first_addr, host);
+        #endif
         }
       #endif
       }

diff --git a/src/src/transports/smtp.h b/src/src/transports/smtp.h
index 8e85294bc337d155234c1fd45d9cb7382fa14164..1768585256bf556b657fc086bdebf7d888121c33 100644 (file)
--- a/src/src/transports/smtp.h
+++ b/src/src/transports/smtp.h
@@ -73,6 +73,9 @@ typedef struct {
   uschar *dkim_sign_headers;
   uschar *dkim_strict;
   #endif
+  #ifdef EXPERIMENTAL_DBL
+  uschar *dbl_host_defer_query;
+  #endif
 } smtp_transport_options_block;
 
 /* Data for reading the private options. */