build: use pkg-config for i18n

[exim.git] / src / src / transport.c

diff --git a/src/src/transport.c b/src/src/transport.c
index d6cedf91171eea5a1f93ecd880a9c6ee9ba912cb..063fda3617f60baed0b671cecec1812b20817d38 100644 (file)
--- a/src/src/transport.c
+++ b/src/src/transport.c
@@ -2,7 +2,7 @@
 *     Exim - an Internet mail transport agent    *
 *************************************************/
 
 *     Exim - an Internet mail transport agent    *
 *************************************************/
 

-/* Copyright (c) The Exim Maintainers 2020 - 2022 */
+/* Copyright (c) The Exim Maintainers 2020 - 2024 */

 /* Copyright (c) University of Cambridge 1995 - 2018 */
 /* See the file NOTICE for conditions of use and distribution. */
 /* SPDX-License-Identifier: GPL-2.0-or-later */
 /* Copyright (c) University of Cambridge 1995 - 2018 */
 /* See the file NOTICE for conditions of use and distribution. */
 /* SPDX-License-Identifier: GPL-2.0-or-later */


@@ -44,7 +44,7 @@ optionlist optionlist_transports[] = {
   { "disable_logging",  opt_bool|opt_public,
                  LOFF(disable_logging) },
   { "driver",           opt_stringptr|opt_public,
   { "disable_logging",  opt_bool|opt_public,
                  LOFF(disable_logging) },
   { "driver",           opt_stringptr|opt_public,

-                 LOFF(driver_name) },
+                 LOFF(drinst.driver_name) },

   { "envelope_to_add",   opt_bool|opt_public,
                  LOFF(envelope_to_add) },
 #ifndef DISABLE_EVENT
   { "envelope_to_add",   opt_bool|opt_public,
                  LOFF(envelope_to_add) },
 #ifndef DISABLE_EVENT


@@ -102,11 +102,12 @@ uschar buf[64];
 
 options_from_list(optionlist_transports, nelem(optionlist_transports), US"TRANSPORTS", NULL);
 
 
 options_from_list(optionlist_transports, nelem(optionlist_transports), US"TRANSPORTS", NULL);
 

-for (transport_info * ti = transports_available; ti->driver_name[0]; ti++)
+for (driver_info * di= (driver_info *)transports_available; di; di = di->next)

   {
   {

-  spf(buf, sizeof(buf), US"_DRIVER_TRANSPORT_%T", ti->driver_name);
+  spf(buf, sizeof(buf), US"_DRIVER_TRANSPORT_%T", di->driver_name);

   builtin_macro_create(buf);
   builtin_macro_create(buf);

-  options_from_list(ti->options, (unsigned)*ti->options_count, US"TRANSPORT", ti->driver_name);
+  options_from_list(di->options, (unsigned)*di->options_count,
+                   US"TRANSPORT", di->driver_name);

   }
 }
 
   }
 }
 


@@ -144,28 +145,68 @@ the work. */
 void
 transport_init(void)
 {
 void
 transport_init(void)
 {

-readconf_driver_init(US"transport",
-  (driver_instance **)(&transports),     /* chain anchor */
-  (driver_info *)transports_available,   /* available drivers */
-  sizeof(transport_info),                /* size of info block */
-  &transport_defaults,                   /* default values for generic options */
-  sizeof(transport_instance),            /* size of instance block */
-  optionlist_transports,                 /* generic options */
-  optionlist_transports_size);
+int old_pool = store_pool;
+store_pool = POOL_PERM;
+  {
+  driver_info ** anchor = (driver_info **) &transports_available;
+  extern transport_info appendfile_transport_info;
+  extern transport_info autoreply_transport_info;
+  extern transport_info lmtp_transport_info;
+  extern transport_info pipe_transport_info;
+  extern transport_info queuefile_transport_info;
+  extern transport_info smtp_transport_info;
+
+  /* Add the transport drivers that are built for static linkage to the
+  list of availables. */
+
+#if defined(TRANSPORT_APPENDFILE) && TRANSPORT_APPENDFILE!=2
+  add_driver_info(anchor, &appendfile_transport_info.drinfo, sizeof(transport_info));
+#endif
+#if defined(TRANSPORT_AUTOREPLY) && TRANSPORT_AUTOREPLY!=2
+  add_driver_info(anchor, &autoreply_transport_info.drinfo, sizeof(transport_info));
+#endif
+#if defined(TRANSPORT_LMTP) && TRANSPORT_LMTP!=2
+  add_driver_info(anchor, &lmtp_transport_info.drinfo, sizeof(transport_info));
+#endif
+#if defined(TRANSPORT_PIPE) && TRANSPORT_PIPE!=2
+  add_driver_info(anchor, &pipe_transport_info.drinfo, sizeof(transport_info));
+#endif
+#if defined(EXPERIMENTAL_QUEUEFILE) && EXPERIMENTAL_QUEUEFILE!=2
+  add_driver_info(anchor, &queuefile_transport_info.drinfo, sizeof(transport_info));
+#endif
+#if defined(TRANSPORT_SMTP) && TRANSPORT_SMTP!=2
+  add_driver_info(anchor, &smtp_transport_info.drinfo, sizeof(transport_info));
+#endif
+  }
+store_pool = old_pool;
+
+/* Read the config file "transports" section, creating a transportsinstance list.
+For any yet-undiscovered driver, check for a loadable module and add it to
+those available. */
+
+readconf_driver_init((driver_instance **)&transports,     /* chain anchor */
+  (driver_info **)&transports_available, /* available drivers */
+  sizeof(transport_info),              /* size of info block */
+  &transport_defaults,                 /* default values for generic options */
+  sizeof(transport_instance),          /* size of instance block */
+  optionlist_transports,               /* generic options */
+  optionlist_transports_size,
+  US"transport");

 
 /* Now scan the configured transports and check inconsistencies. A shadow
 transport is permitted only for local transports. */
 
 
 /* Now scan the configured transports and check inconsistencies. A shadow
 transport is permitted only for local transports. */
 

-for (transport_instance * t = transports; t; t = t->next)
+for (transport_instance * t = transports; t; t = t->drinst.next)

   {
   {

-  if (!t->info->local && t->shadow)
+  transport_info * ti = t->drinst.info;
+  if (!ti->local && t->shadow)

     log_write(0, LOG_PANIC_DIE|LOG_CONFIG,
     log_write(0, LOG_PANIC_DIE|LOG_CONFIG,

-      "shadow transport not allowed on non-local transport %s", t->name);
+      "shadow transport not allowed on non-local transport %s", t->drinst.name);

 
   if (t->body_only && t->headers_only)
     log_write(0, LOG_PANIC_DIE|LOG_CONFIG,
       "%s transport: body_only and headers_only are mutually exclusive",
 
   if (t->body_only && t->headers_only)
     log_write(0, LOG_PANIC_DIE|LOG_CONFIG,
       "%s transport: body_only and headers_only are mutually exclusive",

-      t->name);
+      t->drinst.name);

   }
 }
 
   }
 }
 


@@ -434,10 +475,10 @@ Returns:     TRUE on success, FALSE on failure (with errno preserved)
 */
 
 BOOL
 */
 
 BOOL

-write_chunk(transport_ctx * tctx, uschar *chunk, int len)
+write_chunk(transport_ctx * tctx, const uschar * chunk, int len)

 {
 {

-uschar *start = chunk;
-uschar *end = chunk + len;
+const uschar * start = chunk;
+const uschar * end = chunk + len;

 int mlen = DELIVER_OUT_BUFFER_SIZE - nl_escape_length - 2;
 
 /* The assumption is made that the check string will never stretch over move
 int mlen = DELIVER_OUT_BUFFER_SIZE - nl_escape_length - 2;
 
 /* The assumption is made that the check string will never stretch over move


@@ -474,7 +515,7 @@ if (nl_partial_match >= 0)
 for possible escaping. The code for the non-NL route should be as fast as
 possible. */
 
 for possible escaping. The code for the non-NL route should be as fast as
 possible. */
 

-for (uschar * ptr = start; ptr < end; ptr++)
+for (const uschar * ptr = start; ptr < end; ptr++)

   {
   int ch, len;
 
   {
   int ch, len;
 


@@ -580,7 +621,7 @@ Arguments:
 Returns:            a string
 */
 
 Returns:            a string
 */
 

-uschar *
+const uschar *

 transport_rcpt_address(address_item *addr, BOOL include_affixes)
 {
 uschar *at;
 transport_rcpt_address(address_item *addr, BOOL include_affixes)
 {
 uschar *at;


@@ -704,9 +745,9 @@ Returns:                TRUE on success; FALSE on failure.
 */
 BOOL
 transport_headers_send(transport_ctx * tctx,
 */
 BOOL
 transport_headers_send(transport_ctx * tctx,

-  BOOL (*sendfn)(transport_ctx * tctx, uschar * s, int len))
+  BOOL (*sendfn)(transport_ctx * tctx, const uschar * s, int len))

 {
 {

-const uschar *list;
+const uschar * list;

 transport_instance * tblock = tctx ? tctx->tblock : NULL;
 address_item * addr = tctx ? tctx->addr : NULL;
 
 transport_instance * tblock = tctx ? tctx->tblock : NULL;
 address_item * addr = tctx ? tctx->addr : NULL;
 


@@ -761,15 +802,18 @@ for (header_line * h = header_list; h; h = h->next) if (h->type != htype_old)
 
   if (include_header)
     {
 
   if (include_header)
     {

+    int len;

     if (tblock && tblock->rewrite_rules)
       {
       rmark reset_point = store_mark();
     if (tblock && tblock->rewrite_rules)
       {
       rmark reset_point = store_mark();

-      header_line *hh;
+      header_line * hh;

 
       if ((hh = rewrite_header(h, NULL, NULL, tblock->rewrite_rules,
                  tblock->rewrite_existflags, FALSE)))
        {
 
       if ((hh = rewrite_header(h, NULL, NULL, tblock->rewrite_rules,
                  tblock->rewrite_existflags, FALSE)))
        {

-       if (!sendfn(tctx, hh->text, hh->slen)) return FALSE;
+       len = hh->slen;
+       if (tctx->options & topt_truncate_headers && len > 998) len = 998;
+       if (!sendfn(tctx, hh->text, len)) return FALSE;

        store_reset(reset_point);
        continue;     /* With the next header line */
        }
        store_reset(reset_point);
        continue;     /* With the next header line */
        }


@@ -777,7 +821,9 @@ for (header_line * h = header_list; h; h = h->next) if (h->type != htype_old)
 
     /* Either no rewriting rules, or it didn't get rewritten */
 
 
     /* Either no rewriting rules, or it didn't get rewritten */
 

-    if (!sendfn(tctx, h->text, h->slen)) return FALSE;
+    len = h->slen;
+    if (tctx->options & topt_truncate_headers && len > 998) len = 998;
+    if (!sendfn(tctx, h->text, len)) return FALSE;

     }
 
   /* Header removed */
     }
 
   /* Header removed */


@@ -1038,7 +1084,7 @@ if (tctx->options & topt_use_bdat)
   if (!(tctx->options & topt_no_body))
     {
     if ((fsize = lseek(deliver_datafile, 0, SEEK_END)) < 0) return FALSE;
   if (!(tctx->options & topt_no_body))
     {
     if ((fsize = lseek(deliver_datafile, 0, SEEK_END)) < 0) return FALSE;

-    fsize -= SPOOL_DATA_START_OFFSET;
+    fsize -= spool_data_start_offset(message_id);

     if (size_limit > 0  &&  fsize > size_limit)
       fsize = size_limit;
     size = hsize + fsize;
     if (size_limit > 0  &&  fsize > size_limit)
       fsize = size_limit;
     size = hsize + fsize;


@@ -1096,7 +1142,7 @@ if (  f.spool_file_wireformat
    )
   {
   ssize_t copied = 0;
    )
   {
   ssize_t copied = 0;

-  off_t offset = SPOOL_DATA_START_OFFSET;
+  off_t offset = spool_data_start_offset(message_id);

 
   /* Write out any header data in the buffer */
 
 
   /* Write out any header data in the buffer */
 


@@ -1134,7 +1180,7 @@ if (!(tctx->options & topt_no_body))
 
   nl_check_length = abs(nl_check_length);
   nl_partial_match = 0;
 
   nl_check_length = abs(nl_check_length);
   nl_partial_match = 0;

-  if (lseek(deliver_datafile, SPOOL_DATA_START_OFFSET, SEEK_SET) < 0)
+  if (lseek(deliver_datafile, spool_data_start_offset(message_id), SEEK_SET) < 0)

     return FALSE;
   while (  (len = MIN(DELIVER_IN_BUFFER_SIZE, size)) > 0
        && (len = read(deliver_datafile, deliver_in_buffer, len)) > 0)
     return FALSE;
   while (  (len = MIN(DELIVER_IN_BUFFER_SIZE, size)) > 0
        && (len = read(deliver_datafile, deliver_in_buffer, len)) > 0)


@@ -1492,18 +1538,27 @@ Returns:    nothing
 */
 
 void
 */
 
 void

-transport_update_waiting(host_item *hostlist, uschar *tpname)
+transport_update_waiting(host_item * hostlist, const uschar * tpname)

 {
 {

-const uschar *prevname = US"";
-open_db dbblock;
-open_db *dbm_file;
+const uschar * prevname = US"";
+open_db dbblock, * dbp;
+
+if (!is_new_message_id(message_id))
+  {
+  DEBUG(D_transport) debug_printf("message_id %s is not new format; "
+    "skipping wait-%s database update\n", message_id, tpname);
+  return;
+  }

 
 DEBUG(D_transport) debug_printf("updating wait-%s database\n", tpname);
 
 
 DEBUG(D_transport) debug_printf("updating wait-%s database\n", tpname);
 

-/* Open the database for this transport */
+/* Open the database (or transaction) for this transport */

 
 

-if (!(dbm_file = dbfn_open(string_sprintf("wait-%.200s", tpname),
-                     O_RDWR, &dbblock, TRUE, TRUE)))
+if ( continue_wait_db
+   ? !dbfn_transaction_start(dbp = continue_wait_db)
+   : !(dbp = dbfn_open(string_sprintf("wait-%.200s", tpname),
+                     O_RDWR|O_CREAT, &dbblock, TRUE, TRUE))
+   )

   return;
 
 /* Scan the list of hosts for which this message is waiting, and ensure
   return;
 
 /* Scan the list of hosts for which this message is waiting, and ensure


@@ -1512,7 +1567,7 @@ that the message id is in each host record. */
 for (host_item * host = hostlist; host; host = host->next)
   {
   BOOL already = FALSE;
 for (host_item * host = hostlist; host; host = host->next)
   {
   BOOL already = FALSE;

-  dbdata_wait *host_record;
+  dbdata_wait * host_record;

   int host_length;
   uschar buffer[256];
 
   int host_length;
   uschar buffer[256];
 


@@ -1524,7 +1579,7 @@ for (host_item * host = hostlist; host; host = host->next)
 
   /* Look up the host record; if there isn't one, make an empty one. */
 
 
   /* Look up the host record; if there isn't one, make an empty one. */
 

-  if (!(host_record = dbfn_read(dbm_file, host->name)))
+  if (!(host_record = dbfn_read(dbp, host->name)))

     {
     host_record = store_get(sizeof(dbdata_wait) + MESSAGE_ID_LENGTH, GET_UNTAINTED);
     host_record->count = host_record->sequence = 0;
     {
     host_record = store_get(sizeof(dbdata_wait) + MESSAGE_ID_LENGTH, GET_UNTAINTED);
     host_record->count = host_record->sequence = 0;


@@ -1538,8 +1593,27 @@ for (host_item * host = hostlist; host; host = host->next)
 
   for (uschar * s = host_record->text; s < host_record->text + host_length;
        s += MESSAGE_ID_LENGTH)
 
   for (uschar * s = host_record->text; s < host_record->text + host_length;
        s += MESSAGE_ID_LENGTH)

+    {
+    /* If any ID is seen which is not new-format, wipe the record and
+    any continuations */
+
+    if (!is_new_message_id(s))
+      {
+      DEBUG(D_hints_lookup)
+       debug_printf_indent("NOTE: old or corrupt message-id found in wait=%.200s"
+         " hints DB; deleting records for %s\n", tpname, host->name);
+
+      (void) dbfn_delete(dbp, host->name);
+      for (int i = host_record->sequence - 1; i >= 0; i--)
+       (void) dbfn_delete(dbp,
+                   (sprintf(CS buffer, "%.200s:%d", host->name, i), buffer));
+
+      host_record->count = host_record->sequence = 0;
+      break;
+      }

     if (Ustrncmp(s, message_id, MESSAGE_ID_LENGTH) == 0)
       { already = TRUE; break; }
     if (Ustrncmp(s, message_id, MESSAGE_ID_LENGTH) == 0)
       { already = TRUE; break; }

+    }

 
   /* If we haven't found this message in the main record, search any
   continuation records that exist. */
 
   /* If we haven't found this message in the main record, search any
   continuation records that exist. */


@@ -1548,7 +1622,7 @@ for (host_item * host = hostlist; host; host = host->next)
     {
     dbdata_wait *cont;
     sprintf(CS buffer, "%.200s:%d", host->name, i);
     {
     dbdata_wait *cont;
     sprintf(CS buffer, "%.200s:%d", host->name, i);

-    if ((cont = dbfn_read(dbm_file, buffer)))
+    if ((cont = dbfn_read(dbp, buffer)))

       {
       int clen = cont->count * MESSAGE_ID_LENGTH;
       for (uschar * s = cont->text; s < cont->text + clen; s += MESSAGE_ID_LENGTH)
       {
       int clen = cont->count * MESSAGE_ID_LENGTH;
       for (uschar * s = cont->text; s < cont->text + clen; s += MESSAGE_ID_LENGTH)


@@ -1574,7 +1648,7 @@ for (host_item * host = hostlist; host; host = host->next)
   if (host_record->count >= WAIT_NAME_MAX)
     {
     sprintf(CS buffer, "%.200s:%d", host->name, host_record->sequence);
   if (host_record->count >= WAIT_NAME_MAX)
     {
     sprintf(CS buffer, "%.200s:%d", host->name, host_record->sequence);

-    dbfn_write(dbm_file, buffer, host_record, sizeof(dbdata_wait) + host_length);
+    dbfn_write(dbp, buffer, host_record, sizeof(dbdata_wait) + host_length);

 #ifndef DISABLE_QUEUE_RAMP
     if (f.queue_2stage && queue_fast_ramp && !queue_run_in_order)
       queue_notify_daemon(message_id);
 #ifndef DISABLE_QUEUE_RAMP
     if (f.queue_2stage && queue_fast_ramp && !queue_run_in_order)
       queue_notify_daemon(message_id);


@@ -1603,14 +1677,17 @@ for (host_item * host = hostlist; host; host = host->next)
 
   /* Update the database */
 
 
   /* Update the database */
 

-  dbfn_write(dbm_file, host->name, host_record, sizeof(dbdata_wait) + host_length);
+  dbfn_write(dbp, host->name, host_record, sizeof(dbdata_wait) + host_length);

   DEBUG(D_transport) debug_printf("added %.*s to queue for %s\n",
                                  MESSAGE_ID_LENGTH, message_id, host->name);
   }
 
 /* All now done */
 
   DEBUG(D_transport) debug_printf("added %.*s to queue for %s\n",
                                  MESSAGE_ID_LENGTH, message_id, host->name);
   }
 
 /* All now done */
 

-dbfn_close(dbm_file);
+if (continue_wait_db)
+  dbfn_transaction_commit(dbp);
+else
+  dbfn_close(dbp);

 }
 
 
 }
 
 


@@ -1627,6 +1704,10 @@ another message waiting for the same host. However, it doesn't do this if the
 current continue sequence is greater than the maximum supplied as an argument,
 or greater than the global connection_max_messages, which, if set, overrides.
 
 current continue sequence is greater than the maximum supplied as an argument,
 or greater than the global connection_max_messages, which, if set, overrides.
 

+It is also called if conditions are otherwise right for pipelining a QUIT after
+the message data, since if there is another message waiting we do not want to
+send that QUIT.
+

 Arguments:
   transport_name     name of the transport
   hostname           name of the host
 Arguments:
   transport_name     name of the transport
   hostname           name of the host


@@ -1647,13 +1728,13 @@ typedef struct msgq_s
 } msgq_t;
 
 BOOL
 } msgq_t;
 
 BOOL

-transport_check_waiting(const uschar *transport_name, const uschar *hostname,
-  int local_message_max, uschar *new_message_id, oicf oicf_func, void *oicf_data)
+transport_check_waiting(const uschar * transport_name, const uschar * hostname,
+  int local_message_max, uschar * new_message_id,
+  oicf oicf_func, void * oicf_data)

 {
 {

-dbdata_wait *host_record;
+dbdata_wait * host_record;

 int host_length;
 int host_length;

-open_db dbblock;
-open_db *dbm_file;
+open_db dbblock, * dbp;

 
 int         i;
 struct stat statbuf;
 
 int         i;
 struct stat statbuf;


@@ -1666,7 +1747,7 @@ DEBUG(D_transport)
   acl_level++;
   }
 
   acl_level++;
   }
 

-/* Do nothing if we have hit the maximum number that can be send down one
+/* Do nothing if we have hit the maximum number that can be sent down one

 connection. */
 
 if (connection_max_messages >= 0) local_message_max = connection_max_messages;
 connection. */
 
 if (connection_max_messages >= 0) local_message_max = connection_max_messages;


@@ -1679,17 +1760,24 @@ if (local_message_max > 0 && continue_sequence >= local_message_max)
 
 /* Open the waiting information database. */
 
 
 /* Open the waiting information database. */
 

-if (!(dbm_file = dbfn_open(string_sprintf("wait-%.200s", transport_name),
-                         O_RDWR, &dbblock, TRUE, TRUE)))
+if ( continue_wait_db
+   ? !dbfn_transaction_start(dbp = continue_wait_db)
+   : !(dbp = dbfn_open(string_sprintf("wait-%.200s", transport_name),
+                     O_RDWR, &dbblock, TRUE, TRUE))
+   )
+  {
+  DEBUG(D_transport)
+    debug_printf_indent("no messages waiting for %s\n", hostname);

   goto retfalse;
   goto retfalse;

+  }

 
 /* See if there is a record for this host; if not, there's nothing to do. */
 
 
 /* See if there is a record for this host; if not, there's nothing to do. */
 

-if (!(host_record = dbfn_read(dbm_file, hostname)))
+if (!(host_record = dbfn_read(dbp, hostname)))

   {
   {

-  dbfn_close(dbm_file);
-  DEBUG(D_transport) debug_printf_indent("no messages waiting for %s\n", hostname);
-  goto retfalse;
+  DEBUG(D_transport)
+    debug_printf_indent("no messages waiting for %s\n", hostname);
+  goto dbclose_false;

   }
 
 /* If the data in the record looks corrupt, just log something and
   }
 
 /* If the data in the record looks corrupt, just log something and


@@ -1697,13 +1785,12 @@ don't try to use it. */
 
 if (host_record->count > WAIT_NAME_MAX)
   {
 
 if (host_record->count > WAIT_NAME_MAX)
   {

-  dbfn_close(dbm_file);

   log_write(0, LOG_MAIN|LOG_PANIC, "smtp-wait database entry for %s has bad "
     "count=%d (max=%d)", hostname, host_record->count, WAIT_NAME_MAX);
   log_write(0, LOG_MAIN|LOG_PANIC, "smtp-wait database entry for %s has bad "
     "count=%d (max=%d)", hostname, host_record->count, WAIT_NAME_MAX);

-  goto retfalse;
+  goto dbclose_false;

   }
 
   }
 

-/* Scan the message ids in the record from the end towards the beginning,
+/* Scan the message ids in the record in order

 until one is found for which a spool file actually exists. If the record gets
 emptied, delete it and continue with any continuation records that may exist.
 */
 until one is found for which a spool file actually exists. If the record gets
 emptied, delete it and continue with any continuation records that may exist.
 */


@@ -1716,20 +1803,32 @@ host_length = host_record->count * MESSAGE_ID_LENGTH;
 
 while (1)
   {
 
 while (1)
   {

-  msgq_t      *msgq;
-  int         msgq_count = 0;
-  int         msgq_actual = 0;
-  BOOL        bFound = FALSE;
-  BOOL        bContinuation = FALSE;
+  msgq_t * msgq;
+  int msgq_count = 0, msgq_actual = 0;
+  BOOL bFound = FALSE, bContinuation = FALSE;

 
   /* create an array to read entire message queue into memory for processing  */
 
   msgq = store_get(sizeof(msgq_t) * host_record->count, GET_UNTAINTED);
 
   /* create an array to read entire message queue into memory for processing  */
 
   msgq = store_get(sizeof(msgq_t) * host_record->count, GET_UNTAINTED);

-  msgq_count = host_record->count;
-  msgq_actual = msgq_count;
+  msgq_actual = msgq_count = host_record->count;

 
   for (i = 0; i < host_record->count; ++i)
     {
 
   for (i = 0; i < host_record->count; ++i)
     {

+    /* If any ID is seen which is not new-format, wipe the record and
+    any continuations */
+
+    if (!is_new_message_id(host_record->text + (i * MESSAGE_ID_LENGTH)))
+      {
+      uschar buffer[256];
+      DEBUG(D_hints_lookup)
+       debug_printf_indent("NOTE: old or corrupt message-id found in wait=%.200s"
+         " hints DB; deleting records for %s\n", transport_name, hostname);
+      (void) dbfn_delete(dbp, hostname);
+      for (int j = host_record->sequence - 1; j >= 0; j--)
+       (void) dbfn_delete(dbp,
+                   (sprintf(CS buffer, "%.200s:%d", hostname, j), buffer));
+      goto dbclose_false;
+      }

     msgq[i].bKeep = TRUE;
 
     Ustrncpy_nt(msgq[i].message_id, host_record->text + (i * MESSAGE_ID_LENGTH),
     msgq[i].bKeep = TRUE;
 
     Ustrncpy_nt(msgq[i].message_id, host_record->text + (i * MESSAGE_ID_LENGTH),


@@ -1749,7 +1848,7 @@ while (1)
 
   /* now find the next acceptable message_id */
 
 
   /* now find the next acceptable message_id */
 

-  for (i = msgq_count - 1; i >= 0; --i) if (msgq[i].bKeep)
+  for (i = 0; i < msgq_count; i++) if (msgq[i].bKeep)

     {
     uschar subdir[2];
     uschar * mid = msgq[i].message_id;
     {
     uschar subdir[2];
     uschar * mid = msgq[i].message_id;


@@ -1808,20 +1907,20 @@ while (1)
     for (int i = host_record->sequence - 1; i >= 0 && !newr; i--)
       {
       sprintf(CS buffer, "%.200s:%d", hostname, i);
     for (int i = host_record->sequence - 1; i >= 0 && !newr; i--)
       {
       sprintf(CS buffer, "%.200s:%d", hostname, i);

-      newr = dbfn_read(dbm_file, buffer);
+      newr = dbfn_read(dbp, buffer);

       }
 
     /* If no continuation, delete the current and break the loop */
 
     if (!newr)
       {
       }
 
     /* If no continuation, delete the current and break the loop */
 
     if (!newr)
       {

-      dbfn_delete(dbm_file, hostname);
+      dbfn_delete(dbp, hostname);

       break;
       }
 
     /* Else replace the current with the continuation */
 
       break;
       }
 
     /* Else replace the current with the continuation */
 

-    dbfn_delete(dbm_file, buffer);
+    dbfn_delete(dbp, buffer);

     host_record = newr;
     host_length = host_record->count * MESSAGE_ID_LENGTH;
 
     host_record = newr;
     host_length = host_record->count * MESSAGE_ID_LENGTH;
 


@@ -1837,20 +1936,18 @@ while (1)
 
   if (host_length <= 0)
     {
 
   if (host_length <= 0)
     {

-    dbfn_close(dbm_file);

     DEBUG(D_transport) debug_printf_indent("waiting messages already delivered\n");
     DEBUG(D_transport) debug_printf_indent("waiting messages already delivered\n");

-    goto retfalse;
+    goto dbclose_false;

     }
 
   /* we were not able to find an acceptable message, nor was there a
     }
 
   /* we were not able to find an acceptable message, nor was there a

-   * continuation record.  So bug out, outer logic will clean this up.
-   */
+  continuation record.  So bug out, outer logic will clean this up.
+  */

 
   if (!bContinuation)
     {
     Ustrcpy(new_message_id, message_id);
 
   if (!bContinuation)
     {
     Ustrcpy(new_message_id, message_id);

-    dbfn_close(dbm_file);
-    goto retfalse;
+    goto dbclose_false;

     }
   }            /* we need to process a continuation record */
 
     }
   }            /* we need to process a continuation record */
 


@@ -1862,16 +1959,31 @@ record if required, close the database, and return TRUE. */
 if (host_length > 0)
   {
   host_record->count = host_length/MESSAGE_ID_LENGTH;
 if (host_length > 0)
   {
   host_record->count = host_length/MESSAGE_ID_LENGTH;

-  dbfn_write(dbm_file, hostname, host_record, (int)sizeof(dbdata_wait) + host_length);
+  dbfn_write(dbp, hostname, host_record, (int)sizeof(dbdata_wait) + host_length);

   }
 
   }
 

-dbfn_close(dbm_file);
-DEBUG(D_transport) {acl_level--; debug_printf("transport_check_waiting: TRUE\n"); }
+if (continue_wait_db)
+  dbfn_transaction_commit(dbp);
+else
+  dbfn_close(dbp);
+
+DEBUG(D_transport)
+  {
+  acl_level--;
+  debug_printf("transport_check_waiting: TRUE (found %s)\n", new_message_id);
+  }

 return TRUE;
 
 return TRUE;
 

+dbclose_false:
+  if (continue_wait_db)
+    dbfn_transaction_commit(dbp);
+  else
+    dbfn_close(dbp);
+

 retfalse:
 retfalse:

-DEBUG(D_transport) {acl_level--; debug_printf("transport_check_waiting: FALSE\n"); }
-return FALSE;
+  DEBUG(D_transport)
+    {acl_level--; debug_printf("transport_check_waiting: FALSE\n"); }
+  return FALSE;

 }
 
 /*************************************************
 }
 
 /*************************************************


@@ -1880,8 +1992,8 @@ return FALSE;
 
 /* Just the regain-root-privilege exec portion */
 void
 
 /* Just the regain-root-privilege exec portion */
 void

-transport_do_pass_socket(const uschar *transport_name, const uschar *hostname,
-  const uschar *hostaddress, uschar *id, int socket_fd)
+transport_do_pass_socket(const uschar * transport_name, const uschar * hostname,
+  const uschar * hostaddress, uschar * id, int socket_fd)

 {
 int i = 13;
 const uschar **argv;
 {
 int i = 13;
 const uschar **argv;


@@ -1889,7 +2001,7 @@ const uschar **argv;
 #ifndef DISABLE_TLS
 if (smtp_peer_options & OPTION_TLS) i += 6;
 #endif
 #ifndef DISABLE_TLS
 if (smtp_peer_options & OPTION_TLS) i += 6;
 #endif

-#ifdef EXPERIMENTAL_ESMTP_LIMITS
+#ifndef DISABLE_ESMTP_LIMITS

 if (continue_limit_mail || continue_limit_rcpt || continue_limit_rcptdom)
                                    i += 4;
 #endif
 if (continue_limit_mail || continue_limit_rcpt || continue_limit_rcptdom)
                                    i += 4;
 #endif


@@ -1931,8 +2043,8 @@ if (smtp_peer_options & OPTION_TLS)
     argv[i++] = US"-MCT";
 #endif
 
     argv[i++] = US"-MCT";
 #endif
 

-#ifdef EXPERIMENTAL_ESMTP_LIMITS
-if (continue_limit_rcpt || continue_limit_rcptdom)
+#ifndef DISABLE_ESMTP_LIMITS
+if (continue_limit_mail || continue_limit_rcpt || continue_limit_rcptdom)

   {
   argv[i++] = US"-MCL";
   argv[i++] = string_sprintf("%u", continue_limit_mail);
   {
   argv[i++] = US"-MCL";
   argv[i++] = string_sprintf("%u", continue_limit_mail);


@@ -1986,73 +2098,6 @@ _exit(errno);         /* Note: must be _exit(), NOT exit() */
 
 
 
 
 
 

-/* Fork a new exim process to deliver the message, and do a re-exec, both to
-get a clean delivery process, and to regain root privilege in cases where it
-has been given away.
-
-Arguments:
-  transport_name  to pass to the new process
-  hostname        ditto
-  hostaddress     ditto
-  id              the new message to process
-  socket_fd       the connected socket
-
-Returns:          FALSE if fork fails; TRUE otherwise
-*/
-
-BOOL
-transport_pass_socket(const uschar *transport_name, const uschar *hostname,
-  const uschar *hostaddress, uschar *id, int socket_fd
-#ifdef EXPERIMENTAL_ESMTP_LIMITS
-  , unsigned peer_limit_mail, unsigned peer_limit_rcpt, unsigned peer_limit_rcptdom
-#endif
-  )
-{
-pid_t pid;
-int status;
-
-DEBUG(D_transport) debug_printf("transport_pass_socket entered\n");
-
-#ifdef EXPERIMENTAL_ESMTP_LIMITS
-continue_limit_mail = peer_limit_mail;
-continue_limit_rcpt = peer_limit_rcpt;
-continue_limit_rcptdom = peer_limit_rcptdom;
-#endif
-
-if ((pid = exim_fork(US"continued-transport-interproc")) == 0)
-  {
-  /* Disconnect entirely from the parent process. If we are running in the
-  test harness, wait for a bit to allow the previous process time to finish,
-  write the log, etc., so that the output is always in the same order for
-  automatic comparison. */
-
-  if ((pid = exim_fork(US"continued-transport")) != 0)
-    _exit(EXIT_SUCCESS);
-  testharness_pause_ms(1000);
-
-  transport_do_pass_socket(transport_name, hostname, hostaddress,
-    id, socket_fd);
-  }
-
-/* If the process creation succeeded, wait for the first-level child, which
-immediately exits, leaving the second level process entirely disconnected from
-this one. */
-
-if (pid > 0)
-  {
-  int rc;
-  while ((rc = wait(&status)) != pid && (rc >= 0 || errno != ECHILD));
-  return TRUE;
-  }
-else
-  {
-  DEBUG(D_transport) debug_printf("transport_pass_socket failed to fork: %s\n",
-    strerror(errno));
-  return FALSE;
-  }
-}
-
-

 
 /* Enforce all args untainted, for consistency with a router-sourced pipe
 command, where (because the whole line is passed as one to the tpt) a
 
 /* Enforce all args untainted, for consistency with a router-sourced pipe
 command, where (because the whole line is passed as one to the tpt) a


@@ -2085,18 +2130,18 @@ return FALSE;
 
 /* This function is called when a command line is to be parsed and executed
 directly, without the use of /bin/sh. It is called by the pipe transport,
 
 /* This function is called when a command line is to be parsed and executed
 directly, without the use of /bin/sh. It is called by the pipe transport,

-the queryprogram router, and also from the main delivery code when setting up a
+the queryprogram router, for any ${run } expansion,
+and also from the main delivery code when setting up a

 transport filter process. The code for ETRN also makes use of this; in that
 case, no addresses are passed.
 
 Arguments:
   argvptr            pointer to anchor for argv vector
   cmd                points to the command string (modified IN PLACE)
 transport filter process. The code for ETRN also makes use of this; in that
 case, no addresses are passed.
 
 Arguments:
   argvptr            pointer to anchor for argv vector
   cmd                points to the command string (modified IN PLACE)

-  expand_arguments   true if expansion is to occur
+  flags                     bits for expand-args, allow taint, allow $recipients

   expand_failed      error value to set if expansion fails; not relevant if
                      addr == NULL
   addr               chain of addresses, or NULL
   expand_failed      error value to set if expansion fails; not relevant if
                      addr == NULL
   addr               chain of addresses, or NULL

-  allow_tainted_args as it says; used for ${run}

   etext              text for use in error messages
   errptr             where to put error message if addr is NULL;
                      otherwise it is put in the first address
   etext              text for use in error messages
   errptr             where to put error message if addr is NULL;
                      otherwise it is put in the first address


@@ -2107,8 +2152,8 @@ Returns:             TRUE if all went well; otherwise an error will be
 
 BOOL
 transport_set_up_command(const uschar *** argvptr, const uschar * cmd,
 
 BOOL
 transport_set_up_command(const uschar *** argvptr, const uschar * cmd,

-  BOOL expand_arguments, int expand_failed, address_item * addr,
-  BOOL allow_tainted_args, const uschar * etext, uschar ** errptr)
+  unsigned flags, int expand_failed, address_item * addr,
+  const uschar * etext, uschar ** errptr)

 {
 const uschar ** argv, * s;
 int address_count = 0, argcount = 0, max_args;
 {
 const uschar ** argv, * s;
 int address_count = 0, argcount = 0, max_args;


@@ -2183,10 +2228,10 @@ DEBUG(D_transport)
     debug_printf("  argv[%d] = '%s'\n", i, string_printing(argv[i]));
   }
 
     debug_printf("  argv[%d] = '%s'\n", i, string_printing(argv[i]));
   }
 

-if (expand_arguments)
+if (flags & TSUC_EXPAND_ARGS)

   {
   {

-  BOOL allow_dollar_recipients = addr && addr->parent
-    && Ustrcmp(addr->parent->address, "system-filter") == 0;
+  BOOL allow_dollar_recipients = (flags & TSUC_ALLOW_RECIPIENTS)
+    || (addr && addr->parent && Ustrcmp(addr->parent->address, "system-filter") == 0); /*XXX could we check this at caller? */

 
   for (int i = 0; argv[i]; i++)
     {
 
   for (int i = 0; argv[i]; i++)
     {


@@ -2254,7 +2299,7 @@ if (expand_arguments)
       address_pipe_argv = store_get((address_pipe_max_args+1)*sizeof(uschar *), GET_UNTAINTED);
 
       /* +1 because addr->local_part[0] == '|' since af_force_command is set */
       address_pipe_argv = store_get((address_pipe_max_args+1)*sizeof(uschar *), GET_UNTAINTED);
 
       /* +1 because addr->local_part[0] == '|' since af_force_command is set */

-      s = expand_string(addr->local_part + 1);
+      s = expand_cstring(addr->local_part + 1);

 
       if (!s || !*s)
         {
 
       if (!s || !*s)
         {


@@ -2373,7 +2418,7 @@ if (expand_arguments)
          debug_printf("SPECIFIC TESTSUITE EXEMPTION: tainted arg '%s'\n",
                      expanded_arg);
        }
          debug_printf("SPECIFIC TESTSUITE EXEMPTION: tainted arg '%s'\n",
                      expanded_arg);
        }

-      else if (  !allow_tainted_args
+      else if (  !(flags & TSUC_ALLOW_TAINTED_ARGS)

              && arg_is_tainted(expanded_arg, i, addr, etext, errptr))
        return FALSE;
       argv[i] = expanded_arg;
              && arg_is_tainted(expanded_arg, i, addr, etext, errptr))
        return FALSE;
       argv[i] = expanded_arg;