rework userforward local_part_suffix documentation

[exim.git] / src / src / deliver.c

diff --git a/src/src/deliver.c b/src/src/deliver.c
index c183c86ea906fe518298a988f8ba7225da5752be..10b63397efdadad24bdad409e2562c146b1c6888 100644 (file)
--- a/src/src/deliver.c
+++ b/src/src/deliver.c
@@ -1,10 +1,8 @@
-/* $Cambridge: exim/src/src/deliver.c,v 1.17 2005/06/14 13:48:40 ph10 Exp $ */
-

 /*************************************************
 *     Exim - an Internet mail transport agent    *
 *************************************************/
 
 /*************************************************
 *     Exim - an Internet mail transport agent    *
 *************************************************/
 

-/* Copyright (c) University of Cambridge 1995 - 2005 */
+/* Copyright (c) University of Cambridge 1995 - 2009 */

 /* See the file NOTICE for conditions of use and distribution. */
 
 /* The main code for delivering a message. */
 /* See the file NOTICE for conditions of use and distribution. */
 
 /* The main code for delivering a message. */


@@ -228,11 +226,18 @@ if (addr->next == NULL)
   }
 
 /* For multiple addresses, don't set local part, and leave the domain and
   }
 
 /* For multiple addresses, don't set local part, and leave the domain and

-self_hostname set only if it is the same for all of them. */
+self_hostname set only if it is the same for all of them. It is possible to
+have multiple pipe and file addresses, but only when all addresses have routed
+to the same pipe or file. */

 
 else
   {
   address_item *addr2;
 
 else
   {
   address_item *addr2;

+  if (testflag(addr, af_pfr))
+    {
+    if (testflag(addr, af_file)) address_file = addr->local_part;
+      else if (addr->local_part[0] == '|') address_pipe = addr->local_part;
+    }

   for (addr2 = addr->next; addr2 != NULL; addr2 = addr2->next)
     {
     if (deliver_domain != NULL &&
   for (addr2 = addr->next; addr2 != NULL; addr2 = addr2->next)
     {
     if (deliver_domain != NULL &&


@@ -286,7 +291,7 @@ doesn't always get set automatically. */
 
 if (fd >= 0)
   {
 
 if (fd >= 0)
   {

-  fcntl(fd, F_SETFD, fcntl(fd, F_GETFD) | FD_CLOEXEC);
+  (void)fcntl(fd, F_SETFD, fcntl(fd, F_GETFD) | FD_CLOEXEC);

   if (fchown(fd, exim_uid, exim_gid) < 0)
     {
     *error = US"chown";
   if (fchown(fd, exim_uid, exim_gid) < 0)
     {
     *error = US"chown";


@@ -617,7 +622,7 @@ for (dup = addr_duplicate; dup != NULL; dup = dup->next)
   {
   if (Ustrcmp(addr->unique, dup->unique) == 0)
     {
   {
   if (Ustrcmp(addr->unique, dup->unique) == 0)
     {

-    tree_add_nonrecipient(dup->address);
+    tree_add_nonrecipient(dup->unique);

     child_done(dup, now);
     }
   }
     child_done(dup, now);
     }
   }


@@ -737,17 +742,15 @@ malformed, it won't ever have gone near LDAP.) */
 if (addr->message != NULL)
   {
   addr->message = string_printing(addr->message);
 if (addr->message != NULL)
   {
   addr->message = string_printing(addr->message);

-  if (Ustrstr(addr->message, "failed to expand") != NULL &&
-      (Ustrstr(addr->message, "ldap:") != NULL ||
+  if (((Ustrstr(addr->message, "failed to expand") != NULL) || (Ustrstr(addr->message, "expansion of ") != NULL)) &&
+      (Ustrstr(addr->message, "mysql") != NULL ||
+       Ustrstr(addr->message, "pgsql") != NULL ||
+       Ustrstr(addr->message, "sqlite") != NULL ||
+       Ustrstr(addr->message, "ldap:") != NULL ||

        Ustrstr(addr->message, "ldapdn:") != NULL ||
        Ustrstr(addr->message, "ldapm:") != NULL))
     {
        Ustrstr(addr->message, "ldapdn:") != NULL ||
        Ustrstr(addr->message, "ldapm:") != NULL))
     {

-    uschar *p = Ustrstr(addr->message, "pass=");
-    if (p != NULL)
-      {
-      p += 5;
-      while (*p != 0 && !isspace(*p)) *p++ = 'x';
-      }
+      addr->message = string_sprintf("Temporary internal error");

     }
   }
 
     }
   }
 


@@ -767,7 +770,7 @@ if (addr->return_file >= 0 && addr->return_filename != NULL)
   {
   BOOL return_output = FALSE;
   struct stat statbuf;
   {
   BOOL return_output = FALSE;
   struct stat statbuf;

-  fsync(addr->return_file);
+  (void)EXIMfsync(addr->return_file);

 
   /* If there is no output, do nothing. */
 
 
   /* If there is no output, do nothing. */
 


@@ -798,7 +801,7 @@ if (addr->return_file >= 0 && addr->return_filename != NULL)
           log_write(0, LOG_MAIN, "<%s>: %s transport output: %s",
             addr->address, tb->name, s);
           }
           log_write(0, LOG_MAIN, "<%s>: %s transport output: %s",
             addr->address, tb->name, s);
           }

-        fclose(f);
+        (void)fclose(f);

         }
       }
 
         }
       }
 


@@ -829,7 +832,7 @@ if (addr->return_file >= 0 && addr->return_filename != NULL)
     addr->return_file = -1;
     }
 
     addr->return_file = -1;
     }
 

-  close(addr->return_file);
+  (void)close(addr->return_file);

   }
 
 /* Create the address string for logging. Must not do this earlier, because
   }
 
 /* Create the address string for logging. Must not do this earlier, because


@@ -937,7 +940,8 @@ if (result == OK)
       s = string_append(s, &size, &ptr, 2, US" CV=",
         testflag(addr, af_cert_verified)? "yes":"no");
     if ((log_extra_selector & LX_tls_peerdn) != 0 && addr->peerdn != NULL)
       s = string_append(s, &size, &ptr, 2, US" CV=",
         testflag(addr, af_cert_verified)? "yes":"no");
     if ((log_extra_selector & LX_tls_peerdn) != 0 && addr->peerdn != NULL)

-      s = string_append(s, &size, &ptr, 3, US" DN=\"", addr->peerdn, US"\"");
+      s = string_append(s, &size, &ptr, 3, US" DN=\"",
+        string_printing(addr->peerdn), US"\"");

     #endif
 
     if ((log_extra_selector & LX_smtp_confirmation) != 0 &&
     #endif
 
     if ((log_extra_selector & LX_smtp_confirmation) != 0 &&


@@ -1304,6 +1308,14 @@ else if (tp->expand_gid != NULL)
     }
   }
 
     }
   }
 

+/* If the transport did not set a group, see if the router did. */
+
+if (!gid_set && testflag(addr, af_gid_set))
+  {
+  *gidp = addr->gid;
+  gid_set = TRUE;
+  }
+

 /* Pick up a uid from the transport if one is set. */
 
 if (tp->uid_set) *uidp = tp->uid;
 /* Pick up a uid from the transport if one is set. */
 
 if (tp->uid_set) *uidp = tp->uid;


@@ -1339,20 +1351,13 @@ else if (tp->deliver_as_creator)
     }
   }
 
     }
   }
 

-/* Otherwise see if the address specifies the uid and if so, take its
-initgroups flag. The gid from the address is taken only if the transport hasn't
-set it. In other words, a gid on the transport overrides the gid on the
-address. */
+/* Otherwise see if the address specifies the uid and if so, take it and its
+initgroups flag. */

 
 else if (testflag(addr, af_uid_set))
   {
   *uidp = addr->uid;
   *igfp = testflag(addr, af_initgroups);
 
 else if (testflag(addr, af_uid_set))
   {
   *uidp = addr->uid;
   *igfp = testflag(addr, af_initgroups);

-  if (!gid_set)
-    {
-    *gidp = addr->gid;
-    gid_set = TRUE;
-    }

   }
 
 /* Nothing has specified the uid - default to the Exim user, and group if the
   }
 
 /* Nothing has specified the uid - default to the Exim user, and group if the


@@ -1368,7 +1373,9 @@ else
     }
   }
 
     }
   }
 

-/* If no gid is set, it is a disaster. */
+/* If no gid is set, it is a disaster. We default to the Exim gid only if
+defaulting to the Exim uid. In other words, if the configuration has specified
+a uid, it must also provide a gid. */

 
 if (!gid_set)
   {
 
 if (!gid_set)
   {


@@ -1423,10 +1430,10 @@ int rc = OK;
 int size_limit;
 
 deliver_set_expansions(addr);
 int size_limit;
 
 deliver_set_expansions(addr);

-size_limit = expand_string_integer(tp->message_size_limit);
+size_limit = expand_string_integer(tp->message_size_limit, TRUE);

 deliver_set_expansions(NULL);
 
 deliver_set_expansions(NULL);
 

-if (size_limit < 0)
+if (expand_string_message != NULL)

   {
   rc = DEFER;
   if (size_limit == -1)
   {
   rc = DEFER;
   if (size_limit == -1)


@@ -1487,6 +1494,44 @@ return FALSE;
 
 
 
 
 
 

+/******************************************************
+*      Check for a given header in a header string    *
+******************************************************/
+
+/* This function is used when generating quota warnings. The configuration may
+specify any header lines it likes in quota_warn_message. If certain of them are
+missing, defaults are inserted, so we need to be able to test for the presence
+of a given header.
+
+Arguments:
+  hdr         the required header name
+  hstring     the header string
+
+Returns:      TRUE  the header is in the string
+              FALSE the header is not in the string
+*/
+
+static BOOL
+contains_header(uschar *hdr, uschar *hstring)
+{
+int len = Ustrlen(hdr);
+uschar *p = hstring;
+while (*p != 0)
+  {
+  if (strncmpic(p, hdr, len) == 0)
+    {
+    p += len;
+    while (*p == ' ' || *p == '\t') p++;
+    if (*p == ':') return TRUE;
+    }
+  while (*p != 0 && *p != '\n') p++;
+  if (*p == '\n') p++;
+  }
+return FALSE;
+}
+
+
+

 
 /*************************************************
 *           Perform a local delivery             *
 
 /*************************************************
 *           Perform a local delivery             *


@@ -1578,17 +1623,13 @@ return. */
 
 if (!findugid(addr, tp, &uid, &gid, &use_initgroups)) return;
 
 
 if (!findugid(addr, tp, &uid, &gid, &use_initgroups)) return;
 

-/* See if either the transport or the address specifies a home and/or a current
-working directory. Expand it if necessary. If nothing is set, use "/", for the
-working directory, which is assumed to be a directory to which all users have
-access. It is necessary to be in a visible directory for some operating systems
-when running pipes, as some commands (e.g. "rm" under Solaris 2.5) require
-this. */
-
-deliver_home = (tp->home_dir != NULL)? tp->home_dir :
-               (addr->home_dir != NULL)? addr->home_dir : NULL;
+/* See if either the transport or the address specifies a home directory. A
+home directory set in the address may already be expanded; a flag is set to
+indicate that. In other cases we must expand it. */

 
 

-if (deliver_home != NULL && !testflag(addr, af_home_expanded))
+if ((deliver_home = tp->home_dir) != NULL ||       /* Set in transport, or */
+     ((deliver_home = addr->home_dir) != NULL &&   /* Set in address and */
+       !testflag(addr, af_home_expanded)))         /*   not expanded */

   {
   uschar *rawhome = deliver_home;
   deliver_home = NULL;                      /* in case it contains $home */
   {
   uschar *rawhome = deliver_home;
   deliver_home = NULL;                      /* in case it contains $home */


@@ -1608,8 +1649,15 @@ if (deliver_home != NULL && !testflag(addr, af_home_expanded))
     }
   }
 
     }
   }
 

-working_directory = (tp->current_dir != NULL)? tp->current_dir :
-                    (addr->current_dir != NULL)? addr->current_dir : NULL;
+/* See if either the transport or the address specifies a current directory,
+and if so, expand it. If nothing is set, use the home directory, unless it is
+also unset in which case use "/", which is assumed to be a directory to which
+all users have access. It is necessary to be in a visible directory for some
+operating systems when running pipes, as some commands (e.g. "rm" under Solaris
+2.5) require this. */
+
+working_directory = (tp->current_dir != NULL)?
+  tp->current_dir : addr->current_dir;

 
 if (working_directory != NULL)
   {
 
 if (working_directory != NULL)
   {


@@ -1675,7 +1723,20 @@ if ((pid = fork()) == 0)
   HP-UX doesn't have RLIMIT_CORE; I don't know how to do this in that
   system. Some experimental/developing systems (e.g. GNU/Hurd) may define
   RLIMIT_CORE but not support it in setrlimit(). For such systems, do not
   HP-UX doesn't have RLIMIT_CORE; I don't know how to do this in that
   system. Some experimental/developing systems (e.g. GNU/Hurd) may define
   RLIMIT_CORE but not support it in setrlimit(). For such systems, do not

-  complain if the error is "not supported". */
+  complain if the error is "not supported".
+
+  There are two scenarios where changing the max limit has an effect.  In one,
+  the user is using a .forward and invoking a command of their choice via pipe;
+  for these, we do need the max limit to be 0 unless the admin chooses to
+  permit an increased limit.  In the other, the command is invoked directly by
+  the transport and is under administrator control, thus being able to raise
+  the limit aids in debugging.  So there's no general always-right answer.
+
+  Thus we inhibit core-dumps completely but let individual transports, while
+  still root, re-raise the limits back up to aid debugging.  We make the
+  default be no core-dumps -- few enough people can use core dumps in
+  diagnosis that it's reasonable to make them something that has to be explicitly requested.
+  */

 
   #ifdef RLIMIT_CORE
   struct rlimit rl;
 
   #ifdef RLIMIT_CORE
   struct rlimit rl;


@@ -1702,7 +1763,7 @@ if ((pid = fork()) == 0)
 
   if (addr->transport->setup != NULL)
     {
 
   if (addr->transport->setup != NULL)
     {

-    switch((addr->transport->setup)(addr->transport, addr, NULL,
+    switch((addr->transport->setup)(addr->transport, addr, NULL, uid, gid,

            &(addr->message)))
       {
       case DEFER:
            &(addr->message)))
       {
       case DEFER:


@@ -1728,8 +1789,8 @@ if ((pid = fork()) == 0)
   half - for transports that exec things (e.g. pipe). Then set the required
   gid/uid. */
 
   half - for transports that exec things (e.g. pipe). Then set the required
   gid/uid. */
 

-  close(pfd[pipe_read]);
-  fcntl(pfd[pipe_write], F_SETFD, fcntl(pfd[pipe_write], F_GETFD) |
+  (void)close(pfd[pipe_read]);
+  (void)fcntl(pfd[pipe_write], F_SETFD, fcntl(pfd[pipe_write], F_GETFD) |

     FD_CLOEXEC);
   exim_setugid(uid, gid, use_initgroups,
     string_sprintf("local delivery to %s <%s> transport=%s", addr->local_part,
     FD_CLOEXEC);
   exim_setugid(uid, gid, use_initgroups,
     string_sprintf("local delivery to %s <%s> transport=%s", addr->local_part,


@@ -1794,13 +1855,13 @@ if ((pid = fork()) == 0)
     int local_part_length = Ustrlen(addr2->local_part);
     uschar *s;
 
     int local_part_length = Ustrlen(addr2->local_part);
     uschar *s;
 

-    write(pfd[pipe_write], (void *)&(addr2->transport_return), sizeof(int));
-    write(pfd[pipe_write], (void *)&transport_count, sizeof(transport_count));
-    write(pfd[pipe_write], (void *)&(addr2->flags), sizeof(addr2->flags));
-    write(pfd[pipe_write], (void *)&(addr2->basic_errno), sizeof(int));
-    write(pfd[pipe_write], (void *)&(addr2->more_errno), sizeof(int));
-    write(pfd[pipe_write], (void *)&(addr2->special_action), sizeof(int));
-    write(pfd[pipe_write], (void *)&(addr2->transport),
+    (void)write(pfd[pipe_write], (void *)&(addr2->transport_return), sizeof(int));
+    (void)write(pfd[pipe_write], (void *)&transport_count, sizeof(transport_count));
+    (void)write(pfd[pipe_write], (void *)&(addr2->flags), sizeof(addr2->flags));
+    (void)write(pfd[pipe_write], (void *)&(addr2->basic_errno), sizeof(int));
+    (void)write(pfd[pipe_write], (void *)&(addr2->more_errno), sizeof(int));
+    (void)write(pfd[pipe_write], (void *)&(addr2->special_action), sizeof(int));
+    (void)write(pfd[pipe_write], (void *)&(addr2->transport),

       sizeof(transport_instance *));
 
     /* For a file delivery, pass back the local part, in case the original
       sizeof(transport_instance *));
 
     /* For a file delivery, pass back the local part, in case the original


@@ -1809,8 +1870,8 @@ if ((pid = fork()) == 0)
 
     if (testflag(addr2, af_file))
       {
 
     if (testflag(addr2, af_file))
       {

-      write(pfd[pipe_write], (void *)&local_part_length, sizeof(int));
-      write(pfd[pipe_write], addr2->local_part, local_part_length);
+      (void)write(pfd[pipe_write], (void *)&local_part_length, sizeof(int));
+      (void)write(pfd[pipe_write], addr2->local_part, local_part_length);

       }
 
     /* Now any messages */
       }
 
     /* Now any messages */


@@ -1818,15 +1879,15 @@ if ((pid = fork()) == 0)
     for (i = 0, s = addr2->message; i < 2; i++, s = addr2->user_message)
       {
       int message_length = (s == NULL)? 0 : Ustrlen(s) + 1;
     for (i = 0, s = addr2->message; i < 2; i++, s = addr2->user_message)
       {
       int message_length = (s == NULL)? 0 : Ustrlen(s) + 1;

-      write(pfd[pipe_write], (void *)&message_length, sizeof(int));
-      if (message_length > 0) write(pfd[pipe_write], s, message_length);
+      (void)write(pfd[pipe_write], (void *)&message_length, sizeof(int));
+      if (message_length > 0) (void)write(pfd[pipe_write], s, message_length);

       }
     }
 
   /* OK, this process is now done. Free any cached resources that it opened,
   and close the pipe we were writing down before exiting. */
 
       }
     }
 
   /* OK, this process is now done. Free any cached resources that it opened,
   and close the pipe we were writing down before exiting. */
 

-  close(pfd[pipe_write]);
+  (void)close(pfd[pipe_write]);

   search_tidyup();
   exit(EXIT_SUCCESS);
   }
   search_tidyup();
   exit(EXIT_SUCCESS);
   }


@@ -1845,7 +1906,7 @@ on an empty pipe. We check that a status exists for each address before
 overwriting the address structure. If data is missing, the default DEFER status
 will remain. Afterwards, close the reading end. */
 
 overwriting the address structure. If data is missing, the default DEFER status
 will remain. Afterwards, close the reading end. */
 

-close(pfd[pipe_write]);
+(void)close(pfd[pipe_write]);

 
 for (addr2 = addr; addr2 != NULL; addr2 = addr2->next)
   {
 
 for (addr2 = addr; addr2 != NULL; addr2 = addr2->next)
   {


@@ -1895,7 +1956,7 @@ for (addr2 = addr; addr2 != NULL; addr2 = addr2->next)
     }
   }
 
     }
   }
 

-close(pfd[pipe_read]);
+(void)close(pfd[pipe_read]);

 
 /* Unless shadowing, write all successful addresses immediately to the journal
 file, to ensure they are recorded asap. For homonymic addresses, use the base
 
 /* Unless shadowing, write all successful addresses immediately to the journal
 file, to ensure they are recorded asap. For homonymic addresses, use the base


@@ -1928,7 +1989,7 @@ if (!shadowing)
 
   /* Ensure the journal file is pushed out to disk. */
 
 
   /* Ensure the journal file is pushed out to disk. */
 

-  if (fsync(journal_fd) < 0)
+  if (EXIMfsync(journal_fd) < 0)

     log_write(0, LOG_MAIN|LOG_PANIC, "failed to fsync journal: %s",
       strerror(errno));
   }
     log_write(0, LOG_MAIN|LOG_PANIC, "failed to fsync journal: %s",
       strerror(errno));
   }


@@ -1988,17 +2049,16 @@ if (addr->special_action == SPECIAL_WARN &&
     if (pid > 0)
       {
       FILE *f = fdopen(fd, "wb");
     if (pid > 0)
       {
       FILE *f = fdopen(fd, "wb");

-
-      if (errors_reply_to != NULL)
+      if (errors_reply_to != NULL &&
+          !contains_header(US"Reply-To", warn_message))

         fprintf(f, "Reply-To: %s\n", errors_reply_to);
         fprintf(f, "Reply-To: %s\n", errors_reply_to);

-      fprintf(f, "Auto-Submitted: auto-generated\n");
-      fprintf(f, "From: Mail Delivery System <Mailer-Daemon@%s>\n",
-        qualify_domain_sender);
+      fprintf(f, "Auto-Submitted: auto-replied\n");
+      if (!contains_header(US"From", warn_message)) moan_write_from(f);

       fprintf(f, "%s", CS warn_message);
 
       /* Close and wait for child process to complete, without a timeout. */
 
       fprintf(f, "%s", CS warn_message);
 
       /* Close and wait for child process to complete, without a timeout. */
 

-      fclose(f);
+      (void)fclose(f);

       (void)child_close(pid, 0);
       }
     }
       (void)child_close(pid, 0);
       }
     }


@@ -2077,15 +2137,17 @@ while (addr_local != NULL)
 
   disable_logging = tp->disable_logging;
 
 
   disable_logging = tp->disable_logging;
 

-  /* Check for batched addresses and possible amalgamation. File deliveries can
-  never be batched. Skip all the work if either batch_max <= 1 or there aren't
-  any other addresses for local delivery. */
+  /* Check for batched addresses and possible amalgamation. Skip all the work
+  if either batch_max <= 1 or there aren't any other addresses for local
+  delivery. */

 
 

-  if (!testflag(addr, af_file) && tp->batch_max > 1 && addr_local != NULL)
+  if (tp->batch_max > 1 && addr_local != NULL)

     {
     int batch_count = 1;
     BOOL uses_dom = readconf_depends((driver_instance *)tp, US"domain");
     {
     int batch_count = 1;
     BOOL uses_dom = readconf_depends((driver_instance *)tp, US"domain");

-    BOOL uses_lp = readconf_depends((driver_instance *)tp, US"local_part");
+    BOOL uses_lp = (testflag(addr, af_pfr) &&
+      (testflag(addr, af_file) || addr->local_part[0] == '|')) ||
+      readconf_depends((driver_instance *)tp, US"local_part");

     uschar *batch_id = NULL;
     address_item **anchor = &addr_local;
     address_item *last = addr;
     uschar *batch_id = NULL;
     address_item **anchor = &addr_local;
     address_item *last = addr;


@@ -2114,6 +2176,7 @@ while (addr_local != NULL)
       same transport
       not previously delivered (see comment about 50 lines above)
       same local part if the transport's configuration contains $local_part
       same transport
       not previously delivered (see comment about 50 lines above)
       same local part if the transport's configuration contains $local_part

+        or if this is a file or pipe delivery from a redirection

       same domain if the transport's configuration contains $domain
       same errors address
       same additional headers
       same domain if the transport's configuration contains $domain
       same errors address
       same additional headers


@@ -2127,6 +2190,7 @@ while (addr_local != NULL)
       BOOL ok =
         tp == next->transport &&
         !previously_transported(next, TRUE) &&
       BOOL ok =
         tp == next->transport &&
         !previously_transported(next, TRUE) &&

+        (addr->flags & (af_pfr|af_file)) == (next->flags & (af_pfr|af_file)) &&

         (!uses_lp  || Ustrcmp(next->local_part, addr->local_part) == 0) &&
         (!uses_dom || Ustrcmp(next->domain, addr->domain) == 0) &&
         same_strings(next->p.errors_address, addr->p.errors_address) &&
         (!uses_lp  || Ustrcmp(next->local_part, addr->local_part) == 0) &&
         (!uses_dom || Ustrcmp(next->domain, addr->domain) == 0) &&
         same_strings(next->p.errors_address, addr->p.errors_address) &&


@@ -2247,10 +2311,12 @@ while (addr_local != NULL)
 
         DEBUG(D_retry)
           {
 
         DEBUG(D_retry)
           {

-          debug_printf("retry record exists: age=%d (max=%d)\n",
-            (int)(now - retry_record->time_stamp), retry_data_expire);
-          debug_printf("  time to retry = %d expired = %d\n",
-            (int)(now - retry_record->next_try), retry_record->expired);
+          debug_printf("retry record exists: age=%s ",
+            readconf_printtime(now - retry_record->time_stamp));
+          debug_printf("(max %s)\n", readconf_printtime(retry_data_expire));
+          debug_printf("  time to retry = %s expired = %d\n",
+            readconf_printtime(retry_record->next_try - now),
+            retry_record->expired);

           }
 
         if (queue_running && !deliver_force)
           }
 
         if (queue_running && !deliver_force)


@@ -2270,8 +2336,13 @@ while (addr_local != NULL)
                 retry_record->more_errno);
 
             DEBUG(D_deliver|D_retry)
                 retry_record->more_errno);
 
             DEBUG(D_deliver|D_retry)

+              {

               debug_printf("retry time not reached for %s: "
                 "checking ultimate address timeout\n", addr2->address);
               debug_printf("retry time not reached for %s: "
                 "checking ultimate address timeout\n", addr2->address);

+              debug_printf("  now=%d first_failed=%d next_try=%d expired=%d\n",
+                (int)now, (int)retry_record->first_failed,
+                (int)retry_record->next_try, retry_record->expired);
+              }

 
             if (retry != NULL && retry->rules != NULL)
               {
 
             if (retry != NULL && retry->rules != NULL)
               {


@@ -2279,9 +2350,17 @@ while (addr_local != NULL)
               for (last_rule = retry->rules;
                    last_rule->next != NULL;
                    last_rule = last_rule->next);
               for (last_rule = retry->rules;
                    last_rule->next != NULL;
                    last_rule = last_rule->next);

+              DEBUG(D_deliver|D_retry)
+                debug_printf("  received_time=%d diff=%d timeout=%d\n",
+                  received_time, (int)now - received_time, last_rule->timeout);

               if (now - received_time > last_rule->timeout) ok = TRUE;
               }
               if (now - received_time > last_rule->timeout) ok = TRUE;
               }

-            else ok = TRUE;    /* No rule => timed out */
+            else
+              {
+              DEBUG(D_deliver|D_retry)
+                debug_printf("no retry rule found: assume timed out\n");
+              ok = TRUE;    /* No rule => timed out */
+              }

 
             DEBUG(D_deliver|D_retry)
               {
 
             DEBUG(D_deliver|D_retry)
               {


@@ -2885,7 +2964,7 @@ if (!eop && !done)
 /* Close our end of the pipe, to prevent deadlock if the far end is still
 pushing stuff into it. */
 
 /* Close our end of the pipe, to prevent deadlock if the far end is still
 pushing stuff into it. */
 

-close(fd);
+(void)close(fd);

 p->fd = -1;
 
 /* If we have finished without error, but haven't had data for every address,
 p->fd = -1;
 
 /* If we have finished without error, but haven't had data for every address,


@@ -3564,12 +3643,25 @@ for (delivery_count = 0; addr_remote != NULL; delivery_count++)
     else return_path = new_return_path;
     }
 
     else return_path = new_return_path;
     }
 

+  /* Find the uid, gid, and use_initgroups setting for this transport. Failure
+  logs and sets up error messages, so we just post-process and continue with
+  the next address. */
+
+  if (!findugid(addr, tp, &uid, &gid, &use_initgroups))
+    {
+    remote_post_process(addr, LOG_MAIN|LOG_PANIC, NULL, fallback);
+    continue;
+    }
+

   /* If this transport has a setup function, call it now so that it gets
   run in this process and not in any subprocess. That way, the results of
   /* If this transport has a setup function, call it now so that it gets
   run in this process and not in any subprocess. That way, the results of

-  any setup that are retained by the transport can be reusable. */
+  any setup that are retained by the transport can be reusable. One of the
+  things the setup does is to set the fallback host lists in the addresses.
+  That is why it is called at this point, before the continue delivery
+  processing, because that might use the fallback hosts. */

 
   if (tp->setup != NULL)
 
   if (tp->setup != NULL)

-    (void)((tp->setup)(addr->transport, addr, NULL, NULL));
+    (void)((tp->setup)(addr->transport, addr, NULL, uid, gid, NULL));

 
   /* If this is a run to continue delivery down an already-established
   channel, check that this set of addresses matches the transport and
 
   /* If this is a run to continue delivery down an already-established
   channel, check that this set of addresses matches the transport and


@@ -3645,16 +3737,6 @@ for (delivery_count = 0; addr_remote != NULL; delivery_count++)
 
   transport_filter_argv = NULL;
 
 
   transport_filter_argv = NULL;
 

-  /* Find the uid, gid, and use_initgroups setting for this transport. Failure
-  logs and sets up error messages, so we just post-process and continue with
-  the next address. */
-
-  if (!findugid(addr, tp, &uid, &gid, &use_initgroups))
-    {
-    remote_post_process(addr, LOG_MAIN|LOG_PANIC, NULL, fallback);
-    continue;
-    }
-

   /* Create the pipe for inter-process communication. If pipe creation
   fails, it is probably because the value of remote_max_parallel is so
   large that too many file descriptors for pipes have been created. Arrange
   /* Create the pipe for inter-process communication. If pipe creation
   fails, it is probably because the value of remote_max_parallel is so
   large that too many file descriptors for pipes have been created. Arrange


@@ -3673,9 +3755,9 @@ for (delivery_count = 0; addr_remote != NULL; delivery_count++)
     distinguishes between EOF and no-more-data. */
 
     #ifdef O_NONBLOCK
     distinguishes between EOF and no-more-data. */
 
     #ifdef O_NONBLOCK

-    fcntl(pfd[pipe_read], F_SETFL, O_NONBLOCK);
+    (void)fcntl(pfd[pipe_read], F_SETFL, O_NONBLOCK);

     #else
     #else

-    fcntl(pfd[pipe_read], F_SETFL, O_NDELAY);
+    (void)fcntl(pfd[pipe_read], F_SETFL, O_NDELAY);

     #endif
 
     /* If the maximum number of subprocesses already exist, wait for a process
     #endif
 
     /* If the maximum number of subprocesses already exist, wait for a process


@@ -3707,8 +3789,8 @@ for (delivery_count = 0; addr_remote != NULL; delivery_count++)
 
   if (poffset >= remote_max_parallel)
     {
 
   if (poffset >= remote_max_parallel)
     {

-    close(pfd[pipe_write]);
-    close(pfd[pipe_read]);
+    (void)close(pfd[pipe_write]);
+    (void)close(pfd[pipe_read]);

     remote_post_process(addr, LOG_MAIN|LOG_PANIC,
       US"Unexpectedly no free subprocess slot", fallback);
     continue;
     remote_post_process(addr, LOG_MAIN|LOG_PANIC,
       US"Unexpectedly no free subprocess slot", fallback);
     continue;


@@ -3748,13 +3830,13 @@ for (delivery_count = 0; addr_remote != NULL; delivery_count++)
     a new process that may be forked to do another delivery down the same
     SMTP connection. */
 
     a new process that may be forked to do another delivery down the same
     SMTP connection. */
 

-    fcntl(fd, F_SETFD, fcntl(fd, F_GETFD) | FD_CLOEXEC);
+    (void)fcntl(fd, F_SETFD, fcntl(fd, F_GETFD) | FD_CLOEXEC);

 
     /* Close open file descriptors for the pipes of other processes
     that are running in parallel. */
 
     for (poffset = 0; poffset < remote_max_parallel; poffset++)
 
     /* Close open file descriptors for the pipes of other processes
     that are running in parallel. */
 
     for (poffset = 0; poffset < remote_max_parallel; poffset++)

-      if (parlist[poffset].pid != 0) close(parlist[poffset].fd);
+      if (parlist[poffset].pid != 0) (void)close(parlist[poffset].fd);

 
     /* This process has inherited a copy of the file descriptor
     for the data file, but its file pointer is shared with all the
 
     /* This process has inherited a copy of the file descriptor
     for the data file, but its file pointer is shared with all the


@@ -3764,7 +3846,7 @@ for (delivery_count = 0; addr_remote != NULL; delivery_count++)
     the parent process. There doesn't seem to be any way of doing
     a dup-with-new-file-pointer. */
 
     the parent process. There doesn't seem to be any way of doing
     a dup-with-new-file-pointer. */
 

-    close(deliver_datafile);
+    (void)close(deliver_datafile);

     sprintf(CS spoolname, "%s/input/%s/%s-D", spool_directory, message_subdir,
       message_id);
     deliver_datafile = Uopen(spoolname, O_RDWR | O_APPEND, 0);
     sprintf(CS spoolname, "%s/input/%s/%s-D", spool_directory, message_subdir,
       message_id);
     deliver_datafile = Uopen(spoolname, O_RDWR | O_APPEND, 0);


@@ -3775,7 +3857,7 @@ for (delivery_count = 0; addr_remote != NULL; delivery_count++)
 
     /* Set the close-on-exec flag */
 
 
     /* Set the close-on-exec flag */
 

-    fcntl(deliver_datafile, F_SETFD, fcntl(deliver_datafile, F_GETFD) |
+    (void)fcntl(deliver_datafile, F_SETFD, fcntl(deliver_datafile, F_GETFD) |

       FD_CLOEXEC);
 
     /* Set the uid/gid of this process; bombs out on failure. */
       FD_CLOEXEC);
 
     /* Set the uid/gid of this process; bombs out on failure. */


@@ -3788,7 +3870,7 @@ for (delivery_count = 0; addr_remote != NULL; delivery_count++)
     and run the transport. Afterwards, transport_count will contain the number
     of bytes written. */
 
     and run the transport. Afterwards, transport_count will contain the number
     of bytes written. */
 

-    close(pfd[pipe_read]);
+    (void)close(pfd[pipe_read]);

     set_process_info("delivering %s using %s", message_id, tp->name);
     debug_print_string(tp->debug_string);
     if (!(tp->info->code)(addr->transport, addr)) replicate_status(addr);
     set_process_info("delivering %s using %s", message_id, tp->name);
     debug_print_string(tp->debug_string);
     if (!(tp->info->code)(addr->transport, addr)) replicate_status(addr);


@@ -3817,7 +3899,7 @@ for (delivery_count = 0; addr_remote != NULL; delivery_count++)
       {
       if (h->address == NULL || h->status < hstatus_unusable) continue;
       sprintf(CS big_buffer, "H%c%c%s", h->status, h->why, h->address);
       {
       if (h->address == NULL || h->status < hstatus_unusable) continue;
       sprintf(CS big_buffer, "H%c%c%s", h->status, h->why, h->address);

-      write(fd, big_buffer, Ustrlen(big_buffer+3) + 4);
+      (void)write(fd, big_buffer, Ustrlen(big_buffer+3) + 4);

       }
 
     /* The number of bytes written. This is the same for each address. Even
       }
 
     /* The number of bytes written. This is the same for each address. Even


@@ -3827,7 +3909,7 @@ for (delivery_count = 0; addr_remote != NULL; delivery_count++)
 
     big_buffer[0] = 'S';
     memcpy(big_buffer+1, &transport_count, sizeof(transport_count));
 
     big_buffer[0] = 'S';
     memcpy(big_buffer+1, &transport_count, sizeof(transport_count));

-    write(fd, big_buffer, sizeof(transport_count) + 1);
+    (void)write(fd, big_buffer, sizeof(transport_count) + 1);

 
     /* Information about what happened to each address. Three item types are
     used: an optional 'X' item first, for TLS information, followed by 'R'
 
     /* Information about what happened to each address. Three item types are
     used: an optional 'X' item first, for TLS information, followed by 'R'


@@ -3857,7 +3939,7 @@ for (delivery_count = 0; addr_remote != NULL; delivery_count++)
           sprintf(CS ptr, "%.512s", addr->peerdn);
           while(*ptr++);
           }
           sprintf(CS ptr, "%.512s", addr->peerdn);
           while(*ptr++);
           }

-        write(fd, big_buffer, ptr - big_buffer);
+        (void)write(fd, big_buffer, ptr - big_buffer);

         }
       #endif
 
         }
       #endif
 


@@ -3877,7 +3959,7 @@ for (delivery_count = 0; addr_remote != NULL; delivery_count++)
           sprintf(CS ptr, "%.512s", r->message);
           while(*ptr++);
           }
           sprintf(CS ptr, "%.512s", r->message);
           while(*ptr++);
           }

-        write(fd, big_buffer, ptr - big_buffer);
+        (void)write(fd, big_buffer, ptr - big_buffer);

         }
 
       /* The rest of the information goes in an 'A' item. */
         }
 
       /* The rest of the information goes in an 'A' item. */


@@ -3913,7 +3995,7 @@ for (delivery_count = 0; addr_remote != NULL; delivery_count++)
         memcpy(ptr, &(addr->host_used->port), sizeof(addr->host_used->port));
         ptr += sizeof(addr->host_used->port);
         }
         memcpy(ptr, &(addr->host_used->port), sizeof(addr->host_used->port));
         ptr += sizeof(addr->host_used->port);
         }

-      write(fd, big_buffer, ptr - big_buffer);
+      (void)write(fd, big_buffer, ptr - big_buffer);

       }
 
     /* Add termination flag, close the pipe, and that's it. The character
       }
 
     /* Add termination flag, close the pipe, and that's it. The character


@@ -3923,20 +4005,20 @@ for (delivery_count = 0; addr_remote != NULL; delivery_count++)
 
     big_buffer[0] = 'Z';
     big_buffer[1] = (continue_transport == NULL)? '0' : '1';
 
     big_buffer[0] = 'Z';
     big_buffer[1] = (continue_transport == NULL)? '0' : '1';

-    write(fd, big_buffer, 2);
-    close(fd);
+    (void)write(fd, big_buffer, 2);
+    (void)close(fd);

     exit(EXIT_SUCCESS);
     }
 
   /* Back in the mainline: close the unwanted half of the pipe. */
 
     exit(EXIT_SUCCESS);
     }
 
   /* Back in the mainline: close the unwanted half of the pipe. */
 

-  close(pfd[pipe_write]);
+  (void)close(pfd[pipe_write]);

 
   /* Fork failed; defer with error message */
 
   if (pid < 0)
     {
 
   /* Fork failed; defer with error message */
 
   if (pid < 0)
     {

-    close(pfd[pipe_read]);
+    (void)close(pfd[pipe_read]);

     remote_post_process(addr, LOG_MAIN|LOG_PANIC,
       string_sprintf("fork failed for remote delivery to %s: %s",
         addr->domain, strerror(errno)), fallback);
     remote_post_process(addr, LOG_MAIN|LOG_PANIC,
       string_sprintf("fork failed for remote delivery to %s: %s",
         addr->domain, strerror(errno)), fallback);


@@ -4243,15 +4325,15 @@ introducing newlines. All lines are indented by 4; the initial printing
 position must be set before calling.
 
 This function used always to print the error. Nowadays we want to restrict it
 position must be set before calling.
 
 This function used always to print the error. Nowadays we want to restrict it

-to cases such as SMTP errors from a remote host, and errors from :fail: and
-filter "fail". We no longer pass other information willy-nilly in bounce and
-warning messages. Text in user_message is always output; text in message only
-if the af_pass_message flag is set.
+to cases such as LMTP/SMTP errors from a remote host, and errors from :fail:
+and filter "fail". We no longer pass other information willy-nilly in bounce
+and warning messages. Text in user_message is always output; text in message
+only if the af_pass_message flag is set.

 
 Arguments:
   addr         the address
   f            the FILE to print on
 
 Arguments:
   addr         the address
   f            the FILE to print on

-  s            some leading text
+  t            some leading text

 
 Returns:       nothing
 */
 
 Returns:       nothing
 */


@@ -4260,14 +4342,11 @@ static void
 print_address_error(address_item *addr, FILE *f, uschar *t)
 {
 int count = Ustrlen(t);
 print_address_error(address_item *addr, FILE *f, uschar *t)
 {
 int count = Ustrlen(t);

-uschar *s = (addr->user_message != NULL)? addr->user_message : addr->message;
+uschar *s = testflag(addr, af_pass_message)? addr->message : NULL;

 
 

-if (addr->user_message != NULL)
-  s = addr->user_message;
-else
+if (s == NULL)

   {
   {

-  if (!testflag(addr, af_pass_message) || addr->message == NULL) return;
-  s = addr->message;
+  if (addr->user_message != NULL) s = addr->user_message; else return;

   }
 
 fprintf(f, "\n    %s", t);
   }
 
 fprintf(f, "\n    %s", t);


@@ -4305,16 +4384,13 @@ while (*s != 0)
 /* This function was introduced when the test for duplicate addresses that are
 not pipes, files, or autoreplies was moved from the middle of routing to when
 routing was complete. That was to fix obscure cases when the routing history
 /* This function was introduced when the test for duplicate addresses that are
 not pipes, files, or autoreplies was moved from the middle of routing to when
 routing was complete. That was to fix obscure cases when the routing history

-affects the subsequent routing of identical addresses. If that change has to be
-reversed, this function is no longer needed. For a while, the old code that was
-affected by this change is commented with !!!OLD-DE-DUP!!! so it can be found
-easily.
+affects the subsequent routing of identical addresses. This function is called
+after routing, to check that the final routed addresses are not duplicates.

 
 

-This function is called after routing, to check that the final routed addresses
-are not duplicates. If we detect a duplicate, we remember what it is a
-duplicate of. Note that pipe, file, and autoreply de-duplication is handled
-during routing, so we must leave such "addresses" alone here, as otherwise they
-will incorrectly be discarded.
+If we detect a duplicate, we remember what it is a duplicate of. Note that
+pipe, file, and autoreply de-duplication is handled during routing, so we must
+leave such "addresses" alone here, as otherwise they will incorrectly be
+discarded.

 
 Argument:     address of list anchor
 Returns:      nothing
 
 Argument:     address of list anchor
 Returns:      nothing


@@ -4482,8 +4558,8 @@ if ((rc = spool_read_header(spoolname, TRUE, TRUE)) != spool_read_OK)
     sprintf(CS big_buffer, "%s/input/%s/%s", spool_directory, message_subdir,
       spoolname);
     if (Ustat(big_buffer, &statbuf) == 0)
     sprintf(CS big_buffer, "%s/input/%s/%s", spool_directory, message_subdir,
       spoolname);
     if (Ustat(big_buffer, &statbuf) == 0)

-      log_write(0, LOG_MAIN, "Format error in spool file %s: size=%.30g",
-        spoolname, (double)statbuf.st_size);
+      log_write(0, LOG_MAIN, "Format error in spool file %s: "
+        "size=" OFF_T_FMT, spoolname, statbuf.st_size);

     else log_write(0, LOG_MAIN, "Format error in spool file %s", spoolname);
     }
   else
     else log_write(0, LOG_MAIN, "Format error in spool file %s", spoolname);
     }
   else


@@ -4517,7 +4593,7 @@ if ((rc = spool_read_header(spoolname, TRUE, TRUE)) != spool_read_OK)
       readconf_printtime(keep_malformed));
     }
 
       readconf_printtime(keep_malformed));
     }
 

-  close(deliver_datafile);
+  (void)close(deliver_datafile);

   deliver_datafile = -1;
   return continue_closedown();   /* yields DELIVER_NOT_ATTEMPTED */
   }
   deliver_datafile = -1;
   return continue_closedown();   /* yields DELIVER_NOT_ATTEMPTED */
   }


@@ -4543,7 +4619,7 @@ if (jread != NULL)
     DEBUG(D_deliver) debug_printf("Previously delivered address %s taken from "
       "journal file\n", big_buffer);
     }
     DEBUG(D_deliver) debug_printf("Previously delivered address %s taken from "
       "journal file\n", big_buffer);
     }

-  fclose(jread);
+  (void)fclose(jread);

   /* Panic-dies on error */
   (void)spool_write_header(message_id, SW_DELIVERING, NULL);
   }
   /* Panic-dies on error */
   (void)spool_write_header(message_id, SW_DELIVERING, NULL);
   }


@@ -4558,7 +4634,7 @@ else if (errno != ENOENT)
 
 if (recipients_list == NULL)
   {
 
 if (recipients_list == NULL)
   {

-  close(deliver_datafile);
+  (void)close(deliver_datafile);

   deliver_datafile = -1;
   log_write(0, LOG_MAIN, "Spool error: no recipients for %s", spoolname);
   return continue_closedown();   /* yields DELIVER_NOT_ATTEMPTED */
   deliver_datafile = -1;
   log_write(0, LOG_MAIN, "Spool error: no recipients for %s", spoolname);
   return continue_closedown();   /* yields DELIVER_NOT_ATTEMPTED */


@@ -4618,7 +4694,7 @@ if (deliver_freeze)
           continue_hostname != NULL
         ))
       {
           continue_hostname != NULL
         ))
       {

-      close(deliver_datafile);
+      (void)close(deliver_datafile);

       deliver_datafile = -1;
       log_write(L_skip_delivery, LOG_MAIN, "Message is frozen");
       return continue_closedown();   /* yields DELIVER_NOT_ATTEMPTED */
       deliver_datafile = -1;
       log_write(L_skip_delivery, LOG_MAIN, "Message is frozen");
       return continue_closedown();   /* yields DELIVER_NOT_ATTEMPTED */


@@ -4740,6 +4816,7 @@ else if (system_filter != NULL && process_recipients != RECIP_FAIL_TIMEOUT)
       RDO_REWRITE,
     NULL,                   /* No :include: restriction (not used in filter) */
     NULL,                   /* No sieve vacation directory (not sieve!) */
       RDO_REWRITE,
     NULL,                   /* No :include: restriction (not used in filter) */
     NULL,                   /* No sieve vacation directory (not sieve!) */

+    NULL,                   /* No sieve enotify mailto owner (not sieve!) */

     NULL,                   /* No sieve user address (not sieve!) */
     NULL,                   /* No sieve subaddress (not sieve!) */
     &ugid,                  /* uid/gid data */
     NULL,                   /* No sieve user address (not sieve!) */
     NULL,                   /* No sieve subaddress (not sieve!) */
     &ugid,                  /* uid/gid data */


@@ -4753,7 +4830,7 @@ else if (system_filter != NULL && process_recipients != RECIP_FAIL_TIMEOUT)
 
   if (rc == FF_ERROR || rc == FF_NONEXIST)
     {
 
   if (rc == FF_ERROR || rc == FF_NONEXIST)
     {

-    close(deliver_datafile);
+    (void)close(deliver_datafile);

     deliver_datafile = -1;
     log_write(0, LOG_MAIN|LOG_PANIC, "Error in system filter: %s",
       string_printing(filter_message));
     deliver_datafile = -1;
     log_write(0, LOG_MAIN|LOG_PANIC, "Error in system filter: %s",
       string_printing(filter_message));


@@ -4873,6 +4950,9 @@ else if (system_filter != NULL && process_recipients != RECIP_FAIL_TIMEOUT)
 
     while (p != NULL)
       {
 
     while (p != NULL)
       {

+      if (parent->child_count == SHRT_MAX)
+        log_write(0, LOG_MAIN|LOG_PANIC_DIE, "system filter generated more "
+          "than %d delivery addresses", SHRT_MAX);

       parent->child_count++;
       p->parent = parent;
 
       parent->child_count++;
       p->parent = parent;
 


@@ -5172,7 +5252,20 @@ while (addr_new != NULL)           /* Loop until all addresses dealt with */
 
     if (testflag(addr, af_pfr))
       {
 
     if (testflag(addr, af_pfr))
       {

-      int offset = testflag(addr->parent, af_homonym)? 3:0;
+      /* If an autoreply in a filter could not generate a syntactically valid
+      address, give up forthwith. Set af_ignore_error so that we don't try to
+      generate a bounce. */
+
+      if (testflag(addr, af_bad_reply))
+        {
+        addr->basic_errno = ERRNO_BADADDRESS2;
+        addr->local_part = addr->address;
+        addr->message =
+          US"filter autoreply generated syntactically invalid recipient";
+        setflag(addr, af_ignore_error);
+        (void)post_process_one(addr, FAIL, LOG_MAIN, DTYPE_ROUTER, 0);
+        continue;   /* with the next new address */
+        }

 
       /* If two different users specify delivery to the same pipe or file or
       autoreply, there should be two different deliveries, so build a unique
 
       /* If two different users specify delivery to the same pipe or file or
       autoreply, there should be two different deliveries, so build a unique


@@ -5180,7 +5273,8 @@ while (addr_new != NULL)           /* Loop until all addresses dealt with */
       duplicate testing and recording delivery, and also for retrying. */
 
       addr->unique =
       duplicate testing and recording delivery, and also for retrying. */
 
       addr->unique =

-        string_sprintf("%s:%s", addr->address, addr->parent->unique + offset);
+        string_sprintf("%s:%s", addr->address, addr->parent->unique +
+          (testflag(addr->parent, af_homonym)? 3:0));

 
       addr->address_retry_key = addr->domain_retry_key =
         string_sprintf("T:%s", addr->unique);
 
       addr->address_retry_key = addr->domain_retry_key =
         string_sprintf("T:%s", addr->unique);


@@ -5369,40 +5463,11 @@ while (addr_new != NULL)           /* Loop until all addresses dealt with */
       continue;
       }
 
       continue;
       }
 

-
-    /* !!!OLD-DE-DUP!!!  We used to test for duplicates at this point, in order
-    to save effort on routing duplicate addresses. However, facilities have
-    been added to Exim so that now two identical addresses that are children of
-    other addresses may be routed differently as a result of their previous
-    routing history. For example, different redirect routers may have given
-    them different redirect_router values, but there are other cases too.
-    Therefore, tests for duplicates now take place when routing is complete.
-    This is the old code, kept for a while for the record, and in case this
-    radical change has to be backed out for some reason. */
-
-    #ifdef NEVER
-    /* If it's a duplicate, remember what it's a duplicate of */
-
-    if ((tnode = tree_search(tree_duplicates, addr->unique)) != NULL)
-      {
-      DEBUG(D_deliver|D_route)
-        debug_printf("%s is a duplicate address: discarded\n", addr->unique);
-      addr->dupof = tnode->data.ptr;
-      addr->next = addr_duplicate;
-      addr_duplicate = addr;
-      continue;
-      }
-
-    /* Record this address, so subsequent duplicates get picked up. */
-
-    tree_add_duplicate(addr->unique, addr);
-    #endif
-
-
-

     /* Get the routing retry status, saving the two retry keys (with and
     /* Get the routing retry status, saving the two retry keys (with and

-    without the local part) for subsequent use. Ignore retry records that
-    are too old. */
+    without the local part) for subsequent use. If there is no retry record for
+    the standard address routing retry key, we look for the same key with the
+    sender attached, because this form is used by the smtp transport after a
+    4xx response to RCPT when address_retry_include_sender is true. */

 
     addr->domain_retry_key = string_sprintf("R:%s", addr->domain);
     addr->address_retry_key = string_sprintf("R:%s@%s", addr->local_part,
 
     addr->domain_retry_key = string_sprintf("R:%s", addr->domain);
     addr->address_retry_key = string_sprintf("R:%s@%s", addr->local_part,


@@ -5415,12 +5480,22 @@ while (addr_new != NULL)           /* Loop until all addresses dealt with */
       domain_retry_record = dbfn_read(dbm_file, addr->domain_retry_key);
       if (domain_retry_record != NULL &&
           now - domain_retry_record->time_stamp > retry_data_expire)
       domain_retry_record = dbfn_read(dbm_file, addr->domain_retry_key);
       if (domain_retry_record != NULL &&
           now - domain_retry_record->time_stamp > retry_data_expire)

-        domain_retry_record = NULL;
+        domain_retry_record = NULL;    /* Ignore if too old */

 
       address_retry_record = dbfn_read(dbm_file, addr->address_retry_key);
       if (address_retry_record != NULL &&
           now - address_retry_record->time_stamp > retry_data_expire)
 
       address_retry_record = dbfn_read(dbm_file, addr->address_retry_key);
       if (address_retry_record != NULL &&
           now - address_retry_record->time_stamp > retry_data_expire)

-        address_retry_record = NULL;
+        address_retry_record = NULL;   /* Ignore if too old */
+
+      if (address_retry_record == NULL)
+        {
+        uschar *altkey = string_sprintf("%s:<%s>", addr->address_retry_key,
+          sender_address);
+        address_retry_record = dbfn_read(dbm_file, altkey);
+        if (address_retry_record != NULL &&
+            now - address_retry_record->time_stamp > retry_data_expire)
+          address_retry_record = NULL;   /* Ignore if too old */
+        }

       }
 
     DEBUG(D_deliver|D_retry)
       }
 
     DEBUG(D_deliver|D_retry)


@@ -5452,19 +5527,29 @@ while (addr_new != NULL)           /* Loop until all addresses dealt with */
       (void)post_process_one(addr, DEFER, LOG_MAIN, DTYPE_ROUTER, 0);
       }
 
       (void)post_process_one(addr, DEFER, LOG_MAIN, DTYPE_ROUTER, 0);
       }
 

-    /* If queue_running, defer routing unless no retry data or we've
-    passed the next retry time, or this message is forced. However,
-    if the retry time has expired, allow the routing attempt.
-    If it fails again, the address will be failed. This ensures that
+    /* If we are in a queue run, defer routing unless there is no retry data or
+    we've passed the next retry time, or this message is forced. In other
+    words, ignore retry data when not in a queue run.
+
+    However, if the domain retry time has expired, always allow the routing
+    attempt. If it fails again, the address will be failed. This ensures that

     each address is routed at least once, even after long-term routing
     failures.
 
     If there is an address retry, check that too; just wait for the next
     retry time. This helps with the case when the temporary error on the
     address was really message-specific rather than address specific, since
     each address is routed at least once, even after long-term routing
     failures.
 
     If there is an address retry, check that too; just wait for the next
     retry time. This helps with the case when the temporary error on the
     address was really message-specific rather than address specific, since

-    it allows other messages through. */
+    it allows other messages through.
+
+    We also wait for the next retry time if this is a message sent down an
+    existing SMTP connection (even though that will be forced). Otherwise there
+    will be far too many attempts for an address that gets a 4xx error. In
+    fact, after such an error, we should not get here because, the host should
+    not be remembered as one this message needs. However, there was a bug that
+    used to cause this to  happen, so it is best to be on the safe side. */

 
 

-    else if (!deliver_force && queue_running &&
+    else if (((queue_running && !deliver_force) || continue_hostname != NULL)
+            &&

             ((domain_retry_record != NULL &&
               now < domain_retry_record->next_try &&
               !domain_retry_record->expired)
             ((domain_retry_record != NULL &&
               now < domain_retry_record->next_try &&
               !domain_retry_record->expired)


@@ -5563,12 +5648,16 @@ while (addr_new != NULL)           /* Loop until all addresses dealt with */
         string_sprintf("R:%s", addr->domain), 0);
 
     /* Otherwise, if there is an existing retry record in the database, add
         string_sprintf("R:%s", addr->domain), 0);
 
     /* Otherwise, if there is an existing retry record in the database, add

-    retry items to delete both forms. Since the domain might have been
-    rewritten (expanded to fully qualified) as a result of routing, ensure
-    that the rewritten form is also deleted. */
+    retry items to delete both forms. We must also allow for the possibility
+    of a routing retry that includes the sender address. Since the domain might
+    have been rewritten (expanded to fully qualified) as a result of routing,
+    ensure that the rewritten form is also deleted. */

 
     else if (testflag(addr, af_dr_retry_exists))
       {
 
     else if (testflag(addr, af_dr_retry_exists))
       {

+      uschar *altkey = string_sprintf("%s:<%s>", addr->address_retry_key,
+        sender_address);
+      retry_add_item(addr, altkey, rf_delete);

       retry_add_item(addr, addr->address_retry_key, rf_delete);
       retry_add_item(addr, addr->domain_retry_key, rf_delete);
       if (Ustrcmp(addr->domain, old_domain) != 0)
       retry_add_item(addr, addr->address_retry_key, rf_delete);
       retry_add_item(addr, addr->domain_retry_key, rf_delete);
       if (Ustrcmp(addr->domain, old_domain) != 0)


@@ -5711,11 +5800,6 @@ Ensure they are not set in transports. */
 local_user_gid = (gid_t)(-1);
 local_user_uid = (uid_t)(-1);
 
 local_user_gid = (gid_t)(-1);
 local_user_uid = (uid_t)(-1);
 

-
-/* !!!OLD-DE-DUP!!! The next two statement were introduced when checking for
-duplicates was moved from within routing to afterwards. If that change has to
-be backed out, they should be removed. */
-

 /* Check for any duplicate addresses. This check is delayed until after
 routing, because the flexibility of the routing configuration means that
 identical addresses with different parentage may end up being redirected to
 /* Check for any duplicate addresses. This check is delayed until after
 routing, because the flexibility of the routing configuration means that
 identical addresses with different parentage may end up being redirected to


@@ -5725,7 +5809,6 @@ to) makes this kind of thing not work. */
 do_duplicate_check(&addr_local);
 do_duplicate_check(&addr_remote);
 
 do_duplicate_check(&addr_local);
 do_duplicate_check(&addr_remote);
 

-

 /* When acting as an MUA wrapper, we proceed only if all addresses route to a
 remote transport. The check that they all end up in one transaction happens in
 the do_remote_deliveries() function. */
 /* When acting as an MUA wrapper, we proceed only if all addresses route to a
 remote transport. The check that they all end up in one transaction happens in
 the do_remote_deliveries() function. */


@@ -5845,9 +5928,9 @@ if (addr_local != NULL || addr_remote != NULL)
   that the mode is correct - the group setting doesn't always seem to get
   set automatically. */
 
   that the mode is correct - the group setting doesn't always seem to get
   set automatically. */
 

-  fcntl(journal_fd, F_SETFD, fcntl(journal_fd, F_GETFD) | FD_CLOEXEC);
-  fchown(journal_fd, exim_uid, exim_gid);
-  fchmod(journal_fd, SPOOL_MODE);
+  (void)fcntl(journal_fd, F_SETFD, fcntl(journal_fd, F_GETFD) | FD_CLOEXEC);
+  (void)fchown(journal_fd, exim_uid, exim_gid);
+  (void)fchmod(journal_fd, SPOOL_MODE);

   }
 
 
   }
 
 


@@ -5856,6 +5939,15 @@ deliveries are done first, then remote ones. If ever the problems of how to
 handle fallback transports are figured out, this section can be put into a loop
 for handling fallbacks, though the uid switching will have to be revised. */
 
 handle fallback transports are figured out, this section can be put into a loop
 for handling fallbacks, though the uid switching will have to be revised. */
 

+/* Precompile a regex that is used to recognize a parameter in response
+to an LHLO command, if is isn't already compiled. This may be used on both
+local and remote LMTP deliveries. */
+
+if (regex_IGNOREQUOTA == NULL) regex_IGNOREQUOTA =
+  regex_must_compile(US"\\n250[\\s\\-]IGNOREQUOTA(\\s|\\n|$)", FALSE, TRUE);
+
+/* Handle local deliveries */
+

 if (addr_local != NULL)
   {
   DEBUG(D_deliver|D_transport)
 if (addr_local != NULL)
   {
   DEBUG(D_deliver|D_transport)


@@ -5952,11 +6044,29 @@ set_process_info("tidying up after delivering %s", message_id);
 signal(SIGTERM, SIG_IGN);
 
 /* When we are acting as an MUA wrapper, the smtp transport will either have
 signal(SIGTERM, SIG_IGN);
 
 /* When we are acting as an MUA wrapper, the smtp transport will either have

-succeeded for all addresses, or failed them all. We do not ever want to retry,
-nor do we want to send a bounce message. */
+succeeded for all addresses, or failed them all in normal cases. However, there
+are some setup situations (e.g. when a named port does not exist) that cause an
+immediate exit with deferral of all addresses. Convert those into failures. We
+do not ever want to retry, nor do we want to send a bounce message. */

 
 if (mua_wrapper)
   {
 
 if (mua_wrapper)
   {

+  if (addr_defer != NULL)
+    {
+    address_item *addr, *nextaddr;
+    for (addr = addr_defer; addr != NULL; addr = nextaddr)
+      {
+      log_write(0, LOG_MAIN, "** %s mua_wrapper forced failure for deferred "
+        "delivery", addr->address);
+      nextaddr = addr->next;
+      addr->next = addr_failed;
+      addr_failed = addr;
+      }
+    addr_defer = NULL;
+    }
+
+  /* Now all should either have succeeded or failed. */
+

   if (addr_failed == NULL) final_yield = DELIVER_MUA_SUCCEEDED; else
     {
     uschar *s = (addr_failed->user_message != NULL)?
   if (addr_failed == NULL) final_yield = DELIVER_MUA_SUCCEEDED; else
     {
     uschar *s = (addr_failed->user_message != NULL)?


@@ -6150,9 +6260,8 @@ while (addr_failed != NULL)
 
       if (errors_reply_to != NULL)
         fprintf(f, "Reply-To: %s\n", errors_reply_to);
 
       if (errors_reply_to != NULL)
         fprintf(f, "Reply-To: %s\n", errors_reply_to);

-      fprintf(f, "Auto-Submitted: auto-generated\n");
-      fprintf(f, "From: Mail Delivery System <Mailer-Daemon@%s>\n",
-        qualify_domain_sender);
+      fprintf(f, "Auto-Submitted: auto-replied\n");
+      moan_write_from(f);

       fprintf(f, "To: %s\n", bounce_recipient);
 
       /* Open a template file if one is provided. Log failure to open, but
       fprintf(f, "To: %s\n", bounce_recipient);
 
       /* Open a template file if one is provided. Log failure to open, but


@@ -6292,7 +6401,7 @@ wording. */
           else
             {
             while ((ch = fgetc(fm)) != EOF) fputc(ch, f);
           else
             {
             while ((ch = fgetc(fm)) != EOF) fputc(ch, f);

-            fclose(fm);
+            (void)fclose(fm);

             }
           Uunlink(addr->return_filename);
 
             }
           Uunlink(addr->return_filename);
 


@@ -6345,8 +6454,8 @@ wording. */
             if (emf_text != NULL) fprintf(f, "%s", CS emf_text); else
               {
               fprintf(f,
             if (emf_text != NULL) fprintf(f, "%s", CS emf_text); else
               {
               fprintf(f,

-"------ The body of the message is %.30g characters long; only the first\n"
-"------ %d or so are included here.\n", (double)statbuf.st_size, max);
+"------ The body of the message is " OFF_T_FMT " characters long; only the first\n"
+"------ %d or so are included here.\n", statbuf.st_size, max);

               }
             }
           }
               }
             }
           }


@@ -6365,13 +6474,13 @@ wording. */
         {
         emf_text = next_emf(emf, US"final");
         if (emf_text != NULL) fprintf(f, "%s", CS emf_text);
         {
         emf_text = next_emf(emf, US"final");
         if (emf_text != NULL) fprintf(f, "%s", CS emf_text);

-        fclose(emf);
+        (void)fclose(emf);

         }
 
       /* Close the file, which should send an EOF to the child process
       that is receiving the message. Wait for it to finish. */
 
         }
 
       /* Close the file, which should send an EOF to the child process
       that is receiving the message. Wait for it to finish. */
 

-      fclose(f);
+      (void)fclose(f);

       rc = child_close(pid, 0);     /* Waits for child to close, no timeout */
 
       /* In the test harness, let the child do it's thing first. */
       rc = child_close(pid, 0);     /* Waits for child to close, no timeout */
 
       /* In the test harness, let the child do it's thing first. */


@@ -6455,7 +6564,8 @@ if (addr_defer == NULL)
     else
       {
       if (Uunlink(spoolname) < 0)
     else
       {
       if (Uunlink(spoolname) < 0)

-        log_write(0, LOG_MAIN|LOG_PANIC_DIE, "failed to unlink %s", spoolname);
+        log_write(0, LOG_MAIN|LOG_PANIC_DIE, "failed to unlink %s: %s",
+                 spoolname, strerror(errno));

       }
     }
 
       }
     }
 


@@ -6463,10 +6573,12 @@ if (addr_defer == NULL)
 
   sprintf(CS spoolname, "%s/input/%s/%s-D", spool_directory, message_subdir, id);
   if (Uunlink(spoolname) < 0)
 
   sprintf(CS spoolname, "%s/input/%s/%s-D", spool_directory, message_subdir, id);
   if (Uunlink(spoolname) < 0)

-    log_write(0, LOG_MAIN|LOG_PANIC_DIE, "failed to unlink %s", spoolname);
+    log_write(0, LOG_MAIN|LOG_PANIC_DIE, "failed to unlink %s: %s",
+      spoolname, strerror(errno));

   sprintf(CS spoolname, "%s/input/%s/%s-H", spool_directory, message_subdir, id);
   if (Uunlink(spoolname) < 0)
   sprintf(CS spoolname, "%s/input/%s/%s-H", spool_directory, message_subdir, id);
   if (Uunlink(spoolname) < 0)

-    log_write(0, LOG_MAIN|LOG_PANIC_DIE, "failed to unlink %s", spoolname);
+    log_write(0, LOG_MAIN|LOG_PANIC_DIE, "failed to unlink %s: %s",
+      spoolname, strerror(errno));

 
   /* Log the end of this message, with queue time if requested. */
 
 
   /* Log the end of this message, with queue time if requested. */
 


@@ -6475,6 +6587,9 @@ if (addr_defer == NULL)
       readconf_printtime(time(NULL) - received_time));
   else
     log_write(0, LOG_MAIN, "Completed");
       readconf_printtime(time(NULL) - received_time));
   else
     log_write(0, LOG_MAIN, "Completed");

+
+  /* Unset deliver_freeze so that we won't try to move the spool files further down */
+  deliver_freeze = FALSE;

   }
 
 /* If there are deferred addresses, we are keeping this message because it is
   }
 
 /* If there are deferred addresses, we are keeping this message because it is


@@ -6672,9 +6787,8 @@ else if (addr_defer != (address_item *)(+1))
 
         if (errors_reply_to != NULL)
           fprintf(f, "Reply-To: %s\n", errors_reply_to);
 
         if (errors_reply_to != NULL)
           fprintf(f, "Reply-To: %s\n", errors_reply_to);

-        fprintf(f, "Auto-Submitted: auto-generated\n");
-        fprintf(f, "From: Mail Delivery System <Mailer-Daemon@%s>\n",
-          qualify_domain_sender);
+        fprintf(f, "Auto-Submitted: auto-replied\n");
+        moan_write_from(f);

         fprintf(f, "To: %s\n", recipients);
 
         wmf_text = next_emf(wmf, US"header");
         fprintf(f, "To: %s\n", recipients);
 
         wmf_text = next_emf(wmf, US"header");


@@ -6738,7 +6852,7 @@ else if (addr_defer != (address_item *)(+1))
           {
           wmf_text = next_emf(wmf, US"final");
           if (wmf_text != NULL) fprintf(f, "%s", CS wmf_text);
           {
           wmf_text = next_emf(wmf, US"final");
           if (wmf_text != NULL) fprintf(f, "%s", CS wmf_text);

-          fclose(wmf);
+          (void)fclose(wmf);

           }
         else
           {
           }
         else
           {


@@ -6752,7 +6866,7 @@ else if (addr_defer != (address_item *)(+1))
         /* Close and wait for child process to complete, without a timeout.
         If there's an error, don't update the count. */
 
         /* Close and wait for child process to complete, without a timeout.
         If there's an error, don't update the count. */
 

-        fclose(f);
+        (void)fclose(f);

         if (child_close(pid, 0) == 0)
           {
           warning_count = count;
         if (child_close(pid, 0) == 0)
           {
           warning_count = count;


@@ -6835,7 +6949,7 @@ else if (addr_defer != (address_item *)(+1))
 /* Finished with the message log. If the message is complete, it will have
 been unlinked or renamed above. */
 
 /* Finished with the message log. If the message is complete, it will have
 been unlinked or renamed above. */
 

-if (message_logs) fclose(message_log);
+if (message_logs) (void)fclose(message_log);

 
 /* Now we can close and remove the journal file. Its only purpose is to record
 successfully completed deliveries asap so that this information doesn't get
 
 /* Now we can close and remove the journal file. Its only purpose is to record
 successfully completed deliveries asap so that this information doesn't get


@@ -6849,7 +6963,7 @@ the remove_journal flag. When the journal is left, we also don't move the
 message off the main spool if frozen and the option is set. It should get moved
 at the next attempt, after the journal has been inspected. */
 
 message off the main spool if frozen and the option is set. It should get moved
 at the next attempt, after the journal has been inspected. */
 

-if (journal_fd >= 0) close(journal_fd);
+if (journal_fd >= 0) (void)close(journal_fd);

 
 if (remove_journal)
   {
 
 if (remove_journal)
   {


@@ -6870,7 +6984,7 @@ if (remove_journal)
 will go away. Otherwise the message becomes available for another process
 to try delivery. */
 
 will go away. Otherwise the message becomes available for another process
 to try delivery. */
 

-close(deliver_datafile);
+(void)close(deliver_datafile);

 deliver_datafile = -1;
 DEBUG(D_deliver) debug_printf("end delivery of %s\n", id);
 
 deliver_datafile = -1;
 DEBUG(D_deliver) debug_printf("end delivery of %s\n", id);