doc: string2 not expanded, don't imply otherwise
[exim.git] / src / src / verify.c
index 96c2b66cc0b16a0a95117043a785c24eeedd1f0a..3a8f9b6bc97af3de252cf5fe334668f91eb00f42 100644 (file)
@@ -39,7 +39,7 @@ static tree_node *dnsbl_cache = NULL;
 #define MT_NOT 1
 #define MT_ALL 2
 
-static uschar cutthrough_response(char, uschar **, int);
+static uschar cutthrough_response(int, char, uschar **, int);
 
 
 
@@ -388,7 +388,7 @@ if (addr->transport == cutthrough.addr.transport)
       deliver_domain = addr->domain;
       transport_name = addr->transport->name;
 
-      host_af = (Ustrchr(host->address, ':') == NULL)? AF_INET:AF_INET6;
+      host_af = Ustrchr(host->address, ':') ? AF_INET6 : AF_INET;
 
       if (!smtp_get_interface(tf->interface, host_af, addr, &interface,
              US"callout") ||
@@ -408,10 +408,10 @@ if (addr->transport == cutthrough.addr.transport)
 
        /* Match!  Send the RCPT TO, set done from the response */
        done =
-         smtp_write_command(&ctblock, FALSE, "RCPT TO:<%.1000s>\r\n",
+         smtp_write_command(&ctblock, SCMD_FLUSH, "RCPT TO:<%.1000s>\r\n",
            transport_rcpt_address(addr,
               addr->transport->rcpt_include_affixes)) >= 0 &&
-         cutthrough_response('2', &resp, CUTTHROUGH_DATA_TIMEOUT) == '2';
+         cutthrough_response(cutthrough.fd, '2', &resp, CUTTHROUGH_DATA_TIMEOUT) == '2';
 
        /* This would go horribly wrong if a callout fail was ignored by ACL.
        We punt by abandoning cutthrough on a reject, like the
@@ -429,7 +429,7 @@ if (addr->transport == cutthrough.addr.transport)
          }
        else
          {
-         cancel_cutthrough_connection("recipient rejected");
+         cancel_cutthrough_connection(TRUE, US"recipient rejected");
          if (!resp || errno == ETIMEDOUT)
            {
            HDEBUG(D_verify) debug_printf("SMTP timeout\n");
@@ -459,7 +459,7 @@ if (addr->transport == cutthrough.addr.transport)
       break;   /* host_list */
       }
 if (!done)
-  cancel_cutthrough_connection("incompatible connection");
+  cancel_cutthrough_connection(TRUE, US"incompatible connection");
 return done;
 }
 
@@ -490,6 +490,7 @@ Arguments:
                       vopt_callout_random => do the "random" thing
                       vopt_callout_recipsender => use real sender for recipient
                       vopt_callout_recippmaster => use postmaster for recipient
+                     vopt_callout_hold         => lazy close connection
   se_mailfrom         MAIL FROM address for sender verify; NULL => ""
   pm_mailfrom         if non-NULL, do the postmaster check with this sender
 
@@ -556,7 +557,10 @@ else
 if (cached_callout_lookup(addr, address_key, from_address,
       &options, &pm_mailfrom, &yield, failure_ptr,
       &new_domain_record, &old_domain_cache_result))
+  {
+  cancel_cutthrough_connection(TRUE, US"cache-hit");
   goto END_CALLOUT;
+  }
 
 if (!addr->transport)
   {
@@ -682,18 +686,21 @@ tls_retry_connection:
     SMTP command to send.  If we tried TLS but it failed, try again without
     if permitted */
 
-    if (  (yield = smtp_setup_conn(&sx, FALSE)) == DEFER
+    yield = smtp_setup_conn(&sx, FALSE);
+#ifdef SUPPORT_TLS
+    if (  yield == DEFER
        && addr->basic_errno == ERRNO_TLSFAILURE
        && ob->tls_tempfail_tryclear
        && verify_check_given_host(&ob->hosts_require_tls, host) != OK
        )
       {
-      log_write(0, LOG_MAIN, "TLS session failure:"
-       " callout unencrypted to %s [%s] (not in hosts_require_tls)",
-       host->name, host->address);
+      log_write(0, LOG_MAIN,
+       "%s: callout unencrypted to %s [%s] (not in hosts_require_tls)",
+       addr->message, host->name, host->address);
       addr->transport_return = PENDING_DEFER;
       yield = smtp_setup_conn(&sx, TRUE);
       }
+#endif
     if (yield != OK)
       {
       errno = addr->basic_errno;
@@ -727,8 +734,7 @@ tls_retry_connection:
     sx.send_rset = TRUE;
     sx.completed_addr = FALSE;
 
-    new_domain_record.result =
-      old_domain_cache_result == ccache_reject_mfnull
+    new_domain_record.result = old_domain_cache_result == ccache_reject_mfnull
       ? ccache_reject_mfnull : ccache_accept;
 
     /* Do the random local part check first. Temporarily replace the recipient
@@ -754,9 +760,12 @@ tls_retry_connection:
        }
 #endif
 
-      /* This would be ok for 1st rcpt of a cutthrough (XXX do we have a count?) , but no way to
-      handle a subsequent because of the RSET.  So refuse to support any. */
-      cancel_cutthrough_connection("random-recipient");
+      /* This would be ok for 1st rcpt of a cutthrough (the case handled here;
+      subsequents are done in cutthrough_multi()), but no way to
+      handle a subsequent because of the RSET vaporising the MAIL FROM.
+      So refuse to support any.  Most cutthrough use will not involve
+      random_local_part, so no loss. */
+      cancel_cutthrough_connection(TRUE, US"random-recipient");
 
       addr->address = string_sprintf("%s@%.1000s",
                                    random_local_part, rcpt_domain);
@@ -795,18 +804,18 @@ tls_retry_connection:
            XXX We don't care about that for postmaster_full.  Should we? */
 
            if ((done =
-             smtp_write_command(&sx.outblock, FALSE, "RSET\r\n") >= 0 &&
+             smtp_write_command(&sx.outblock, SCMD_FLUSH, "RSET\r\n") >= 0 &&
              smtp_read_response(&sx.inblock, sx.buffer, sizeof(sx.buffer),
                '2', callout)))
              break;
 
            HDEBUG(D_acl|D_v)
-             debug_printf("problem after random/rset/mfrom; reopen conn\n");
+             debug_printf_indent("problem after random/rset/mfrom; reopen conn\n");
            random_local_part = NULL;
 #ifdef SUPPORT_TLS
            tls_close(FALSE, TRUE);
 #endif
-           HDEBUG(D_transport|D_acl|D_v) debug_printf("  SMTP(close)>>\n");
+           HDEBUG(D_transport|D_acl|D_v) debug_printf_indent("  SMTP(close)>>\n");
            (void)close(sx.inblock.sock);
            sx.inblock.sock = sx.outblock.sock = -1;
 #ifndef DISABLE_EVENT
@@ -885,10 +894,10 @@ tls_retry_connection:
       /* Could possibly shift before main verify, just above, and be ok
       for cutthrough.  But no way to handle a subsequent rcpt, so just
       refuse any */
-      cancel_cutthrough_connection("postmaster verify");
-      HDEBUG(D_acl|D_v) debug_printf("Cutthrough cancelled by presence of postmaster verify\n");
+      cancel_cutthrough_connection(TRUE, US"postmaster verify");
+      HDEBUG(D_acl|D_v) debug_printf_indent("Cutthrough cancelled by presence of postmaster verify\n");
 
-      done = smtp_write_command(&sx.outblock, FALSE, "RSET\r\n") >= 0
+      done = smtp_write_command(&sx.outblock, SCMD_FLUSH, "RSET\r\n") >= 0
           && smtp_read_response(&sx.inblock, sx.buffer,
                                sizeof(sx.buffer), '2', callout);
 
@@ -912,7 +921,7 @@ tls_retry_connection:
          done = TRUE;
        else
          done = (options & vopt_callout_fullpm) != 0
-             && smtp_write_command(&sx.outblock, FALSE,
+             && smtp_write_command(&sx.outblock, SCMD_FLUSH,
                            "RCPT TO:<postmaster>\r\n") >= 0
              && smtp_read_response(&sx.inblock, sx.buffer,
                            sizeof(sx.buffer), '2', callout);
@@ -1001,8 +1010,10 @@ no_conn:
 
     /* Cutthrough - on a successful connect and recipient-verify with
     use-sender and we are 1st rcpt and have no cutthrough conn so far
-    here is where we want to leave the conn open */
-    if (  cutthrough.delivery
+    here is where we want to leave the conn open.  Ditto for a lazy-close
+    verify. */
+
+    if (  (cutthrough.delivery || options & vopt_callout_hold)
        && rcpt_count == 1
        && done
        && yield == OK
@@ -1014,14 +1025,29 @@ no_conn:
        && !sx.lmtp
        )
       {
-      HDEBUG(D_acl|D_v) debug_printf("holding verify callout open for cutthrough delivery\n");
-
-      cutthrough.fd = sx.outblock.sock;        /* We assume no buffer in use in the outblock */
-      cutthrough.nrcpt = 1;
-      cutthrough.interface = interface;
-      cutthrough.host = *host;
-      cutthrough.addr = *addr;         /* Save the address_item for later logging */
-      cutthrough.addr.next =     NULL;
+      HDEBUG(D_acl|D_v) debug_printf_indent("holding verify callout open for %s\n",
+       cutthrough.delivery
+       ? "cutthrough delivery" : "potential further verifies and delivery");
+
+      cutthrough.callout_hold_only = !cutthrough.delivery;
+      cutthrough.is_tls =      tls_out.active >= 0;
+      cutthrough.fd =  sx.outblock.sock;       /* We assume no buffer in use in the outblock */
+      cutthrough.nrcpt =       1;
+      cutthrough.transport =   addr->transport->name;
+      cutthrough.interface =   interface;
+      cutthrough.snd_port =    sending_port;
+      cutthrough.peer_options =        smtp_peer_options;
+      cutthrough.host =                *host;
+       {
+       int oldpool = store_pool;
+       store_pool = POOL_PERM;
+       cutthrough.snd_ip = string_copy(sending_ip_address);
+       cutthrough.host.name = string_copy(host->name);
+       cutthrough.host.address = string_copy(host->address);
+       store_pool = oldpool;
+       }
+      cutthrough.addr =                *addr;          /* Save the address_item for later logging */
+      cutthrough.addr.next =   NULL;
       cutthrough.addr.host_used = &cutthrough.host;
       if (addr->parent)
         *(cutthrough.addr.parent = store_get(sizeof(address_item))) =
@@ -1034,12 +1060,12 @@ no_conn:
       }
     else
       {
-      /* Ensure no cutthrough on multiple address verifies */
+      /* Ensure no cutthrough on multiple verifies that were incompatible */
       if (options & vopt_callout_recipsender)
-        cancel_cutthrough_connection("not usable for cutthrough");
+        cancel_cutthrough_connection(TRUE, US"not usable for cutthrough");
       if (sx.send_quit)
        {
-       (void) smtp_write_command(&sx.outblock, FALSE, "QUIT\r\n");
+       (void) smtp_write_command(&sx.outblock, SCMD_FLUSH, "QUIT\r\n");
 
        /* Wait a short time for response, and discard it */
        smtp_read_response(&sx.inblock, sx.buffer, sizeof(sx.buffer),
@@ -1051,7 +1077,7 @@ no_conn:
 #ifdef SUPPORT_TLS
        tls_close(FALSE, TRUE);
 #endif
-       HDEBUG(D_transport|D_acl|D_v) debug_printf("  SMTP(close)>>\n");
+       HDEBUG(D_transport|D_acl|D_v) debug_printf_indent("  SMTP(close)>>\n");
        (void)close(sx.inblock.sock);
        sx.inblock.sock = sx.outblock.sock = -1;
 #ifndef DISABLE_EVENT
@@ -1128,7 +1154,7 @@ int rc;
 get rewritten. */
 
 addr2 = *addr;
-HDEBUG(D_acl) debug_printf("----------- %s cutthrough setup ------------\n",
+HDEBUG(D_acl) debug_printf_indent("----------- %s cutthrough setup ------------\n",
   rcpt_count > 1 ? "more" : "start");
 rc = verify_address(&addr2, NULL,
        vopt_is_recipient | vopt_callout_recipsender | vopt_callout_no_cache,
@@ -1136,7 +1162,7 @@ rc = verify_address(&addr2, NULL,
        NULL, NULL, NULL);
 addr->message = addr2.message;
 addr->user_message = addr2.user_message;
-HDEBUG(D_acl) debug_printf("----------- end cutthrough setup ------------\n");
+HDEBUG(D_acl) debug_printf_indent("----------- end cutthrough setup ------------\n");
 return rc;
 }
 
@@ -1151,7 +1177,7 @@ if(cutthrough.fd < 0)
 
 if(
 #ifdef SUPPORT_TLS
-   (tls_out.active == cutthrough.fd) ? tls_write(FALSE, ctblock.buffer, n) :
+   tls_out.active == cutthrough.fd ? tls_write(FALSE, ctblock.buffer, n, FALSE) :
 #endif
    send(cutthrough.fd, ctblock.buffer, n, 0) > 0
   )
@@ -1161,7 +1187,7 @@ if(
   return TRUE;
 }
 
-HDEBUG(D_transport|D_acl) debug_printf("cutthrough_send failed: %s\n", strerror(errno));
+HDEBUG(D_transport|D_acl) debug_printf_indent("cutthrough_send failed: %s\n", strerror(errno));
 return FALSE;
 }
 
@@ -1182,20 +1208,27 @@ return TRUE;
 }
 
 /* Buffered output of counted data block.   Return boolean success */
-BOOL
+static BOOL
 cutthrough_puts(uschar * cp, int n)
 {
 if (cutthrough.fd < 0)       return TRUE;
 if (_cutthrough_puts(cp, n)) return TRUE;
-cancel_cutthrough_connection("transmit failed");
+cancel_cutthrough_connection(TRUE, US"transmit failed");
 return FALSE;
 }
 
+void
+cutthrough_data_puts(uschar * cp, int n)
+{
+if (cutthrough.delivery) (void) cutthrough_puts(cp, n);
+return;
+}
+
 
 static BOOL
 _cutthrough_flush_send(void)
 {
-int n= ctblock.ptr-ctblock.buffer;
+int n = ctblock.ptr - ctblock.buffer;
 
 if(n>0)
   if(!cutthrough_send(n))
@@ -1209,21 +1242,28 @@ BOOL
 cutthrough_flush_send(void)
 {
 if (_cutthrough_flush_send()) return TRUE;
-cancel_cutthrough_connection("transmit failed");
+cancel_cutthrough_connection(TRUE, US"transmit failed");
 return FALSE;
 }
 
 
-BOOL
+static BOOL
 cutthrough_put_nl(void)
 {
 return cutthrough_puts(US"\r\n", 2);
 }
 
 
+void
+cutthrough_data_put_nl(void)
+{
+cutthrough_data_puts(US"\r\n", 2);
+}
+
+
 /* Get and check response from cutthrough target */
 static uschar
-cutthrough_response(char expect, uschar ** copy, int timeout)
+cutthrough_response(int fd, char expect, uschar ** copy, int timeout)
 {
 smtp_inblock inblock;
 uschar inbuffer[4096];
@@ -1233,12 +1273,12 @@ inblock.buffer = inbuffer;
 inblock.buffersize = sizeof(inbuffer);
 inblock.ptr = inbuffer;
 inblock.ptrend = inbuffer;
-inblock.sock = cutthrough.fd;
+inblock.sock = fd;
 /* this relies on (inblock.sock == tls_out.active) */
 if(!smtp_read_response(&inblock, responsebuffer, sizeof(responsebuffer), expect, timeout))
-  cancel_cutthrough_connection("target timeout on read");
+  cancel_cutthrough_connection(TRUE, US"target timeout on read");
 
-if(copy != NULL)
+if(copy)
   {
   uschar * cp;
   *copy = cp = string_copy(responsebuffer);
@@ -1256,21 +1296,21 @@ return responsebuffer[0];
 BOOL
 cutthrough_predata(void)
 {
-if(cutthrough.fd < 0)
+if(cutthrough.fd < 0 || cutthrough.callout_hold_only)
   return FALSE;
 
-HDEBUG(D_transport|D_acl|D_v) debug_printf("  SMTP>> DATA\n");
+HDEBUG(D_transport|D_acl|D_v) debug_printf_indent("  SMTP>> DATA\n");
 cutthrough_puts(US"DATA\r\n", 6);
 cutthrough_flush_send();
 
 /* Assume nothing buffered.  If it was it gets ignored. */
-return cutthrough_response('3', NULL, CUTTHROUGH_DATA_TIMEOUT) == '3';
+return cutthrough_response(cutthrough.fd, '3', NULL, CUTTHROUGH_DATA_TIMEOUT) == '3';
 }
 
 
-/* fd and tctx args only to match write_chunk() */
+/* tctx arg only to match write_chunk() */
 static BOOL
-cutthrough_write_chunk(int fd, transport_ctx * tctx, uschar * s, int len)
+cutthrough_write_chunk(transport_ctx * tctx, uschar * s, int len)
 {
 uschar * s2;
 while(s && (s2 = Ustrchr(s, '\n')))
@@ -1291,61 +1331,75 @@ cutthrough_headers_send(void)
 {
 transport_ctx tctx;
 
-if(cutthrough.fd < 0)
+if(cutthrough.fd < 0 || cutthrough.callout_hold_only)
   return FALSE;
 
 /* We share a routine with the mainline transport to handle header add/remove/rewrites,
    but having a separate buffered-output function (for now)
 */
-HDEBUG(D_acl) debug_printf("----------- start cutthrough headers send -----------\n");
+HDEBUG(D_acl) debug_printf_indent("----------- start cutthrough headers send -----------\n");
 
+tctx.u.fd = cutthrough.fd;
 tctx.tblock = cutthrough.addr.transport;
 tctx.addr = &cutthrough.addr;
 tctx.check_string = US".";
 tctx.escape_string = US"..";
+/*XXX check under spool_files_wireformat.  Might be irrelevant */
 tctx.options = topt_use_crlf;
 
-if (!transport_headers_send(cutthrough.fd, &tctx, &cutthrough_write_chunk))
+if (!transport_headers_send(&tctx, &cutthrough_write_chunk))
   return FALSE;
 
-HDEBUG(D_acl) debug_printf("----------- done cutthrough headers send ------------\n");
+HDEBUG(D_acl) debug_printf_indent("----------- done cutthrough headers send ------------\n");
 return TRUE;
 }
 
 
 static void
-close_cutthrough_connection(const char * why)
+close_cutthrough_connection(const uschar * why)
 {
-if(cutthrough.fd >= 0)
+int fd = cutthrough.fd;
+if(fd >= 0)
   {
   /* We could be sending this after a bunch of data, but that is ok as
      the only way to cancel the transfer in dataphase is to drop the tcp
      conn before the final dot.
   */
   ctblock.ptr = ctbuffer;
-  HDEBUG(D_transport|D_acl|D_v) debug_printf("  SMTP>> QUIT\n");
+  HDEBUG(D_transport|D_acl|D_v) debug_printf_indent("  SMTP>> QUIT\n");
   _cutthrough_puts(US"QUIT\r\n", 6);   /* avoid recursion */
   _cutthrough_flush_send();
+  cutthrough.fd = -1;                  /* avoid recursion via read timeout */
 
   /* Wait a short time for response, and discard it */
-  cutthrough_response('2', NULL, 1);
+  cutthrough_response(fd, '2', NULL, 1);
 
-  #ifdef SUPPORT_TLS
+#ifdef SUPPORT_TLS
   tls_close(FALSE, TRUE);
-  #endif
-  HDEBUG(D_transport|D_acl|D_v) debug_printf("  SMTP(close)>>\n");
-  (void)close(cutthrough.fd);
-  cutthrough.fd = -1;
-  HDEBUG(D_acl) debug_printf("----------- cutthrough shutdown (%s) ------------\n", why);
+#endif
+  HDEBUG(D_transport|D_acl|D_v) debug_printf_indent("  SMTP(close)>>\n");
+  (void)close(fd);
+  HDEBUG(D_acl) debug_printf_indent("----------- cutthrough shutdown (%s) ------------\n", why);
   }
 ctblock.ptr = ctbuffer;
 }
 
 void
-cancel_cutthrough_connection(const char * why)
+cancel_cutthrough_connection(BOOL close_noncutthrough_verifies, const uschar * why)
+{
+if (cutthrough.delivery || close_noncutthrough_verifies)
+  close_cutthrough_connection(why);
+cutthrough.delivery = cutthrough.callout_hold_only = FALSE;
+}
+
+
+void
+release_cutthrough_connection(const uschar * why)
 {
-close_cutthrough_connection(why);
-cutthrough.delivery = FALSE;
+if (cutthrough.fd < 0) return;
+HDEBUG(D_acl) debug_printf_indent("release cutthrough conn: %s\n", why);
+cutthrough.fd = -1;
+cutthrough.delivery = cutthrough.callout_hold_only = FALSE;
 }
 
 
@@ -1361,7 +1415,7 @@ cutthrough_finaldot(void)
 {
 uschar res;
 address_item * addr;
-HDEBUG(D_transport|D_acl|D_v) debug_printf("  SMTP>> .\n");
+HDEBUG(D_transport|D_acl|D_v) debug_printf_indent("  SMTP>> .\n");
 
 /* Assume data finshed with new-line */
 if(  !cutthrough_puts(US".", 1)
@@ -1370,7 +1424,7 @@ if(  !cutthrough_puts(US".", 1)
   )
   return cutthrough.addr.message;
 
-res = cutthrough_response('2', &cutthrough.addr.message, CUTTHROUGH_DATA_TIMEOUT);
+res = cutthrough_response(cutthrough.fd, '2', &cutthrough.addr.message, CUTTHROUGH_DATA_TIMEOUT);
 for (addr = &cutthrough.addr; addr; addr = addr->next)
   {
   addr->message = cutthrough.addr.message;
@@ -1378,7 +1432,7 @@ for (addr = &cutthrough.addr; addr; addr = addr->next)
     {
     case '2':
       delivery_log(LOG_MAIN, addr, (int)'>', NULL);
-      close_cutthrough_connection("delivered");
+      close_cutthrough_connection(US"delivered");
       break;
 
     case '4':
@@ -1463,7 +1517,7 @@ va_list ap;
 
 va_start(ap, format);
 if (smtp_out && (f == smtp_out))
-  smtp_vprintf(format, ap);
+  smtp_vprintf(format, FALSE, ap);
 else
   vfprintf(f, format, ap);
 va_end(ap);
@@ -1801,7 +1855,7 @@ while (addr_new)
                  dnssec_domains = &ob->dnssec;
                  }
 
-                (void)host_find_bydns(host, NULL, flags, NULL, NULL, NULL,
+                (void) host_find_bydns(host, NULL, flags, NULL, NULL, NULL,
                  dnssec_domains, NULL, NULL);
                }
               }
@@ -1877,7 +1931,7 @@ while (addr_new)
         }
       respond_printf(f, "%s\n", cr);
       }
-    cancel_cutthrough_connection("routing hard fail");
+    cancel_cutthrough_connection(TRUE, US"routing hard fail");
 
     if (!full_info)
       {
@@ -1916,7 +1970,7 @@ while (addr_new)
         }
       respond_printf(f, "%s\n", cr);
       }
-    cancel_cutthrough_connection("routing soft fail");
+    cancel_cutthrough_connection(TRUE, US"routing soft fail");
 
     if (!full_info)
       {
@@ -1989,7 +2043,7 @@ while (addr_new)
       /* If stopped because more than one new address, cannot cutthrough */
 
       if (addr_new && addr_new->next)
-       cancel_cutthrough_connection("multiple addresses from routing");
+       cancel_cutthrough_connection(TRUE, US"multiple addresses from routing");
 
       yield = OK;
       goto out;
@@ -2239,18 +2293,16 @@ verify_check_header_names_ascii(uschar **msgptr)
 header_line *h;
 uschar *colon, *s;
 
-for (h = header_list; h != NULL; h = h->next)
+for (h = header_list; h; h = h->next)
   {
-   colon = Ustrchr(h->text, ':');
-   for(s = h->text; s < colon; s++)
-     {
-        if ((*s < 33) || (*s > 126))
-        {
-                *msgptr = string_sprintf("Invalid character in header \"%.*s\" found",
-                                         colon - h->text, h->text);
-                return FAIL;
-        }
-     }
+  colon = Ustrchr(h->text, ':');
+  for(s = h->text; s < colon; s++)
+    if ((*s < 33) || (*s > 126))
+      {
+      *msgptr = string_sprintf("Invalid character in header \"%.*s\" found",
+                            colon - h->text, h->text);
+      return FAIL;
+      }
   }
 return OK;
 }