X-Git-Url: https://git.exim.org/exim.git/blobdiff_plain/077ba9434716afd74c87b8100f745bb0fcc092e4..5aba2a452669cc19defd5732e2512984ecd1603e:/src/src/transport.c diff --git a/src/src/transport.c b/src/src/transport.c index d222adce8..9737acc58 100644 --- a/src/src/transport.c +++ b/src/src/transport.c @@ -2,7 +2,7 @@ * Exim - an Internet mail transport agent * *************************************************/ -/* Copyright (c) University of Cambridge 1995 - 2014 */ +/* Copyright (c) University of Cambridge 1995 - 2016 */ /* See the file NOTICE for conditions of use and distribution. */ /* General functions concerned with transportation, and generic options for all @@ -66,6 +66,10 @@ optionlist optionlist_transports[] = { (void *)offsetof(transport_instance, driver_name) }, { "envelope_to_add", opt_bool|opt_public, (void *)(offsetof(transport_instance, envelope_to_add)) }, +#ifndef DISABLE_EVENT + { "event_action", opt_stringptr | opt_public, + (void *)offsetof(transport_instance, event_action) }, +#endif { "group", opt_expand_gid|opt_public, (void *)offsetof(transport_instance, gid) }, { "headers_add", opt_stringptr|opt_public|opt_rep_str, @@ -80,6 +84,8 @@ optionlist optionlist_transports[] = { (void *)offsetof(transport_instance, home_dir) }, { "initgroups", opt_bool|opt_public, (void *)offsetof(transport_instance, initgroups) }, + { "max_parallel", opt_stringptr|opt_public, + (void *)offsetof(transport_instance, max_parallel) }, { "message_size_limit", opt_stringptr|opt_public, (void *)offsetof(transport_instance, message_size_limit) }, { "rcpt_include_affixes", opt_bool|opt_public, @@ -94,10 +100,6 @@ optionlist optionlist_transports[] = { (void *)offsetof(transport_instance, shadow_condition) }, { "shadow_transport", opt_stringptr|opt_public, (void *)offsetof(transport_instance, shadow) }, -#ifdef EXPERIMENTAL_TPDA - { "tpda_delivery_action",opt_stringptr | opt_public, - (void *)offsetof(transport_instance, tpda_delivery_action) }, -#endif { "transport_filter", opt_stringptr|opt_public, (void *)offsetof(transport_instance, filter_command) }, { "transport_filter_timeout", opt_time|opt_public, @@ -628,23 +630,22 @@ that means they were rewritten, or are a record of envelope rewriting, or were removed (e.g. Bcc). If remove_headers is not null, skip any headers that match any entries therein. It is a colon-sep list; expand the items separately and squash any empty ones. -Then check addr->p.remove_headers too, provided that addr is not NULL. */ +Then check addr->prop.remove_headers too, provided that addr is not NULL. */ for (h = header_list; h != NULL; h = h->next) if (h->type != htype_old) { int i; - uschar *list = remove_headers; + const uschar *list = remove_headers; BOOL include_header = TRUE; - for (i = 0; i < 2; i++) /* For remove_headers && addr->p.remove_headers */ + for (i = 0; i < 2; i++) /* For remove_headers && addr->prop.remove_headers */ { if (list) { int sep = ':'; /* This is specified as a colon-separated list */ uschar *s, *ss; - uschar buffer[128]; - while ((s = string_nextinlist(&list, &sep, buffer, sizeof(buffer)))) + while ((s = string_nextinlist(&list, &sep, NULL, 0))) { int len; @@ -654,7 +655,7 @@ for (h = header_list; h != NULL; h = h->next) if (h->type != htype_old) errno = ERRNO_CHHEADER_FAIL; return FALSE; } - len = Ustrlen(s); + len = s ? Ustrlen(s) : 0; if (strncmpic(h->text, s, len) != 0) continue; ss = h->text + len; while (*ss == ' ' || *ss == '\t') ss++; @@ -662,7 +663,7 @@ for (h = header_list; h != NULL; h = h->next) if (h->type != htype_old) } if (s != NULL) { include_header = FALSE; break; } } - if (addr != NULL) list = addr->p.remove_headers; + if (addr != NULL) list = addr->prop.remove_headers; } /* If this header is to be output, try to rewrite it if there are rewriting @@ -710,7 +711,7 @@ Headers added to an address by a router are guaranteed to end with a newline. if (addr) { int i; - header_line *hprev = addr->p.extra_headers; + header_line *hprev = addr->prop.extra_headers; header_line *hnext; for (i = 0; i < 2; i++) { @@ -741,7 +742,7 @@ if (add_headers) int sep = '\n'; uschar * s; - while ((s = string_nextinlist(&add_headers, &sep, NULL, 0))) + while ((s = string_nextinlist(CUSS &add_headers, &sep, NULL, 0))) if (!(s = expand_string(s))) { if (!expand_string_forcedfail) @@ -915,7 +916,7 @@ if ((options & topt_no_headers) == 0) /* Then the message's headers. Don't write any that are flagged as "old"; that means they were rewritten, or are a record of envelope rewriting, or were removed (e.g. Bcc). If remove_headers is not null, skip any headers that - match any entries therein. Then check addr->p.remove_headers too, provided that + match any entries therein. Then check addr->prop.remove_headers too, provided that addr is not NULL. */ if (!transport_headers_send(addr, fd, add_headers, remove_headers, &write_chunk, use_crlf, rewrite_rules, rewrite_existflags)) @@ -932,7 +933,8 @@ if ((options & topt_no_body) == 0) { nl_check_length = abs(nl_check_length); nl_partial_match = 0; - lseek(deliver_datafile, SPOOL_DATA_START_OFFSET, SEEK_SET); + if (lseek(deliver_datafile, SPOOL_DATA_START_OFFSET, SEEK_SET) < 0) + return FALSE; while ((len = read(deliver_datafile, deliver_in_buffer, DELIVER_IN_BUFFER_SIZE)) > 0) { @@ -1016,7 +1018,6 @@ char sbuf[2048]; int sread = 0; int wwritten = 0; uschar *dkim_signature = NULL; -off_t size = 0; /* If we can't sign, just call the original function. */ @@ -1026,8 +1027,9 @@ if (!(dkim_private_key && dkim_domain && dkim_selector)) check_string, escape_string, rewrite_rules, rewrite_existflags); -(void)string_format(dkim_spool_name, 256, "%s/input/%s/%s-%d-K", - spool_directory, message_subdir, message_id, (int)getpid()); +(void)string_format(dkim_spool_name, sizeof(dkim_spool_name), + "%s/input/%s/%s/%s-%d-K", + spool_directory, queue_name, message_subdir, message_id, (int)getpid()); if ((dkim_fd = Uopen(dkim_spool_name, O_RDWR|O_CREAT|O_TRUNC, SPOOL_MODE)) < 0) { @@ -1104,12 +1106,6 @@ if (dkim_private_key && dkim_domain && dkim_selector) } } - /* Fetch file positition (the size) */ - size = lseek(dkim_fd,0,SEEK_CUR); - -/* Rewind file */ -lseek(dkim_fd, 0, SEEK_SET); - #ifdef HAVE_LINUX_SENDFILE /* We can use sendfile() to shove the file contents to the socket. However only if we don't use TLS, @@ -1117,8 +1113,13 @@ lseek(dkim_fd, 0, SEEK_SET); before the data finally hits the socket. */ if (tls_out.active != fd) { + off_t size = lseek(dkim_fd, 0, SEEK_END); /* Fetch file size */ ssize_t copied = 0; off_t offset = 0; + + /* Rewind file */ + lseek(dkim_fd, 0, SEEK_SET); + while(copied >= 0 && offset < size) copied = sendfile(fd, dkim_fd, &offset, size - offset); if (copied < 0) @@ -1126,42 +1127,47 @@ if (tls_out.active != fd) save_errno = errno; rc = FALSE; } - goto CLEANUP; } +else + #endif -/* Send file down the original fd */ -while((sread = read(dkim_fd, sbuf, 2048)) > 0) { - char *p = sbuf; - /* write the chunk */ + /* Rewind file */ + lseek(dkim_fd, 0, SEEK_SET); - while (sread) + /* Send file down the original fd */ + while((sread = read(dkim_fd, sbuf, 2048)) > 0) { + char *p = sbuf; + /* write the chunk */ + + while (sread) + { #ifdef SUPPORT_TLS - wwritten = tls_out.active == fd - ? tls_write(FALSE, US p, sread) - : write(fd, p, sread); + wwritten = tls_out.active == fd + ? tls_write(FALSE, US p, sread) + : write(fd, p, sread); #else - wwritten = write(fd, p, sread); + wwritten = write(fd, p, sread); #endif - if (wwritten == -1) - { - /* error, bail out */ - save_errno = errno; - rc = FALSE; - goto CLEANUP; + if (wwritten == -1) + { + /* error, bail out */ + save_errno = errno; + rc = FALSE; + goto CLEANUP; + } + p += wwritten; + sread -= wwritten; } - p += wwritten; - sread -= wwritten; } - } -if (sread == -1) - { - save_errno = errno; - rc = FALSE; - goto CLEANUP; + if (sread == -1) + { + save_errno = errno; + rc = FALSE; + } } CLEANUP: @@ -1201,7 +1207,7 @@ transport_write_message(address_item *addr, int fd, int options, BOOL use_crlf; BOOL last_filter_was_NL = TRUE; int rc, len, yield, fd_read, fd_write, save_errno; -int pfd[2]; +int pfd[2] = {-1, -1}; pid_t filter_pid, write_pid; transport_filter_timed_out = FALSE; @@ -1209,7 +1215,10 @@ transport_filter_timed_out = FALSE; /* If there is no filter command set up, call the internal function that does the actual work, passing it the incoming fd, and return its result. */ -if (transport_filter_argv == NULL) +if ( !transport_filter_argv + || !*transport_filter_argv + || !**transport_filter_argv + ) return internal_transport_write_message(addr, fd, options, size_limit, add_headers, remove_headers, check_string, escape_string, rewrite_rules, rewrite_existflags); @@ -1243,8 +1252,8 @@ yield = FALSE; write_pid = (pid_t)(-1); (void)fcntl(fd, F_SETFD, fcntl(fd, F_GETFD) | FD_CLOEXEC); -filter_pid = child_open(transport_filter_argv, NULL, 077, &fd_write, &fd_read, - FALSE); +filter_pid = child_open(USS transport_filter_argv, NULL, 077, + &fd_write, &fd_read, FALSE); (void)fcntl(fd, F_SETFD, fcntl(fd, F_GETFD) & ~FD_CLOEXEC); if (filter_pid < 0) goto TIDY_UP; /* errno set */ @@ -1482,7 +1491,7 @@ void transport_update_waiting(host_item *hostlist, uschar *tpname) { uschar buffer[256]; -uschar *prevname = US""; +const uschar *prevname = US""; host_item *host; open_db dbblock; open_db *dbm_file; @@ -1623,20 +1632,34 @@ Arguments: as set by the caller transport new_message_id set to the message id of a waiting message more set TRUE if there are yet more messages waiting + oicf_func function to call to validate if it is ok to send + to this message_id from the current instance. + oicf_data opaque data for oicf_func Returns: TRUE if new_message_id set; FALSE otherwise */ +typedef struct msgq_s +{ + uschar message_id [MESSAGE_ID_LENGTH + 1]; + BOOL bKeep; +} msgq_t; + BOOL -transport_check_waiting(uschar *transport_name, uschar *hostname, - int local_message_max, uschar *new_message_id, BOOL *more) +transport_check_waiting(const uschar *transport_name, const uschar *hostname, + int local_message_max, uschar *new_message_id, BOOL *more, oicf oicf_func, void *oicf_data) { dbdata_wait *host_record; -int host_length, path_len; +int host_length; open_db dbblock; open_db *dbm_file; uschar buffer[256]; +int i; +uschar spool_dir [PATH_MAX]; +uschar spool_file [PATH_MAX]; +struct stat statbuf; + *more = FALSE; DEBUG(D_transport) @@ -1665,8 +1688,7 @@ if (dbm_file == NULL) return FALSE; /* See if there is a record for this host; if not, there's nothing to do. */ -host_record = dbfn_read(dbm_file, hostname); -if (host_record == NULL) +if (!(host_record = dbfn_read(dbm_file, hostname))) { dbfn_close(dbm_file); DEBUG(D_transport) debug_printf("no messages waiting for %s\n", hostname); @@ -1689,58 +1711,109 @@ until one is found for which a spool file actually exists. If the record gets emptied, delete it and continue with any continuation records that may exist. */ -host_length = host_record->count * MESSAGE_ID_LENGTH; +/* For Bug 1141, I refactored this major portion of the routine, it is risky +but the 1 off will remain without it. This code now allows me to SKIP over +a message I do not want to send out on this run. */ -/* Loop to handle continuation host records in the database */ +sprintf(CS spool_dir, "%s/input/", spool_directory); -for (;;) +host_length = host_record->count * MESSAGE_ID_LENGTH; + +while (1) { - BOOL found = FALSE; + msgq_t *msgq; + int msgq_count = 0; + int msgq_actual = 0; + BOOL bFound = FALSE; + BOOL bContinuation = FALSE; + + /* create an array to read entire message queue into memory for processing */ - sprintf(CS buffer, "%s/input/", spool_directory); - path_len = Ustrlen(buffer); + msgq = (msgq_t*) malloc(sizeof(msgq_t) * host_record->count); + msgq_count = host_record->count; + msgq_actual = msgq_count; - for (host_length -= MESSAGE_ID_LENGTH; host_length >= 0; - host_length -= MESSAGE_ID_LENGTH) + for (i = 0; i < host_record->count; ++i) { - struct stat statbuf; - Ustrncpy(new_message_id, host_record->text + host_length, + msgq[i].bKeep = TRUE; + + Ustrncpy(msgq[i].message_id, host_record->text + (i * MESSAGE_ID_LENGTH), MESSAGE_ID_LENGTH); - new_message_id[MESSAGE_ID_LENGTH] = 0; + msgq[i].message_id[MESSAGE_ID_LENGTH] = 0; + } + /* first thing remove current message id if it exists */ + + for (i = 0; i < msgq_count; ++i) + if (Ustrcmp(msgq[i].message_id, message_id) == 0) + { + msgq[i].bKeep = FALSE; + break; + } + + /* now find the next acceptable message_id */ + + for (i = msgq_count - 1; i >= 0; --i) if (msgq[i].bKeep) + { if (split_spool_directory) - sprintf(CS(buffer + path_len), "%c/%s-D", new_message_id[5], new_message_id); + snprintf(CS spool_file, sizeof(spool_file), "%s/%s/%c/%s-D", + spool_dir, queue_name, msgq[i].message_id[5], msgq[i].message_id); else - sprintf(CS(buffer + path_len), "%s-D", new_message_id); - - /* The listed message may be the one we are currently processing. If - so, we want to remove it from the list without doing anything else. - If not, do a stat to see if it is an existing message. If it is, break - the loop to handle it. No need to bother about locks; as this is all - "hint" processing, it won't matter if it doesn't exist by the time exim - actually tries to deliver it. */ + snprintf(CS spool_file, sizeof(spool_file), "%s/%s/%s-D", + spool_dir, queue_name, msgq[i].message_id); - if (Ustrcmp(new_message_id, message_id) != 0 && - Ustat(buffer, &statbuf) == 0) + if (Ustat(spool_file, &statbuf) != 0) + msgq[i].bKeep = FALSE; + else if (!oicf_func || oicf_func(msgq[i].message_id, oicf_data)) { - found = TRUE; + Ustrcpy(new_message_id, msgq[i].message_id); + msgq[i].bKeep = FALSE; + bFound = TRUE; break; } } - /* If we have removed all the message ids from the record delete the record. - If there is a continuation record, fetch it and remove it from the file, - as it will be rewritten as the main record. Repeat in the case of an - empty continuation. */ + /* re-count */ + for (msgq_actual = 0, i = 0; i < msgq_count; ++i) + if (msgq[i].bKeep) + msgq_actual++; + + /* reassemble the host record, based on removed message ids, from in + memory queue */ + + if (msgq_actual <= 0) + { + host_length = 0; + host_record->count = 0; + } + else + { + host_length = msgq_actual * MESSAGE_ID_LENGTH; + host_record->count = msgq_actual; + + if (msgq_actual < msgq_count) + { + int new_count; + for (new_count = 0, i = 0; i < msgq_count; ++i) + if (msgq[i].bKeep) + Ustrncpy(&host_record->text[new_count++ * MESSAGE_ID_LENGTH], + msgq[i].message_id, MESSAGE_ID_LENGTH); + + host_record->text[new_count * MESSAGE_ID_LENGTH] = 0; + } + } + +/* Jeremy: check for a continuation record, this code I do not know how to +test but the code should work */ while (host_length <= 0) { int i; - dbdata_wait *newr = NULL; + dbdata_wait * newr = NULL; /* Search for a continuation */ - for (i = host_record->sequence - 1; i >= 0 && newr == NULL; i--) + for (i = host_record->sequence - 1; i >= 0 && !newr; i--) { sprintf(CS buffer, "%.200s:%d", hostname, i); newr = dbfn_read(dbm_file, buffer); @@ -1748,7 +1821,7 @@ for (;;) /* If no continuation, delete the current and break the loop */ - if (newr == NULL) + if (!newr) { dbfn_delete(dbm_file, hostname); break; @@ -1759,11 +1832,15 @@ for (;;) dbfn_delete(dbm_file, buffer); host_record = newr; host_length = host_record->count * MESSAGE_ID_LENGTH; - } - /* If we found an existing message, break the continuation loop. */ + bContinuation = TRUE; + } - if (found) break; + if (bFound) /* Usual exit from main loop */ + { + free (msgq); + break; + } /* If host_length <= 0 we have emptied a record and not found a good message, and there are no continuation records. Otherwise there is a continuation @@ -1775,7 +1852,20 @@ for (;;) DEBUG(D_transport) debug_printf("waiting messages already delivered\n"); return FALSE; } - } + + /* we were not able to find an acceptable message, nor was there a + * continuation record. So bug out, outer logic will clean this up. + */ + + if (!bContinuation) + { + Ustrcpy(new_message_id, message_id); + dbfn_close(dbm_file); + return FALSE; + } + + free(msgq); + } /* we need to process a continuation record */ /* Control gets here when an existing message has been encountered; its id is in new_message_id, and host_length is the revised length of the @@ -1785,6 +1875,7 @@ record if required, close the database, and return TRUE. */ if (host_length > 0) { host_record->count = host_length/MESSAGE_ID_LENGTH; + dbfn_write(dbm_file, hostname, host_record, (int)sizeof(dbdata_wait) + host_length); *more = TRUE; } @@ -1793,8 +1884,6 @@ dbfn_close(dbm_file); return TRUE; } - - /************************************************* * Deliver waiting message down same socket * *************************************************/ @@ -1814,8 +1903,8 @@ Returns: FALSE if fork fails; TRUE otherwise */ BOOL -transport_pass_socket(uschar *transport_name, uschar *hostname, - uschar *hostaddress, uschar *id, int socket_fd) +transport_pass_socket(const uschar *transport_name, const uschar *hostname, + const uschar *hostaddress, uschar *id, int socket_fd) { pid_t pid; int status; @@ -1825,7 +1914,7 @@ DEBUG(D_transport) debug_printf("transport_pass_socket entered\n"); if ((pid = fork()) == 0) { int i = 16; - uschar **argv; + const uschar **argv; /* Disconnect entirely from the parent process. If we are running in the test harness, wait for a bit to allow the previous process time to finish, @@ -1838,12 +1927,15 @@ if ((pid = fork()) == 0) /* Set up the calling arguments; use the standard function for the basics, but we have a number of extras that may be added. */ - argv = child_exec_exim(CEE_RETURN_ARGV, TRUE, &i, FALSE, 0); + argv = CUSS child_exec_exim(CEE_RETURN_ARGV, TRUE, &i, FALSE, 0); + + if (*queue_name) + { + argv[i++] = US"-MCG"; + argv[i++] = queue_name; + } - #ifdef EXPERIMENTAL_DSN - /* Call with the dsn flag */ if (smtp_use_dsn) argv[i++] = US"-MCD"; - #endif if (smtp_authenticated) argv[i++] = US"-MCA"; @@ -1862,9 +1954,9 @@ if ((pid = fork()) == 0) } argv[i++] = US"-MC"; - argv[i++] = transport_name; - argv[i++] = hostname; - argv[i++] = hostaddress; + argv[i++] = US transport_name; + argv[i++] = US hostname; + argv[i++] = US hostaddress; argv[i++] = string_sprintf("%d", continue_sequence + 1); argv[i++] = id; argv[i++] = NULL; @@ -1918,7 +2010,7 @@ case, no addresses are passed. Arguments: argvptr pointer to anchor for argv vector - cmd points to the command string + cmd points to the command string (modified IN PLACE) expand_arguments true if expansion is to occur expand_failed error value to set if expansion fails; not relevant if addr == NULL @@ -1932,11 +2024,12 @@ Returns: TRUE if all went well; otherwise an error will be */ BOOL -transport_set_up_command(uschar ***argvptr, uschar *cmd, BOOL expand_arguments, - int expand_failed, address_item *addr, uschar *etext, uschar **errptr) +transport_set_up_command(const uschar ***argvptr, uschar *cmd, + BOOL expand_arguments, int expand_failed, address_item *addr, + uschar *etext, uschar **errptr) { address_item *ad; -uschar **argv; +const uschar **argv; uschar *s, *ss; int address_count = 0; int argcount = 0; @@ -1970,7 +2063,7 @@ while (*s != 0 && argcount < max_args) if (*s != 0) s++; *ss++ = 0; } - else argv[argcount++] = string_dequote(&s); + else argv[argcount++] = string_copy(string_dequote(CUSS &s)); while (isspace(*s)) s++; } @@ -2099,7 +2192,8 @@ if (expand_arguments) if (*s != 0) s++; *ss++ = 0; } - else address_pipe_argv[address_pipe_argcount++] = string_dequote(&s); + else address_pipe_argv[address_pipe_argcount++] = + string_copy(string_dequote(CUSS &s)); while (isspace(*s)) s++; /* strip space after arg */ } @@ -2167,9 +2261,9 @@ if (expand_arguments) else { - uschar *expanded_arg; + const uschar *expanded_arg; enable_dollar_recipients = allow_dollar_recipients; - expanded_arg = expand_string(argv[i]); + expanded_arg = expand_cstring(argv[i]); enable_dollar_recipients = FALSE; if (expanded_arg == NULL)