X-Git-Url: https://git.exim.org/exim.git/blobdiff_plain/41313d92e0f157dacfa758993e7fc76e291b0415..04403ab0b5778126d98d5c9dc8064405688129e0:/src/src/queue.c diff --git a/src/src/queue.c b/src/src/queue.c index d8c276013..ac7aad1a0 100644 --- a/src/src/queue.c +++ b/src/src/queue.c @@ -2,7 +2,7 @@ * Exim - an Internet mail transport agent * *************************************************/ -/* Copyright (c) University of Cambridge 1995 - 2015 */ +/* Copyright (c) University of Cambridge 1995 - 2018 */ /* See the file NOTICE for conditions of use and distribution. */ /* Functions that operate on the input queue. */ @@ -12,49 +12,20 @@ -/* The number of nodes to use for the bottom-up merge sort when a list of queue -items is to be ordered. The code for this sort was contributed as a patch by -Michael Haardt. */ - -#define LOG2_MAXNODES 32 -/* Routines with knowlege of spool layout */ - -static void -spool_pname_buf(uschar * buf, int len) -{ -snprintf(CS buf, len, "%s/input/%s", spool_directory, queue_name); -} -static uschar * -spool_dname(const uschar * purpose, uschar * subdir) -{ -return string_sprintf("%s/%s/%s/%s", - spool_directory, purpose, queue_name, subdir); -} - -uschar * -spool_sname(const uschar * purpose, uschar * subdir) -{ -return string_sprintf("%s%s%s%s%s", - purpose, - *queue_name ? "/" : "", queue_name, - *message_subdir ? "/" : "", message_subdir); -} +#ifndef COMPILE_UTILITY -uschar * -spool_fname(const uschar * purpose, uschar * subdir, uschar * fname, uschar * suffix) -{ -return string_sprintf("%s/%s/%s/%s/%s%s", - spool_directory, purpose, queue_name, subdir, fname, suffix); -} +/* The number of nodes to use for the bottom-up merge sort when a list of queue +items is to be ordered. The code for this sort was contributed as a patch by +Michael Haardt. */ +#define LOG2_MAXNODES 32 -#ifndef COMPILE_UTILITY /************************************************* * Helper sort function for queue_get_spool_list * *************************************************/ @@ -75,9 +46,12 @@ merge_queue_lists(queue_filename *a, queue_filename *b) queue_filename *first = NULL; queue_filename **append = &first; -while (a != NULL && b != NULL) +while (a && b) { - if (Ustrcmp(a->text, b->text) < 0) + int d; + if ((d = Ustrncmp(a->text, b->text, 6)) == 0) + d = Ustrcmp(a->text + 14, b->text + 14); + if (d < 0) { *append = a; append= &a->next; @@ -91,7 +65,7 @@ while (a != NULL && b != NULL) } } -*append=((a != NULL)? a : b); +*append = a ? a : b; return first; } @@ -160,8 +134,11 @@ according to the bits of the flags variable. Get a collection of bits from the current time. Use the bottom 16 and just keep re-using them if necessary. When not randomizing, initialize the sublists for the bottom-up merge sort. */ -if (randomize) resetflags = time(NULL) & 0xFFFF; - else for (i = 0; i < LOG2_MAXNODES; i++) root[i] = NULL; +if (randomize) + resetflags = time(NULL) & 0xFFFF; +else + for (i = 0; i < LOG2_MAXNODES; i++) + root[i] = NULL; /* If processing the full queue, or just the top-level, start at the base directory, and initialize the first subdirectory name (as none). Otherwise, @@ -173,7 +150,8 @@ if (subdiroffset <= 0) subdirs[0] = 0; *subcount = 0; } -else i = subdiroffset; +else + i = subdiroffset; /* Set up prototype for the directory name. */ @@ -203,7 +181,7 @@ for (; i <= *subcount; i++) /* Now scan the directory. */ - while ((ent = readdir(dd)) != NULL) + while ((ent = readdir(dd))) { uschar *name = US ent->d_name; int len = Ustrlen(name); @@ -228,7 +206,7 @@ for (; i <= *subcount; i++) Ustrcmp(name + SPOOL_NAME_LENGTH - 2, "-H") == 0) { queue_filename *next = - store_get(sizeof(queue_filename) + Ustrlen(name)); + store_get(sizeof(queue_filename) + Ustrlen(name), is_tainted(name)); Ustrcpy(next->text, name); next->dir_uschar = subdirchar; @@ -239,15 +217,15 @@ for (; i <= *subcount; i++) to store the number with each item. */ if (randomize) - { - if (yield == NULL) + if (!yield) { next->next = NULL; yield = last = next; } else { - if (flags == 0) flags = resetflags; + if (flags == 0) + flags = resetflags; if ((flags & 1) == 0) { next->next = yield; @@ -261,27 +239,23 @@ for (; i <= *subcount; i++) } flags = flags >> 1; } - } /* Otherwise do a bottom-up merge sort based on the name. */ else { - int j; next->next = NULL; - for (j = 0; j < LOG2_MAXNODES; j++) - { - if (root[j] != NULL) + for (int j = 0; j < LOG2_MAXNODES; j++) + if (root[j]) { next = merge_queue_lists(next, root[j]); - root[j] = (j == LOG2_MAXNODES - 1)? next : NULL; + root[j] = j == LOG2_MAXNODES - 1 ? next : NULL; } else { root[j] = next; break; } - } } } } @@ -313,10 +287,8 @@ for (; i <= *subcount; i++) /* If we have just scanned the base directory, and subdiroffset is 0, we do not want to continue scanning the sub-directories. */ - else - { - if (subdiroffset == 0) break; - } + else if (subdiroffset == 0) + break; } /* Loop for multiple subdirectories */ /* When using a bottom-up merge sort, do the final merging of the sublists. @@ -367,14 +339,18 @@ Returns: nothing void queue_run(uschar *start_id, uschar *stop_id, BOOL recurse) { -BOOL force_delivery = queue_run_force || deliver_selectstring != NULL || +BOOL force_delivery = f.queue_run_force || deliver_selectstring != NULL || deliver_selectstring_sender != NULL; const pcre *selectstring_regex = NULL; const pcre *selectstring_regex_sender = NULL; uschar *log_detail = NULL; int subcount = 0; -int i; uschar subdirs[64]; +pid_t qpid[4] = {0}; /* Parallelism factor for q2stage 1st phase */ + +#ifdef MEASURE_TIMING +report_time_since(×tamp_startup, US"queue_run start"); +#endif /* Cancel any specific queue domains. Turn off the flag that causes SMTP deliveries not to happen, unless doing a 2-stage queue run, when the SMTP flag @@ -384,10 +360,10 @@ on TCP/IP channels have queue_run_pid set, but not queue_running. */ queue_domains = NULL; queue_smtp_domains = NULL; -queue_smtp = queue_2stage; +f.queue_smtp = f.queue_2stage; queue_run_pid = getpid(); -queue_running = TRUE; +f.queue_running = TRUE; /* Log the true start of a queue run, and fancy options */ @@ -396,36 +372,26 @@ if (!recurse) uschar extras[8]; uschar *p = extras; - if (queue_2stage) *p++ = 'q'; - if (queue_run_first_delivery) *p++ = 'i'; - if (queue_run_force) *p++ = 'f'; - if (deliver_force_thaw) *p++ = 'f'; - if (queue_run_local) *p++ = 'l'; + if (f.queue_2stage) *p++ = 'q'; + if (f.queue_run_first_delivery) *p++ = 'i'; + if (f.queue_run_force) *p++ = 'f'; + if (f.deliver_force_thaw) *p++ = 'f'; + if (f.queue_run_local) *p++ = 'l'; *p = 0; p = big_buffer; - sprintf(CS p, "pid=%d", (int)queue_run_pid); - while (*p != 0) p++; + p += sprintf(CS p, "pid=%d", (int)queue_run_pid); if (extras[0] != 0) - { - sprintf(CS p, " -q%s", extras); - while (*p != 0) p++; - } + p += sprintf(CS p, " -q%s", extras); - if (deliver_selectstring != NULL) - { - sprintf(CS p, " -R%s %s", deliver_selectstring_regex? "r" : "", + if (deliver_selectstring) + p += sprintf(CS p, " -R%s %s", f.deliver_selectstring_regex? "r" : "", deliver_selectstring); - while (*p != 0) p++; - } - if (deliver_selectstring_sender != NULL) - { - sprintf(CS p, " -S%s %s", deliver_selectstring_sender_regex? "r" : "", + if (deliver_selectstring_sender) + p += sprintf(CS p, " -S%s %s", f.deliver_selectstring_sender_regex? "r" : "", deliver_selectstring_sender); - while (*p != 0) p++; - } log_detail = string_copy(big_buffer); if (*queue_name) @@ -437,10 +403,10 @@ if (!recurse) /* If deliver_selectstring is a regex, compile it. */ -if (deliver_selectstring != NULL && deliver_selectstring_regex) +if (deliver_selectstring && f.deliver_selectstring_regex) selectstring_regex = regex_must_compile(deliver_selectstring, TRUE, FALSE); -if (deliver_selectstring_sender != NULL && deliver_selectstring_sender_regex) +if (deliver_selectstring_sender && f.deliver_selectstring_sender_regex) selectstring_regex_sender = regex_must_compile(deliver_selectstring_sender, TRUE, FALSE); @@ -454,17 +420,16 @@ any messages therein), and then repeats for any subdirectories that were found. When the first argument of queue_get_spool_list() is 0, it scans the top directory, fills in subdirs, and sets subcount. The order of the directories is then randomized after the first time through, before they are scanned in -subsqeuent iterations. +subsequent iterations. When the first argument of queue_get_spool_list() is -1 (for queue_run_in_ order), it scans all directories and makes a single message list. */ -for (i = (queue_run_in_order? -1 : 0); - i <= (queue_run_in_order? -1 : subcount); +for (int i = queue_run_in_order ? -1 : 0; + i <= (queue_run_in_order ? -1 : subcount); i++) { - queue_filename *f; - void *reset_point1 = store_get(0); + rmark reset_point1 = store_mark(); DEBUG(D_queue_run) { @@ -476,9 +441,9 @@ for (i = (queue_run_in_order? -1 : 0); debug_printf("queue running subdirectory '%c'\n", subdirs[i]); } - for (f = queue_get_spool_list(i, subdirs, &subcount, !queue_run_in_order); - f != NULL; - f = f->next) + for (queue_filename * fq = queue_get_spool_list(i, subdirs, &subcount, + !queue_run_in_order); + fq; fq = fq->next) { pid_t pid; int status; @@ -489,10 +454,8 @@ for (i = (queue_run_in_order? -1 : 0); /* Unless deliveries are forced, if deliver_queue_load_max is non-negative, check that the load average is low enough to permit deliveries. */ - if (!queue_run_force && deliver_queue_load_max >= 0) - { - load_average = os_getloadavg(); - if (load_average > deliver_queue_load_max) + if (!f.queue_run_force && deliver_queue_load_max >= 0) + if ((load_average = os_getloadavg()) > deliver_queue_load_max) { log_write(L_queue_run, LOG_MAIN, "Abandon queue run: %s (load %.2f, max %.2f)", log_detail, @@ -502,25 +465,44 @@ for (i = (queue_run_in_order? -1 : 0); break; } else - { DEBUG(D_load) debug_printf("load average = %.2f max = %.2f\n", (double)load_average/1000.0, (double)deliver_queue_load_max/1000.0); - } + + /* If initial of a 2-phase run, maintain a set of child procs + to get disk parallelism */ + + if (f.queue_2stage && !queue_run_in_order) + { + int i; + if (qpid[nelem(qpid) - 1]) + { + DEBUG(D_queue_run) debug_printf("q2stage waiting for child\n"); + waitpid(qpid[0], NULL, 0); + DEBUG(D_queue_run) debug_printf("q2stage reaped child %d\n", (int)qpid[0]); + for (i = 0; i < nelem(qpid) - 1; i++) qpid[i] = qpid[i+1]; + qpid[i] = 0; + } + else + for (i = 0; qpid[i]; ) i++; + DEBUG(D_queue_run) debug_printf("q2stage forking\n"); + if ((qpid[i] = fork())) + continue; /* parent loops around */ + DEBUG(D_queue_run) debug_printf("q2stage child\n"); } /* Skip this message unless it's within the ID limits */ - if (stop_id != NULL && Ustrncmp(f->text, stop_id, MESSAGE_ID_LENGTH) > 0) - continue; - if (start_id != NULL && Ustrncmp(f->text, start_id, MESSAGE_ID_LENGTH) < 0) - continue; + if (stop_id && Ustrncmp(fq->text, stop_id, MESSAGE_ID_LENGTH) > 0) + goto go_around; + if (start_id && Ustrncmp(fq->text, start_id, MESSAGE_ID_LENGTH) < 0) + goto go_around; /* Check that the message still exists */ - message_subdir[0] = f->dir_uschar; - if (Ustat(spool_fname(US"input", message_subdir, f->text, US""), &statbuf) < 0) - continue; + message_subdir[0] = fq->dir_uschar; + if (Ustat(spool_fname(US"input", message_subdir, fq->text, US""), &statbuf) < 0) + goto go_around; /* There are some tests that require the reading of the header file. Ensure the store used is scavenged afterwards so that this process doesn't keep @@ -528,26 +510,26 @@ for (i = (queue_run_in_order? -1 : 0); delivering, but it's cheaper than forking a delivery process for each message when many are not going to be delivered. */ - if (deliver_selectstring != NULL || deliver_selectstring_sender != NULL || - queue_run_first_delivery) + if (deliver_selectstring || deliver_selectstring_sender || + f.queue_run_first_delivery) { BOOL wanted = TRUE; - BOOL orig_dont_deliver = dont_deliver; - void *reset_point2 = store_get(0); + BOOL orig_dont_deliver = f.dont_deliver; + rmark reset_point2 = store_mark(); /* Restore the original setting of dont_deliver after reading the header, so that a setting for a particular message doesn't force it for any that follow. If the message is chosen for delivery, the header is read again in the deliver_message() function, in a subprocess. */ - if (spool_read_header(f->text, FALSE, TRUE) != spool_read_OK) continue; - dont_deliver = orig_dont_deliver; + if (spool_read_header(fq->text, FALSE, TRUE) != spool_read_OK) goto go_around; + f.dont_deliver = orig_dont_deliver; /* Now decide if we want to deliver this message. As we have read the header file, we might as well do the freeze test now, and save forking another process. */ - if (deliver_freeze && !deliver_force_thaw) + if (f.deliver_freeze && !f.deliver_force_thaw) { log_write(L_skip_delivery, LOG_MAIN, "Message is frozen"); wanted = FALSE; @@ -555,46 +537,47 @@ for (i = (queue_run_in_order? -1 : 0); /* Check first_delivery in the case when there are no message logs. */ - else if (queue_run_first_delivery && !deliver_firsttime) + else if (f.queue_run_first_delivery && !f.deliver_firsttime) { - DEBUG(D_queue_run) debug_printf("%s: not first delivery\n", f->text); + DEBUG(D_queue_run) debug_printf("%s: not first delivery\n", fq->text); wanted = FALSE; } - /* Check for a matching address if deliver_selectstring[_sender} is set. + /* Check for a matching address if deliver_selectstring[_sender] is set. If so, we do a fully delivery - don't want to omit other addresses since their routing might trigger re-writing etc. */ /* Sender matching */ - else if (deliver_selectstring_sender != NULL && - !(deliver_selectstring_sender_regex? - (pcre_exec(selectstring_regex_sender, NULL, CS sender_address, - Ustrlen(sender_address), 0, PCRE_EOPT, NULL, 0) >= 0) - : - (strstric(sender_address, deliver_selectstring_sender, FALSE) - != NULL))) + else if ( deliver_selectstring_sender + && !(f.deliver_selectstring_sender_regex + ? (pcre_exec(selectstring_regex_sender, NULL, + CS sender_address, Ustrlen(sender_address), 0, PCRE_EOPT, + NULL, 0) >= 0) + : (strstric(sender_address, deliver_selectstring_sender, FALSE) + != NULL) + ) ) { DEBUG(D_queue_run) debug_printf("%s: sender address did not match %s\n", - f->text, deliver_selectstring_sender); + fq->text, deliver_selectstring_sender); wanted = FALSE; } /* Recipient matching */ - else if (deliver_selectstring != NULL) + else if (deliver_selectstring) { int i; for (i = 0; i < recipients_count; i++) { uschar *address = recipients_list[i].address; - if ((deliver_selectstring_regex? - (pcre_exec(selectstring_regex, NULL, CS address, - Ustrlen(address), 0, PCRE_EOPT, NULL, 0) >= 0) - : - (strstric(address, deliver_selectstring, FALSE) != NULL)) - && - tree_search(tree_nonrecipients, address) == NULL) + if ( (f.deliver_selectstring_regex + ? (pcre_exec(selectstring_regex, NULL, CS address, + Ustrlen(address), 0, PCRE_EOPT, NULL, 0) >= 0) + : (strstric(address, deliver_selectstring, FALSE) != NULL) + ) + && tree_search(tree_nonrecipients, address) == NULL + ) break; } @@ -602,15 +585,16 @@ for (i = (queue_run_in_order? -1 : 0); { DEBUG(D_queue_run) debug_printf("%s: no recipient address matched %s\n", - f->text, deliver_selectstring); + fq->text, deliver_selectstring); wanted = FALSE; } } /* Recover store used when reading the header */ + spool_clear_header_globals(); store_reset(reset_point2); - if (!wanted) continue; /* With next message */ + if (!wanted) goto go_around; /* With next message */ } /* OK, got a message we want to deliver. Create a pipe which will @@ -623,10 +607,8 @@ for (i = (queue_run_in_order? -1 : 0); pretty cheap. */ if (pipe(pfd) < 0) - { log_write(0, LOG_MAIN|LOG_PANIC_DIE, "failed to create pipe in queue " "runner process %d: %s", queue_run_pid, strerror(errno)); - } queue_run_pipe = pfd[pipe_write]; /* To ensure it gets passed on. */ /* Make sure it isn't stdin. This seems unlikely, but just to be on the @@ -651,15 +633,19 @@ for (i = (queue_run_in_order? -1 : 0); /* Now deliver the message; get the id by cutting the -H off the file name. The return of the process is zero if a delivery was attempted. */ - set_process_info("running queue: %s", f->text); - f->text[SPOOL_NAME_LENGTH-2] = 0; + set_process_info("running queue: %s", fq->text); + fq->text[SPOOL_NAME_LENGTH-2] = 0; +#ifdef MEASURE_TIMING + report_time_since(×tamp_startup, US"queue msg selected"); +#endif + if ((pid = fork()) == 0) { int rc; - if (running_in_test_harness) millisleep(100); + testharness_pause_ms(100); (void)close(pfd[pipe_read]); - rc = deliver_message(f->text, force_delivery, FALSE); - _exit(rc == DELIVER_NOT_ATTEMPTED); + rc = deliver_message(fq->text, force_delivery, FALSE); + exim_underbar_exit(rc == DELIVER_NOT_ATTEMPTED); } if (pid < 0) log_write(0, LOG_MAIN|LOG_PANIC_DIE, "fork of delivery process from " @@ -669,22 +655,20 @@ for (i = (queue_run_in_order? -1 : 0); then wait for the first level process to terminate. */ (void)close(pfd[pipe_write]); - set_process_info("running queue: waiting for %s (%d)", f->text, pid); + set_process_info("running queue: waiting for %s (%d)", fq->text, pid); while (wait(&status) != pid); /* A zero return means a delivery was attempted; turn off the force flag for any subsequent calls unless queue_force is set. */ - if ((status & 0xffff) == 0) force_delivery = queue_run_force; + if (!(status & 0xffff)) force_delivery = f.queue_run_force; /* If the process crashed, tell somebody */ - else if ((status & 0x00ff) != 0) - { + else if (status & 0x00ff) log_write(0, LOG_MAIN|LOG_PANIC, "queue run: process %d crashed with signal %d while delivering %s", - (int)pid, status & 0x00ff, f->text); - } + (int)pid, status & 0x00ff, fq->text); /* Before continuing, wait till the pipe gets closed at the far end. This tells us that any children created by the delivery to re-use any SMTP @@ -692,33 +676,46 @@ for (i = (queue_run_in_order? -1 : 0); the mere fact that read() unblocks is enough. */ set_process_info("running queue: waiting for children of %d", pid); - if (read(pfd[pipe_read], buffer, sizeof(buffer)) > 0) - log_write(0, LOG_MAIN|LOG_PANIC, "queue run: unexpected data on pipe"); + if ((status = read(pfd[pipe_read], buffer, sizeof(buffer))) != 0) + log_write(0, LOG_MAIN|LOG_PANIC, status > 0 ? + "queue run: unexpected data on pipe" : "queue run: error on pipe: %s", + strerror(errno)); (void)close(pfd[pipe_read]); set_process_info("running queue"); + /* If initial of a 2-phase run, we are a child - so just exit */ + if (f.queue_2stage && !queue_run_in_order) + exim_exit(EXIT_SUCCESS, US"2-phase child"); + /* If we are in the test harness, and this is not the first of a 2-stage queue run, update fudged queue times. */ - if (running_in_test_harness && !queue_2stage) + if (f.running_in_test_harness && !f.queue_2stage) { - uschar *fqtnext = Ustrchr(fudged_queue_times, '/'); - if (fqtnext != NULL) fudged_queue_times = fqtnext + 1; + uschar * fqtnext = Ustrchr(fudged_queue_times, '/'); + if (fqtnext) fudged_queue_times = fqtnext + 1; } + + + continue; + + go_around: + /* If initial of a 2-phase run, we are a child - so just exit */ + if (f.queue_2stage && !queue_run_in_order) + exim_exit(EXIT_SUCCESS, US"2-phase child"); } /* End loop for list of messages */ + tree_nonrecipients = NULL; store_reset(reset_point1); /* Scavenge list of messages */ /* If this was the first time through for random order processing, and sub-directories have been found, randomize their order if necessary. */ if (i == 0 && subcount > 1 && !queue_run_in_order) - { - int j; - for (j = 1; j <= subcount; j++) + for (int j = 1; j <= subcount; j++) { - int r = random_number(100); - if (r >= 50) + int r; + if ((r = random_number(100)) >= 50) { int k = (r % subcount) + 1; int x = subdirs[j]; @@ -726,15 +723,27 @@ for (i = (queue_run_in_order? -1 : 0); subdirs[k] = x; } } - } } /* End loop for multiple directories */ /* If queue_2stage is true, we do it all again, with the 2stage flag turned off. */ -if (queue_2stage) +if (f.queue_2stage) { - queue_2stage = FALSE; + + /* wait for last children */ + for (int i = 0; i < nelem(qpid); i++) + if (qpid[i]) + { + DEBUG(D_queue_run) debug_printf("q2stage reaped child %d\n", (int)qpid[i]); + waitpid(qpid[i], NULL, 0); + } + else break; + +#ifdef MEASURE_TIMING + report_time_since(×tamp_startup, US"queue_run 1st phase done"); +#endif + f.queue_2stage = FALSE; queue_run(start_id, stop_id, TRUE); } @@ -758,26 +767,39 @@ if (!recurse) /* Called as a result of -bpc Arguments: none -Returns: nothing +Returns: count */ -void +unsigned queue_count(void) { int subcount; -int count = 0; -queue_filename *f = NULL; +unsigned count = 0; uschar subdirs[64]; -f = queue_get_spool_list( - -1, /* entire queue */ - subdirs, /* for holding sub list */ - &subcount, /* for subcount */ - FALSE); /* not random */ -for (; f != NULL; f = f->next) count++; -fprintf(stdout, "%d\n", count); + +for (queue_filename * f = queue_get_spool_list( + -1, /* entire queue */ + subdirs, /* for holding sub list */ + &subcount, /* for subcount */ + FALSE); /* not random */ + f; f = f->next) count++; +return count; } +#define QUEUE_SIZE_AGE 60 /* update rate for queue_size */ + +unsigned +queue_count_cached(void) +{ +time_t now; +if ((now = time(NULL)) >= queue_size_next) + { + queue_size = queue_count(); + queue_size_next = now + (f.running_in_test_harness ? 3 : QUEUE_SIZE_AGE); + } +return queue_size; +} /************************************************ * List extra deliveries * @@ -793,11 +815,12 @@ Argument: points to the tree node Returns: nothing */ -static void queue_list_extras(tree_node *p) +static void +queue_list_extras(tree_node *p) { -if (p->left != NULL) queue_list_extras(p->left); +if (p->left) queue_list_extras(p->left); if (!p->data.val) printf(" +D %s\n", p->name); -if (p->right != NULL) queue_list_extras(p->right); +if (p->right) queue_list_extras(p->right); } @@ -828,11 +851,10 @@ Returns: nothing void queue_list(int option, uschar **list, int count) { -int i; int subcount; int now = (int)time(NULL); -void *reset_point; -queue_filename *f = NULL; +rmark reset_point; +queue_filename * qf = NULL; uschar subdirs[64]; /* If given a list of messages, build a chain containing their ids. */ @@ -840,14 +862,14 @@ uschar subdirs[64]; if (count > 0) { queue_filename *last = NULL; - for (i = 0; i < count; i++) + for (int i = 0; i < count; i++) { queue_filename *next = - store_get(sizeof(queue_filename) + Ustrlen(list[i]) + 2); + store_get(sizeof(queue_filename) + Ustrlen(list[i]) + 2, is_tainted(list[i])); sprintf(CS next->text, "%s-H", list[i]); next->dir_uschar = '*'; next->next = NULL; - if (i == 0) f = next; else last->next = next; + if (i == 0) qf = next; else last->next = next; last = next; } } @@ -855,7 +877,7 @@ if (count > 0) /* Otherwise get a list of the entire queue, in order if necessary. */ else - f = queue_get_spool_list( + qf = queue_get_spool_list( -1, /* entire queue */ subdirs, /* for holding sub list */ &subcount, /* for subcount */ @@ -866,29 +888,30 @@ if (option >= 8) option -= 8; /* Now scan the chain and print information, resetting store used each time. */ -reset_point = store_get(0); - -for (; f != NULL; f = f->next) +for (; + qf && (reset_point = store_mark()); + spool_clear_header_globals(), store_reset(reset_point), qf = qf->next + ) { int rc, save_errno; int size = 0; BOOL env_read; - store_reset(reset_point); message_size = 0; - message_subdir[0] = f->dir_uschar; - rc = spool_read_header(f->text, FALSE, count <= 0); - if (rc == spool_read_notopen && errno == ENOENT && count <= 0) continue; + message_subdir[0] = qf->dir_uschar; + rc = spool_read_header(qf->text, FALSE, count <= 0); + if (rc == spool_read_notopen && errno == ENOENT && count <= 0) + continue; save_errno = errno; env_read = (rc == spool_read_OK || rc == spool_read_hdrerror); if (env_read) { - int ptr; + int i, ptr; FILE *jread; struct stat statbuf; - uschar * fname = spool_fname(US"input", message_subdir, f->text, US""); + uschar * fname = spool_fname(US"input", message_subdir, qf->text, US""); ptr = Ustrlen(fname)-1; fname[ptr] = 'D'; @@ -899,7 +922,7 @@ for (; f != NULL; f = f->next) if (Ustat(fname, &statbuf) == 0) size = message_size + statbuf.st_size - SPOOL_DATA_START_OFFSET + 1; - i = (now - received_time)/60; /* minutes on queue */ + i = (now - received_time.tv_sec)/60; /* minutes on queue */ if (i > 90) { i = (i + 30)/60; @@ -910,8 +933,7 @@ for (; f != NULL; f = f->next) /* Collect delivered addresses from any J file */ fname[ptr] = 'J'; - jread = Ufopen(fname, "rb"); - if (jread != NULL) + if ((jread = Ufopen(fname, "rb"))) { while (Ufgets(big_buffer, big_buffer_size, jread) != NULL) { @@ -924,12 +946,12 @@ for (; f != NULL; f = f->next) } fprintf(stdout, "%s ", string_format_size(size, big_buffer)); - for (i = 0; i < 16; i++) fputc(f->text[i], stdout); + for (int i = 0; i < 16; i++) fputc(qf->text[i], stdout); - if (env_read && sender_address != NULL) + if (env_read && sender_address) { printf(" <%s>", sender_address); - if (sender_set_untrusted) printf(" (%s)", originator_login); + if (f.sender_set_untrusted) printf(" (%s)", originator_login); } if (rc != spool_read_OK) @@ -938,7 +960,7 @@ for (; f != NULL; f = f->next) if (save_errno == ERRNO_SPOOLFORMAT) { struct stat statbuf; - uschar * fname = spool_fname(US"input", message_subdir, f->text, US""); + uschar * fname = spool_fname(US"input", message_subdir, qf->text, US""); if (Ustat(fname, &statbuf) == 0) printf("*** spool format error: size=" OFF_T_FMT " ***", @@ -953,22 +975,22 @@ for (; f != NULL; f = f->next) } } - if (deliver_freeze) printf(" *** frozen ***"); + if (f.deliver_freeze) printf(" *** frozen ***"); printf("\n"); - if (recipients_list != NULL) + if (recipients_list) { - for (i = 0; i < recipients_count; i++) + for (int i = 0; i < recipients_count; i++) { tree_node *delivered = tree_search(tree_nonrecipients, recipients_list[i].address); if (!delivered || option != 1) - printf(" %s %s\n", (delivered != NULL)? "D":" ", - recipients_list[i].address); - if (delivered != NULL) delivered->data.val = TRUE; + printf(" %s %s\n", + delivered ? "D" : " ", recipients_list[i].address); + if (delivered) delivered->data.val = TRUE; } - if (option == 2 && tree_nonrecipients != NULL) + if (option == 2 && tree_nonrecipients) queue_list_extras(tree_nonrecipients); printf("\n"); } @@ -998,7 +1020,6 @@ Returns: FALSE if there was any problem BOOL queue_action(uschar *id, int action, uschar **argv, int argc, int recipients_arg) { -int i, j; BOOL yield = TRUE; BOOL removed = FALSE; struct passwd *pw; @@ -1017,10 +1038,10 @@ done. Only admin users may read the spool files. */ if (action >= MSG_SHOW_BODY) { - int fd, i, rc; + int fd, rc; uschar *subdirectory, *suffix; - if (!admin_user) + if (!f.admin_user) { printf("Permission denied\n"); return FALSE; @@ -1048,9 +1069,9 @@ if (action >= MSG_SHOW_BODY) suffix = US""; } - for (i = 0; i < 2; i++) + for (int i = 0; i < 2; i++) { - message_subdir[0] = split_spool_directory == (i == 0) ? id[5] : 0; + set_subdir_str(message_subdir, id, i); if ((fd = Uopen(spool_fname(subdirectory, message_subdir, id, suffix), O_RDONLY, 0)) >= 0) break; @@ -1082,7 +1103,7 @@ if ((deliver_datafile = spool_open_datafile(id)) < 0) { yield = FALSE; printf("Spool data file for %s does not exist\n", id); - if (action != MSG_REMOVE || !admin_user) return FALSE; + if (action != MSG_REMOVE || !f.admin_user) return FALSE; printf("Continuing, to ensure all files removed\n"); } else @@ -1107,7 +1128,7 @@ if (spool_read_header(spoolname, TRUE, FALSE) != spool_read_OK) printf("Spool read error for %s: %s\n", spoolname, strerror(errno)); else printf("Spool format error for %s\n", spoolname); - if (action != MSG_REMOVE || !admin_user) + if (action != MSG_REMOVE || !f.admin_user) { (void)close(deliver_datafile); deliver_datafile = -1; @@ -1121,7 +1142,7 @@ message. Only admin users may freeze/thaw, add/cancel recipients, or otherwise mess about, but the original sender is permitted to remove a message. That's why we leave this check until after the headers are read. */ -if (!admin_user && (action != MSG_REMOVE || real_uid != originator_uid)) +if (!f.admin_user && (action != MSG_REMOVE || real_uid != originator_uid)) { printf("Permission denied\n"); (void)close(deliver_datafile); @@ -1142,22 +1163,26 @@ if (action != MSG_SHOW_COPY) printf("Message %s ", id); switch(action) { case MSG_SHOW_COPY: - deliver_in_buffer = store_malloc(DELIVER_IN_BUFFER_SIZE); - deliver_out_buffer = store_malloc(DELIVER_OUT_BUFFER_SIZE); - transport_write_message(NULL, 1, 0, 0, NULL, NULL, NULL, NULL, NULL, 0); - break; + { + transport_ctx tctx = {{0}}; + deliver_in_buffer = store_malloc(DELIVER_IN_BUFFER_SIZE); + deliver_out_buffer = store_malloc(DELIVER_OUT_BUFFER_SIZE); + tctx.u.fd = 1; + (void) transport_write_message(&tctx, 0); + break; + } case MSG_FREEZE: - if (deliver_freeze) + if (f.deliver_freeze) { yield = FALSE; printf("is already frozen\n"); } else { - deliver_freeze = TRUE; - deliver_manual_thaw = FALSE; + f.deliver_freeze = TRUE; + f.deliver_manual_thaw = FALSE; deliver_frozen_at = time(NULL); if (spool_write_header(id, SW_MODIFYING, &errmsg) >= 0) { @@ -1174,15 +1199,15 @@ switch(action) case MSG_THAW: - if (!deliver_freeze) + if (!f.deliver_freeze) { yield = FALSE; printf("is not frozen\n"); } else { - deliver_freeze = FALSE; - deliver_manual_thaw = TRUE; + f.deliver_freeze = FALSE; + f.deliver_manual_thaw = TRUE; if (spool_write_header(id, SW_MODIFYING, &errmsg) >= 0) { printf("is no longer frozen\n"); @@ -1211,7 +1236,7 @@ switch(action) suffix[2] = 0; message_subdir[0] = id[5]; - for (j = 0; j < 2; message_subdir[0] = 0, j++) + for (int j = 0; j < 2; message_subdir[0] = 0, j++) { uschar * fname = spool_fname(US"msglog", message_subdir, id, US""); @@ -1231,7 +1256,7 @@ switch(action) DEBUG(D_any) debug_printf(" (ok)\n"); } - for (i = 0; i < 3; i++) + for (int i = 0; i < 3; i++) { uschar * fname; @@ -1263,6 +1288,38 @@ switch(action) else printf("has been removed or did not exist\n"); if (removed) { +#ifndef DISABLE_EVENT + if (event_action) for (int i = 0; i < recipients_count; i++) + { + tree_node *delivered = + tree_search(tree_nonrecipients, recipients_list[i].address); + if (!delivered) + { + uschar * save_local = deliver_localpart; + const uschar * save_domain = deliver_domain; + uschar * addr = recipients_list[i].address, * errmsg = NULL; + int start, end, dom; + + if (!parse_extract_address(addr, &errmsg, &start, &end, &dom, TRUE)) + log_write(0, LOG_MAIN|LOG_PANIC, + "failed to parse address '%.100s'\n: %s", addr, errmsg); + else + { + deliver_localpart = + string_copyn(addr+start, dom ? (dom-1) - start : end - start); + deliver_domain = dom + ? CUS string_copyn(addr+dom, end - dom) : CUS""; + + event_raise(event_action, US"msg:fail:internal", + string_sprintf("message removed by %s", username)); + + deliver_localpart = save_local; + deliver_domain = save_domain; + } + } + } + (void) event_raise(event_action, US"msg:complete", NULL); +#endif log_write(0, LOG_MAIN, "removed by %s", username); log_write(0, LOG_MAIN, "Completed"); } @@ -1270,15 +1327,22 @@ switch(action) } + case MSG_SETQUEUE: + /* The global "queue_name_dest" is used as destination, "queue_name" + as source */ + + spool_move_message(id, message_subdir, US"", US""); + break; + + case MSG_MARK_ALL_DELIVERED: - for (i = 0; i < recipients_count; i++) - { + for (int i = 0; i < recipients_count; i++) tree_add_nonrecipient(recipients_list[i].address); - } + if (spool_write_header(id, SW_MODIFYING, &errmsg) >= 0) { printf("has been modified\n"); - for (i = 0; i < recipients_count; i++) + for (int i = 0; i < recipients_count; i++) log_write(0, LOG_MAIN, "address <%s> marked delivered by %s", recipients_list[i].address, username); } @@ -1349,6 +1413,7 @@ switch(action) } else if (action == MSG_MARK_DELIVERED) { + int i; for (i = 0; i < recipients_count; i++) if (Ustrcmp(recipients_list[i].address, recipient) == 0) break; if (i >= recipients_count) @@ -1377,7 +1442,6 @@ switch(action) } if (yield) - { if (spool_write_header(id, SW_MODIFYING, &errmsg) >= 0) printf("has been modified\n"); else @@ -1385,7 +1449,6 @@ switch(action) yield = FALSE; printf("- while %s: %s\n", doing, errmsg); } - } break; } @@ -1393,8 +1456,11 @@ switch(action) /* Closing the datafile releases the lock and permits other processes to operate on the message (if it still exists). */ -(void)close(deliver_datafile); -deliver_datafile = -1; +if (deliver_datafile >= 0) + { + (void)close(deliver_datafile); + deliver_datafile = -1; + } return yield; } @@ -1414,37 +1480,63 @@ Returns: nothing void queue_check_only(void) { -BOOL *set; int sep = 0; struct stat statbuf; -const uschar *s; -uschar *ss, *name; -uschar buffer[1024]; +const uschar * s = queue_only_file; +uschar * ss; -if (queue_only_file == NULL) return; +if (s) + while ((ss = string_nextinlist(&s, &sep, NULL, 0))) + if (Ustrncmp(ss, "smtp", 4) == 0) + { + ss += 4; + if (Ustat(ss, &statbuf) == 0) + { + f.queue_smtp = TRUE; + DEBUG(D_receive) debug_printf("queue_smtp set because %s exists\n", ss); + } + } + else + if (Ustat(ss, &statbuf) == 0) + { + queue_only = TRUE; + DEBUG(D_receive) debug_printf("queue_only set because %s exists\n", ss); + } +} -s = queue_only_file; -while ((ss = string_nextinlist(&s, &sep, buffer, sizeof(buffer))) != NULL) - { - if (Ustrncmp(ss, "smtp", 4) == 0) - { - name = US"queue_smtp"; - set = &queue_smtp; - ss += 4; - } - else - { - name = US"queue_only"; - set = &queue_only; - } - if (Ustat(ss, &statbuf) == 0) - { - *set = TRUE; - DEBUG(D_receive) debug_printf("%s set because %s exists\n", name, ss); - } + +/******************************************************************************/ +/******************************************************************************/ + +#ifdef EXPERIMENTAL_QUEUE_RAMP +void +queue_notify_daemon(const uschar * msgid) +{ +uschar buf[MESSAGE_ID_LENGTH + 2]; +int fd; + +DEBUG(D_queue_run) debug_printf("%s: %s\n", __FUNCTION__, msgid); + +buf[0] = NOTIFY_MSG_QRUN; +memcpy(buf+1, msgid, MESSAGE_ID_LENGTH+1); + +if ((fd = socket(AF_UNIX, SOCK_DGRAM, 0)) >= 0) + { + struct sockaddr_un sun = {.sun_family = AF_UNIX}; + int len = offsetof(struct sockaddr_un, sun_path) + 1 + + snprintf(sun.sun_path+1, sizeof(sun.sun_path)-1, "%s", + NOTIFIER_SOCKET_NAME); + sun.sun_path[0] = 0; + + if (sendto(fd, buf, sizeof(buf), 0, &sun, len) < 0) + DEBUG(D_queue_run) + debug_printf("%s: sendto %s\n", __FUNCTION__, strerror(errno)); + close(fd); } +else DEBUG(D_queue_run) debug_printf(" socket: %s\n", strerror(errno)); } +#endif #endif /*!COMPILE_UTILITY*/