X-Git-Url: https://git.exim.org/exim.git/blobdiff_plain/1d28cc061677bd07d9bed48dd84bd5c590247043..107077d7fd6736711bf5cd980221723401d37c51:/src/src/receive.c diff --git a/src/src/receive.c b/src/src/receive.c index 9bf834aaf..8a400caf0 100644 --- a/src/src/receive.c +++ b/src/src/receive.c @@ -2,7 +2,7 @@ * Exim - an Internet mail transport agent * *************************************************/ -/* Copyright (c) The Exim Maintainers 2020 - 2022 */ +/* Copyright (c) The Exim Maintainers 2020 - 2023 */ /* Copyright (c) University of Cambridge 1995 - 2018 */ /* See the file NOTICE for conditions of use and distribution. */ /* SPDX-License-Identifier: GPL-2.0-or-later */ @@ -514,7 +514,7 @@ Returns: nothing */ void -receive_add_recipient(uschar *recipient, int pno) +receive_add_recipient(uschar * recipient, int pno) { if (recipients_count >= recipients_list_max) { @@ -1156,7 +1156,7 @@ Returns: the SMTP response */ static uschar * -handle_lost_connection(uschar *s) +handle_lost_connection(uschar * s) { log_write(L_lost_incoming_connection | L_smtp_connection, LOG_MAIN, "%s lost while reading message data%s", smtp_get_connection_info(), s); @@ -1230,9 +1230,9 @@ Returns: nothing */ static void -add_acl_headers(int where, uschar *acl_name) +add_acl_headers(int where, uschar * acl_name) { -header_line *last_received = NULL; +header_line * last_received = NULL; switch(where) { @@ -1254,15 +1254,22 @@ if (acl_removed_headers) for (header_line * h = header_list; h; h = h->next) if (h->type != htype_old) { - const uschar * list = acl_removed_headers; + const uschar * list = acl_removed_headers, * s; int sep = ':'; /* This is specified as a colon-separated list */ - uschar *s; + /* If a list element has a leading '^' then it is an RE for + the whole header, else just a header name. */ while ((s = string_nextinlist(&list, &sep, NULL, 0))) - if (header_testname(h, s, Ustrlen(s), FALSE)) + if ( ( *s == '^' + && regex_match( + regex_must_compile(s, MCS_CACHEABLE, FALSE), + h->text, h->slen, NULL) + ) + || header_testname(h, s, Ustrlen(s), FALSE) + ) { h->type = htype_old; - DEBUG(D_receive|D_acl) debug_printf_indent(" %s", h->text); + DEBUG(D_receive|D_acl) debug_printf_indent(" %s", h->text); } } acl_removed_headers = NULL; @@ -1372,6 +1379,8 @@ if (f.tcp_in_fastopen && !f.tcp_in_fastopen_logged) } if (sender_ident) g = string_append(g, 2, US" U=", sender_ident); +if (LOGGING(connection_id)) + g = string_fmt_append(g, " Ci=%lu", connection_id); if (received_protocol) g = string_append(g, 2, US" P=", received_protocol); if (LOGGING(pipelining) && f.smtp_in_pipelining_advertised) @@ -1686,7 +1695,9 @@ int error_rc = error_handling == ERRORS_SENDER int header_size = 256; int had_zero = 0; int prevlines_length = 0; -const int id_resolution = BASE_62 == 62 ? 5000 : 10000; +const int id_resolution = BASE_62 == 62 && !host_number_string ? 1 + : BASE_62 != 62 && host_number_string ? 4 + : 2; int ptr = 0; @@ -2658,7 +2669,7 @@ if (extract_recip) that this has happened, in order to give a better error if there are no recipients left. */ - else if (recipient != NULL) + else if (recipient) { if (tree_search(tree_nonrecipients, recipient) == NULL) receive_add_recipient(recipient, -1); @@ -2668,7 +2679,7 @@ if (extract_recip) /* Move on past this address */ - s = ss + (*ss? 1:0); + s = ss + (*ss ? 1 : 0); while (isspace(*s)) s++; } /* Next address */ @@ -2686,41 +2697,37 @@ if (extract_recip) } /* Now build the unique message id. This has changed several times over the -lifetime of Exim. This description was rewritten for Exim 4.14 (February 2003). -Retaining all the history in the comment has become too unwieldy - read -previous release sources if you want it. - -The message ID has 3 parts: tttttt-pppppp-ss. Each part is a number in base 62. -The first part is the current time, in seconds. The second part is the current -pid. Both are large enough to hold 32-bit numbers in base 62. The third part -can hold a number in the range 0-3843. It used to be a computed sequence -number, but is now the fractional component of the current time in units of -1/2000 of a second (i.e. a value in the range 0-1999). After a message has been -received, Exim ensures that the timer has ticked at the appropriate level -before proceeding, to avoid duplication if the pid happened to be re-used -within the same time period. It seems likely that most messages will take at -least half a millisecond to be received, so no delay will normally be -necessary. At least for some time... +lifetime of Exim, and is changing for Exim 4.97. +The previous change was in about 2003. + +Detail for the pre-4.97 version is here in [square-brackets]. + +The message ID has 3 parts: tttttt-ppppppppppp-ssss (6, 11, 4 - total 23 with +the dashes). Each part is a number in base 62. +[ tttttt-pppppp-ss 6, 6, 2 => 16 ] -There is a modification when localhost_number is set. Formerly this was allowed -to be as large as 255. Now it is restricted to the range 0-16, and the final -component of the message id becomes (localhost_number * 200) + fractional time -in units of 1/200 of a second (i.e. a value in the range 0-3399). +The first part is the current time, in seconds. Six chars is enough until +year 3700 with case-sensitive filesystes, but will run out in 2038 on +case-insensitive ones (Cygwin, Darwin - where we have to use base-36. +Both of those are in the "unsupported" bucket, so ignore for now). -Some not-really-Unix operating systems use case-insensitive file names (Darwin, -Cygwin). For these, we have to use base 36 instead of base 62. Luckily, this -still allows the tttttt field to hold a large enough number to last for some -more decades, and the final two-digit field can hold numbers up to 1295, which -is enough for milliseconds (instead of 1/2000 of a second). +The second part is the current pid, and supports 64b [31b] PIDs. -However, the pppppp field cannot hold a 32-bit pid, but it can hold a 31-bit -pid, so it is probably safe because pids have to be positive. The -localhost_number is restricted to 0-10 for these hosts, and when it is set, the -final field becomes (localhost_number * 100) + fractional time in centiseconds. +The third part holds sub-second time, plus (when localhost_number is set) +the host number multiplied by a number large enough to keep it away from +the time portion. Host numbers are restricted to the range 0-16. +The time resolution is variously 1, 2 or 4 microseconds [0.5 or 1 ms] +depending on the use of localhost_nubmer and of case-insensitive filesystems. + +After a message has been received, Exim ensures that the timer has ticked at the +appropriate level before proceeding, to avoid duplication if the pid happened to +be re-used within the same time period. It seems likely that most messages will +take at least half a millisecond to be received, so no delay will normally be +necessary. At least for some time... -Note that string_base62() returns its data in a static storage block, so it -must be copied before calling string_base62() again. It always returns exactly -6 characters. +Note that string_base62_XX() returns its data in a static storage block, so it +must be copied before calling string_base62_XXX) again. It always returns exactly +11 (_64) or 6 (_32) characters. There doesn't seem to be anything in the RFC which requires a message id to start with a letter, but Smail was changed to ensure this. The external form of @@ -2733,27 +2740,35 @@ checking that a string is in this format must be updated in a corresponding way. It appears in the initializing code in exim.c. The macro MESSAGE_ID_LENGTH must also be changed to reflect the correct string length. The queue-sort code needs to know the layout. Then, of course, other programs that rely on the -message id format will need updating too. */ +message id format will need updating too (inc. at least exim_msgdate). */ -Ustrncpy(message_id, string_base62((long int)(message_id_tv.tv_sec)), 6); -message_id[6] = '-'; -Ustrncpy(message_id + 7, string_base62((long int)getpid()), 6); +Ustrncpy(message_id, string_base62_32((long int)(message_id_tv.tv_sec)), MESSAGE_ID_TIME_LEN); +message_id[MESSAGE_ID_TIME_LEN] = '-'; +Ustrncpy(message_id + MESSAGE_ID_TIME_LEN + 1, + string_base62_64((long int)getpid()), + MESSAGE_ID_PID_LEN + ); /* Deal with the case where the host number is set. The value of the number was checked when it was read, to ensure it isn't too big. */ if (host_number_string) - sprintf(CS(message_id + MESSAGE_ID_LENGTH - 3), "-%2s", - string_base62((long int)( - host_number * (1000000/id_resolution) + - message_id_tv.tv_usec/id_resolution)) + 4); + sprintf(CS(message_id + MESSAGE_ID_TIME_LEN + 1 + MESSAGE_ID_PID_LEN), + "-%" str(MESSAGE_ID_SUBTIME_LEN) "s", + string_base62_32((long int)( + host_number * (1000000/id_resolution) + + message_id_tv.tv_usec/id_resolution)) + + (6 - MESSAGE_ID_SUBTIME_LEN) + ); /* Host number not set: final field is just the fractional time at an appropriate resolution. */ else - sprintf(CS(message_id + MESSAGE_ID_LENGTH - 3), "-%2s", - string_base62((long int)(message_id_tv.tv_usec/id_resolution)) + 4); + sprintf(CS(message_id + MESSAGE_ID_TIME_LEN + 1 + MESSAGE_ID_PID_LEN), + "-%" str(MESSAGE_ID_SUBTIME_LEN) "s", + string_base62_32((long int)(message_id_tv.tv_usec/id_resolution)) + + (6 - MESSAGE_ID_SUBTIME_LEN)); /* Add the current message id onto the current process info string if it will fit. */ @@ -3136,9 +3151,8 @@ if (cutthrough.cctx.sock >= 0 && cutthrough.delivery) sender_address, sender_fullhost ? "H=" : "", sender_fullhost ? sender_fullhost : US"", sender_ident ? "U=" : "", sender_ident ? sender_ident : US""); - message_id[0] = 0; /* Indicate no message accepted */ smtp_reply = US"550 Too many \"Received\" headers - suspected mail loop"; - goto TIDYUP; /* Skip to end of function */ + goto NOT_ACCEPTED; /* Skip to end of function */ } received_header_gen(); add_acl_headers(ACL_WHERE_RCPT, US"MAIL or RCPT"); @@ -3185,7 +3199,7 @@ spool_data_file = fdopen(data_fd, "w+"); lock_data.l_type = F_WRLCK; lock_data.l_whence = SEEK_SET; lock_data.l_start = 0; -lock_data.l_len = SPOOL_DATA_START_OFFSET; +lock_data.l_len = spool_data_start_offset(message_id); if (fcntl(data_fd, F_SETLK, &lock_data) < 0) log_write(0, LOG_MAIN|LOG_PANIC_DIE, "Cannot lock %s (%d): %s", spool_name, @@ -3235,12 +3249,12 @@ if (!ferror(spool_data_file) && !(receive_feof)() && message_ended != END_DOT) case END_EOF: if (smtp_input) { - Uunlink(spool_name); /* Lose data file when closed */ + Uunlink(spool_name); /* Lose data file when closed */ cancel_cutthrough_connection(TRUE, US"sender closed connection"); - message_id[0] = 0; /* Indicate no message accepted */ + message_id[0] = 0; /* Indicate no message_accepted */ smtp_reply = handle_lost_connection(US""); smtp_yield = FALSE; - goto TIDYUP; /* Skip to end of function */ + goto TIDYUP; /* Skip to end of function */ } break; @@ -3265,12 +3279,11 @@ if (!ferror(spool_data_file) && !(receive_feof)() && message_ended != END_DOT) if (smtp_input) { smtp_reply = US"552 Message size exceeds maximum permitted"; - message_id[0] = 0; /* Indicate no message accepted */ - goto TIDYUP; /* Skip to end of function */ + goto NOT_ACCEPTED; /* Skip to end of function */ } else { - fseek(spool_data_file, (long int)SPOOL_DATA_START_OFFSET, SEEK_SET); + fseek(spool_data_file, (long int)spool_data_start_offset(message_id), SEEK_SET); give_local_error(ERRMESS_TOOBIG, string_sprintf("message too big (max=%d)", thismessage_size_limit), US"message rejected: ", error_rc, spool_data_file, header_list); @@ -3284,8 +3297,7 @@ if (!ferror(spool_data_file) && !(receive_feof)() && message_ended != END_DOT) Uunlink(spool_name); /* Lose the data file when closed */ cancel_cutthrough_connection(TRUE, US"sender protocol error"); smtp_reply = US""; /* Response already sent */ - message_id[0] = 0; /* Indicate no message accepted */ - goto TIDYUP; /* Skip to end of function */ + goto NOT_ACCEPTED; /* Skip to end of function */ } } @@ -3326,13 +3338,12 @@ if (fflush(spool_data_file) == EOF || ferror(spool_data_file) || smtp_reply = US"451 Error while writing spool file"; receive_swallow_smtp(); } - message_id[0] = 0; /* Indicate no message accepted */ - goto TIDYUP; /* Skip to end of function */ + goto NOT_ACCEPTED; /* Skip to end of function */ } else { - fseek(spool_data_file, (long int)SPOOL_DATA_START_OFFSET, SEEK_SET); + fseek(spool_data_file, (long int)spool_data_start_offset(message_id), SEEK_SET); give_local_error(ERRMESS_IOERR, msg, US"", error_rc, spool_data_file, header_list); /* Does not return */ @@ -3373,7 +3384,7 @@ if (extract_recip && (bad_addresses || recipients_count == 0)) log_write(0, LOG_MAIN|LOG_PANIC, "%s %s found in headers", message_id, bad_addresses ? "bad addresses" : "no recipients"); - fseek(spool_data_file, (long int)SPOOL_DATA_START_OFFSET, SEEK_SET); + fseek(spool_data_file, (long int)spool_data_start_offset(message_id), SEEK_SET); /* If configured to send errors to the sender, but this fails, force a failure error code. We use a special one for no recipients so that it @@ -3440,7 +3451,7 @@ if (!received_header->text) /* Non-cutthrough case */ /* Set the value of message_body_size for the DATA ACL and for local_scan() */ message_body_size = (fstat(data_fd, &statbuf) == 0)? - statbuf.st_size - SPOOL_DATA_START_OFFSET : -1; + statbuf.st_size - spool_data_start_offset(message_id) : -1; /* If an ACL from any RCPT commands set up any warning headers to add, do so now, before running the DATA ACL. */ @@ -3449,7 +3460,7 @@ if (!received_header->text) /* Non-cutthrough case */ } else message_body_size = (fstat(data_fd, &statbuf) == 0)? - statbuf.st_size - SPOOL_DATA_START_OFFSET : -1; + statbuf.st_size - spool_data_start_offset(message_id) : -1; /* If an ACL is specified for checking things at this stage of reception of a message, run it, unless all the recipients were removed by "discard" in earlier @@ -3560,8 +3571,7 @@ else if (smtp_handle_acl_fail(ACL_WHERE_DKIM, rc, user_msg, log_msg) != 0) smtp_yield = FALSE; /* No more messages after dropped connection */ smtp_reply = US""; /* Indicate reply already sent */ - message_id[0] = 0; /* Indicate no message accepted */ - goto TIDYUP; /* Skip to end of function */ + goto NOT_ACCEPTED; /* Skip to end of function */ } } else @@ -3642,10 +3652,7 @@ else ? US"accepted" : US"accepted for some recipients"); if (recipients_count == 0) - { - message_id[0] = 0; /* Indicate no message accepted */ - goto TIDYUP; - } + goto NOT_ACCEPTED; } else prdr_requested = FALSE; @@ -3679,8 +3686,7 @@ else if (smtp_handle_acl_fail(ACL_WHERE_DATA, rc, user_msg, log_msg) != 0) smtp_yield = FALSE; /* No more messages after dropped connection */ smtp_reply = US""; /* Indicate reply already sent */ - message_id[0] = 0; /* Indicate no message accepted */ - goto TIDYUP; /* Skip to end of function */ + goto NOT_ACCEPTED; /* Skip to end of function */ } } } @@ -3733,7 +3739,7 @@ else /* Does not return */ else { - fseek(spool_data_file, (long int)SPOOL_DATA_START_OFFSET, SEEK_SET); + fseek(spool_data_file, (long int)spool_data_start_offset(message_id), SEEK_SET); give_local_error(ERRMESS_LOCAL_ACL, user_msg, US"message rejected by non-SMTP ACL: ", error_rc, spool_data_file, header_list); @@ -3765,7 +3771,7 @@ version supplied with Exim always accepts, but this is a hook for sysadmins to supply their own checking code. The local_scan() function is run even when all the recipients have been discarded. */ -lseek(data_fd, (long int)SPOOL_DATA_START_OFFSET, SEEK_SET); +lseek(data_fd, (long int)spool_data_start_offset(message_id), SEEK_SET); /* Arrange to catch crashes in local_scan(), so that the -D file gets deleted, and the incident gets logged. */ @@ -3855,10 +3861,10 @@ the spool file gets corrupted. Ensure that all recipients are qualified. */ if (rc == LOCAL_SCAN_ACCEPT) { if (local_scan_data) - for (uschar * s = local_scan_data; *s != 0; s++) if (*s == '\n') *s = ' '; - for (int i = 0; i < recipients_count; i++) + for (uschar * s = local_scan_data; *s; s++) if (*s == '\n') *s = ' '; + for (recipient_item * r = recipients_list; + r < recipients_list + recipients_count; r++) { - recipient_item *r = recipients_list + i; r->address = rewrite_address_qualify(r->address, TRUE); if (r->errors_to) r->errors_to = rewrite_address_qualify(r->errors_to, TRUE); @@ -3907,27 +3913,25 @@ else break; } - g = string_append(NULL, 2, US"F=", - sender_address[0] == 0 ? US"<>" : sender_address); + g = string_append(NULL, 2, US"F=", *sender_address ? sender_address : US"<>"); g = add_host_info_for_log(g); - log_write(0, LOG_MAIN|LOG_REJECT, "%s %srejected by local_scan(): %.256s", - string_from_gstring(g), istemp, string_printing(errmsg)); + log_write(0, LOG_MAIN|LOG_REJECT, "%Y %srejected by local_scan(): %.256s", + g, istemp, string_printing(errmsg)); if (smtp_input) if (!smtp_batched_input) { smtp_respond(smtp_code, 3, TRUE, errmsg); - message_id[0] = 0; /* Indicate no message accepted */ smtp_reply = US""; /* Indicate reply already sent */ - goto TIDYUP; /* Skip to end of function */ + goto NOT_ACCEPTED; /* Skip to end of function */ } else moan_smtp_batch(NULL, "%s %s", smtp_code, errmsg); /* Does not return */ else { - fseek(spool_data_file, (long int)SPOOL_DATA_START_OFFSET, SEEK_SET); + fseek(spool_data_file, (long int)spool_data_start_offset(message_id), SEEK_SET); give_local_error(ERRMESS_LOCAL_SCAN, errmsg, US"message rejected by local scan code: ", error_rc, spool_data_file, header_list); @@ -3942,6 +3946,19 @@ signal(SIGTERM, SIG_IGN); signal(SIGINT, SIG_IGN); #endif /* HAVE_LOCAL_SCAN */ +/* If we are faking a reject or defer, avoid sennding a DSN for the +actually-accepted message */ + +if (fake_response != OK) + for (recipient_item * r = recipients_list; + r < recipients_list + recipients_count; r++) + { + DEBUG(D_receive) if (r->dsn_flags & (rf_notify_success | rf_notify_delay)) + debug_printf("DSN: clearing flags due to fake-response for message\n"); + r->dsn_flags = r->dsn_flags & ~(rf_notify_success | rf_notify_delay) + | rf_notify_never; + } + /* Ensure the first time flag is set in the newly-received message. */ @@ -3950,7 +3967,7 @@ f.deliver_firsttime = TRUE; #ifdef EXPERIMENTAL_BRIGHTMAIL if (bmi_run == 1) { /* rewind data file */ - lseek(data_fd, (long int)SPOOL_DATA_START_OFFSET, SEEK_SET); + lseek(data_fd, (long int)spool_data_start_offset(message_id), SEEK_SET); bmi_verdicts = bmi_process_message(header_list, data_fd); } #endif @@ -3997,12 +4014,11 @@ else if (smtp_input) { smtp_reply = US"451 Error in writing spool file"; - message_id[0] = 0; /* Indicate no message accepted */ - goto TIDYUP; + goto NOT_ACCEPTED; } else { - fseek(spool_data_file, (long int)SPOOL_DATA_START_OFFSET, SEEK_SET); + fseek(spool_data_file, (long int)spool_data_start_offset(message_id), SEEK_SET); give_local_error(ERRMESS_IOERR, errmsg, US"", error_rc, spool_data_file, header_list); /* Does not return */ @@ -4028,12 +4044,11 @@ if (fflush(spool_data_file)) if (smtp_input) { smtp_reply = US"451 Error in writing spool file"; - message_id[0] = 0; /* Indicate no message accepted */ - goto TIDYUP; + goto NOT_ACCEPTED; } else { - fseek(spool_data_file, (long int)SPOOL_DATA_START_OFFSET, SEEK_SET); + fseek(spool_data_file, (long int)spool_data_start_offset(message_id), SEEK_SET); give_local_error(ERRMESS_IOERR, errmsg, US"", error_rc, spool_data_file, header_list); /* Does not return */ @@ -4041,7 +4056,7 @@ if (fflush(spool_data_file)) } fstat(data_fd, &statbuf); -msg_size += statbuf.st_size - SPOOL_DATA_START_OFFSET + 1; +msg_size += statbuf.st_size - spool_data_start_offset(message_id) + 1; /* Generate a "message received" log entry. We do this by building up a dynamic string as required. We log the arrival of a new message while the @@ -4055,7 +4070,7 @@ g = string_get(256); g = string_append(g, 2, fake_response == FAIL ? US"(= " : US"<= ", - sender_address[0] == 0 ? US"<>" : sender_address); + *sender_address ? sender_address : US"<>"); if (message_reference) g = string_append(g, 2, US" R=", message_reference); @@ -4214,7 +4229,8 @@ if (message_logs && !blackholed_by) } else { - uschar *now = tod_stamp(tod_log); + uschar * now = tod_stamp(tod_log); + /* Drop the initial "<= " */ fprintf(message_log, "%s Received from %s\n", now, g->s+3); if (f.deliver_freeze) fprintf(message_log, "%s frozen by %s\n", now, frozen_by); @@ -4266,10 +4282,10 @@ if ( smtp_input && sender_host_address && !f.sender_host_notsocket /* Re-use the log line workspace */ - g->ptr = 0; + gstring_reset(g); g = string_cat(g, US"SMTP connection lost after final dot"); g = add_host_info_for_log(g); - log_write(0, LOG_MAIN, "%s", string_from_gstring(g)); + log_write(0, LOG_MAIN, "%Y", g); /* Delete the files for this aborted message. */ @@ -4335,7 +4351,7 @@ if(!smtp_reply) log_write(0, LOG_MAIN | (LOGGING(received_recipients) ? LOG_RECIPIENTS : 0) | (LOGGING(received_sender) ? LOG_SENDER : 0), - "%s", g->s); + "%Y", g); /* Log any control actions taken by an ACL or local_scan(). */ @@ -4378,6 +4394,11 @@ if this happens? We can at least log it; if it is observed on some platform then we can think about properly declaring the message not-received. */ +goto TIDYUP; + +NOT_ACCEPTED: +message_id[0] = 0; /* Indicate no message accepted */ + TIDYUP: process_info[process_info_len] = 0; /* Remove message id */ if (spool_data_file && cutthrough_done == NOT_TRIED)