From 9d4319dfec653f43b64562c8f31b87f2890365b2 Mon Sep 17 00:00:00 2001 From: Jeremy Harris Date: Sun, 12 Apr 2015 19:18:26 +0100 Subject: [PATCH] smtp input --- TODO | 14 +++-- doc/doc-txt/experimental-spec.txt | 15 +++++ src/src/dns.c | 17 ++++++ src/src/exim.c | 4 ++ src/src/functions.h | 1 + src/src/globals.c | 2 +- src/src/parse.c | 10 +--- src/src/smtp_in.c | 90 +++++++++++++++------------- src/src/utf8.c | 10 +--- test/confs/4201 | 60 +++++++++++++++++++ test/log/4201 | 9 +++ test/runtest | 6 +- test/scripts/4200-International/4201 | 64 ++++++++++++++++++++ test/stdout/4201 | 70 ++++++++++++++++++++++ 14 files changed, 308 insertions(+), 64 deletions(-) create mode 100644 test/confs/4201 create mode 100644 test/log/4201 create mode 100644 test/scripts/4200-International/4201 create mode 100644 test/stdout/4201 diff --git a/TODO b/TODO index d2d31ef07..6cce9a6f5 100644 --- a/TODO +++ b/TODO @@ -15,23 +15,25 @@ destination supports the SMTPUTF8 extension ====================== to-Alabel convert of helo name +- smtp transport ++ An "international" flag on the message? ++ An is-international expansion condition? ++ helo-time option handling -conversion of utf-8 domains on input rfc5890 -- deconversion on forwarding -- deconversion for trace headers +++ conversion of utf-8 domains for DNS rfc5890 +-- MSA mode: convert on forward? + dsn handling rfc6533 logging - international msg - presentation of local-part in log -- a log option? encoding of local_part -encoding transform string-expansions + Recieved-by header tracking info - WITH protocol types get UTF8 prefix +- use for logging also forwarding checks rfc6530 7.1 -3- - rcpt-time rejects get 533 mailbox name not allowed @@ -39,9 +41,11 @@ forwarding checks rfc6530 7.1 -3- - bounces (see dsn handling) -expansions for to- and from-Alabel ? bug1567 +++ expansions for to- and from-Alabel ? bug1567 enhanced status codes? rfc5248++ VRFY EXPN + +non-smtp input diff --git a/doc/doc-txt/experimental-spec.txt b/doc/doc-txt/experimental-spec.txt index 738f02cce..59dd44ea1 100644 --- a/doc/doc-txt/experimental-spec.txt +++ b/doc/doc-txt/experimental-spec.txt @@ -1278,6 +1278,21 @@ RFCs 6530, 6533, 5890 Compile with EXPERIMENTAL_INTERNATIONAL and libidn. +Main config option smtputf8_advertise_hosts, default '*', +a host list. If this matches the sending host and +accept_8bitmime is true (the default) then the ESMTP option +SMTPUTF8 will be advertised. + +If the sender specifies the SMTPUTF8 option on a MAIL command +international handling for the message is enabled and +the expansion variable $message_smtputf8 will have value TRUE. + +The option allow_utf8_domains is set to true for this +message, but all DNS lookups are converted to a-label form. + +Log lines and Received-by: header lines will aquire a "utf8" +prefix on the 'with' element, eg. utf8esmtp. + Expansion operators: ${utf8_domain_to_alabel:str} ${utf8_domain_from_alabel:str} diff --git a/src/src/dns.c b/src/src/dns.c index a2f430993..f1619f4a4 100644 --- a/src/src/dns.c +++ b/src/src/dns.c @@ -576,6 +576,23 @@ if (previous != NULL) return previous->data.val; } +#ifdef EXPERIMENTAL_INTERNATIONAL +/* Convert all names to a-label form before doing lookup */ + { + uschar * alabel; + uschar * errstr = NULL; + if ((alabel = string_domain_utf8_to_alabel(name, &errstr)), errstr) + { + DEBUG(D_dns) + debug_printf("DNS name '%s' utf8 conversion to alabel failed: %s", name, + errstr); + host_find_failed_syntax = TRUE; + return DNS_NOMATCH; + } + name = alabel; + } +#endif + /* If configured, check the hygene of the name passed to lookup. Otherwise, although DNS lookups may give REFUSED at the lower level, some resolvers turn this into TRY_AGAIN, which is silly. Give a NOMATCH return, since such diff --git a/src/src/exim.c b/src/src/exim.c index f6d9be6ef..121c6c2e3 100644 --- a/src/src/exim.c +++ b/src/src/exim.c @@ -3705,6 +3705,10 @@ is equivalent to the ability to modify a setuid binary! This needs to happen before we read the main configuration. */ init_lookup_list(); +#ifdef EXPERIMENTAL_INTERNATIONAL +if (running_in_test_harness) smtputf8_advertise_hosts = NULL; +#endif + /* Read the main runtime configuration data; this gives up if there is a failure. It leaves the configuration file open so that the subsequent configuration data for delivery can be read if needed. */ diff --git a/src/src/functions.h b/src/src/functions.h index ac93c1635..fdd629228 100644 --- a/src/src/functions.h +++ b/src/src/functions.h @@ -422,6 +422,7 @@ extern const uschar *string_printing2(const uschar *, BOOL); extern uschar *string_split_message(uschar *); extern uschar *string_unprinting(uschar *); #ifdef EXPERIMENTAL_INTERNATIONAL +extern uschar *string_address_utf8_to_alabel(uschar *, uschar **, int *); extern uschar *string_domain_alabel_to_utf8(const uschar *, uschar **); extern uschar *string_domain_utf8_to_alabel(const uschar *, uschar **); extern uschar *string_localpart_alabel_to_utf8(const uschar *, uschar **); diff --git a/src/src/globals.c b/src/src/globals.c index cb93a0192..2cbafcdce 100644 --- a/src/src/globals.c +++ b/src/src/globals.c @@ -1274,7 +1274,7 @@ int smtp_rlr_threshold = INT_MAX; BOOL smtp_use_pipelining = FALSE; BOOL smtp_use_size = FALSE; #ifdef EXPERIMENTAL_INTERNATIONAL -uschar *smtputf8_advertise_hosts = US"*"; +uschar *smtputf8_advertise_hosts = US"*"; /* overridden under test-harness */ #endif #ifdef WITH_CONTENT_SCAN diff --git a/src/src/parse.c b/src/src/parse.c index ff814e738..a648f755a 100644 --- a/src/src/parse.c +++ b/src/src/parse.c @@ -550,9 +550,7 @@ read_addr_spec(uschar *s, uschar *t, int term, uschar **errorptr, { s = read_local_part(s, t, errorptr, FALSE); if (*errorptr == NULL) - { if (*s != term) - { if (*s != '@') *errorptr = string_sprintf("\"@\" or \".\" expected after \"%s\"", t); else @@ -562,8 +560,6 @@ if (*errorptr == NULL) *domainptr = t; s = read_domain(s, t, errorptr); } - } - } return s; } @@ -744,7 +740,7 @@ if (*s == '<') while (bracket_count-- > 0) if (*s++ != '>') { *errorptr = (s[-1] == 0)? US"'>' missing at end of address" : - string_sprintf("malformed address: %.32s may not follow %.*s", + string_sprintf("malformed address A: %.32s may not follow %.*s", s-1, s - (uschar *)mailbox - 1, mailbox); goto PARSE_FAILED; } @@ -797,7 +793,7 @@ if (*s != 0) } else { - *errorptr = string_sprintf("malformed address: %.32s may not follow %.*s", + *errorptr = string_sprintf("malformed address B: %.32s may not follow %.*s", s, s - (uschar *)mailbox, mailbox); goto PARSE_FAILED; } @@ -817,7 +813,7 @@ if (*end - *start > ADDRESS_MAXLENGTH) return NULL; } -return (uschar *)yield; +return yield; /* Use goto (via the macro FAILED) to get to here from a variety of places. We might have an empty address in a group - the caller can choose to ignore diff --git a/src/src/smtp_in.c b/src/src/smtp_in.c index 8086e9456..a0e44d89a 100644 --- a/src/src/smtp_in.c +++ b/src/src/smtp_in.c @@ -3828,10 +3828,8 @@ while (done <= 0) (char *)mail_args < (char *)env_mail_type_list + sizeof(env_mail_type_list); mail_args++ ) - { if (strcmpic(name, mail_args->name) == 0) break; - } if (mail_args->need_value && strcmpic(value, US"") == 0) break; @@ -3859,16 +3857,17 @@ while (done <= 0) and "7BIT" as body types, but take no action. */ case ENV_MAIL_OPT_BODY: if (accept_8bitmime) { - if (strcmpic(value, US"8BITMIME") == 0) { + if (strcmpic(value, US"8BITMIME") == 0) body_8bitmime = 8; - } else if (strcmpic(value, US"7BIT") == 0) { + else if (strcmpic(value, US"7BIT") == 0) body_8bitmime = 7; - } else { + else + { body_8bitmime = 0; done = synprot_error(L_smtp_syntax_error, 501, NULL, US"invalid data for BODY"); goto COMMAND_LOOP; - } + } DEBUG(D_receive) debug_printf("8BITMIME: %d\n", body_8bitmime); break; } @@ -3880,35 +3879,43 @@ while (done <= 0) is included only if configured in at build time. */ case ENV_MAIL_OPT_RET: - if (dsn_advertised) { + if (dsn_advertised) + { /* Check if RET has already been set */ - if (dsn_ret > 0) { + if (dsn_ret > 0) + { synprot_error(L_smtp_syntax_error, 501, NULL, US"RET can be specified once only"); goto COMMAND_LOOP; - } - dsn_ret = (strcmpic(value, US"HDRS") == 0)? dsn_ret_hdrs : - (strcmpic(value, US"FULL") == 0)? dsn_ret_full : 0; + } + dsn_ret = strcmpic(value, US"HDRS") == 0 + ? dsn_ret_hdrs + : strcmpic(value, US"FULL") == 0 + ? dsn_ret_full + : 0; DEBUG(D_receive) debug_printf("DSN_RET: %d\n", dsn_ret); /* Check for invalid invalid value, and exit with error */ - if (dsn_ret == 0) { + if (dsn_ret == 0) + { synprot_error(L_smtp_syntax_error, 501, NULL, US"Value for RET is invalid"); goto COMMAND_LOOP; - } - } + } + } break; case ENV_MAIL_OPT_ENVID: - if (dsn_advertised) { + if (dsn_advertised) + { /* Check if the dsn envid has been already set */ - if (dsn_envid != NULL) { + if (dsn_envid != NULL) + { synprot_error(L_smtp_syntax_error, 501, NULL, US"ENVID can be specified once only"); goto COMMAND_LOOP; - } + } dsn_envid = string_copy(value); DEBUG(D_receive) debug_printf("DSN_ENVID: %s\n", dsn_envid); - } + } break; /* Handle the AUTH extension. If the value given is not "<>" and either @@ -3948,34 +3955,34 @@ while (done <= 0) switch (rc) { case OK: - if (authenticated_by == NULL || - authenticated_by->mail_auth_condition == NULL || - expand_check_condition(authenticated_by->mail_auth_condition, - authenticated_by->name, US"authenticator")) - break; /* Accept the AUTH */ - - ignore_msg = US"server_mail_auth_condition failed"; - if (authenticated_id != NULL) - ignore_msg = string_sprintf("%s: authenticated ID=\"%s\"", - ignore_msg, authenticated_id); + if (authenticated_by == NULL || + authenticated_by->mail_auth_condition == NULL || + expand_check_condition(authenticated_by->mail_auth_condition, + authenticated_by->name, US"authenticator")) + break; /* Accept the AUTH */ + + ignore_msg = US"server_mail_auth_condition failed"; + if (authenticated_id != NULL) + ignore_msg = string_sprintf("%s: authenticated ID=\"%s\"", + ignore_msg, authenticated_id); /* Fall through */ case FAIL: - authenticated_sender = NULL; - log_write(0, LOG_MAIN, "ignoring AUTH=%s from %s (%s)", - value, host_and_ident(TRUE), ignore_msg); - break; + authenticated_sender = NULL; + log_write(0, LOG_MAIN, "ignoring AUTH=%s from %s (%s)", + value, host_and_ident(TRUE), ignore_msg); + break; /* Should only get DEFER or ERROR here. Put back terminator overrides for error message */ default: - value[-1] = '='; - name[-1] = ' '; - (void)smtp_handle_acl_fail(ACL_WHERE_MAILAUTH, rc, user_msg, - log_msg); - goto COMMAND_LOOP; + value[-1] = '='; + name[-1] = ' '; + (void)smtp_handle_acl_fail(ACL_WHERE_MAILAUTH, rc, user_msg, + log_msg); + goto COMMAND_LOOP; } } break; @@ -3990,7 +3997,7 @@ while (done <= 0) #ifdef EXPERIMENTAL_INTERNATIONAL case ENV_MAIL_OPT_UTF8: if (smtputf8_advertised) - message_smtputf8 = TRUE; + message_smtputf8 = allow_utf8_domains = TRUE; break; #endif /* Unknown option. Stick back the terminator characters and break @@ -4025,9 +4032,10 @@ while (done <= 0) /* Now extract the address, first applying any SMTP-time rewriting. The TRUE flag allows "<>" as a sender address. */ - raw_sender = ((rewrite_existflags & rewrite_smtp) != 0)? - rewrite_one(smtp_cmd_data, rewrite_smtp, NULL, FALSE, US"", - global_rewrite_rules) : smtp_cmd_data; + raw_sender = rewrite_existflags & rewrite_smtp + ? rewrite_one(smtp_cmd_data, rewrite_smtp, NULL, FALSE, US"", + global_rewrite_rules) + : smtp_cmd_data; /* rfc821_domains = TRUE; << no longer needed */ raw_sender = diff --git a/src/src/utf8.c b/src/src/utf8.c index 9a2b8656e..32d2eaed0 100644 --- a/src/src/utf8.c +++ b/src/src/utf8.c @@ -78,9 +78,6 @@ size_t p_len = ucs4_len*4; /* this multiplier is pure guesswork */ uschar * res = store_get(p_len+5); int rc; -DEBUG(D_expand) debug_printf("l_u2a: ulen %d plen %d\n", ucs4_len, p_len); -DEBUG(D_expand) for (rc = 0; rc < ucs4_len; rc++) debug_printf("%08x ", p[rc]); - res[0] = 'x'; res[1] = 'n'; res[2] = res[3] = '-'; if ((rc = punycode_encode(ucs4_len, p, NULL, &p_len, res+4)) != PUNYCODE_SUCCESS) @@ -90,10 +87,7 @@ if ((rc = punycode_encode(ucs4_len, p, NULL, &p_len, res+4)) != PUNYCODE_SUCCESS if (err) *err = US punycode_strerror(rc); return NULL; } -DEBUG(D_expand) debug_printf("l_u2a: plen %d\n", p_len); p_len += 4; -DEBUG(D_expand) for (rc = 0; rc < p_len; rc++) debug_printf("%02x ", res[rc]); -DEBUG(D_expand) debug_printf("\n"); free(p); res[p_len] = '\0'; return res; @@ -114,9 +108,8 @@ if (alabel[0] != 'x' || alabel[1] != 'n' || alabel[2] != '-' || alabel[3] != '-' if (err) *err = US"bad alabel prefix"; return NULL; } -p_len -= 4; -DEBUG(D_expand) debug_printf("l_a2u: plen %d\n", p_len); +p_len -= 4; p = (punycode_uint *) store_get((p_len+1) * sizeof(*p)); if ((rc = punycode_decode(p_len, CCS alabel+4, &p_len, p, NULL)) != PUNYCODE_SUCCESS) @@ -124,7 +117,6 @@ if ((rc = punycode_decode(p_len, CCS alabel+4, &p_len, p, NULL)) != PUNYCODE_SUC if (err) *err = US punycode_strerror(rc); return NULL; } -DEBUG(D_expand) debug_printf("l_a2u: dlen %d\n", p_len); s = stringprep_ucs4_to_utf8(p, p_len, NULL, &p_len); res = string_copyn(s, p_len); diff --git a/test/confs/4201 b/test/confs/4201 new file mode 100644 index 000000000..7d9af4bf3 --- /dev/null +++ b/test/confs/4201 @@ -0,0 +1,60 @@ +# Exim test configuration 4201 + +exim_path = EXIM_PATH +host_lookup_order = bydns +spool_directory = DIR/spool +log_file_path = DIR/spool/log/%slog +gecos_pattern = "" +gecos_name = CALLER_NAME + +# ----- Main settings ----- + +domainlist local_domains = test.ex + +acl_smtp_rcpt = check_recipient +trusted_users = CALLER +log_selector = +received_recipients + +queue_only +queue_run_in_order + +smtputf8_advertise_hosts = * + + +# ----- ACL ----- + +begin acl + +check_recipient: + accept hosts = : + accept domains = +local_domains + deny message = relay not permitted + +# ----- Routers ----- + +begin routers + +fail_remote_domains: + driver = redirect + domains = ! +local_domains + data = :fail: unrouteable mail domain "$domain" + +localuser: + driver = redirect + data = :blackhole: + +# ----- Transports ----- + +begin transports + +local_delivery: + driver = appendfile + delivery_date_add + envelope_to_add + file = DIR/test-mail/$local_part + headers_add = "X-body-linecount: $body_linecount\n\ + X-message-linecount: $message_linecount\n\ + X-received-count: $received_count" + return_path_add + +# End diff --git a/test/log/4201 b/test/log/4201 new file mode 100644 index 000000000..29ce53d48 --- /dev/null +++ b/test/log/4201 @@ -0,0 +1,9 @@ +1999-03-02 09:44:33 exim x.yz daemon started: pid=pppp, no queue runs, listening for SMTP on port 1225 +1999-03-02 09:44:33 10HmaX-0005vi-00 <= someone@some.domain H=(client) [127.0.0.1] P=esmtp S=sss for userx@test.ex +1999-03-02 09:44:33 10HmaY-0005vi-00 <= ليهمابتكلموشعربي؟@czech.Pročprostěnemluvíčesky.com H=(client) [127.0.0.1] P=esmtp S=sss for userx@test.ex +1999-03-02 09:44:33 Start queue run: pid=pppp -qq +1999-03-02 09:44:33 10HmaX-0005vi-00 => :blackhole: R=localuser +1999-03-02 09:44:33 10HmaX-0005vi-00 Completed +1999-03-02 09:44:33 10HmaY-0005vi-00 => :blackhole: R=localuser +1999-03-02 09:44:33 10HmaY-0005vi-00 Completed +1999-03-02 09:44:33 End queue run: pid=pppp -qq diff --git a/test/runtest b/test/runtest index 2baf2cafd..43ae1d42f 100755 --- a/test/runtest +++ b/test/runtest @@ -844,7 +844,6 @@ RESET_AFTER_EXTRA_LINE_READ: next if /^SSL info: unknown state/; next if /^SSL info: SSLv2\/v3 write client hello A/; next if /^SSL info: SSLv3 read server key exchange A/; - } # ======== stderr ======== @@ -1011,6 +1010,9 @@ RESET_AFTER_EXTRA_LINE_READ: next if /in\shosts_require_dane\?\sno\s\(option\sunset\)/x; + # Experimental_International + next if / in smtputf8_advertise_hosts\? no \(option unset\)/; + # Skip some lines that Exim puts out at the start of debugging output # because they will be different in different binaries. @@ -1027,6 +1029,8 @@ RESET_AFTER_EXTRA_LINE_READ: /^Fixed never_users:/ || /^Size of off_t:/ ); + + } next; diff --git a/test/scripts/4200-International/4201 b/test/scripts/4200-International/4201 new file mode 100644 index 000000000..bac040f9b --- /dev/null +++ b/test/scripts/4200-International/4201 @@ -0,0 +1,64 @@ +# Internationalised mail: smtp +# Exim test configuration 4200 +# +exim -DSERVER=server -bd -oX PORT_D +**** +# +# +# Basic smtp input, no delivery +client 127.0.0.1 PORT_D +??? 220 +EHLO client +??? 250- +??? 250-SIZE +??? 250-8BITMIME +??? 250-PIPELINING +??? 250-SMTPUTF8 +??? 250 HELP +MAIL FROM: SMTPUTF8 +??? 250 +RCPT TO: +??? 250 +DATA +??? 354 +Subject: test + +body +. +??? 250 +QUIT +??? 221 +**** +# +# +# utf-8 from, Basic smtp input, no delivery +client 127.0.0.1 PORT_D +??? 220 +EHLO client +??? 250- +??? 250-SIZE +??? 250-8BITMIME +??? 250-PIPELINING +??? 250-SMTPUTF8 +??? 250 HELP +MAIL FROM: <ليهمابتكلموشعربي؟@czech.Pročprostěnemluvíčesky.com> SMTPUTF8 +??? 250 +RCPT TO: +??? 250 +DATA +??? 354 +Subject: test + +body +. +??? 250 +QUIT +??? 221 +**** +# +# +killdaemon +exim -DSERVER=server -qq +**** +no_msglog_check + diff --git a/test/stdout/4201 b/test/stdout/4201 new file mode 100644 index 000000000..b37028d59 --- /dev/null +++ b/test/stdout/4201 @@ -0,0 +1,70 @@ +Connecting to 127.0.0.1 port 1225 ... connected +??? 220 +<<< 220 the.local.host.name ESMTP Exim x.yz Tue, 2 Mar 1999 09:44:33 +0000 +>>> EHLO client +??? 250- +<<< 250-the.local.host.name Hello client [127.0.0.1] +??? 250-SIZE +<<< 250-SIZE 52428800 +??? 250-8BITMIME +<<< 250-8BITMIME +??? 250-PIPELINING +<<< 250-PIPELINING +??? 250-SMTPUTF8 +<<< 250-SMTPUTF8 +??? 250 HELP +<<< 250 HELP +>>> MAIL FROM: SMTPUTF8 +??? 250 +<<< 250 OK +>>> RCPT TO: +??? 250 +<<< 250 Accepted +>>> DATA +??? 354 +<<< 354 Enter message, ending with "." on a line by itself +>>> Subject: test +>>> +>>> body +>>> . +??? 250 +<<< 250 OK id=10HmaX-0005vi-00 +>>> QUIT +??? 221 +<<< 221 the.local.host.name closing connection +End of script +Connecting to 127.0.0.1 port 1225 ... connected +??? 220 +<<< 220 the.local.host.name ESMTP Exim x.yz Tue, 2 Mar 1999 09:44:33 +0000 +>>> EHLO client +??? 250- +<<< 250-the.local.host.name Hello client [127.0.0.1] +??? 250-SIZE +<<< 250-SIZE 52428800 +??? 250-8BITMIME +<<< 250-8BITMIME +??? 250-PIPELINING +<<< 250-PIPELINING +??? 250-SMTPUTF8 +<<< 250-SMTPUTF8 +??? 250 HELP +<<< 250 HELP +>>> MAIL FROM: <ليهمابتكلموشعربي؟@czech.Pročprostěnemluvíčesky.com> SMTPUTF8 +??? 250 +<<< 250 OK +>>> RCPT TO: +??? 250 +<<< 250 Accepted +>>> DATA +??? 354 +<<< 354 Enter message, ending with "." on a line by itself +>>> Subject: test +>>> +>>> body +>>> . +??? 250 +<<< 250 OK id=10HmaY-0005vi-00 +>>> QUIT +??? 221 +<<< 221 the.local.host.name closing connection +End of script -- 2.30.2