From 0d81dabc92972f340421d0f80fc04156215e2eb8 Mon Sep 17 00:00:00 2001 From: Jeremy Harris Date: Thu, 4 May 2017 16:59:46 +0100 Subject: [PATCH] CHUNKING / wire-format spool: use block-copies for receiption --- src/src/functions.h | 3 + src/src/globals.c | 2 + src/src/globals.h | 2 + src/src/receive.c | 50 +++++++------- src/src/smtp_in.c | 109 ++++++++++++++++++++--------- src/src/tls-gnu.c | 156 +++++++++++++++++++++++++----------------- src/src/tls-openssl.c | 150 ++++++++++++++++++++++++---------------- 7 files changed, 296 insertions(+), 176 deletions(-) diff --git a/src/src/functions.h b/src/src/functions.h index 201111623..7bbc23aea 100644 --- a/src/src/functions.h +++ b/src/src/functions.h @@ -56,6 +56,7 @@ extern int tls_feof(void); extern int tls_ferror(void); extern void tls_free_cert(void **); extern int tls_getc(unsigned); +extern uschar *tls_getbuf(unsigned *); extern void tls_get_cache(void); extern int tls_import_cert(const uschar *, void **); extern int tls_read(BOOL, uschar *, size_t); @@ -104,6 +105,7 @@ extern int auth_xtextdecode(uschar *, uschar **); extern uschar *b64encode(uschar *, int); extern int b64decode(uschar *, uschar **); extern int bdat_getc(unsigned); +extern uschar *bdat_getbuf(unsigned *); extern int bdat_ungetc(int); extern void bdat_flush_data(void); @@ -404,6 +406,7 @@ extern BOOL smtp_get_interface(uschar *, int, address_item *, uschar **, uschar *); extern BOOL smtp_get_port(uschar *, address_item *, int *, uschar *); extern int smtp_getc(unsigned); +extern uschar *smtp_getbuf(unsigned *); extern void smtp_get_cache(void); extern int smtp_handle_acl_fail(int, int, uschar *, uschar *); extern void smtp_log_no_mail(void); diff --git a/src/src/globals.c b/src/src/globals.c index f722fab12..46db4f373 100644 --- a/src/src/globals.c +++ b/src/src/globals.c @@ -184,8 +184,10 @@ stand-alone tests. */ #ifndef STAND_ALONE int (*lwr_receive_getc)(unsigned) = stdin_getc; +uschar * (*lwr_receive_getbuf)(unsigned *) = NULL; int (*lwr_receive_ungetc)(int) = stdin_ungetc; int (*receive_getc)(unsigned) = stdin_getc; +uschar * (*receive_getbuf)(unsigned *) = NULL; void (*receive_get_cache)(void)= NULL; int (*receive_ungetc)(int) = stdin_ungetc; int (*receive_feof)(void) = stdin_feof; diff --git a/src/src/globals.h b/src/src/globals.h index e31517bf4..63c9c29c7 100644 --- a/src/src/globals.h +++ b/src/src/globals.h @@ -137,8 +137,10 @@ extern uschar *dsn_advertise_hosts; /* host for which TLS is advertised */ incoming TCP/IP. */ extern int (*lwr_receive_getc)(unsigned); +extern uschar * (*lwr_receive_getbuf)(unsigned *); extern int (*lwr_receive_ungetc)(int); extern int (*receive_getc)(unsigned); +extern uschar * (*receive_getbuf)(unsigned *); extern void (*receive_get_cache)(void); extern int (*receive_ungetc)(int); extern int (*receive_feof)(void); diff --git a/src/src/receive.c b/src/src/receive.c index b03ab71ed..28da47f83 100644 --- a/src/src/receive.c +++ b/src/src/receive.c @@ -1026,32 +1026,34 @@ int ch; DEBUG(D_receive) debug_printf("CHUNKING: writing spoolfile in wire format\n"); spool_file_wireformat = TRUE; -/* Unfortunately cannot use sendfile() even if not TLS -as that requires (on linux) mmap-like operations on the input fd. - -XXX but worthwhile doing a block interface to the bdat_getc buffer -in the future */ - -for (;;) switch (ch = bdat_getc(GETC_BUFFER_UNLIMITED)) +for (;;) { - case EOF: return END_EOF; - case EOD: return END_DOT; - case ERR: return END_PROTOCOL; + if (chunking_data_left > 0) + { + unsigned len = MAX(chunking_data_left, thismessage_size_limit - message_size + 1); + uschar * buf = bdat_getbuf(&len); - default: - message_size++; -/*XXX not done: -linelength -max_received_linelength -body_linecount -body_zerocount -*/ - if (fout) - { - if (fputc(ch, fout) == EOF) return END_WERROR; - if (message_size > thismessage_size_limit) return END_SIZE; - } - break; + message_size += len; + if (fout && fwrite(buf, len, 1, fout) != 1) return END_WERROR; + } + else switch (ch = bdat_getc(GETC_BUFFER_UNLIMITED)) + { + case EOF: return END_EOF; + case EOD: return END_DOT; + case ERR: return END_PROTOCOL; + + default: + message_size++; + /*XXX not done: + linelength + max_received_linelength + body_linecount + body_zerocount + */ + if (fout && fputc(ch, fout) == EOF) return END_WERROR; + break; + } + if (message_size > thismessage_size_limit) return END_SIZE; } /*NOTREACHED*/ } diff --git a/src/src/smtp_in.c b/src/src/smtp_in.c index 3c40a5c61..01c12caf6 100644 --- a/src/src/smtp_in.c +++ b/src/src/smtp_in.c @@ -400,6 +400,44 @@ log_write(L_smtp_incomplete_transaction, LOG_MAIN|LOG_SENDER|LOG_RECIPIENTS, +/* Refill the buffer, and notify DKIM verification code. +Return false for error or EOF. +*/ + +static BOOL +smtp_refill(unsigned lim) +{ +int rc, save_errno; +if (!smtp_out) return FALSE; +fflush(smtp_out); +if (smtp_receive_timeout > 0) alarm(smtp_receive_timeout); + +/* Limit amount read, so non-message data is not fed to DKIM */ + +rc = read(fileno(smtp_in), smtp_inbuffer, MIN(IN_BUFFER_SIZE, lim)); +save_errno = errno; +alarm(0); +if (rc <= 0) + { + /* Must put the error text in fixed store, because this might be during + header reading, where it releases unused store above the header. */ + if (rc < 0) + { + smtp_had_error = save_errno; + smtp_read_error = string_copy_malloc( + string_sprintf(" (error: %s)", strerror(save_errno))); + } + else smtp_had_eof = 1; + return FALSE; + } +#ifndef DISABLE_DKIM +dkim_exim_verify_feed(smtp_inbuffer, rc); +#endif +smtp_inend = smtp_inbuffer + rc; +smtp_inptr = smtp_inbuffer; +return TRUE; +} + /************************************************* * SMTP version of getc() * *************************************************/ @@ -417,39 +455,28 @@ int smtp_getc(unsigned lim) { if (smtp_inptr >= smtp_inend) - { - int rc, save_errno; - if (!smtp_out) return EOF; - fflush(smtp_out); - if (smtp_receive_timeout > 0) alarm(smtp_receive_timeout); - - /* Limit amount read, so non-message data is not fed to DKIM */ - - rc = read(fileno(smtp_in), smtp_inbuffer, MIN(IN_BUFFER_SIZE, lim)); - save_errno = errno; - alarm(0); - if (rc <= 0) - { - /* Must put the error text in fixed store, because this might be during - header reading, where it releases unused store above the header. */ - if (rc < 0) - { - smtp_had_error = save_errno; - smtp_read_error = string_copy_malloc( - string_sprintf(" (error: %s)", strerror(save_errno))); - } - else smtp_had_eof = 1; + if (!smtp_refill(lim)) return EOF; - } -#ifndef DISABLE_DKIM - dkim_exim_verify_feed(smtp_inbuffer, rc); -#endif - smtp_inend = smtp_inbuffer + rc; - smtp_inptr = smtp_inbuffer; - } return *smtp_inptr++; } +uschar * +smtp_getbuf(unsigned * len) +{ +unsigned size; +uschar * buf; + +if (smtp_inptr >= smtp_inend) + if (!smtp_refill(*len)) + { *len = 0; return NULL; } + +if ((size = smtp_inend - smtp_inptr) > *len) size = *len; +buf = smtp_inptr; +smtp_inptr += size; +*len = size; +return buf; +} + void smtp_get_cache(void) { @@ -493,6 +520,7 @@ for(;;) return lwr_receive_getc(chunking_data_left--); receive_getc = lwr_receive_getc; + receive_getbuf = lwr_receive_getbuf; receive_ungetc = lwr_receive_ungetc; #ifndef DISABLE_DKIM dkim_save = dkim_collect_input; @@ -594,6 +622,7 @@ next_cmd: } receive_getc = bdat_getc; + receive_getbuf = bdat_getbuf; receive_ungetc = bdat_ungetc; #ifndef DISABLE_DKIM dkim_collect_input = dkim_save; @@ -604,14 +633,28 @@ next_cmd: } } +uschar * +bdat_getbuf(unsigned * len) +{ +uschar * buf; + +if (chunking_data_left <= 0) + { *len = 0; return NULL; } + +if (*len > chunking_data_left) *len = chunking_data_left; +buf = lwr_receive_getbuf(len); /* Either smtp_getbuf or tls_getbuf */ +chunking_data_left -= *len; +return buf; +} + void bdat_flush_data(void) { -while (chunking_data_left > 0) - if (lwr_receive_getc(chunking_data_left--) < 0) - break; +unsigned n = chunking_data_left; +(void) bdat_getbuf(&n); receive_getc = lwr_receive_getc; +receive_getbuf = lwr_receive_getbuf; receive_ungetc = lwr_receive_ungetc; if (chunking_state != CHUNKING_LAST) @@ -2332,6 +2375,7 @@ if (!(smtp_inbuffer = (uschar *)malloc(IN_BUFFER_SIZE))) log_write(0, LOG_MAIN|LOG_PANIC_DIE, "malloc() failed for SMTP input buffer"); receive_getc = smtp_getc; +receive_getbuf = smtp_getbuf; receive_get_cache = smtp_get_cache; receive_ungetc = smtp_ungetc; receive_feof = smtp_feof; @@ -4975,6 +5019,7 @@ while (done <= 0) (int)chunking_state, chunking_data_left); lwr_receive_getc = receive_getc; + lwr_receive_getbuf = receive_getbuf; lwr_receive_ungetc = receive_ungetc; receive_getc = bdat_getc; receive_ungetc = bdat_ungetc; diff --git a/src/src/tls-gnu.c b/src/src/tls-gnu.c index 8836bb259..d623d8e4a 100644 --- a/src/src/tls-gnu.c +++ b/src/src/tls-gnu.c @@ -1885,6 +1885,7 @@ and initialize appropriately. */ state->xfer_buffer = store_malloc(ssl_xfer_buffer_size); receive_getc = tls_getc; +receive_getbuf = tls_getbuf; receive_get_cache = tls_get_cache; receive_ungetc = tls_ungetc; receive_feof = tls_feof; @@ -2154,6 +2155,75 @@ if ((state_server.session == NULL) && (state_client.session == NULL)) +static BOOL +tls_refill(unsigned lim) +{ +exim_gnutls_state_st * state = &state_server; +ssize_t inbytes; + +DEBUG(D_tls) debug_printf("Calling gnutls_record_recv(%p, %p, %u)\n", + state->session, state->xfer_buffer, ssl_xfer_buffer_size); + +if (smtp_receive_timeout > 0) alarm(smtp_receive_timeout); +inbytes = gnutls_record_recv(state->session, state->xfer_buffer, + MIN(ssl_xfer_buffer_size, lim)); +alarm(0); + +/* Timeouts do not get this far; see command_timeout_handler(). + A zero-byte return appears to mean that the TLS session has been + closed down, not that the socket itself has been closed down. Revert to + non-TLS handling. */ + +if (sigalrm_seen) + { + DEBUG(D_tls) debug_printf("Got tls read timeout\n"); + state->xfer_error = 1; + return FALSE; + } + +else if (inbytes == 0) + { + DEBUG(D_tls) debug_printf("Got TLS_EOF\n"); + + receive_getc = smtp_getc; + receive_getbuf = smtp_getbuf; + receive_get_cache = smtp_get_cache; + receive_ungetc = smtp_ungetc; + receive_feof = smtp_feof; + receive_ferror = smtp_ferror; + receive_smtp_buffered = smtp_buffered; + + gnutls_deinit(state->session); + gnutls_certificate_free_credentials(state->x509_cred); + + state->session = NULL; + state->tlsp->active = -1; + state->tlsp->bits = 0; + state->tlsp->certificate_verified = FALSE; + tls_channelbinding_b64 = NULL; + state->tlsp->cipher = NULL; + state->tlsp->peercert = NULL; + state->tlsp->peerdn = NULL; + + return FALSE; + } + +/* Handle genuine errors */ + +else if (inbytes < 0) + { + record_io_error(state, (int) inbytes, US"recv", NULL); + state->xfer_error = 1; + return FALSE; + } +#ifndef DISABLE_DKIM +dkim_exim_verify_feed(state->xfer_buffer, inbytes); +#endif +state->xfer_buffer_hwm = (int) inbytes; +state->xfer_buffer_lwm = 0; +return TRUE; +} + /************************************************* * TLS version of getc * *************************************************/ @@ -2171,77 +2241,41 @@ Returns: the next character or EOF int tls_getc(unsigned lim) { -exim_gnutls_state_st *state = &state_server; -if (state->xfer_buffer_lwm >= state->xfer_buffer_hwm) - { - ssize_t inbytes; - - DEBUG(D_tls) debug_printf("Calling gnutls_record_recv(%p, %p, %u)\n", - state->session, state->xfer_buffer, ssl_xfer_buffer_size); - - if (smtp_receive_timeout > 0) alarm(smtp_receive_timeout); - inbytes = gnutls_record_recv(state->session, state->xfer_buffer, - MIN(ssl_xfer_buffer_size, lim)); - alarm(0); - - /* Timeouts do not get this far; see command_timeout_handler(). - A zero-byte return appears to mean that the TLS session has been - closed down, not that the socket itself has been closed down. Revert to - non-TLS handling. */ - - if (sigalrm_seen) - { - DEBUG(D_tls) debug_printf("Got tls read timeout\n"); - state->xfer_error = 1; - return EOF; - } - - else if (inbytes == 0) - { - DEBUG(D_tls) debug_printf("Got TLS_EOF\n"); - - receive_getc = smtp_getc; - receive_get_cache = smtp_get_cache; - receive_ungetc = smtp_ungetc; - receive_feof = smtp_feof; - receive_ferror = smtp_ferror; - receive_smtp_buffered = smtp_buffered; +exim_gnutls_state_st * state = &state_server; - gnutls_deinit(state->session); - gnutls_certificate_free_credentials(state->x509_cred); +if (state->xfer_buffer_lwm >= state->xfer_buffer_hwm) + if (!tls_refill(lim)) + return state->xfer_error ? EOF : smtp_getc(lim); - state->session = NULL; - state->tlsp->active = -1; - state->tlsp->bits = 0; - state->tlsp->certificate_verified = FALSE; - tls_channelbinding_b64 = NULL; - state->tlsp->cipher = NULL; - state->tlsp->peercert = NULL; - state->tlsp->peerdn = NULL; +/* Something in the buffer; return next uschar */ - return smtp_getc(lim); - } +return state->xfer_buffer[state->xfer_buffer_lwm++]; +} - /* Handle genuine errors */ +uschar * +tls_getbuf(unsigned * len) +{ +exim_gnutls_state_st * state = &state_server; +unsigned size; +uschar * buf; - else if (inbytes < 0) +if (state->xfer_buffer_lwm >= state->xfer_buffer_hwm) + if (!tls_refill(*len)) { - record_io_error(state, (int) inbytes, US"recv", NULL); - state->xfer_error = 1; - return EOF; + if (!state->xfer_error) return smtp_getbuf(len); + *len = 0; + return NULL; } -#ifndef DISABLE_DKIM - dkim_exim_verify_feed(state->xfer_buffer, inbytes); -#endif - state->xfer_buffer_hwm = (int) inbytes; - state->xfer_buffer_lwm = 0; - } -/* Something in the buffer; return next uschar */ - -return state->xfer_buffer[state->xfer_buffer_lwm++]; +if ((size = state->xfer_buffer_hwm - state->xfer_buffer_lwm) > *len) + size = *len; +buf = &state->xfer_buffer[state->xfer_buffer_lwm]; +state->xfer_buffer_lwm += size; +*len = size; +return buf; } + void tls_get_cache() { diff --git a/src/src/tls-openssl.c b/src/src/tls-openssl.c index ead41c037..f17d94b5f 100644 --- a/src/src/tls-openssl.c +++ b/src/src/tls-openssl.c @@ -2016,6 +2016,7 @@ ssl_xfer_buffer_lwm = ssl_xfer_buffer_hwm = 0; ssl_xfer_eof = ssl_xfer_error = 0; receive_getc = tls_getc; +receive_getbuf = tls_getbuf; receive_get_cache = tls_get_cache; receive_ungetc = tls_ungetc; receive_feof = tls_feof; @@ -2352,6 +2353,74 @@ return OK; +static BOOL +tls_refill(unsigned lim) +{ +int error; +int inbytes; + +DEBUG(D_tls) debug_printf("Calling SSL_read(%p, %p, %u)\n", server_ssl, + ssl_xfer_buffer, ssl_xfer_buffer_size); + +if (smtp_receive_timeout > 0) alarm(smtp_receive_timeout); +inbytes = SSL_read(server_ssl, CS ssl_xfer_buffer, + MIN(ssl_xfer_buffer_size, lim)); +error = SSL_get_error(server_ssl, inbytes); +alarm(0); + +/* SSL_ERROR_ZERO_RETURN appears to mean that the SSL session has been +closed down, not that the socket itself has been closed down. Revert to +non-SSL handling. */ + +if (error == SSL_ERROR_ZERO_RETURN) + { + DEBUG(D_tls) debug_printf("Got SSL_ERROR_ZERO_RETURN\n"); + + receive_getc = smtp_getc; + receive_getbuf = smtp_getbuf; + receive_get_cache = smtp_get_cache; + receive_ungetc = smtp_ungetc; + receive_feof = smtp_feof; + receive_ferror = smtp_ferror; + receive_smtp_buffered = smtp_buffered; + + SSL_free(server_ssl); + server_ssl = NULL; + tls_in.active = -1; + tls_in.bits = 0; + tls_in.cipher = NULL; + tls_in.peerdn = NULL; + tls_in.sni = NULL; + + return FALSE; + } + +/* Handle genuine errors */ + +else if (error == SSL_ERROR_SSL) + { + ERR_error_string(ERR_get_error(), ssl_errstring); + log_write(0, LOG_MAIN, "TLS error (SSL_read): %s", ssl_errstring); + ssl_xfer_error = 1; + return FALSE; + } + +else if (error != SSL_ERROR_NONE) + { + DEBUG(D_tls) debug_printf("Got SSL error %d\n", error); + ssl_xfer_error = 1; + return FALSE; + } + +#ifndef DISABLE_DKIM +dkim_exim_verify_feed(ssl_xfer_buffer, inbytes); +#endif +ssl_xfer_buffer_hwm = inbytes; +ssl_xfer_buffer_lwm = 0; +return TRUE; +} + + /************************************************* * TLS version of getc * *************************************************/ @@ -2369,74 +2438,37 @@ int tls_getc(unsigned lim) { if (ssl_xfer_buffer_lwm >= ssl_xfer_buffer_hwm) - { - int error; - int inbytes; - - DEBUG(D_tls) debug_printf("Calling SSL_read(%p, %p, %u)\n", server_ssl, - ssl_xfer_buffer, ssl_xfer_buffer_size); + if (!tls_refill(lim)) + return ssl_xfer_error ? EOF : smtp_getc(lim); - if (smtp_receive_timeout > 0) alarm(smtp_receive_timeout); - inbytes = SSL_read(server_ssl, CS ssl_xfer_buffer, - MIN(ssl_xfer_buffer_size, lim)); - error = SSL_get_error(server_ssl, inbytes); - alarm(0); - - /* SSL_ERROR_ZERO_RETURN appears to mean that the SSL session has been - closed down, not that the socket itself has been closed down. Revert to - non-SSL handling. */ - - if (error == SSL_ERROR_ZERO_RETURN) - { - DEBUG(D_tls) debug_printf("Got SSL_ERROR_ZERO_RETURN\n"); - - receive_getc = smtp_getc; - receive_get_cache = smtp_get_cache; - receive_ungetc = smtp_ungetc; - receive_feof = smtp_feof; - receive_ferror = smtp_ferror; - receive_smtp_buffered = smtp_buffered; - - SSL_free(server_ssl); - server_ssl = NULL; - tls_in.active = -1; - tls_in.bits = 0; - tls_in.cipher = NULL; - tls_in.peerdn = NULL; - tls_in.sni = NULL; - - return smtp_getc(lim); - } +/* Something in the buffer; return next uschar */ - /* Handle genuine errors */ +return ssl_xfer_buffer[ssl_xfer_buffer_lwm++]; +} - else if (error == SSL_ERROR_SSL) - { - ERR_error_string(ERR_get_error(), ssl_errstring); - log_write(0, LOG_MAIN, "TLS error (SSL_read): %s", ssl_errstring); - ssl_xfer_error = 1; - return EOF; - } +uschar * +tls_getbuf(unsigned * len) +{ +unsigned size; +uschar * buf; - else if (error != SSL_ERROR_NONE) +if (ssl_xfer_buffer_lwm >= ssl_xfer_buffer_hwm) + if (!tls_refill(*len)) { - DEBUG(D_tls) debug_printf("Got SSL error %d\n", error); - ssl_xfer_error = 1; - return EOF; + if (!ssl_xfer_error) return smtp_getbuf(len); + *len = 0; + return NULL; } -#ifndef DISABLE_DKIM - dkim_exim_verify_feed(ssl_xfer_buffer, inbytes); -#endif - ssl_xfer_buffer_hwm = inbytes; - ssl_xfer_buffer_lwm = 0; - } - -/* Something in the buffer; return next uschar */ - -return ssl_xfer_buffer[ssl_xfer_buffer_lwm++]; +if ((size = ssl_xfer_buffer_hwm - ssl_xfer_buffer_lwm) > *len) + size = *len; +buf = &ssl_xfer_buffer[ssl_xfer_buffer_lwm]; +ssl_xfer_buffer_lwm += size; +*len = size; +return buf; } + void tls_get_cache() { -- 2.30.2