commit bd5cdd5f2ddf4e9c26ff9353d15324dd0bbda37c
Author: Jeremy Harris <jgh146exb@wizmail.org>
Date:   Mon Feb 12 17:29:22 2018 +0000

    ACL: Enforce non-usability of control=utf8_downconvert in MAIL ACL.  Bug 2239

diff --git a/src/src/acl.c b/src/src/acl.c
index 750fd2da..95ee2667 100644
--- a/src/src/acl.c
+++ b/src/src/acl.c
@@ -533,7 +533,9 @@ static control_def controls_list[] = {
   },
 #ifdef SUPPORT_I18N
 [CONTROL_UTF8_DOWNCONVERT] =
-  { US"utf8_downconvert",        TRUE, 0 }
+  { US"utf8_downconvert",        TRUE,
+    (unsigned) ~((1<<ACL_WHERE_RCPT) | (1<<ACL_WHERE_VRFY))
+  }
 #endif
 };
 

commit 695a7097d5f737350c6bbb007f9f8a3b7cad5c24
Author: Jeremy Harris <jgh146exb@wizmail.org>
Date:   Sat Feb 17 16:53:27 2018 +0000

    Fix memory leak during multi-message reception using STARTTLS
    
    Reported-by: Wolfgang Breyha

diff --git a/src/src/tls-gnu.c b/src/src/tls-gnu.c
index fc3aba59..58f50510 100644
--- a/src/src/tls-gnu.c
+++ b/src/src/tls-gnu.c
@@ -2202,9 +2202,10 @@ gnutls_certificate_free_credentials(state->x509_cred);
 
 
 state->tlsp->active = -1;
+if (state->xfer_buffer) store_free(state->xfer_buffer);
 memcpy(state, &exim_gnutls_state_init, sizeof(exim_gnutls_state_init));
 
-if ((state_server.session == NULL) && (state_client.session == NULL))
+if (!state_server.session && !state_client.session)
   {
   gnutls_global_deinit();
   exim_gnutls_base_init_done = FALSE;
diff --git a/src/src/tls-openssl.c b/src/src/tls-openssl.c
index b225eb76..03b9023d 100644
--- a/src/src/tls-openssl.c
+++ b/src/src/tls-openssl.c
@@ -2073,7 +2073,7 @@ DEBUG(D_tls)
    smtp_read_response()/ip_recv().
    Hence no need to duplicate for _in and _out.
  */
-ssl_xfer_buffer = store_malloc(ssl_xfer_buffer_size);
+if (!ssl_xfer_buffer) ssl_xfer_buffer = store_malloc(ssl_xfer_buffer_size);
 ssl_xfer_buffer_lwm = ssl_xfer_buffer_hwm = 0;
 ssl_xfer_eof = ssl_xfer_error = 0;
 

commit 2a8bf6ec7eb5ef9760bae9c8b104966d25cfb510
Author: Wolfgang Breyha <wbreyha@gmx.net>
Date:   Mon Feb 19 18:27:55 2018 +0000

    OpenSSL: Fix memory leak during multi-message connections using STARTTLS
    
    Reported-by: Wolfgang Breyha
    Fix-by: Wolfgang Breyha, with additions from Jeremy Harris

diff --git a/src/src/daemon.c b/src/src/daemon.c
index 37fefd4b..f7a27477 100644
--- a/src/src/daemon.c
+++ b/src/src/daemon.c
@@ -653,7 +653,7 @@ if (pid == 0)
         the data structures if necessary. */
 
 #ifdef SUPPORT_TLS
-        tls_close(TRUE, FALSE);
+        tls_close(TRUE, TLS_NO_SHUTDOWN);
 #endif
 
         /* Reset SIGHUP and SIGCHLD in the child in both cases. */
diff --git a/src/src/deliver.c b/src/src/deliver.c
index 9f9990be..2dfa9e38 100644
--- a/src/src/deliver.c
+++ b/src/src/deliver.c
@@ -4990,7 +4990,7 @@ all pipes, so I do not see a reason to use non-blocking IO here
   if (cutthrough.fd >= 0 && cutthrough.callout_hold_only)
     {
 #ifdef SUPPORT_TLS
-    tls_close(FALSE, FALSE);
+    tls_close(FALSE, TLS_NO_SHUTDOWN);
 #endif
     (void) close(cutthrough.fd);
     release_cutthrough_connection(US"passed to transport proc");
diff --git a/src/src/exim.c b/src/src/exim.c
index d71af0ac..b18d3688 100644
--- a/src/src/exim.c
+++ b/src/src/exim.c
@@ -543,7 +543,7 @@ close_unwanted(void)
 if (smtp_input)
   {
   #ifdef SUPPORT_TLS
-  tls_close(TRUE, FALSE);      /* Shut down the TLS library */
+  tls_close(TRUE, TLS_NO_SHUTDOWN);      /* Shut down the TLS library */
   #endif
   (void)close(fileno(smtp_in));
   (void)close(fileno(smtp_out));
diff --git a/src/src/functions.h b/src/src/functions.h
index e50fa6f9..53891162 100644
--- a/src/src/functions.h
+++ b/src/src/functions.h
@@ -50,7 +50,7 @@ extern int     tls_client_start(int, host_item *, address_item *,
 		dns_answer *,
 # endif
 		uschar **);
-extern void    tls_close(BOOL, BOOL);
+extern void    tls_close(BOOL, int);
 extern BOOL    tls_could_read(void);
 extern int     tls_export_cert(uschar *, size_t, void *);
 extern int     tls_feof(void);
diff --git a/src/src/macros.h b/src/src/macros.h
index 764c65b8..5f9c7422 100644
--- a/src/src/macros.h
+++ b/src/src/macros.h
@@ -991,5 +991,10 @@ enum { FILTER_UNSET, FILTER_FORWARD, FILTER_EXIM, FILTER_SIEVE };
 #define UTF8_VERT_2DASH		"\xE2\x95\x8E"
 
 
+/* Options on tls_close */
+#define TLS_NO_SHUTDOWN		0
+#define TLS_SHUTDOWN_NOWAIT	1
+#define TLS_SHUTDOWN_WAIT	2
+
 
 /* End of macros.h */
diff --git a/src/src/smtp_in.c b/src/src/smtp_in.c
index 2603da25..3339f9a1 100644
--- a/src/src/smtp_in.c
+++ b/src/src/smtp_in.c
@@ -3732,7 +3732,7 @@ else
   smtp_printf("221 %s closing connection\r\n", FALSE, smtp_active_hostname);
 
 #ifdef SUPPORT_TLS
-tls_close(TRUE, TRUE);
+tls_close(TRUE, TLS_SHUTDOWN_NOWAIT);
 #endif
 
 log_write(L_smtp_connection, LOG_MAIN, "%s closed by QUIT",
@@ -5417,7 +5417,7 @@ while (done <= 0)
 	smtp_printf("554 Security failure\r\n", FALSE);
 	break;
       }
-    tls_close(TRUE, TRUE);
+    tls_close(TRUE, TLS_SHUTDOWN_NOWAIT);
     break;
     #endif
 
diff --git a/src/src/tls-gnu.c b/src/src/tls-gnu.c
index 58f50510..ffadd96b 100644
--- a/src/src/tls-gnu.c
+++ b/src/src/tls-gnu.c
@@ -2180,12 +2180,15 @@ return OK;
 daemon, to shut down the TLS library, without actually doing a shutdown (which
 would tamper with the TLS session in the parent process).
 
-Arguments:   TRUE if gnutls_bye is to be called
+Arguments:
+  shutdown	1 if TLS close-alert is to be sent,
+ 		2 if also response to be waited for
+
 Returns:     nothing
 */
 
 void
-tls_close(BOOL is_server, BOOL shutdown)
+tls_close(BOOL is_server, int shutdown)
 {
 exim_gnutls_state_st *state = is_server ? &state_server : &state_client;
 
@@ -2193,8 +2196,12 @@ if (!state->tlsp || state->tlsp->active < 0) return;  /* TLS was not active */
 
 if (shutdown)
   {
-  DEBUG(D_tls) debug_printf("tls_close(): shutting down TLS\n");
-  gnutls_bye(state->session, GNUTLS_SHUT_WR);
+  DEBUG(D_tls) debug_printf("tls_close(): shutting down TLS%s\n",
+    shutdown > 1 ? " (with response-wait)" : "");
+
+  alarm(2);
+  gnutls_bye(state->session, shutdown > 1 ? GNUTLS_SHUT_RDWR : GNUTLS_SHUT_WR);
+  alarm(0);
   }
 
 gnutls_deinit(state->session);
diff --git a/src/src/tls-openssl.c b/src/src/tls-openssl.c
index 03b9023d..da7eb8a4 100644
--- a/src/src/tls-openssl.c
+++ b/src/src/tls-openssl.c
@@ -152,6 +152,7 @@ typedef struct tls_ext_ctx_cb {
   uschar *certificate;
   uschar *privatekey;
   BOOL is_server;
+  STACK_OF(X509_NAME) * acceptable_certnames;
 #ifndef DISABLE_OCSP
   STACK_OF(X509) *verify_stack;		/* chain for verifying the proof */
   union {
@@ -1488,6 +1489,7 @@ cbinfo = store_malloc(sizeof(tls_ext_ctx_cb));
 cbinfo->certificate = certificate;
 cbinfo->privatekey = privatekey;
 cbinfo->is_server = host==NULL;
+cbinfo->acceptable_certnames = NULL;
 #ifndef DISABLE_OCSP
 cbinfo->verify_stack = NULL;
 if (!host)
@@ -1732,6 +1734,9 @@ chain_from_pem_file(const uschar * file, STACK_OF(X509) * verify_stack)
 BIO * bp;
 X509 * x;
 
+while (sk_X509_num(verify_stack) > 0)
+  X509_free(sk_X509_pop(verify_stack));
+
 if (!(bp = BIO_new_file(CS file, "r"))) return FALSE;
 while ((x = PEM_read_bio_X509(bp, NULL, 0, NULL)))
   sk_X509_push(verify_stack, x);
@@ -1741,7 +1746,8 @@ return TRUE;
 
 
 
-/* Called by both client and server startup
+/* Called by both client and server startup; on the server possibly
+repeated after a Server Name Indication.
 
 Arguments:
   sctx          SSL_CTX* to initialise
@@ -1791,6 +1797,7 @@ if (expcerts && *expcerts)
 	{ file = NULL; dir = expcerts; }
       else
 	{
+	/*XXX somewhere down here we leak memory per-STARTTLS, on a multi-message conn, server-side */
 	file = expcerts; dir = NULL;
 #ifndef DISABLE_OCSP
 	/* In the server if we will be offering an OCSP proof, load chain from
@@ -1831,11 +1838,21 @@ if (expcerts && *expcerts)
       */
       if (file)
 	{
-	STACK_OF(X509_NAME) * names = SSL_load_client_CA_file(CS file);
+	tls_ext_ctx_cb * cbinfo = host
+	  ? client_static_cbinfo : server_static_cbinfo;
+	STACK_OF(X509_NAME) * names;
 
+	if ((names = cbinfo->acceptable_certnames))
+	  {
+	  sk_X509_NAME_pop_free(names, X509_NAME_free);
+	  cbinfo->acceptable_certnames = NULL;
+	  }
+	names = SSL_load_client_CA_file(CS file);
+
+	SSL_CTX_set_client_CA_list(sctx, names);
 	DEBUG(D_tls) debug_printf("Added %d certificate authorities.\n",
 				    sk_X509_NAME_num(names));
-	SSL_CTX_set_client_CA_list(sctx, names);
+	cbinfo->acceptable_certnames = names;
 	}
       }
     }
@@ -2446,7 +2463,16 @@ if (error == SSL_ERROR_ZERO_RETURN)
   receive_ferror = smtp_ferror;
   receive_smtp_buffered = smtp_buffered;
 
+  if (SSL_get_shutdown(server_ssl) == SSL_RECEIVED_SHUTDOWN)
+ 	SSL_shutdown(server_ssl);
+
+  sk_X509_pop_free(server_static_cbinfo->verify_stack, X509_free);
+  sk_X509_NAME_pop_free(server_static_cbinfo->acceptable_certnames, X509_NAME_free);
   SSL_free(server_ssl);
+  SSL_CTX_free(server_ctx);
+  server_static_cbinfo->verify_stack = NULL;
+  server_static_cbinfo->acceptable_certnames = NULL;
+  server_ctx = NULL;
   server_ssl = NULL;
   tls_in.active = -1;
   tls_in.bits = 0;
@@ -2680,15 +2706,19 @@ return len;
 daemon, to shut down the TLS library, without actually doing a shutdown (which
 would tamper with the SSL session in the parent process).
 
-Arguments:   TRUE if SSL_shutdown is to be called
+Arguments:
+  shutdown	1 if TLS close-alert is to be sent,
+ 		2 if also response to be waited for
+
 Returns:     nothing
 
 Used by both server-side and client-side TLS.
 */
 
 void
-tls_close(BOOL is_server, BOOL shutdown)
+tls_close(BOOL is_server, int shutdown)
 {
+SSL_CTX **ctxp = is_server ? &server_ctx : &client_ctx;
 SSL **sslp = is_server ? &server_ssl : &client_ssl;
 int *fdp = is_server ? &tls_in.active : &tls_out.active;
 
@@ -2696,13 +2726,38 @@ if (*fdp < 0) return;  /* TLS was not active */
 
 if (shutdown)
   {
-  DEBUG(D_tls) debug_printf("tls_close(): shutting down SSL\n");
-  SSL_shutdown(*sslp);
+  int rc;
+  DEBUG(D_tls) debug_printf("tls_close(): shutting down TLS%s\n",
+    shutdown > 1 ? " (with response-wait)" : "");
+
+  if (  (rc = SSL_shutdown(*sslp)) == 0	/* send "close notify" alert */
+     && shutdown > 1)
+    {
+    alarm(2);
+    rc = SSL_shutdown(*sslp);		/* wait for response */
+    alarm(0);
+    }
+
+  if (rc < 0) DEBUG(D_tls)
+    {
+    ERR_error_string(ERR_get_error(), ssl_errstring);
+    debug_printf("SSL_shutdown: %s\n", ssl_errstring);
+    }
+  }
+
+if (is_server)
+  {
+  sk_X509_pop_free(server_static_cbinfo->verify_stack, X509_free);
+  sk_X509_NAME_pop_free(server_static_cbinfo->acceptable_certnames,
+    X509_NAME_free);
+  server_static_cbinfo->verify_stack = NULL;
+  server_static_cbinfo->acceptable_certnames = NULL;
   }
 
+SSL_CTX_free(*ctxp);
 SSL_free(*sslp);
+*ctxp = NULL;
 *sslp = NULL;
-
 *fdp = -1;
 }
 
diff --git a/src/src/transports/smtp.c b/src/src/transports/smtp.c
index 5d351dd7..acfec506 100644
--- a/src/src/transports/smtp.c
+++ b/src/src/transports/smtp.c
@@ -2233,7 +2233,7 @@ if (sx->send_quit)
   (void)smtp_write_command(&sx->outblock, SCMD_FLUSH, "QUIT\r\n");
 
 #ifdef SUPPORT_TLS
-tls_close(FALSE, TRUE);
+tls_close(FALSE, TLS_SHUTDOWN_NOWAIT);
 #endif
 
 /* Close the socket, and return the appropriate value, first setting
@@ -2667,7 +2667,7 @@ for (fd_bits = 3; fd_bits; )
     if ((rc = read(pfd[0], buf, bsize)) <= 0)
       {
       fd_bits = 0;
-      tls_close(FALSE, TRUE);
+      tls_close(FALSE, TLS_SHUTDOWN_NOWAIT);
       }
     else
       {
@@ -3457,7 +3457,7 @@ if (sx.completed_addr && sx.ok && sx.send_quit)
 	  a new EHLO. If we don't get a good response, we don't attempt to pass
 	  the socket on. */
 
-	  tls_close(FALSE, TRUE);
+	  tls_close(FALSE, TLS_SHUTDOWN_WAIT);
 	  smtp_peer_options = smtp_peer_options_wrap;
 	  sx.ok = !sx.smtps
 	    && smtp_write_command(&sx.outblock, SCMD_FLUSH,
@@ -3522,7 +3522,7 @@ propagate it from the initial
 	    close(pfd[0]);
 	    /* tidy the inter-proc to disconn the proxy proc */
 	    waitpid(pid, NULL, 0);
-	    tls_close(FALSE, FALSE);
+	    tls_close(FALSE, TLS_NO_SHUTDOWN);
 	    (void)close(sx.inblock.sock);
 	    continue_transport = NULL;
 	    continue_hostname = NULL;
@@ -3568,7 +3568,7 @@ if (sx.send_quit) (void)smtp_write_command(&sx.outblock, SCMD_FLUSH, "QUIT\r\n")
 END_OFF:
 
 #ifdef SUPPORT_TLS
-tls_close(FALSE, TRUE);
+tls_close(FALSE, TLS_SHUTDOWN_NOWAIT);
 #endif
 
 /* Close the socket, and return the appropriate value, first setting
@@ -4540,7 +4540,7 @@ retry_non_continued:
     if (tls_out.active == fd)
       {
       (void) tls_write(FALSE, US"QUIT\r\n", 6, FALSE);
-      tls_close(FALSE, TRUE);
+      tls_close(FALSE, TLS_SHUTDOWN_NOWAIT);
       }
     else
 #else
diff --git a/src/src/verify.c b/src/src/verify.c
index 5486ca23..544069bb 100644
--- a/src/src/verify.c
+++ b/src/src/verify.c
@@ -821,7 +821,7 @@ tls_retry_connection:
 	      debug_printf_indent("problem after random/rset/mfrom; reopen conn\n");
 	    random_local_part = NULL;
 #ifdef SUPPORT_TLS
-	    tls_close(FALSE, TRUE);
+	    tls_close(FALSE, TLS_SHUTDOWN_NOWAIT);
 #endif
 	    HDEBUG(D_transport|D_acl|D_v) debug_printf_indent("  SMTP(close)>>\n");
 	    (void)close(sx.inblock.sock);
@@ -1088,7 +1088,7 @@ no_conn:
       if (sx.inblock.sock >= 0)
 	{
 #ifdef SUPPORT_TLS
-	tls_close(FALSE, TRUE);
+	tls_close(FALSE, TLS_SHUTDOWN_NOWAIT);
 #endif
 	HDEBUG(D_transport|D_acl|D_v) debug_printf_indent("  SMTP(close)>>\n");
 	(void)close(sx.inblock.sock);
@@ -1389,7 +1389,7 @@ if(fd >= 0)
   cutthrough_response(fd, '2', NULL, 1);
 
 #ifdef SUPPORT_TLS
-  tls_close(FALSE, TRUE);
+  tls_close(FALSE, TLS_SHUTDOWN_NOWAIT);
 #endif
   HDEBUG(D_transport|D_acl|D_v) debug_printf_indent("  SMTP(close)>>\n");
   (void)close(fd);

commit 8b17525025ccd7af1299c9fee2bd43595ab3de09
Author: Jeremy Harris <jgh146exb@wizmail.org>
Date:   Thu Feb 22 11:26:34 2018 +0000

    Fix exim_dbmbuild to permit directoryless filenames.  Bug 2242
    
    Broken-by: 0a6c178c6c

diff --git a/src/src/exim_dbmbuild.c b/src/src/exim_dbmbuild.c
index 2dcc40f3..fb5317b8 100644
--- a/src/src/exim_dbmbuild.c
+++ b/src/src/exim_dbmbuild.c
@@ -210,6 +210,8 @@ Ustrcat(temp_dbmname, ".dbmbuild_temp");
 Ustrcpy(dirname, temp_dbmname);
 if ((bptr = Ustrrchr(dirname, '/')))
   *bptr = '\0';
+else
+  Ustrcpy(dirname, ".");
 
 /* It is apparently necessary to open with O_RDWR for this to work
 with gdbm-1.7.3, though no reading is actually going to be done. */

commit 4cdbbcada0318fdc8db463fbf203a4afa1914d92
Author: Jeremy Harris <jgh146exb@wizmail.org>
Date:   Thu Feb 22 23:52:17 2018 +0000

    OpenSSL: revert needless free of certificate list.  The library handles it internally.
    
    Reported-by: Torsten Tributh

diff --git a/src/src/tls-openssl.c b/src/src/tls-openssl.c
index da7eb8a4..addda713 100644
--- a/src/src/tls-openssl.c
+++ b/src/src/tls-openssl.c
@@ -152,7 +152,6 @@ typedef struct tls_ext_ctx_cb {
   uschar *certificate;
   uschar *privatekey;
   BOOL is_server;
-  STACK_OF(X509_NAME) * acceptable_certnames;
 #ifndef DISABLE_OCSP
   STACK_OF(X509) *verify_stack;		/* chain for verifying the proof */
   union {
@@ -1489,7 +1488,6 @@ cbinfo = store_malloc(sizeof(tls_ext_ctx_cb));
 cbinfo->certificate = certificate;
 cbinfo->privatekey = privatekey;
 cbinfo->is_server = host==NULL;
-cbinfo->acceptable_certnames = NULL;
 #ifndef DISABLE_OCSP
 cbinfo->verify_stack = NULL;
 if (!host)
@@ -1840,19 +1838,11 @@ if (expcerts && *expcerts)
 	{
 	tls_ext_ctx_cb * cbinfo = host
 	  ? client_static_cbinfo : server_static_cbinfo;
-	STACK_OF(X509_NAME) * names;
-
-	if ((names = cbinfo->acceptable_certnames))
-	  {
-	  sk_X509_NAME_pop_free(names, X509_NAME_free);
-	  cbinfo->acceptable_certnames = NULL;
-	  }
-	names = SSL_load_client_CA_file(CS file);
+	STACK_OF(X509_NAME) * names = SSL_load_client_CA_file(CS file);
 
 	SSL_CTX_set_client_CA_list(sctx, names);
 	DEBUG(D_tls) debug_printf("Added %d certificate authorities.\n",
 				    sk_X509_NAME_num(names));
-	cbinfo->acceptable_certnames = names;
 	}
       }
     }
@@ -2467,11 +2457,9 @@ if (error == SSL_ERROR_ZERO_RETURN)
  	SSL_shutdown(server_ssl);
 
   sk_X509_pop_free(server_static_cbinfo->verify_stack, X509_free);
-  sk_X509_NAME_pop_free(server_static_cbinfo->acceptable_certnames, X509_NAME_free);
   SSL_free(server_ssl);
   SSL_CTX_free(server_ctx);
   server_static_cbinfo->verify_stack = NULL;
-  server_static_cbinfo->acceptable_certnames = NULL;
   server_ctx = NULL;
   server_ssl = NULL;
   tls_in.active = -1;
@@ -2748,10 +2736,7 @@ if (shutdown)
 if (is_server)
   {
   sk_X509_pop_free(server_static_cbinfo->verify_stack, X509_free);
-  sk_X509_NAME_pop_free(server_static_cbinfo->acceptable_certnames,
-    X509_NAME_free);
   server_static_cbinfo->verify_stack = NULL;
-  server_static_cbinfo->acceptable_certnames = NULL;
   }
 
 SSL_CTX_free(*ctxp);

commit ae06ddc47aa43bb5e2dcbe2643e41649d1947a9d
Author: Jeremy Harris <jgh146exb@wizmail.org>
Date:   Sat Feb 24 20:04:25 2018 +0000

    I18N: Fix utf8_downconvert propagation through a redirect router

diff --git a/src/src/routers/redirect.c b/src/src/routers/redirect.c
index c823f023..0f9839a3 100644
--- a/src/src/routers/redirect.c
+++ b/src/src/routers/redirect.c
@@ -465,8 +465,9 @@ while (generated)
     }
 
 #ifdef SUPPORT_I18N
-    next->prop.utf8_msg = string_is_utf8(next->address)
-      || (sender_address && string_is_utf8(sender_address));
+    if (!next->prop.utf8_msg)
+      next->prop.utf8_msg = string_is_utf8(next->address)
+        || (sender_address && string_is_utf8(sender_address));
 #endif
 
   DEBUG(D_route)
@@ -567,9 +568,9 @@ addr_prop.remove_headers = NULL;
 addr_prop.srs_sender = NULL;
 #endif
 #ifdef SUPPORT_I18N
-addr_prop.utf8_msg = FALSE;	/*XXX should we not copy this from the parent? */
-addr_prop.utf8_downcvt = FALSE;
-addr_prop.utf8_downcvt_maybe = FALSE;
+addr_prop.utf8_msg = addr->prop.utf8_msg;
+addr_prop.utf8_downcvt = addr->prop.utf8_downcvt;
+addr_prop.utf8_downcvt_maybe = addr->prop.utf8_downcvt_maybe;
 #endif
 
 

commit d4ddcd11d9158cef132253f33b76f28a0bdb3bba
Author: Jeremy Harris <jgh146exb@wizmail.org>
Date:   Sun Mar 11 19:09:19 2018 +0000

    Fix ldap lookups for zero-length attribute value.  Bug 2251
    Broken-by: acec9514b1

diff --git a/src/src/lookups/ldap.c b/src/src/lookups/ldap.c
index 235af0f9..06db734c 100644
--- a/src/src/lookups/ldap.c
+++ b/src/src/lookups/ldap.c
@@ -884,13 +884,13 @@ while ((rc = ldap_result(lcp->ld, msgid, 0, timeoutptr, &result)) ==
   result = NULL;
   }            /* End "while" loop for multiple results */
 
-/* Terminate the dynamic string that we have built and reclaim unused store */
+/* Terminate the dynamic string that we have built and reclaim unused store.
+In the odd case of a single attribute with zero-length value, allocate
+an empty string. */
 
-if (data)
-  {
-  (void) string_from_gstring(data);
-  store_reset(data->s + data->ptr + 1);
-  }
+if (!data) data = string_get(1);
+(void) string_from_gstring(data);
+store_reset(data->s + data->ptr + 1);
 
 /* Copy the last dn into eldap_dn */
 

commit fc35a505f8b412be5cf09bf587c237f3316a2bfe
Author: Jeremy Harris <jgh146exb@wizmail.org>
Date:   Tue Mar 13 13:52:26 2018 +0000

    Mark variables unused before release of store in the daemon loop

diff --git a/src/src/daemon.c b/src/src/daemon.c
index f7a27477..259c21fe 100644
--- a/src/src/daemon.c
+++ b/src/src/daemon.c
@@ -563,18 +563,13 @@ if (pid == 0)
 
     /* Reclaim up the store used in accepting this message */
 
-    return_path = sender_address = NULL;
-    authenticated_sender = NULL;
-    sending_ip_address = NULL;
-    deliver_host_address = deliver_host =
-    deliver_domain_orig = deliver_localpart_orig = NULL;
-    dnslist_domain = dnslist_matched = NULL;
-    callout_address = NULL;
-#ifndef DISABLE_DKIM
-    dkim_cur_signer = NULL;
-#endif
-    acl_var_m = NULL;
-    store_reset(reset_point);
+      {
+      int r = receive_messagecount;
+      BOOL q = queue_only_policy;
+      smtp_reset(reset_point);
+      queue_only_policy = q;
+      receive_messagecount = r;
+      }
 
     /* If queue_only is set or if there are too many incoming connections in
     existence, session_local_queue_only will be TRUE. If it is not, check
diff --git a/src/src/functions.h b/src/src/functions.h
index 53891162..ccd3d581 100644
--- a/src/src/functions.h
+++ b/src/src/functions.h
@@ -416,6 +416,7 @@ extern void    smtp_log_no_mail(void);
 extern void    smtp_message_code(uschar **, int *, uschar **, uschar **, BOOL);
 extern void    smtp_proxy_tls(uschar *, size_t, int *, int);
 extern BOOL    smtp_read_response(smtp_inblock *, uschar *, int, int, int);
+extern void    smtp_reset(void *);
 extern void    smtp_respond(uschar *, int, BOOL, uschar *);
 extern void    smtp_notquit_exit(uschar *, uschar *, uschar *, ...);
 extern void    smtp_port_for_connect(host_item *, int);
diff --git a/src/src/smtp_in.c b/src/src/smtp_in.c
index 3339f9a1..8bbcafb1 100644
--- a/src/src/smtp_in.c
+++ b/src/src/smtp_in.c
@@ -1954,13 +1954,13 @@ return TRUE;
 *************************************************/
 
 /* This function is called whenever the SMTP session is reset from
-within either of the setup functions.
+within either of the setup functions; also from the daemon loop.
 
 Argument:   the stacking pool storage reset point
 Returns:    nothing
 */
 
-static void
+void
 smtp_reset(void *reset_point)
 {
 recipients_list = NULL;
@@ -2004,9 +2004,8 @@ bmi_verdicts = NULL;
 #endif
 dnslist_domain = dnslist_matched = NULL;
 #ifndef DISABLE_DKIM
-dkim_signers = NULL;
-dkim_disable_verify = FALSE;
-dkim_collect_input = FALSE;
+dkim_cur_signer = dkim_signers = NULL;
+dkim_disable_verify = dkim_collect_input = FALSE;
 #endif
 dsn_ret = 0;
 dsn_envid = NULL;
@@ -2015,10 +2014,7 @@ deliver_host = deliver_host_address = NULL;	/* Can be set by ACL */
 prdr_requested = FALSE;
 #endif
 #ifdef EXPERIMENTAL_SPF
-spf_header_comment = NULL;
-spf_received = NULL;
-spf_result = NULL;
-spf_smtp_comment = NULL;
+spf_header_comment = spf_received = spf_result = spf_smtp_comment = NULL;
 #endif
 #ifdef SUPPORT_I18N
 message_smtputf8 = FALSE;

commit 1f3a53b0af67aac909b2ba6ce1244a8a64d9a718
Author: Jeremy Harris <jgh146exb@wizmail.org>
Date:   Tue Mar 13 16:27:54 2018 +0000

    Mark variables unused before release of store in the queue-runner loop

diff --git a/src/src/functions.h b/src/src/functions.h
index ccd3d581..dfe8ff57 100644
--- a/src/src/functions.h
+++ b/src/src/functions.h
@@ -430,13 +430,14 @@ extern int     smtp_write_command(smtp_outblock *, int, const char *, ...) PRINT
 extern int     spam(const uschar **);
 extern FILE   *spool_mbox(unsigned long *, const uschar *, uschar **);
 #endif
-extern BOOL    spool_move_message(uschar *, uschar *, uschar *, uschar *);
+extern void    spool_clear_header_globals(void);
 extern uschar *spool_dname(const uschar *, uschar *);
 extern uschar *spool_fname(const uschar *, const uschar *, const uschar *, const uschar *);
-extern uschar *spool_sname(const uschar *, uschar *);
+extern BOOL    spool_move_message(uschar *, uschar *, uschar *, uschar *);
 extern int     spool_open_datafile(uschar *);
 extern int     spool_open_temp(uschar *);
 extern int     spool_read_header(uschar *, BOOL, BOOL);
+extern uschar *spool_sname(const uschar *, uschar *);
 extern int     spool_write_header(uschar *, int, uschar **);
 extern int     stdin_getc(unsigned);
 extern int     stdin_feof(void);
diff --git a/src/src/queue.c b/src/src/queue.c
index 09a149e9..e613bfac 100644
--- a/src/src/queue.c
+++ b/src/src/queue.c
@@ -601,9 +601,7 @@ for (i = queue_run_in_order ? -1 : 0;
 
       /* Recover store used when reading the header */
 
-      received_protocol = NULL;
-      sender_address = sender_ident = NULL;
-      authenticated_id = authenticated_sender = NULL;
+      spool_clear_header_globals();
       store_reset(reset_point2);
       if (!wanted) continue;      /* With next message */
       }
diff --git a/src/src/spool_in.c b/src/src/spool_in.c
index 2a99c63d..d4db37de 100644
--- a/src/src/spool_in.c
+++ b/src/src/spool_in.c
@@ -206,49 +206,13 @@ return TRUE;
 
 
 
-/*************************************************
-*             Read spool header file             *
-*************************************************/
-
-/* This function reads a spool header file and places the data into the
-appropriate global variables. The header portion is always read, but header
-structures are built only if read_headers is set true. It isn't, for example,
-while generating -bp output.
-
-It may be possible for blocks of nulls (binary zeroes) to get written on the
-end of a file if there is a system crash during writing. It was observed on an
-earlier version of Exim that omitted to fsync() the files - this is thought to
-have been the cause of that incident, but in any case, this code must be robust
-against such an event, and if such a file is encountered, it must be treated as
-malformed.
-
-As called from deliver_message() (at least) we are running as root.
-
-Arguments:
-  name          name of the header file, including the -H
-  read_headers  TRUE if in-store header structures are to be built
-  subdir_set    TRUE is message_subdir is already set
-
-Returns:        spool_read_OK        success
-                spool_read_notopen   open failed
-                spool_read_enverror  error in the envelope portion
-                spool_read_hdrerror  error in the header portion
-*/
-
-int
-spool_read_header(uschar *name, BOOL read_headers, BOOL subdir_set)
-{
-FILE *f = NULL;
-int n;
-int rcount = 0;
-long int uid, gid;
-BOOL inheader = FALSE;
-uschar *p;
-
 /* Reset all the global variables to their default values. However, there is
 one exception. DO NOT change the default value of dont_deliver, because it may
 be forced by an external setting. */
 
+void
+spool_clear_header_globals(void)
+{
 acl_var_c = acl_var_m = NULL;
 authenticated_id = NULL;
 authenticated_sender = NULL;
@@ -328,6 +292,53 @@ message_utf8_downconvert = 0;
 
 dsn_ret = 0;
 dsn_envid = NULL;
+}
+
+
+/*************************************************
+*             Read spool header file             *
+*************************************************/
+
+/* This function reads a spool header file and places the data into the
+appropriate global variables. The header portion is always read, but header
+structures are built only if read_headers is set true. It isn't, for example,
+while generating -bp output.
+
+It may be possible for blocks of nulls (binary zeroes) to get written on the
+end of a file if there is a system crash during writing. It was observed on an
+earlier version of Exim that omitted to fsync() the files - this is thought to
+have been the cause of that incident, but in any case, this code must be robust
+against such an event, and if such a file is encountered, it must be treated as
+malformed.
+
+As called from deliver_message() (at least) we are running as root.
+
+Arguments:
+  name          name of the header file, including the -H
+  read_headers  TRUE if in-store header structures are to be built
+  subdir_set    TRUE is message_subdir is already set
+
+Returns:        spool_read_OK        success
+                spool_read_notopen   open failed
+                spool_read_enverror  error in the envelope portion
+                spool_read_hdrerror  error in the header portion
+*/
+
+int
+spool_read_header(uschar *name, BOOL read_headers, BOOL subdir_set)
+{
+FILE *f = NULL;
+int n;
+int rcount = 0;
+long int uid, gid;
+BOOL inheader = FALSE;
+uschar *p;
+
+/* Reset all the global variables to their default values. However, there is
+one exception. DO NOT change the default value of dont_deliver, because it may
+be forced by an external setting. */
+
+spool_clear_header_globals();
 
 /* Generate the full name and open the file. If message_subdir is already
 set, just look in the given directory. Otherwise, look in both the split

commit b9aa0a847979a8b7e917c25a734c4e176c52be59
Author: Jeremy Harris <jgh146exb@wizmail.org>
Date:   Thu Mar 15 14:23:04 2018 +0000

    Mark variables that are unused before release of store in the receive message loop

diff --git a/src/src/acl.c b/src/src/acl.c
index 95ee2667..23c28c73 100644
--- a/src/src/acl.c
+++ b/src/src/acl.c
@@ -4475,7 +4475,7 @@ switch (where)
   }
 
 deliver_domain = deliver_localpart = deliver_address_data =
-  sender_address_data = NULL;
+  deliver_domain_data = sender_address_data = NULL;
 
 /* A DISCARD response is permitted only for message ACLs, excluding the PREDATA
 ACL, which is really in the middle of an SMTP command. */
diff --git a/src/src/smtp_in.c b/src/src/smtp_in.c
index 8bbcafb1..a96b4e96 100644
--- a/src/src/smtp_in.c
+++ b/src/src/smtp_in.c
@@ -1987,8 +1987,8 @@ active_local_sender_retain = local_sender_retain;    /* Can be set by ACL */
 sending_ip_address = NULL;
 return_path = sender_address = NULL;
 sender_data = NULL;				     /* Can be set by ACL */
-deliver_localpart_orig = NULL;
-deliver_domain_orig = NULL;
+deliver_localpart_parent = deliver_localpart_orig = NULL;
+deliver_domain_parent = deliver_domain_orig = NULL;
 callout_address = NULL;
 submission_name = NULL;                              /* Can be set by ACL */
 raw_sender = NULL;                  /* After SMTP rewrite, before qualifying */

commit aea78c24dfc83958336dc3cb3418cea5ca8d221b
Author: Jeremy Harris <jgh146exb@wizmail.org>
Date:   Sat Mar 17 15:19:08 2018 +0000

    DMARC: add variables to list of those now-unused at the tail of the SMTP per-message loop

diff --git a/src/src/smtp_in.c b/src/src/smtp_in.c
index a96b4e96..527d3706 100644
--- a/src/src/smtp_in.c
+++ b/src/src/smtp_in.c
@@ -2016,6 +2016,12 @@ prdr_requested = FALSE;
 #ifdef EXPERIMENTAL_SPF
 spf_header_comment = spf_received = spf_result = spf_smtp_comment = NULL;
 #endif
+#ifdef EXPERIMENTAL_DMARC
+dmarc_has_been_checked = dmarc_disable_verify = dmarc_enable_forensic = FALSE;
+dmarc_ar_header = dmarc_domain_policy = dmarc_forensic_sender =
+dmarc_history_file = dmarc_status = dmarc_status_text =
+dmarc_tld_file = dmarc_used_domain = NULL;
+#endif
 #ifdef SUPPORT_I18N
 message_smtputf8 = FALSE;
 #endif

commit e06f773b7024ad7025fbb3dab2a7c073746d431f
Author: Jeremy Harris <jgh146exb@wizmail.org>
Date:   Sat Mar 17 23:55:45 2018 +0000

    Fix heavy-pipeline SMTP command input corruption.  Bug 2250

diff --git a/src/src/smtp_in.c b/src/src/smtp_in.c
index 527d3706..88bd9391 100644
--- a/src/src/smtp_in.c
+++ b/src/src/smtp_in.c
@@ -359,9 +359,6 @@ rc = smtp_getc(GETC_BUFFER_UNLIMITED);
 if (rc < 0) return TRUE;      /* End of file or error */
 
 smtp_ungetc(rc);
-rc = smtp_inend - smtp_inptr;
-if (rc > 150) rc = 150;
-smtp_inptr[rc] = 0;
 return FALSE;
 }
 
@@ -439,9 +436,10 @@ if (!smtp_out) return FALSE;
 fflush(smtp_out);
 if (smtp_receive_timeout > 0) alarm(smtp_receive_timeout);
 
-/* Limit amount read, so non-message data is not fed to DKIM */
+/* Limit amount read, so non-message data is not fed to DKIM.
+Take care to not touch the safety NUL at the end of the buffer. */
 
-rc = read(fileno(smtp_in), smtp_inbuffer, MIN(IN_BUFFER_SIZE, lim));
+rc = read(fileno(smtp_in), smtp_inbuffer, MIN(IN_BUFFER_SIZE-1, lim));
 save_errno = errno;
 alarm(0);
 if (rc <= 0)
@@ -2429,10 +2427,12 @@ else
     (sender_host_address ? protocols : protocols_local) [pnormal];
 
 /* Set up the buffer for inputting using direct read() calls, and arrange to
-call the local functions instead of the standard C ones. */
+call the local functions instead of the standard C ones.  Place a NUL at the
+end of the buffer to safety-stop C-string reads from it. */
 
 if (!(smtp_inbuffer = US malloc(IN_BUFFER_SIZE)))
   log_write(0, LOG_MAIN|LOG_PANIC_DIE, "malloc() failed for SMTP input buffer");
+smtp_inbuffer[IN_BUFFER_SIZE-1] = '\0';
 
 receive_getc = smtp_getc;
 receive_getbuf = smtp_getbuf;
@@ -5684,7 +5684,7 @@ while (done <= 0)
     if (smtp_inend >= smtp_inbuffer + IN_BUFFER_SIZE)
       smtp_inend = smtp_inbuffer + IN_BUFFER_SIZE - 1;
     c = smtp_inend - smtp_inptr;
-    if (c > 150) c = 150;
+    if (c > 150) c = 150;	/* limit logged amount */
     smtp_inptr[c] = 0;
     incomplete_transaction_log(US"sync failure");
     log_write(0, LOG_MAIN|LOG_REJECT, "SMTP protocol synchronization error "

commit e863ac56f091041d8bf86acbb1ebb682440712ee
Author: Jeremy Harris <jgh146exb@wizmail.org>
Date:   Sun Mar 18 18:48:13 2018 +0000

    Unbreak DMARC
    
    Broken-by: aea78c24df

diff --git a/src/src/smtp_in.c b/src/src/smtp_in.c
index 88bd9391..d866886d 100644
--- a/src/src/smtp_in.c
+++ b/src/src/smtp_in.c
@@ -2018,7 +2018,7 @@ spf_header_comment = spf_received = spf_result = spf_smtp_comment = NULL;
 dmarc_has_been_checked = dmarc_disable_verify = dmarc_enable_forensic = FALSE;
 dmarc_ar_header = dmarc_domain_policy = dmarc_forensic_sender =
 dmarc_history_file = dmarc_status = dmarc_status_text =
-dmarc_tld_file = dmarc_used_domain = NULL;
+dmarc_used_domain = NULL;
 #endif
 #ifdef SUPPORT_I18N
 message_smtputf8 = FALSE;

commit 88fc7b4382a3e98948e085dd7581dd80b801bca4
Author: Jeremy Harris <jgh146exb@wizmail.org>
Date:   Tue Mar 20 17:54:47 2018 +0000

    Fix pipe transport to not use a socket-only syscall.  Bug 2257
    
    Broken-by: 42055a3385

diff --git a/src/src/macros.h b/src/src/macros.h
index 5f9c7422..242ef37f 100644
--- a/src/src/macros.h
+++ b/src/src/macros.h
@@ -853,6 +853,7 @@ enum {
 #define topt_use_bdat		0x100  /* prepend chunks with RFC3030 BDAT header */
 #define topt_output_string	0x200  /* create string rather than write to fd */
 #define topt_continuation	0x400  /* do not reset buffer */
+#define topt_not_socket		0x800  /* cannot do socket-only syscalls */
 
 /* Options for smtp_write_command */
 
diff --git a/src/src/transport.c b/src/src/transport.c
index 47da45fd..74103ef4 100644
--- a/src/src/transport.c
+++ b/src/src/transport.c
@@ -245,7 +245,8 @@ for (i = 0; i < 100; i++)
 	tls_out.active == fd ? tls_write(FALSE, block, len, more) :
 #endif
 #ifdef MSG_MORE
-	more ? send(fd, block, len, MSG_MORE) :
+	more && !(tctx->options & topt_not_socket)
+	  ? send(fd, block, len, MSG_MORE) :
 #endif
 	write(fd, block, len);
     save_errno = errno;
diff --git a/src/src/transports/appendfile.c b/src/src/transports/appendfile.c
index ece1da51..6350445c 100644
--- a/src/src/transports/appendfile.c
+++ b/src/src/transports/appendfile.c
@@ -945,9 +945,7 @@ copy_mbx_message(int to_fd, int from_fd, off_t saved_size)
 int used;
 off_t size;
 struct stat statbuf;
-transport_ctx tctx = {{0}};
-
-tctx.u.fd = to_fd;
+transport_ctx tctx = { .u={.fd = to_fd}, .options = topt_not_socket };
 
 /* If the current mailbox size is zero, write a header block */
 
@@ -2921,12 +2919,12 @@ at initialization time. */
 if (yield == OK)
   {
   transport_ctx tctx = {
-    {fd},
-    tblock,
-    addr,
-    ob->check_string,
-    ob->escape_string,
-    ob->options
+    .u = {.fd=fd},
+    .tblock = tblock,
+    .addr = addr,
+    .check_string = ob->check_string,
+    .escape_string = ob->escape_string,
+    .options =  ob->options | topt_not_socket
   };
   if (!transport_write_message(&tctx, 0))
     yield = DEFER;
diff --git a/src/src/transports/autoreply.c b/src/src/transports/autoreply.c
index bf31951f..2ac06caa 100644
--- a/src/src/transports/autoreply.c
+++ b/src/src/transports/autoreply.c
@@ -694,15 +694,17 @@ if (return_message)
     :
     US"------ This is a copy of the message, including all the headers.\n";
   transport_ctx tctx = {
-    {fileno(f)},
-    tblock,
-    addr,
-    NULL, NULL,
-    (tblock->body_only ? topt_no_headers : 0) |
-    (tblock->headers_only ? topt_no_body : 0) |
-    (tblock->return_path_add ? topt_add_return_path : 0) |
-    (tblock->delivery_date_add ? topt_add_delivery_date : 0) |
-    (tblock->envelope_to_add ? topt_add_envelope_to : 0)
+    .u = {.fd = fileno(f)},
+    .tblock = tblock,
+    .addr = addr,
+    .check_string = NULL,
+    .escape_string =  NULL,
+    .options = (tblock->body_only ? topt_no_headers : 0)
+    	| (tblock->headers_only ? topt_no_body : 0)
+    	| (tblock->return_path_add ? topt_add_return_path : 0)
+    	| (tblock->delivery_date_add ? topt_add_delivery_date : 0)
+    	| (tblock->envelope_to_add ? topt_add_envelope_to : 0)
+	| topt_not_socket
   };
 
   if (bounce_return_size_limit > 0 && !tblock->headers_only)
diff --git a/src/src/transports/pipe.c b/src/src/transports/pipe.c
index 0361cc81..81327c6d 100644
--- a/src/src/transports/pipe.c
+++ b/src/src/transports/pipe.c
@@ -566,12 +566,11 @@ const uschar *envlist = ob->environment;
 uschar *cmd, *ss;
 uschar *eol = ob->use_crlf ? US"\r\n" : US"\n";
 transport_ctx tctx = {
-  {0},
-  tblock,
-  addr,
-  ob->check_string,
-  ob->escape_string,
-  ob->options /* set at initialization time */
+  .tblock = tblock,
+  .addr = addr,
+  .check_string = ob->check_string,
+  .escape_string = ob->escape_string,
+  ob->options | topt_not_socket /* set at initialization time */
 };
 
 DEBUG(D_transport) debug_printf("%s transport entered\n", tblock->name);

commit f81d6431fae4191b8851fba9345c4b0ed22d5650
Author: Jeremy Harris <jgh146exb@wizmail.org>
Date:   Wed Mar 21 11:34:22 2018 +0000

    Pipe transport, part two.  Bug 2257

diff --git a/src/src/transport.c b/src/src/transport.c
index 74103ef4..d7670e0c 100644
--- a/src/src/transport.c
+++ b/src/src/transport.c
@@ -263,7 +263,8 @@ for (i = 0; i < 100; i++)
 	tls_out.active == fd ? tls_write(FALSE, block, len, more) :
 #endif
 #ifdef MSG_MORE
-	more ? send(fd, block, len, MSG_MORE) :
+	more && !(tctx->options & topt_not_socket)
+	  ? send(fd, block, len, MSG_MORE) :
 #endif
 	write(fd, block, len);
 

commit e85aad604e8dba48dd9f2dbe9644ca8a46f3137c
Author: Jeremy Harris <jgh146exb@wizmail.org>
Date:   Fri Mar 23 12:18:53 2018 +0000

    Fix spool_wireformat final-dot on LMTP transport.  Bug 2258
    
    Broken-by: 328c5688db

diff --git a/src/src/transport.c b/src/src/transport.c
index d7670e0c..d35d25f6 100644
--- a/src/src/transport.c
+++ b/src/src/transport.c
@@ -1132,9 +1132,10 @@ if (!(tctx->options & topt_no_body))
   if (len != 0) return FALSE;
   }
 
-/* Finished with the check string */
+/* Finished with the check string, and spool-format consideration */
 
 nl_check_length = nl_escape_length = 0;
+spool_file_wireformat = FALSE;
 
 /* If requested, add a terminating "." line (SMTP output). */
 
@@ -1401,6 +1402,7 @@ filter was not NL, insert a NL to make the SMTP protocol work. */
 if (yield)
   {
   nl_check_length = nl_escape_length = 0;
+  spool_file_wireformat = FALSE;
   if (  tctx->options & topt_end_dot
      && ( last_filter_was_NL
         ? !write_chunk(tctx, US".\n", 2)
