Appendfile: release regex-match store every thousand files. Bug 3047

diff --git a/doc/doc-txt/ChangeLog b/doc/doc-txt/ChangeLog
index 14cd45d85d9af943e64f13f1de8442028593ac1d..e258966d8497c6470d0e7b521244d6a10b0b6c74 100644 (file)
--- a/doc/doc-txt/ChangeLog
+++ b/doc/doc-txt/ChangeLog
@@ -93,6 +93,13 @@ JH/19 Fix TLS startup. When the last expansion done before the initiation of a
       parts.  It could also potentially affect tls_privatekeys.  The underlyding
       coding errors go back to 4.90 but were only exposed in 4.97.
 
       parts.  It could also potentially affect tls_privatekeys.  The underlyding
       coding errors go back to 4.90 but were only exposed in 4.97.
 

+JH/20 Bug 3047: A recent (somewhere between 10.34 and 10.42) version of the
+      pcre2 library starting allocating 20kB rather than 112 bytes per match
+      call, which broke the 2GB total limitation on Exim's memory management
+      when a user had over 104207 messages stored and the appendfile
+      maildir_quota_directory_regex option is in use.  Release the allocated
+      memory every thosand files to avoid this.
+

 
 Exim version 4.97
 -----------------
 
 Exim version 4.97
 -----------------

diff --git a/src/src/exim.c b/src/src/exim.c
index c53b98186ba95a297317a4bad85739cc214f8b02..e5b72f11ae8442603507279b8681b86e8e5d03d2 100644 (file)
--- a/src/src/exim.c
+++ b/src/src/exim.c
@@ -49,6 +49,8 @@ optimize out the tail recursion and so not make them too expensive. */
 static void *
 function_store_malloc(PCRE2_SIZE size, void * tag)
 {
 static void *
 function_store_malloc(PCRE2_SIZE size, void * tag)
 {

+if (size > INT_MAX)
+  log_write(0, LOG_MAIN|LOG_PANIC_DIE, "excessive memory alloc request");

 return store_malloc((int)size);
 }
 
 return store_malloc((int)size);
 }
 


@@ -63,12 +65,15 @@ if (block) store_free(block);
 static void *
 function_store_get(PCRE2_SIZE size, void * tag)
 {
 static void *
 function_store_get(PCRE2_SIZE size, void * tag)
 {

+if (size > INT_MAX)
+  log_write(0, LOG_MAIN|LOG_PANIC_DIE, "excessive memory alloc request");

 return store_get((int)size, GET_UNTAINTED);    /* loses track of taint */
 }
 
 static void
 function_store_nullfree(void * block, void * tag)
 {
 return store_get((int)size, GET_UNTAINTED);    /* loses track of taint */
 }
 
 static void
 function_store_nullfree(void * block, void * tag)
 {

+/* We cannot free memory allocated using store_get() */

 }
 
 
 }
 
 

diff --git a/src/src/transports/appendfile.c b/src/src/transports/appendfile.c
index ec41ca035a660ac595891de62beddf139314427e..91b35307901e5eba28d6f9cf3dcfb735947d22fb 100644 (file)
--- a/src/src/transports/appendfile.c
+++ b/src/src/transports/appendfile.c
@@ -153,6 +153,10 @@ static const char *mailbox_formats[] = {
   (!ob->quota_warn_threshold_is_percent || ob->quota_value > 0))
 
 
   (!ob->quota_warn_threshold_is_percent || ob->quota_value > 0))
 
 

+/* Free memory allocated by PCRE2 every so often, because a recent version
+is now using 20kB for every match call */
+
+#define RESET_STORE_FILECNT    1000

 
 /*************************************************
 *              Setup entry point                 *
 
 /*************************************************
 *              Setup entry point                 *


@@ -661,13 +665,14 @@ Returns:        the sum of the sizes of the stattable files
 off_t
 check_dir_size(const uschar * dirname, int * countptr, const pcre2_code * re)
 {
 off_t
 check_dir_size(const uschar * dirname, int * countptr, const pcre2_code * re)
 {

-DIR *dir;
+DIR * dir;

 off_t sum = 0;
 off_t sum = 0;

-int count = *countptr;
+int count = *countptr, lcount = RESET_STORE_FILECNT;
+rmark reset_point = store_mark();

 
 if (!(dir = exim_opendir(dirname))) return 0;
 
 
 if (!(dir = exim_opendir(dirname))) return 0;
 

-for (struct dirent *ent; ent = readdir(dir); )
+for (struct dirent * ent; ent = readdir(dir); )

   {
   uschar * path, * name = US ent->d_name;
   struct stat statbuf;
   {
   uschar * path, * name = US ent->d_name;
   struct stat statbuf;


@@ -675,6 +680,11 @@ for (struct dirent *ent; ent = readdir(dir); )
   if (Ustrcmp(name, ".") == 0 || Ustrcmp(name, "..") == 0) continue;
 
   count++;
   if (Ustrcmp(name, ".") == 0 || Ustrcmp(name, "..") == 0) continue;
 
   count++;

+  if (--lcount == 0)
+    {
+    store_reset(reset_point); reset_point = store_mark();
+    lcount = RESET_STORE_FILECNT;
+    }

 
   /* If there's a regex, try to find the size using it */
 
 
   /* If there's a regex, try to find the size using it */
 


@@ -726,6 +736,7 @@ DEBUG(D_transport)
   debug_printf("check_dir_size: dir=%s sum=" OFF_T_FMT " count=%d\n", dirname,
     sum, count);
 
   debug_printf("check_dir_size: dir=%s sum=" OFF_T_FMT " count=%d\n", dirname,
     sum, count);
 

+store_reset(reset_point);

 *countptr = count;
 return sum;
 }
 *countptr = count;
 return sum;
 }