From: Jeremy Harris <jgh146exb@wizmail.org>
Date: Thu, 25 Jan 2024 17:48:43 +0000 (+0000)
Subject: Appendfile: release regex-match store every thousand files.  Bug 3047
X-Git-Tag: exim-4.98-RC0~83
X-Git-Url: https://git.exim.org/exim.git/commitdiff_plain/b4e7527561f1c68b821d5cf25efe29ae63d1d434

Appendfile: release regex-match store every thousand files.  Bug 3047
---

diff --git a/doc/doc-txt/ChangeLog b/doc/doc-txt/ChangeLog
index 14cd45d85..e258966d8 100644
--- a/doc/doc-txt/ChangeLog
+++ b/doc/doc-txt/ChangeLog
@@ -93,6 +93,13 @@ JH/19 Fix TLS startup. When the last expansion done before the initiation of a
       parts.  It could also potentially affect tls_privatekeys.  The underlyding
       coding errors go back to 4.90 but were only exposed in 4.97.
 
+JH/20 Bug 3047: A recent (somewhere between 10.34 and 10.42) version of the
+      pcre2 library starting allocating 20kB rather than 112 bytes per match
+      call, which broke the 2GB total limitation on Exim's memory management
+      when a user had over 104207 messages stored and the appendfile
+      maildir_quota_directory_regex option is in use.  Release the allocated
+      memory every thosand files to avoid this.
+
 
 Exim version 4.97
 -----------------
diff --git a/src/src/exim.c b/src/src/exim.c
index c53b98186..e5b72f11a 100644
--- a/src/src/exim.c
+++ b/src/src/exim.c
@@ -49,6 +49,8 @@ optimize out the tail recursion and so not make them too expensive. */
 static void *
 function_store_malloc(PCRE2_SIZE size, void * tag)
 {
+if (size > INT_MAX)
+  log_write(0, LOG_MAIN|LOG_PANIC_DIE, "excessive memory alloc request");
 return store_malloc((int)size);
 }
 
@@ -63,12 +65,15 @@ if (block) store_free(block);
 static void *
 function_store_get(PCRE2_SIZE size, void * tag)
 {
+if (size > INT_MAX)
+  log_write(0, LOG_MAIN|LOG_PANIC_DIE, "excessive memory alloc request");
 return store_get((int)size, GET_UNTAINTED);	/* loses track of taint */
 }
 
 static void
 function_store_nullfree(void * block, void * tag)
 {
+/* We cannot free memory allocated using store_get() */
 }
 
 
diff --git a/src/src/transports/appendfile.c b/src/src/transports/appendfile.c
index ec41ca035..91b353079 100644
--- a/src/src/transports/appendfile.c
+++ b/src/src/transports/appendfile.c
@@ -153,6 +153,10 @@ static const char *mailbox_formats[] = {
   (!ob->quota_warn_threshold_is_percent || ob->quota_value > 0))
 
 
+/* Free memory allocated by PCRE2 every so often, because a recent version
+is now using 20kB for every match call */
+
+#define RESET_STORE_FILECNT	1000
 
 /*************************************************
 *              Setup entry point                 *
@@ -661,13 +665,14 @@ Returns:        the sum of the sizes of the stattable files
 off_t
 check_dir_size(const uschar * dirname, int * countptr, const pcre2_code * re)
 {
-DIR *dir;
+DIR * dir;
 off_t sum = 0;
-int count = *countptr;
+int count = *countptr, lcount = RESET_STORE_FILECNT;
+rmark reset_point = store_mark();
 
 if (!(dir = exim_opendir(dirname))) return 0;
 
-for (struct dirent *ent; ent = readdir(dir); )
+for (struct dirent * ent; ent = readdir(dir); )
   {
   uschar * path, * name = US ent->d_name;
   struct stat statbuf;
@@ -675,6 +680,11 @@ for (struct dirent *ent; ent = readdir(dir); )
   if (Ustrcmp(name, ".") == 0 || Ustrcmp(name, "..") == 0) continue;
 
   count++;
+  if (--lcount == 0)
+    {
+    store_reset(reset_point); reset_point = store_mark();
+    lcount = RESET_STORE_FILECNT;
+    }
 
   /* If there's a regex, try to find the size using it */
 
@@ -726,6 +736,7 @@ DEBUG(D_transport)
   debug_printf("check_dir_size: dir=%s sum=" OFF_T_FMT " count=%d\n", dirname,
     sum, count);
 
+store_reset(reset_point);
 *countptr = count;
 return sum;
 }