Release packaging & scripting improvements.

* Make the .xz tarball variant too, and work harder on compressing our
  files for distribution.
  + The .xz files have gained more positive feedback than any other part
    of the 4.89 release.
* Drop usercodes from tarball
  + We shouldn't be embedding own-system-specifc ownership information
    into software release tarballs.  That's for local system backups,
    not distribution.
* Script for the size/checksums
  + We include checksums in the mail; this gets the format fixed and not
    including checksums-of-signatures, etc.  I've also experimented with
    including the size, so let's script that to be portably generated.
* Better tarball signing script
  + Automatically find the signing directory (if not already in it)
  + Sign all files, properly skipping existing .asc files
  + Find the signing key from git config, if available, else error out
    (Nigel is not on the hook as the default victim now)
  + Show what we're doing as we do it

All changes made on the original `release_4_89` branch with
`RELEASE EXPERIMENT` subject tags.

diff --git a/release-process/scripts/mk_exim_release b/release-process/scripts/mk_exim_release
index f6cd33c7e69cee3c5284858563c86d65dd693273..34fe77c6726b1b95680893a6ec34e24c36641bb5 100755 (executable)
--- a/release-process/scripts/mk_exim_release
+++ b/release-process/scripts/mk_exim_release
@@ -361,21 +361,25 @@ sub create_tar_files {
         }
     }
 
         }
     }
 

+    # See also environment variables set in main, tuning compression levels
+    my @COMPRESSIONS = (
+        # compressors-dict-key, file-extension, flags-as-string
+        [ "gzip", "gz", "--gzip" ],
+        [ "bzip2", "bz2", "--bzip2" ],
+        [ "lzip", "lz", "--lzip" ],
+        [ "xz", "xz", "--xz" ],
+    );
+

     foreach my $dir ( glob( File::Spec->catdir( $pkgdirs, ( 'exim*-' . $context->{release} ) ) ) ) {
         my $dirname = ( File::Spec->splitdir($dir) )[-1];
     foreach my $dir ( glob( File::Spec->catdir( $pkgdirs, ( 'exim*-' . $context->{release} ) ) ) ) {
         my $dirname = ( File::Spec->splitdir($dir) )[-1];

-        if ($context->{compressors}{gzip}) {
-            print "Creating: ${pkgs}/${dirname}.tar.gz\n" if ($verbose || $debug);
-            system("$tar cf  ${pkgs}/${dirname}.tar.gz  --gzip  -C ${pkgdirs} ${dirname}")
-        }
-        if ($context->{compressors}{bzip2}) {
-            print "Creating: ${pkgs}/${dirname}.tar.bz2\n" if ($verbose || $debug);
-            system("$tar cf  ${pkgs}/${dirname}.tar.bz2 --bzip2 -C ${pkgdirs} ${dirname}")
-        }
-        if ($context->{compressors}{lzip}) {
-            print "Creating: ${pkgs}/${dirname}.tar.lz\n" if ($verbose || $debug);
-            system("$tar cf  ${pkgs}/${dirname}.tar.lz  --lzip  -C ${pkgdirs} ${dirname}")
+        foreach my $comp (@COMPRESSIONS) {
+            my ($compkey, $extension, $flags) = @{$comp};
+            next unless $context->{compressors}{$compkey};
+            print "Creating: ${pkgs}/${dirname}.tar.${extension}\n" if ($verbose || $debug);
+            system("$tar cf  ${pkgs}/${dirname}.tar.${extension} ${flags} --numeric-owner -C ${pkgdirs} ${dirname}");

         }
     }
         }
     }

+

 }
 
 # ------------------------------------------------------------------
 }
 
 # ------------------------------------------------------------------


@@ -394,6 +398,7 @@ MAIN: {
         compressors => {
                 gzip    => 1,
                 bzip2   => 1,
         compressors => {
                 gzip    => 1,
                 bzip2   => 1,

+                xz      => 1,

                 lzip    => 0,
         },
         build_docs   => 1,
                 lzip    => 0,
         },
         build_docs   => 1,


@@ -402,6 +407,13 @@ MAIN: {
     my $delete;
     my $cleanup = 1;
     ##$ENV{'PATH'} = '/opt/local/bin:' . $ENV{'PATH'};
     my $delete;
     my $cleanup = 1;
     ##$ENV{'PATH'} = '/opt/local/bin:' . $ENV{'PATH'};

+    # We are creating files for mass distribution, so work harder to make smaller files.
+    $ENV{'GZIP'} = '-9';
+    $ENV{'BZIP2'} = '-9';
+    # xz documents minimum file sizes for levels higher than -6 to be useful and each
+    # requires more RAM on the decompressing system.  Exim tarball currently 24MiB so
+    # using -8.
+    $ENV{'XZ_DEFAULTS'} = '-8';

 
     GetOptions(
         'directory=s'   => \$context->{directory},
 
     GetOptions(
         'directory=s'   => \$context->{directory},

diff --git a/release-process/scripts/sign_exim_packages b/release-process/scripts/sign_exim_packages
index 4ed614f196cd5e7bbbdee30fb8e8658f5bb9d4d4..bd02d1183d9a15c465b6e8218730204070dbcf32 100755 (executable)
--- a/release-process/scripts/sign_exim_packages
+++ b/release-process/scripts/sign_exim_packages
@@ -1,17 +1,53 @@
-#!/bin/sh
+#!/bin/sh -eu
+
+# gpg signs all *.tar.* files under the release directory.
+# Invoke from that dir, or let the script try to figure it out for you.
+
+# Key used is from env var EXIM_KEY; if git config finds user.signingkey, then
+# that is the default.  You can set this per-repo with:
+#    git config --local user.signingkey SOME_IDENTIFIER

 #
 #

-# gpg signs all *.tar.* files under a given directory
-# key used set from env var EXIM_KEY, script defaults that to Nigel's.
+# If not set in git config then you _MUST_ set the env var.
+

 # woe betide the poor sod who does not use a gpg agent, so has
 # to enter their password for every file...
 # woe betide the poor sod who does not use a gpg agent, so has
 # to enter their password for every file...

-#

 
 

-dir=${1:?start directory}
-
-: ${EXIM_KEY:=nigel@exim.org}
+if repo_signing_key="$(git config user.signingkey)"; then
+       : "${EXIM_KEY:=$repo_signing_key}"
+else
+       : "${EXIM_KEY:?Need a PGP key uid to sign with}"
+fi

 
 

+: "${GPG_COMMAND:=gpg}"

 umask 022
 
 umask 022
 

-find "$dir" \
-    -type f -name '*.tar.*' \
-    -exec gpg  --local-user ${EXIM_KEY} --detach-sig --armor {} \;
+cd_to() { echo "Working in: $1"; cd "$1"; }
+
+okay=false
+if [ -d ../../release-process ] && [ "${PWD##*/}" = "pkgs" ]; then
+       okay=true # we are in right dir
+elif [ -d release-process ]; then
+       b="$(find . -maxdepth 1 -name 'exim-packaging-*' | sort | tail -n 1)"
+       if [ ".$b" != "." ]; then
+               cd_to "$b/pkgs"
+               okay=true
+       fi
+fi
+if ! $okay; then
+       if [ -d "${1:?need a directory to look in}" ]; then
+               cd_to "$1"
+               shift
+       else
+               printf "%s: %s\n" >&2 "$(basename "$0")" "where should I be looking"
+               exit 1
+       fi
+fi
+
+# Assumes no whitespace (strictly, $IFS) in filenames, which we're okay with
+set $(find . -name '*.asc' -prune -o -type f -print | cut -c 3- | sort)
+
+for FILE
+do
+       echo "Signing: $FILE"
+       ${GPG_COMMAND} --local-user "${EXIM_KEY}" --detach-sig --armor "$FILE"
+done

diff --git a/release-process/scripts/stats_for_email b/release-process/scripts/stats_for_email
new file mode 100755 (executable)
index 0000000..0eb0c29
--- /dev/null
+++ b/release-process/scripts/stats_for_email
@@ -0,0 +1,28 @@
+#!/bin/sh -eu
+
+okay=false
+if [ -d ../../release-process ] && [ "${PWD##*/}" = "pkgs" ]; then
+       okay=true # we are in right dir
+elif [ -d release-process ]; then
+       b="$(find . -maxdepth 1 -name 'exim-packaging-*' | sort | tail -n 1)"
+       if [ ".$b" != "." ]; then
+               cd "$b/pkgs"
+               okay=true
+       fi
+fi
+if ! $okay; then
+       if [ -d "${1:?need a directory to look in}" ]; then
+               cd "$1"
+               shift
+       else
+               printf "%s: %s\n" >&2 "$(basename "$0")" "where should I be looking"
+               exit 1
+       fi
+fi
+
+set $(find "${1:-.}" -name '*.asc' -prune -o -type f -print | cut -c 3- | sort)
+
+# stat(1) formats are non-portable BSD vs GNU
+perl -le 'print "SIZE($_)= @{[-s $_]}" foreach @ARGV' "$@"
+echo
+openssl dgst -sha256 "$@"