From 00f7a87b04290db615ec29584e0554928fca81c7 Mon Sep 17 00:00:00 2001 From: Phil Pennock Date: Thu, 2 Feb 2017 15:38:14 -0500 Subject: [PATCH] Release packaging & scripting improvements. * Make the .xz tarball variant too, and work harder on compressing our files for distribution. + The .xz files have gained more positive feedback than any other part of the 4.89 release. * Drop usercodes from tarball + We shouldn't be embedding own-system-specifc ownership information into software release tarballs. That's for local system backups, not distribution. * Script for the size/checksums + We include checksums in the mail; this gets the format fixed and not including checksums-of-signatures, etc. I've also experimented with including the size, so let's script that to be portably generated. * Better tarball signing script + Automatically find the signing directory (if not already in it) + Sign all files, properly skipping existing .asc files + Find the signing key from git config, if available, else error out (Nigel is not on the hook as the default victim now) + Show what we're doing as we do it All changes made on the original `release_4_89` branch with `RELEASE EXPERIMENT` subject tags. --- release-process/scripts/mk_exim_release | 34 ++++++++----- release-process/scripts/sign_exim_packages | 56 ++++++++++++++++++---- release-process/scripts/stats_for_email | 28 +++++++++++ 3 files changed, 97 insertions(+), 21 deletions(-) create mode 100755 release-process/scripts/stats_for_email diff --git a/release-process/scripts/mk_exim_release b/release-process/scripts/mk_exim_release index f6cd33c7e..34fe77c67 100755 --- a/release-process/scripts/mk_exim_release +++ b/release-process/scripts/mk_exim_release @@ -361,21 +361,25 @@ sub create_tar_files { } } + # See also environment variables set in main, tuning compression levels + my @COMPRESSIONS = ( + # compressors-dict-key, file-extension, flags-as-string + [ "gzip", "gz", "--gzip" ], + [ "bzip2", "bz2", "--bzip2" ], + [ "lzip", "lz", "--lzip" ], + [ "xz", "xz", "--xz" ], + ); + foreach my $dir ( glob( File::Spec->catdir( $pkgdirs, ( 'exim*-' . $context->{release} ) ) ) ) { my $dirname = ( File::Spec->splitdir($dir) )[-1]; - if ($context->{compressors}{gzip}) { - print "Creating: ${pkgs}/${dirname}.tar.gz\n" if ($verbose || $debug); - system("$tar cf ${pkgs}/${dirname}.tar.gz --gzip -C ${pkgdirs} ${dirname}") - } - if ($context->{compressors}{bzip2}) { - print "Creating: ${pkgs}/${dirname}.tar.bz2\n" if ($verbose || $debug); - system("$tar cf ${pkgs}/${dirname}.tar.bz2 --bzip2 -C ${pkgdirs} ${dirname}") - } - if ($context->{compressors}{lzip}) { - print "Creating: ${pkgs}/${dirname}.tar.lz\n" if ($verbose || $debug); - system("$tar cf ${pkgs}/${dirname}.tar.lz --lzip -C ${pkgdirs} ${dirname}") + foreach my $comp (@COMPRESSIONS) { + my ($compkey, $extension, $flags) = @{$comp}; + next unless $context->{compressors}{$compkey}; + print "Creating: ${pkgs}/${dirname}.tar.${extension}\n" if ($verbose || $debug); + system("$tar cf ${pkgs}/${dirname}.tar.${extension} ${flags} --numeric-owner -C ${pkgdirs} ${dirname}"); } } + } # ------------------------------------------------------------------ @@ -394,6 +398,7 @@ MAIN: { compressors => { gzip => 1, bzip2 => 1, + xz => 1, lzip => 0, }, build_docs => 1, @@ -402,6 +407,13 @@ MAIN: { my $delete; my $cleanup = 1; ##$ENV{'PATH'} = '/opt/local/bin:' . $ENV{'PATH'}; + # We are creating files for mass distribution, so work harder to make smaller files. + $ENV{'GZIP'} = '-9'; + $ENV{'BZIP2'} = '-9'; + # xz documents minimum file sizes for levels higher than -6 to be useful and each + # requires more RAM on the decompressing system. Exim tarball currently 24MiB so + # using -8. + $ENV{'XZ_DEFAULTS'} = '-8'; GetOptions( 'directory=s' => \$context->{directory}, diff --git a/release-process/scripts/sign_exim_packages b/release-process/scripts/sign_exim_packages index 4ed614f19..bd02d1183 100755 --- a/release-process/scripts/sign_exim_packages +++ b/release-process/scripts/sign_exim_packages @@ -1,17 +1,53 @@ -#!/bin/sh +#!/bin/sh -eu + +# gpg signs all *.tar.* files under the release directory. +# Invoke from that dir, or let the script try to figure it out for you. + +# Key used is from env var EXIM_KEY; if git config finds user.signingkey, then +# that is the default. You can set this per-repo with: +# git config --local user.signingkey SOME_IDENTIFIER # -# gpg signs all *.tar.* files under a given directory -# key used set from env var EXIM_KEY, script defaults that to Nigel's. +# If not set in git config then you _MUST_ set the env var. + # woe betide the poor sod who does not use a gpg agent, so has # to enter their password for every file... -# -dir=${1:?start directory} - -: ${EXIM_KEY:=nigel@exim.org} +if repo_signing_key="$(git config user.signingkey)"; then + : "${EXIM_KEY:=$repo_signing_key}" +else + : "${EXIM_KEY:?Need a PGP key uid to sign with}" +fi +: "${GPG_COMMAND:=gpg}" umask 022 -find "$dir" \ - -type f -name '*.tar.*' \ - -exec gpg --local-user ${EXIM_KEY} --detach-sig --armor {} \; +cd_to() { echo "Working in: $1"; cd "$1"; } + +okay=false +if [ -d ../../release-process ] && [ "${PWD##*/}" = "pkgs" ]; then + okay=true # we are in right dir +elif [ -d release-process ]; then + b="$(find . -maxdepth 1 -name 'exim-packaging-*' | sort | tail -n 1)" + if [ ".$b" != "." ]; then + cd_to "$b/pkgs" + okay=true + fi +fi +if ! $okay; then + if [ -d "${1:?need a directory to look in}" ]; then + cd_to "$1" + shift + else + printf "%s: %s\n" >&2 "$(basename "$0")" "where should I be looking" + exit 1 + fi +fi + +# Assumes no whitespace (strictly, $IFS) in filenames, which we're okay with +set $(find . -name '*.asc' -prune -o -type f -print | cut -c 3- | sort) + +for FILE +do + echo "Signing: $FILE" + ${GPG_COMMAND} --local-user "${EXIM_KEY}" --detach-sig --armor "$FILE" +done diff --git a/release-process/scripts/stats_for_email b/release-process/scripts/stats_for_email new file mode 100755 index 000000000..0eb0c2981 --- /dev/null +++ b/release-process/scripts/stats_for_email @@ -0,0 +1,28 @@ +#!/bin/sh -eu + +okay=false +if [ -d ../../release-process ] && [ "${PWD##*/}" = "pkgs" ]; then + okay=true # we are in right dir +elif [ -d release-process ]; then + b="$(find . -maxdepth 1 -name 'exim-packaging-*' | sort | tail -n 1)" + if [ ".$b" != "." ]; then + cd "$b/pkgs" + okay=true + fi +fi +if ! $okay; then + if [ -d "${1:?need a directory to look in}" ]; then + cd "$1" + shift + else + printf "%s: %s\n" >&2 "$(basename "$0")" "where should I be looking" + exit 1 + fi +fi + +set $(find "${1:-.}" -name '*.asc' -prune -o -type f -print | cut -c 3- | sort) + +# stat(1) formats are non-portable BSD vs GNU +perl -le 'print "SIZE($_)= @{[-s $_]}" foreach @ARGV' "$@" +echo +openssl dgst -sha256 "$@" -- 2.30.2