From: Phil Pennock Date: Mon, 9 Apr 2018 02:28:56 +0000 (-0400) Subject: Added util/renew-opendmarc-tlds.sh script to renew PSL X-Git-Tag: exim-4_91~15 X-Git-Url: https://git.exim.org/users/heiko/exim.git/commitdiff_plain/ddd16464764026559f8efe414ca6ac05406618a4 Added util/renew-opendmarc-tlds.sh script to renew PSL --- diff --git a/doc/doc-txt/ChangeLog b/doc/doc-txt/ChangeLog index 5e9d2afb7..9fc466365 100644 --- a/doc/doc-txt/ChangeLog +++ b/doc/doc-txt/ChangeLog @@ -203,6 +203,9 @@ JH/36 Fix reinitialisation of DKIM logging variable between messages. JH/37 Bug 2255: Revert the disable of the OpenSSL session caching. This triggered odd behaviour from Outlook Express clients. +PP/03 Add util/renew-opendmarc-tlds.sh script for safe renewal of public + suffix list. + Exim version 4.90 ----------------- diff --git a/doc/doc-txt/NewStuff b/doc/doc-txt/NewStuff index ea190aafb..c7889223f 100644 --- a/doc/doc-txt/NewStuff +++ b/doc/doc-txt/NewStuff @@ -46,6 +46,7 @@ Version 4.91 Authentication-Results: header. 13. EXPERIMENTAL_ARC. See the experimental.spec file. + See also new util/renew-opendmarc-tlds.sh script for use with DMARC/ARC. 14: A dane:fail event, intended to facilitate reporting. diff --git a/doc/doc-txt/experimental-spec.txt b/doc/doc-txt/experimental-spec.txt index 4e244ac5f..0eeb22758 100644 --- a/doc/doc-txt/experimental-spec.txt +++ b/doc/doc-txt/experimental-spec.txt @@ -436,6 +436,7 @@ dmarc_tld_file Defines the location of a text file of valid during domain parsing. Maintained by Mozilla, the most current version can be downloaded from a link at http://publicsuffix.org/list/. + See also util/renew-opendmarc-tlds.sh script. Optional: dmarc_history_file Defines the location of a file to log results diff --git a/src/util/renew-opendmarc-tlds.sh b/src/util/renew-opendmarc-tlds.sh new file mode 100755 index 000000000..c276fcd5f --- /dev/null +++ b/src/util/renew-opendmarc-tlds.sh @@ -0,0 +1,117 @@ +#!/bin/sh -eu +# +# Short version of this script: +# curl -f -o /var/cache/exim/opendmarc.tlds https://publicsuffix.org/list/public_suffix_list.dat +# but run as Exim runtime user, writing to a place it can write to, and with +# sanity checks and atomic replacement. +# +# For now, we deliberately leave the invalid file around for analysis +# with . suffix. +# +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~8< cut here >8~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# +# Create a cron-job as the Exim run-time user to invoke this daily, with a +# single parameter, 'cron'. Eg: +# +# 3 4 * * * /usr/local/sbin/renew-opendmarc-tlds.sh cron +# +# That will, at 3 minutes past the 4th hour (in whatever timezone cron is +# running it) invoke this script with 'cron'; we will then sleep between 10 and +# 50 seconds, before continuing. +# +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~8< cut here >8~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# +# This should be "pretty portable"; the only things it depends upon are: +# * a POSIX shell which additionally implements 'local' (dash works) +# * the 'curl' command; change the fetch_candidate() function to replace that +# * the 'stat' command, to get the size of a file; change size_of() if need be +# * the 'hexdump' command and /dev/urandom existing +# + used when invoked with 'cron', to avoid retrieving on a minute boundary +# and contending with many other automated systems. +# + with bash/zsh, can replace with: $(( 10 + ( RANDOM % 40 ) )) +# + on Debian/Ubuntu systems, hexdump is in the 'bsdmainutils' package. + +# Consider putting an email address inside the parentheses, something like +# noc@example.org or other reachable address, so that if something goes wrong +# and the server operators need to step in, they can see from logs who to +# contact instead of just blocking your IP: +readonly CurlUserAgent='renew-opendmarc-tlds/0.1 (distributed with Exim)' + +# change this to your Exim run-time user (exim -n -bP exim_user) : +readonly RuntimeUser='_exim' + +# Do not make this a directory which untrusted users can write to: +readonly StateDir='/var/cache/exim' + +readonly URL='https://publicsuffix.org/list/public_suffix_list.dat' + +readonly TargetShortFile='opendmarc.tlds' + +# When replacing, new file must be at least this percentage the size of +# the old one or it's an error: +readonly MinNewSizeRation=90 + +# Each of these regexps must be matched by the file, or it's an error: +readonly MustExistRegexps=' + ^ac\.uk$ + ^org$ + ^tech$ + ' + +# =======================8< end of configuration >8======================= + +set -eu + +readonly FullTargetPath="${StateDir}/${TargetShortFile}" +readonly WorkingFile="${FullTargetPath}.$$" + +progname="$(basename "$0")" +note() { printf >&2 '%s: %s\n' "$progname" "$*"; } +die() { note "$@"; exit 1; } + +# guard against stomping on file-permissions +[ ".$(id -un)" = ".${RuntimeUser:?}" ] || \ + die "must be invoked as ${RuntimeUser}" + +fetch_candidate() { + curl --user-agent "$CurlUserAgent" -fSs -o "${WorkingFile}" "${URL}" +} + +size_of() { + stat -c %s "$1" +} + +sanity_check_candidate() { + local new_size prev_size re + new_size="$(size_of "$WorkingFile")" + + for re in $MustExistRegexps; do + grep -qs "$re" -- "$WorkingFile" || \ + die "regexp $re not found in $WorkingFile" + done + + if ! prev_size="$(size_of "$FullTargetPath")"; then + note "missing previous file, can't size-compare: $FullTargetPath" + # We're sane by definition, probably initial fetch, and the + # stat failure and this note will be printed. That's fine; if + # a cron invocation is missing the file then something has gone + # badly wrong. + return 0 + fi + local ratio + ratio=$(expr $new_size \* 100 / $prev_size) + if [ $ratio -lt $MinNewSizeRation ]; then + die "New $TargetShortFile candidate only ${ratio}% size of old; $new_size vs $prev_size" + fi +} + +if [ "${1:-.}" = "cron" ]; then + shift + # Don't pull on-the-minute, wait for off-cycle-peak + sleep $(( ($(dd if=/dev/urandom bs=1 count=1 2>/dev/null | hexdump -e '1/1 "%u"') % 40) + 10)) +fi + +umask 022 +fetch_candidate +sanity_check_candidate +mv -- "$WorkingFile" "$FullTargetPath"