From 725fd3eeb8665d5f862245cd4f7655bb90f679f0 Mon Sep 17 00:00:00 2001 From: Darren 'Tadgy' Austin Date: Thu, 21 May 2026 17:44:16 +0100 Subject: [PATCH 01/10] Add mirror script. --- mirror | 377 +++++++++++++++++++++++++++++++++ do-mirroring => mirror-wrapper | 0 2 files changed, 377 insertions(+) create mode 100755 mirror rename do-mirroring => mirror-wrapper (100%) diff --git a/mirror b/mirror new file mode 100755 index 0000000..c8436ab --- /dev/null +++ b/mirror @@ -0,0 +1,377 @@ +#!/bin/bash +# Version: 0.4.0 +# shellcheck disable=SC2034 + +# Script config. +DEBUG="echo" # Set to 'echo' for debugging, or ':' for none +SLEEP="5m" # Sleep time between syncs when there was an error in the sync +ERRORS_MAX="3" # After this many errors in the sync tries, give up +MAX_PROC="3600" # If a run takes longer than this time (in seconds), a new run is started +MAX_RUNS="3" # Maximum number of runs + +# Sync config. +IPV4="5.101.171.215" +DATADIR="/data/depository" +RSYNC_COMMAND="/usr/bin/rsync" +RSYNC_REMOTE_OPTIONS=("--address=$IPV4" '--no-motd' '--contimeout=30' '--timeout=60 -4 -aH' '--chmod=go-w,+rX' '--partial' '--partial-dir=.rsync-tmp' '--delete-delay' '--delay-updates') +RSYNC_LOCAL_OPTIONS=('-aH' '--chmod=go-w,+rX' '--partial' '--partial-dir=.rsync-tmp' '--delay-updates') +# RSYNC_VERBOSE=('--verbose' '--human-readable') +# RSYNC_VERBOSE=('--progress' '--verbose' '--stats' '--human-readable') +LFTP_COMMAND="/usr/bin/lftp" +LFTP_OPTIONS="set net:socket-bind-ipv4 $IPV4 && set xfer:log false && set net:timeout 30s && set xfer:timeout 60s && set mirror:parallel-directories true && mirror --continue --delete --parallel=10" +# LFTP_VERBOSE=('--verbose=3') + +# Copy the new ISO into place when it's released. +# Only set this when a new Slackware release is imminent. +#NEW_SLACK_VERSION="15.1" + +# +# Remotes +# +# Mirror info for people/alien +ALIEN_TYPE="rsync" +ALIEN_MIRROR="bear.alienbase.nl" +ALIEN_MODULE="mirrors/people/alien/" +ALIEN_DESTDIR="people/alien/" +ALIEN_OPTIONS=() +ALIEN_FILTER=() + +# Mirror info for people/alien-current-iso +CURRENTISO_TYPE="rsync" +CURRENTISO_MIRROR="bear.alienbase.nl" +CURRENTISO_MODULE="mirrors/slackware/" +CURRENTISO_DESTDIR="people/alien-current-iso/" +CURRENTISO_OPTIONS=() +CURRENTISO_FILTER=('--include=slackware*-current-iso/***' '--exclude=*') + +# Mirror info for people/alien-kde +KTOWN_TYPE="rsync" +KTOWN_MIRROR="slackware.nl" +KTOWN_MODULE="mirrors/alien-kde/" +KTOWN_DESTDIR="people/alien-kde/" +KTOWN_OPTIONS=() +KTOWN_FILTER=() + +# Mirror info for liveslak/ +LIVESLAK_TYPE="rsync" +LIVESLAK_MIRROR="bear.alienbase.nl" +LIVESLAK_MODULE="mirrors/slackware-live/" +LIVESLAK_DESTDIR="liveslak/" +LIVESLAK_OPTIONS=() +LIVESLAK_FILTER=() + +# Mirror info for porteus +PORTEUS_TYPE="rsync" +PORTEUS_MIRROR="mirrors.dotsrc.org" +PORTEUS_MODULE="porteus/" +PORTEUS_DESTDIR="porteus/" +PORTEUS_OPTIONS=() +PORTEUS_FILTER=() + +# Mirror info for people/r0ni +R0NI_TYPE="rsync" +R0NI_MIRROR="slackware.lngn.net" +R0NI_MODULE="slackware-lngn-net/" +R0NI_DESTDIR="people/r0ni/" +R0NI_OPTIONS=() +R0NI_FILTER=() + +# Mirror info for people/rworkman +RWORKMAN_TYPE="rsync" +RWORKMAN_MIRROR="harrier.slackbuilds.org" +RWORKMAN_MODULE="all-public/rworkman/" +RWORKMAN_DESTDIR="people/rworkman/" +RWORKMAN_OPTIONS=() +RWORKMAN_FILTER=('--exclude=index.php' '--exclude=favicon.ico') + +# Mirror info for Salix +SALIX_TYPE="rsync" +SALIX_MIRROR="rsync.salixos.org" +SALIX_MODULE="salix/" +SALIX_DESTDIR="salix/" +SALIX_OPTIONS=() +SALIX_FILTER=('--exclude=/sbo') + +# Mirror info for slackbuilds.org +SBO_TYPE="rsync" +SBO_MIRROR="slackbuilds.org" +SBO_MODULE="slackbuilds/" +SBO_DESTDIR="slackbuilds.org/" +SBO_OPTIONS=() +SBO_FILTER=() + +# Mirror info for slackware +SLACKTREE_TYPE="rsync" +SLACKTREE_MIRROR="ftp.osuosl.org" +SLACKTREE_MODULE="slackware/" +SLACKTREE_DESTDIR="slackware/" +SLACKTREE_OPTIONS=() +SLACKTREE_FILTER=('--exclude=/*-iso' '--exclude=/slackware-pre*' '--exclude=/slackware-[1234789].*' '--exclude=/slackware-1[012].*') + +# Mirror info for Zenwalk +ZENWALK_TYPE="rsync" +ZENWALK_MIRROR="download.zenwalk.org" +ZENWALK_MODULE="zenwalk/" +ZENWALK_DESTDIR="zenwalk/" +ZENWALK_OPTIONS=() +ZENWALK_FILTER=('--exclude=/download' '--exclude=/x86_64/people') + +# +# Locals +# +# Mirror info for the multilib cumulative tree. +MULTILIBARCHIVE_TYPE="rsync" +MULTILIBARCHIVE_LOCAL="people/alien/multilib/" +MULTILIBARCHIVE_DESTDIR="cumulative/multilib/" +MULTILIBARCHIVE_OPTIONS=() +MULTILIBARCHIVE_FILTER=('--exclude=source/') + +# Mirror info for Slackware cumulative tree. +SLACKARCHIVE_TYPE="rsync" +SLACKARCHIVE_LOCAL="slackware/" +SLACKARCHIVE_DESTDIR="cumulative/" +SLACKARCHIVE_OPTIONS=() +SLACKARCHIVE_FILTER=('--exclude=source/' '--include=/slackware-15.0' '--include=/slackware64-15.0' '--include=/slackware-current' '--include=/slackware64-current' '--exclude=/*') + +# +# lftp example +# +# EXAMPLE_TYPE="lftp" +# EXAMPLE_MIRROR="http://example.com" +# EXAMPLE_MODULE="/" +# EXAMPLE_DESTDIR="example/" +# EXAMPLE_OPTIONS=() +# EXAMPLE_FILTER=('-X' 'exclude-item/') + +# +# Inactive/dead projects +# +# SLACKWARELOONG Dead repository address +# SLACKDCE Dead repository address +# ALPHAGEEK Alphageek died +# MICROLINUX Project dead - informed by maintaner +# ALIENOPENVZ Project dead - informed by alienBOB +# ALIENARM Project dead - informed by alienBOB +# SLACKONLY Dead repository address +# SLACKY Dead repository address +# SLACKAR Dead repository address +# SLARM64 Project dead + +####################################################################################################################################### + +# Only allow one copy of the script to run at any time. +# shellcheck disable=SC2154 +if [[ "$FLOCK" != "$0" ]]; then + # shellcheck disable=SC2093 + exec env FLOCK="$0" flock -E 10 -e -n "$0" "$0" "$@" + ERR="$?" + if (( ERR == 10 )); then + # File is locked, exit now. + exit 0 + elif (( ERR > 0 )); then + echo "ERROR: flock execution error" >&2 + exit 1 + fi +fi + +# Move to the depository directory. +cd "$DATADIR" >/dev/null 2>&1 || { + echo "ERROR: $DATADIR does not exist." >&2 + exit 1 +} + +# Variables +declare -a LFTP_LIST RSYNC_LIST + +# Parse command line. +while [[ -n "$1" ]]; do + case "$1" in + -h|-help|--help) + echo "Usage: ${0##*/} [mirror-ID1 mirror-ID2 ...]" + exit 0 + ;; + *) + ITEM="${1^^}" + [[ ! -v ${ITEM}_DESTDIR ]] && { + echo "ERROR: '$1' is not a valid mirror ID" >&2 + exit 1 + } + TYPE="${ITEM}_TYPE" + if [[ "${!TYPE}" == "rsync" ]]; then + RSYNC_LIST+=("${ITEM}") + elif [[ "${!TYPE}" == "lftp" ]]; then + LFTP_LIST+=("${!ITEM}") + else + echo "ERROR: invalid type of mirroring for '${ITEM}'" >&2 + exit 1 + fi + shift + ;; + esac +done + +# What should be sync'd, if not provided on the command line. +(( ${#RSYNC_LIST[@]} == 0 )) && { + # Main tree. + [[ $(printf "%(%H)T") =~ .* ]] && RSYNC_LIST+=('SLACKTREE') + + # Other remotes. + [[ $(printf "%(%H)T") =~ (02|08|14|20) ]] && RSYNC_LIST+=('ALIEN') + [[ $(printf "%(%H)T") =~ (02|08|14|20) ]] && RSYNC_LIST+=('CURRENTISO') + [[ $(printf "%(%H)T") =~ (02|08|14|20) ]] && RSYNC_LIST+=('KTOWN') + [[ $(printf "%(%H)T") =~ (02|08|14|20) ]] && RSYNC_LIST+=('LIVESLAK') + [[ $(printf "%(%H)T") =~ (02|08|14|20) ]] && RSYNC_LIST+=('PORTEUS') + [[ $(printf "%(%H)T") =~ (02|08|14|20) ]] && RSYNC_LIST+=('R0NI') + [[ $(printf "%(%H)T") =~ (02|08|14|20) ]] && RSYNC_LIST+=('RWORKMAN') + [[ $(printf "%(%H)T") =~ (02|08|14|20) ]] && RSYNC_LIST+=('SALIX') + [[ $(printf "%(%H)T") =~ (02|08|14|20) ]] && RSYNC_LIST+=('SBO') + [[ $(printf "%(%H)T") =~ (02|08|14|20) ]] && RSYNC_LIST+=('ZENWALK') + + # Locals. + [[ $(printf "%(%H)T") =~ .* ]] && RSYNC_LIST+=('SLACKARCHIVE') + [[ $(printf "%(%H)T") =~ (02|08|14|20) ]] && RSYNC_LIST+=('MULTILIBARCHIVE') +} +(( ${#LFTP_LIST[@]} == 0 )) && { + # Nothing to sync here currently + : +} + +# Mirror a new Slackware version. +[[ -v NEW_SLACK_VERSION ]] && { + "$DEBUG" "DEBUG: Checking for new Slackware release at $(date)." >&2 + #export NEW_SLACK_VERSION + /opt/bin/mirror-new-slackware-release "$NEW_SLACK_VERSION" + "$DEBUG" "DEBUG: Finished checking for new release at $(date)." >&2 +} + +# Exit now if there's nothing to sync. +(( ${#RSYNC_LIST[@]} == 0 )) && (( ${#LFTP_LIST[@]} == 0 )) && { + exit 0 +} + +echo "INFO: Sync list: ${RSYNC_LIST[*]} ${LFTP_LIST[*]}" >&2 +START_TIME="$(printf "%(%s)T")" +for RUN in $(seq -s' ' 1 "$MAX_RUNS"); do + # Mirrors synced using rsync + echo "INFO: Begining rsync run $RUN at $(printf "%(%c)T")" >&2 + for VAR in "${RSYNC_LIST[@]}"; do + TYPE="${VAR}_TYPE" + MIRROR="${VAR}_MIRROR" + LOCAL="${VAR}_LOCAL" + MODULE="${VAR}_MODULE" + PASSWORD="${VAR}_PASSWORD" + DESTDIR="${VAR}_DESTDIR" + OPTIONS="${VAR}_OPTIONS[@]" + FILTER="${VAR}_FILTER[@]" + SRC="" + RSYNC_OPTS="" + ERR_COUNT=0 + + if [[ "${!TYPE}" != "rsync" ]]; then + echo "WARNING: Wrong type set for '$VAR' when in RSYNC_LIST." >&2 + continue + elif [ -n "${!LOCAL}" ] && [ -n "${!MIRROR}" ]; then + echo "WARNING: Mutually exclusive LOCAL and MIRROR set for '$VAR'." >&2 + continue + elif [ -z "${!LOCAL}" ] && [ -z "${!MIRROR}" ]; then + echo "WARNING: No LOCAL or MIRROR set for '$VAR'." >&2 + continue + elif [ -n "${!MIRROR}" ] && [ -z "${!MODULE}" ]; then + echo "WARNING: No MODULE set for '$VAR'." >&2 + continue + elif [ -n "${!LOCAL}" ]; then + # Use a local path. + SRC="${!LOCAL}" + RSYNC_OPTS="RSYNC_LOCAL_OPTIONS[@]" + else + # Use a remote mirror+module. + SRC="${!MIRROR}::${!MODULE}" + RSYNC_OPTS="RSYNC_REMOTE_OPTIONS[@]" + fi + + echo "INFO: Begining processing of '$VAR' (run $RUN) at $(printf "%(%c)T")." >&2 + "$DEBUG" "DEBUG: Command line: RSYNC_PASSWORD=\"XXX\" $RSYNC_COMMAND ${!RSYNC_OPTS} ${RSYNC_VERBOSE[*]} ${!OPTIONS} ${!FILTER} $SRC ${!DESTDIR}" >&2 + while :; do + RSYNC_PASSWORD="${!PASSWORD}" "$RSYNC_COMMAND" "${!RSYNC_OPTS}" "${RSYNC_VERBOSE[@]}" "${!OPTIONS}" "${!FILTER}" "$SRC" "${!DESTDIR}" >&2 + ERR_CODE="$?" + case "$ERR_CODE" in + '0') + break + ;; + *) + (( ERR_COUNT++ )) + if (( ERR_COUNT < ERRORS_MAX )); then + echo "WARNING: Failed try $ERR_COUNT with error code: $ERR_CODE -- retrying in $SLEEP." >&2 + sleep "$SLEEP" + continue + else + echo "WARNING: Failed try $ERR_COUNT with error code: $ERR_CODE -- giving up." >&2 + break + fi + ;; + esac + done + echo "INFO: Finished processing of $VAR at $(printf "%(%c)T")." >&2 + done + + # Mirrors synced using lftp + echo "INFO: Begining lftp run $RUN at $(printf "%(%c)T")" >&2 + for VAR in "${LFTP_LIST[@]}"; do + TYPE="${VAR}_TYPE" + MIRROR="${VAR}_MIRROR" + LOCAL="${VAR}_LOCAL" + MODULE="${VAR}_MODULE" + DESTDIR="${VAR}_DESTDIR" + OPTIONS="${VAR}_OPTIONS[@]" + FILTER="${VAR}_FILTER[@]" + ERR_COUNT=0 + + if [[ "${!TYPE}" != "lftp" ]]; then + echo "WARNING: Wrong type set for '$VAR' when in LFTP_LIST." >&2 + continue + elif [ -n "${!LOCAL}" ] && [ -n "${!MIRROR}" ]; then + echo "WARNING: Mutually exclusive LOCAL and MIRROR set for '$VAR'." >&2 + continue + elif [ -z "${!LOCAL}" ] && [ -z "${!MIRROR}" ]; then + echo "WARNING: No LOCAL or MIRROR set for '$VAR'." >&2 + continue + elif [ -n "${!MIRROR}" ] && [ -z "${!MODULE}" ]; then + echo "WARNING: No MODULE set for '$VAR'." >&2 + continue + fi + + echo "INFO: Begining processing of '$VAR' (run $RUN) at $(printf "%(%c)T")." >&2 + "$DEBUG" "DEBUG: Command line: $LFTP_COMMAND -c \"$LFTP_OPTIONS ${LFTP_VERBOSE[*]} ${!FILTER} ${!MIRROR}${!MODULE} ${!DESTDIR}\"" >&2 + while :; do + "$LFTP_COMMAND" -c "$LFTP_OPTIONS ${LFTP_VERBOSE[*]} ${!FILTER} ${!MIRROR}/${!MODULE} ${!DESTDIR}" >&2 + ERR_CODE="$?" + case "$ERR_CODE" in + '0') + break + ;; + *) + (( ERR_COUNT++ )) + if (( ERR_COUNT < ERRORS_MAX )); then + echo "WARNING: Failed try $ERR_COUNT with error code: $ERR_CODE -- retrying in $SLEEP." >&2 + sleep "$SLEEP" + continue + else + echo "WARNING: Failed try $ERR_COUNT with error code: $ERR_CODE -- giving up." >&2 + break + fi + ;; + esac + done + echo "INFO: Finished processing of $VAR at $(printf "%(%c)T")." >&2 + done + FINISH_TIME="$(printf "%(%s)T")" + if (( (FINISH_TIME - START_TIME) > MAX_PROC )); then + echo "WARNING: Max processing time (${MAX_PROC}s) exceeded - re-running syncs." >&2 + continue + else + echo "INFO: Finished run $RUN at $(printf "%(%c)T")" >&2 + break + fi +done + +exit 0 diff --git a/do-mirroring b/mirror-wrapper similarity index 100% rename from do-mirroring rename to mirror-wrapper From 1ced1c286da6b21e94633b2cae7960929692438c Mon Sep 17 00:00:00 2001 From: Darren 'Tadgy' Austin Date: Thu, 21 May 2026 18:27:10 +0100 Subject: [PATCH 02/10] Add more output to mirror. --- mirror | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mirror b/mirror index c8436ab..9b7b530 100755 --- a/mirror +++ b/mirror @@ -13,7 +13,7 @@ MAX_RUNS="3" # Maximum number of runs IPV4="5.101.171.215" DATADIR="/data/depository" RSYNC_COMMAND="/usr/bin/rsync" -RSYNC_REMOTE_OPTIONS=("--address=$IPV4" '--no-motd' '--contimeout=30' '--timeout=60 -4 -aH' '--chmod=go-w,+rX' '--partial' '--partial-dir=.rsync-tmp' '--delete-delay' '--delay-updates') +RSYNC_REMOTE_OPTIONS=('-4' "--address=$IPV4" '--no-motd' '--contimeout=30' '--timeout=60' '-aH' '--chmod=go-w,+rX' '--partial' '--partial-dir=.rsync-tmp' '--delete-delay' '--delay-updates') RSYNC_LOCAL_OPTIONS=('-aH' '--chmod=go-w,+rX' '--partial' '--partial-dir=.rsync-tmp' '--delay-updates') # RSYNC_VERBOSE=('--verbose' '--human-readable') # RSYNC_VERBOSE=('--progress' '--verbose' '--stats' '--human-readable') @@ -313,6 +313,7 @@ for RUN in $(seq -s' ' 1 "$MAX_RUNS"); do done echo "INFO: Finished processing of $VAR at $(printf "%(%c)T")." >&2 done + echo "INFO: End of rsync run $RUN at $(printf "%(%c)T")" >&2 # Mirrors synced using lftp echo "INFO: Begining lftp run $RUN at $(printf "%(%c)T")" >&2 @@ -364,6 +365,8 @@ for RUN in $(seq -s' ' 1 "$MAX_RUNS"); do done echo "INFO: Finished processing of $VAR at $(printf "%(%c)T")." >&2 done + echo "INFO: End of lftp run $RUN at $(printf "%(%c)T")" >&2 + FINISH_TIME="$(printf "%(%s)T")" if (( (FINISH_TIME - START_TIME) > MAX_PROC )); then echo "WARNING: Max processing time (${MAX_PROC}s) exceeded - re-running syncs." >&2 From b5038f2291844e38a872418644429e225d783b6d Mon Sep 17 00:00:00 2001 From: Darren 'Tadgy' Austin Date: Thu, 21 May 2026 18:27:45 +0100 Subject: [PATCH 03/10] Update mirror-new-slackware-release.gpg. --- mirror-new-slackware-release.gpg | Bin 1153 -> 1178 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/mirror-new-slackware-release.gpg b/mirror-new-slackware-release.gpg index c76e350ee853ab10e5137fa32740df9669a16f29..992a9f4c068d05f0e2de6153fd454643028dae43 100644 GIT binary patch literal 1178 zcmV;L1ZDe-4Fm}T3S;&?Vltip(Erlv0blsICjO_an9vcQW^oE+)8-*IAo-SbFUX&K ztHnYKxWTm=`2RnBD};oKItY-H^~UpA%tdW~&r6X!9I3<5H(G-q^xky;uP*pA{^$_< zs9O%KO@|;3JXPGTPkoR&z9Xj(@Osa*Q$p=#49&#ZarrfOo6sL!*fy^|Fp2vr- zWQbo^-gCe$$VkzrW!3YlQ`Y`9`s~K(IH%J*ksZ0QFdr3H7Np zg}9YnL|KZ2qZ;LS+B}n zNb5xGaJ?|hiu8d2E4JjhVdaah$K#$lbyy1Yf^vjzOI`W+sQxLi$IKTv+K0+L6kHG8 z%`PY58GOiQpvjk+r{e_`03F9hbgufrYR*lI-{|BGgqplR-eRKm#h$a6EjcPyMtobC zq-VT6yd?s{$b4nAr=>Zhb@J74vn+#p`zi*yTz9_5@3D_HA|+#H%2z1o-RlE6hT45t zOzvOw2}JB{Ud-Y~hP_S8QkD69iWZ1Es!fUFB&js1Hp!Zsy6!kCrNlHv$wa@3;%2xV zi%o*p^|OSrWZO-C1uaIDh!V40oXY0;WktGgB1iTW%pyJwnK2+54`mRIKk4<(izc#2 z?^QHNc0{lAgtIvv#%$N>OzNzOgOZQlq<#O(` zU!8AzW<`Yi*;kQ~lCzjahmnm;!I%P=Go#42XnG6lqM9fpJgK s-s%%jam-8}(rWEUWL1%^a_yN|l0%lm2#*c2gR{4rLKkFc$K|Ij#6(_KH~;_u literal 1153 zcmV-{1b+LB4Fm}T2+T)O)sxmA6YtXM0Y+>)PjJS?!3imiEbag(t1m;>>{SfOr-x}t zyP&nZvJ>dyP(MA$Nw4zQ#l3L#ZE9pNsRFLXZ!9u+nSNtQg5XJ}Soazeh*V3|xZTg-mMW`}h%!qdg6Wy0D}J)HceofEq};}eJvUd=i6?;Kd|Dr1q;3zf`c zz6(B%EoCCPx5U=~Zo9xqGED+PjxD8;w77kyq(|sVWiER634`Zak zi5Eg^zu}4G9!)Q9ai4wAI@dXLEB{60fu6{KL!!4kAL(#*aU8B%9Sb5qlJC`{C#@Uo zz@x(UXk}rG{Cxu4bwgvYq4o$OZdWXd>g0$vK@>a7$--5Di1QmO7pLh0ux2f*ilTS4 zhau3Wx#R32<+^E|KhvEnkOqE0Te=Fzs2j5Ypruak+Vt#WLHF_!l7+_005I%4qpw9K zKKqt{yC07tvu+e%wXm(uSU{oX_-7ur^jQ6EbMfya103uv}$BouL4eo4K^tt06ApLpQ>7n;zY`j+jf7!V?sMto#^Dj>w{I%}91xszB ziu`{Zn@eK2#%%&Cl2D;;`aF>~+aYis3Ha#JwavdZaXR9gqsHhk{a+@};2GLnzfcFM zPk&Djz2Q%6jiIwi+t4&PNO1?h!>fQ8WPc(Ra~~rx#q3S8B6L3(Rhhdx zjQHu3@$RE%Z-}Ht4lpU%87d+=)j0J7Xn2r}SdW`5Zm3@*&D}o)8@WV*77K4A1oQ`* zYM2OxbhH2i$)Tnw3x1e7>?+&SApQ;q#M=XJOe6YasuOTC$`da+obg+HsYXWohJ@w# T4FrJ1Z>2%?bau3IdJSzzo Date: Thu, 21 May 2026 19:19:41 +0100 Subject: [PATCH 04/10] Update the mirror-wrapper script. --- mirror-wrapper | 79 ++++++++++++++++++++++---------------------------- 1 file changed, 35 insertions(+), 44 deletions(-) diff --git a/mirror-wrapper b/mirror-wrapper index 05ba79d..1bf5c96 100755 --- a/mirror-wrapper +++ b/mirror-wrapper @@ -1,6 +1,6 @@ #!/bin/bash -# Version: 0.1.0 -# Copyright (c) 2023: +# Version: 0.1.1 +# Copyright (c) 2023-2026: # Darren 'Tadgy' Austin # Licensed under the terms of the GNU General Public License version 3. # @@ -10,7 +10,25 @@ COMMAND="/opt/bin/mirror" LOGSDIR="/var/log/duplication/mirroring/$(printf "%(%Y/%m)T")" LOGFILE="$(printf "%(%Y%m%d-%H%M%S)T")-$$" -MIN_LOGFILE_SIZE="550" +MIN_LOGFILE_SIZE="550" # Used to prevent unnecessary emails - only messages over this size are sent. +# Where from/to to send emails. Comment for no emailing. +EMAIL_FROM="\"Server: ${HOSTNAME%%.*}\" " +EMAIL_TO=("Systems' Administrator ") + +# Functions +notify() { + if [[ -n "$EMAIL_FROM" ]] && (( "${#EMAIL_TO[@]}" != 0 )); then + printf "%s: %s\\n%s: %s\\n%s:\\n%s\\n" "Exit code" "$ERR" "Logfile" "$LOGSDIR/$LOGFILE.xz" "Output" "$(<"$LOGSDIR/$LOGFILE")" | mail -r "$EMAIL_FROM" -s "Mirroring $1" "${EMAIL_TO[@]}" >/dev/null 2>&1 || { + printf "%s: %s\\n" "${0##*/}" "mail command failed" >&2 + return 1 + } + else + printf "%s: %s\\n" "${0##*/}" "no sender and/or recipient configured for mail delivery" >&2 + return 1 + fi + + return 0 +} # Only allow one copy of the script to run at any time. # shellcheck disable=SC2154 @@ -27,21 +45,19 @@ if [[ "$FLOCK" != "$0" ]]; then fi fi -# Source the mail configuration. -source /etc/mail.conf "backups" 2>/dev/null || { - printf "%s: %s\\n" "${0##*/}" "Failed to source /etc/mail.conf" >&2 - exit 1 -} +# Logs are only for root. +umask 027 # Make sure the logs directory exists. -mkdir -p "$LOGSDIR" 2>/dev/null || { - printf "%s: %s\\n" "${0##*/}" "Failed to create logs directory" >&2 +# shellcheck disable=SC2174 +mkdir -p -m 750 "$LOGSDIR" 2>/dev/null || { + printf "%s: %s\\n" "${0##*/}" "Failed to create logs directory '$LOGSDIR'" >&2 exit 1 } # Make sure the command to do the work is runnable. [[ ! -x "$COMMAND" ]] && { - printf "%s: '%s' %s\\n" "${0##*/}" "$COMMAND" "is not executable" >&2 + printf "%s: %s\\n" "${0##*/}" "'$COMMAND' is not executable" >&2 exit 1 } @@ -49,42 +65,17 @@ mkdir -p "$LOGSDIR" 2>/dev/null || { "$COMMAND" "$@" >"$LOGSDIR/$LOGFILE" 2>&1 ERR="$?" +# Compress the log to save some space. +xz -9 "$LOGSDIR/$LOGFILE" 2>/dev/null || printf "%s: %s\\n" "${0##*/}" "failed to compress '$LOGSDIR/$LOGFILE'" >&2 + # Tell the sysadmin what went on. if (( "$ERR" == 0 )); then - # Only email a mirroring report if it's not a Slackware tree only hourly run. + # Only email a mirroring report if it's not the hourly Slackware tree only run. # The size of the log file determines if it gets emailed. - (( $(stat --printf="%s" "$LOGSDIR/$LOGFILE") > MIN_LOGFILE_SIZE )) && { - if [[ -n "${EMAIL_TO[*]}" ]]; then - mailx "${MAILX_ARGS[@]}" -S "from=$EMAIL_FROM" -s "Mirroring report" "${EMAIL_TO[@]}" <<-EOF 2>/dev/null || \ - { printf "%s: %s\\n" "${0##*/}" "mailx command failed" >&2; RET=1; } - Exit code: $ERR - Logfile: $LOGSDIR/$LOGFILE.gz - Output: - $(< "$LOGSDIR/$LOGFILE") - EOF - else - printf "%s: %s\\n" "${0##*/}" "no recipient configured for mail delivery" >&2 - RET=1 - fi - } + (( $(stat --printf="%s" "$LOGSDIR/$LOGFILE") > MIN_LOGFILE_SIZE )) && notify "report" + exit "$?" else # Mirroring failed, tell the admin. - [[ -x /opt/bin/pushover-client ]] && /opt/bin/pushover-client "mirroring" -p -1 -m "Mirroring failure" - if [[ -n "${EMAIL_TO[*]}" ]]; then - mailx "${MAILX_ARGS[@]}" -S "from=$EMAIL_FROM" -s "Mirroring failure" "${EMAIL_TO[@]}" <<-EOF 2>/dev/null || \ - { printf "%s: %s\\n" "${0##*/}" "mailx command failed" >&2; RET=1; } - Exit code: $ERR - Logfile: $LOGSDIR/$LOGFILE.gz - Output: - $(< "$LOGSDIR/$LOGFILE") - EOF - else - printf "%s: %s\\n" "${0##*/}" "no recipient configured for mail delivery" >&2 - RET=1 - fi + notify "failure" || [[ -x /opt/bin/pushover-client ]] && /opt/bin/pushover-client "mirroring" -p -1 -s "Mirroring failure" -m "Check log in email" + exit 1 fi - -# Compress the log to save some space. -gzip -9 "$LOGSDIR/$LOGFILE" 2>/dev/null || { printf "%s: %s '%s'\\n" "${0##*/}" "failed to compress" "$LOGSDIR/$LOGFILE" >&2; RET=1; } - -exit "${RET:-0}" From 51131002cf31f59399656e0855acb88ac58b3ec6 Mon Sep 17 00:00:00 2001 From: Darren 'Tadgy' Austin Date: Thu, 21 May 2026 19:21:23 +0100 Subject: [PATCH 05/10] Update version. --- mirror-wrapper | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mirror-wrapper b/mirror-wrapper index 1bf5c96..ab8b078 100755 --- a/mirror-wrapper +++ b/mirror-wrapper @@ -1,5 +1,5 @@ #!/bin/bash -# Version: 0.1.1 +# Version: 0.2.0 # Copyright (c) 2023-2026: # Darren 'Tadgy' Austin # Licensed under the terms of the GNU General Public License version 3. From be4d10eeba23723c5c3c1ea7a4c5aa349c2b2120 Mon Sep 17 00:00:00 2001 From: Darren 'Tadgy' Austin Date: Thu, 21 May 2026 19:23:48 +0100 Subject: [PATCH 06/10] Don't emit an error when there's no email sender and/or recipient. --- mirror-wrapper | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/mirror-wrapper b/mirror-wrapper index ab8b078..c810829 100755 --- a/mirror-wrapper +++ b/mirror-wrapper @@ -1,5 +1,5 @@ #!/bin/bash -# Version: 0.2.0 +# Version: 0.2.1 # Copyright (c) 2023-2026: # Darren 'Tadgy' Austin # Licensed under the terms of the GNU General Public License version 3. @@ -17,15 +17,12 @@ EMAIL_TO=("Systems' Administrator ") # Functions notify() { - if [[ -n "$EMAIL_FROM" ]] && (( "${#EMAIL_TO[@]}" != 0 )); then + [[ -n "$EMAIL_FROM" ]] && (( "${#EMAIL_TO[@]}" != 0 )) && { printf "%s: %s\\n%s: %s\\n%s:\\n%s\\n" "Exit code" "$ERR" "Logfile" "$LOGSDIR/$LOGFILE.xz" "Output" "$(<"$LOGSDIR/$LOGFILE")" | mail -r "$EMAIL_FROM" -s "Mirroring $1" "${EMAIL_TO[@]}" >/dev/null 2>&1 || { printf "%s: %s\\n" "${0##*/}" "mail command failed" >&2 return 1 } - else - printf "%s: %s\\n" "${0##*/}" "no sender and/or recipient configured for mail delivery" >&2 - return 1 - fi + } return 0 } From 2eb0c75bdcb92a6bcb014faf3c943b3ffe7e6a79 Mon Sep 17 00:00:00 2001 From: Darren 'Tadgy' Austin Date: Thu, 21 May 2026 19:30:34 +0100 Subject: [PATCH 07/10] In mirror-wrapper, compress the logs /after/ they are emailed. --- mirror-wrapper | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/mirror-wrapper b/mirror-wrapper index c810829..d9de8a5 100755 --- a/mirror-wrapper +++ b/mirror-wrapper @@ -62,17 +62,19 @@ mkdir -p -m 750 "$LOGSDIR" 2>/dev/null || { "$COMMAND" "$@" >"$LOGSDIR/$LOGFILE" 2>&1 ERR="$?" -# Compress the log to save some space. -xz -9 "$LOGSDIR/$LOGFILE" 2>/dev/null || printf "%s: %s\\n" "${0##*/}" "failed to compress '$LOGSDIR/$LOGFILE'" >&2 - # Tell the sysadmin what went on. if (( "$ERR" == 0 )); then # Only email a mirroring report if it's not the hourly Slackware tree only run. # The size of the log file determines if it gets emailed. (( $(stat --printf="%s" "$LOGSDIR/$LOGFILE") > MIN_LOGFILE_SIZE )) && notify "report" - exit "$?" + ERR="$?" else # Mirroring failed, tell the admin. notify "failure" || [[ -x /opt/bin/pushover-client ]] && /opt/bin/pushover-client "mirroring" -p -1 -s "Mirroring failure" -m "Check log in email" - exit 1 + ERR="1" fi + +# Compress the log to save some space. +xz -9 "$LOGSDIR/$LOGFILE" 2>/dev/null || printf "%s: %s\\n" "${0##*/}" "failed to compress '$LOGSDIR/$LOGFILE'" >&2 + +exit "$ERR" From 7f8c5df154754695e97c8243337770a48abe6183 Mon Sep 17 00:00:00 2001 From: Darren 'Tadgy' Austin Date: Thu, 21 May 2026 19:35:23 +0100 Subject: [PATCH 08/10] Update sbosrcarch wrapper script. --- sbosrcarch-wrapper | 80 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100755 sbosrcarch-wrapper diff --git a/sbosrcarch-wrapper b/sbosrcarch-wrapper new file mode 100755 index 0000000..55fe5fa --- /dev/null +++ b/sbosrcarch-wrapper @@ -0,0 +1,80 @@ +#!/bin/bash +# Version: 0.2.0 +# Copyright (c) 2023-2026: +# Darren 'Tadgy' Austin +# Licensed under the terms of the GNU General Public License version 3. +# +# Wrapper around /opt/bin/sbosrcarch to keep a log of the session, and email it. + +# Configuration. +COMMAND="/opt/bin/sbosrcarch" +LOGSDIR="/var/log/duplication/sbosrcarch/$(printf "%(%Y/%m)T")" +LOGFILE="$(printf "%(%Y%m%d-%H%M%S)T")-$$" +# Where from/to to send emails. Comment for no emailing. +EMAIL_FROM="\"Server: ${HOSTNAME%%.*}\" " +EMAIL_TO=("Systems' Administrator ") + +# Functions +notify() { + [[ -n "$EMAIL_FROM" ]] && (( "${#EMAIL_TO[@]}" != 0 )) && { + printf "%s: %s\\n%s: %s\\n%s:\\n%s\\n" "Exit code" "$ERR" "Logfile" "$LOGSDIR/$LOGFILE.xz" "Output" "$(<"$LOGSDIR/$LOGFILE")" | mail -r "$EMAIL_FROM" -s "SboSrcArch $1" "${EMAIL_TO[@]}" >/dev/null 2>&1 || { + printf "%s: %s\\n" "${0##*/}" "mail command failed" >&2 + return 1 + } + } + + return 0 +} + +# Only allow one copy of the script to run at any time. +# shellcheck disable=SC2154 +if [[ "$FLOCK" != "$0" ]]; then + # shellcheck disable=SC2093 + exec env FLOCK="$0" flock -E 10 -e -n "$0" "$0" "$@" + ERR="$?" + if (( ERR == 10 )); then + # File is locked, exit now. + exit 0 + elif (( ERR > 0 )); then + printf "%s: %s\\n" "${0##*/}" "flock execution error" >&2 + exit 1 + fi +fi + +# Logs are only for root. +umask 027 + +# Make sure the logs directory exists. +# shellcheck disable=SC2174 +mkdir -p -m 750 "$LOGSDIR" 2>/dev/null || { + printf "%s: %s\\n" "${0##*/}" "Failed to create logs directory '$LOGSDIR'" >&2 + exit 1 +} + +# Make sure the command to do the work is runnable. +[[ ! -x "$COMMAND" ]] && { + printf "%s: %s\\n" "${0##*/}" "'$COMMAND' is not executable" >&2 + exit 1 +} + +# Do the sbosrcarch work. +"$COMMAND" "${@:-update}" >"$LOGSDIR/$LOGFILE" 2>&1 +ERR="$?" +printf "\\n" >>"$LOGSDIR/$LOGFILE" +"$COMMAND" "status" >>"$LOGSDIR/$LOGFILE" 2>&1 + +# Tell the sysadmin what went on. +if (( "$ERR" == 0 )); then + # Send a report. + notify "report" + ERR="$?" +else + # sbosrcarch failed, tell the admin. + notify "failure" || [[ -x /opt/bin/pushover-client ]] && /opt/bin/pushover-client "mirroring" -p -1 -s "SBoSrcArch failure" -m "Check log in email" + ERR="1" +fi + +# Compress the log to save some space. +xz -9 "$LOGSDIR/$LOGFILE" 2>/dev/null || printf "%s: %s\\n" "${0##*/}" "failed to compress '$LOGSDIR/$LOGFILE'" >&2 + +exit "$ERR" From ce4a6f7e664779f485c0e503635593228f2bba0b Mon Sep 17 00:00:00 2001 From: Darren 'Tadgy' Austin Date: Thu, 21 May 2026 19:36:40 +0100 Subject: [PATCH 09/10] Remove do-sbosrcarch. --- do-sbosrcarch | 87 --------------------------------------------------- 1 file changed, 87 deletions(-) delete mode 100755 do-sbosrcarch diff --git a/do-sbosrcarch b/do-sbosrcarch deleted file mode 100755 index 32cb460..0000000 --- a/do-sbosrcarch +++ /dev/null @@ -1,87 +0,0 @@ -#!/bin/bash -# Version: 0.1.0 -# Copyright (c) 2023: -# Darren 'Tadgy' Austin -# Licensed under the terms of the GNU General Public License version 3. -# -# Wrapper around /opt/bin/sbosrcarch to keep a log of the session, and email it. - -# Configuration. -COMMAND="/opt/bin/sbosrcarch" -LOGSDIR="/var/log/duplication/sbosrcarch/$(printf "%(%Y/%m)T")" -LOGFILE="$(printf "%(%Y%m%d-%H%M%S)T")-$$" - -# Only allow one copy of the script to run at any time. -# shellcheck disable=SC2154 -if [[ "$FLOCK" != "$0" ]]; then - # shellcheck disable=SC2093 - exec env FLOCK="$0" flock -E 10 -e -n "$0" "$0" "$@" - ERR="$?" - if (( ERR == 10 )); then - # File is locked, exit now. - exit 0 - elif (( ERR > 0 )); then - printf "%s: %s\\n" "${0##*/}" "flock execution error" >&2 - exit 1 - fi -fi - -# Source the mail configuration. -source /etc/mail.conf "sbosrcarch" 2>/dev/null || { - printf "%s: %s\\n" "${0##*/}" "Failed to source /etc/mail.conf" >&2 - exit 1 -} - -# Make sure the logs directory exists. -mkdir -p "$LOGSDIR" 2>/dev/null || { - printf "%s: %s\\n" "${0##*/}" "Failed to create logs directory" >&2 - exit 1 -} - -# Make sure the command to do the work is runnable. -[[ ! -x "$COMMAND" ]] && { - printf "%s: '%s' %s\\n" "${0##*/}" "$COMMAND" "is not executable" >&2 - exit 1 -} - -# Do the sbosrcarch work. -"$COMMAND" "${@:-update}" >"$LOGSDIR/$LOGFILE" 2>&1 -ERR="$?" -printf "\\n" >>"$LOGSDIR/$LOGFILE" -"$COMMAND" "status" >>"$LOGSDIR/$LOGFILE" 2>&1 - -# Tell the sysadmin what went on. -if (( "$ERR" == 0 )); then - if [[ -n "${EMAIL_TO[*]}" ]]; then - mailx "${MAILX_ARGS[@]}" -S "from=$EMAIL_FROM" -s "SBosrcarch report" "${EMAIL_TO[@]}" <<-EOF 2>/dev/null || \ - { printf "%s: %s\\n" "${0##*/}" "mailx command failed" >&2; RET=1; } - Exit code: $ERR - Logfile: $LOGSDIR/$LOGFILE.gz - Output: - $(< "$LOGSDIR/$LOGFILE") - EOF - else - printf "%s: %s\\n" "${0##*/}" "no recipient configured for mail delivery" >&2 - RET=1 - fi -else - # Updating failed, tell the admin. - [[ -x "/opt/bin/pushover" ]] && CONFIG_FILE="mirroring" /opt/bin/pushover -T "SBosrcarch" -p '-1' -m "SBosrcarch failed" - if [[ -n "${EMAIL_TO[*]}" ]]; then - mailx "${MAILX_ARGS[@]}" -S "from=$EMAIL_FROM" -s "SBosrcarch failure" "${EMAIL_TO[@]}" <<-EOF 2>/dev/null || \ - { printf "%s: %s\\n" "${0##*/}" "mailx command failed" >&2; RET=1; } - Exit code: $ERR - Logfile: $LOGSDIR/$LOGFILE.gz - Output: - $(< "$LOGSDIR/$LOGFILE") - EOF - else - printf "%s: %s\\n" "${0##*/}" "no recipient configured for mail delivery" >&2 - RET=1 - fi -fi - -# Compress the log to save some space. -gzip -9 "$LOGSDIR/$LOGFILE" 2>/dev/null || { printf "%s: %s '%s'\\n" "${0##*/}" "failed to compress" "$LOGSDIR/$LOGFILE" >&2; RET=1; } - -exit "${RET:-0}" From 2a341ba7656a8cb756bd13df70314253a21af884 Mon Sep 17 00:00:00 2001 From: Darren 'Tadgy' Austin Date: Thu, 21 May 2026 20:43:08 +0100 Subject: [PATCH 10/10] Add copyright to mirror script. --- mirror | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/mirror b/mirror index 9b7b530..965c594 100755 --- a/mirror +++ b/mirror @@ -1,5 +1,8 @@ #!/bin/bash # Version: 0.4.0 +# Copyright (c) 2026: +# Darren 'Tadgy' Austin +# Licensed under the terms of the GNU General Public License version 3. # shellcheck disable=SC2034 # Script config. @@ -250,8 +253,9 @@ done } echo "INFO: Sync list: ${RSYNC_LIST[*]} ${LFTP_LIST[*]}" >&2 -START_TIME="$(printf "%(%s)T")" for RUN in $(seq -s' ' 1 "$MAX_RUNS"); do + START_TIME="$(printf "%(%s)T")" + # Mirrors synced using rsync echo "INFO: Begining rsync run $RUN at $(printf "%(%c)T")" >&2 for VAR in "${RSYNC_LIST[@]}"; do