Heavy modifications to {mirror,sbosrcarch}-wrapper scripts.

mirror

@@ -40,6 +40,14 @@ ALIEN_DESTDIR="people/alien/"
 ALIEN_OPTIONS=()
 ALIEN_FILTER=()
 
+# Mirror info for bonslack
+BONSLACK_TYPE="rsync"
+BONSLACK_MIRROR="bonslack.org"
+BONSLACK_MODULE="bonslack/"
+BONSLACK_DESTDIR="bonslack/"
+BONSLACK_OPTIONS=()
+BONSLACK_FILTER=('--exclude=/.status.html*' '--exclude=/.template' '--exclude=/HEADER.html' '--exclude=status.html*' '--exclude=etc/shadow')
+
 # Mirror info for people/alien-current-iso
 CURRENTISO_TYPE="rsync"
 CURRENTISO_MIRROR="bear.alienbase.nl"
@@ -114,7 +122,7 @@ SLACKTREE_FILTER=('--exclude=/*-iso' '--exclude=/slackware-pre*' '--exclude=/sla
 
 # Mirror info for Zenwalk
 ZENWALK_TYPE="rsync"
-ZENWALK_MIRROR="download.zenwalk.org"
+ZENWALK_MIRROR="ssh.zenwalk.org"
 ZENWALK_MODULE="zenwalk/"
 ZENWALK_DESTDIR="zenwalk/"
 ZENWALK_OPTIONS=()
@@ -135,7 +143,14 @@ SLACKARCHIVE_TYPE="rsync"
 SLACKARCHIVE_LOCAL="slackware/"
 SLACKARCHIVE_DESTDIR="cumulative/"
 SLACKARCHIVE_OPTIONS=()
-SLACKARCHIVE_FILTER=('--exclude=source/' '--include=/slackware-15.0' '--include=/slackware64-15.0' '--include=/slackware-current' '--include=/slackware64-current' '--exclude=/*')
+SLACKARCHIVE_FILTER=('--exclude=source/' '--include=/slackware-1[5-9].*' '--include=/slackware-[2-9][0-9].*' '--include=/slackware64-1[5-9].*' '--include=/slackware64-[2-9][0-9].*' '--include=/slackware-current' '--include=/slackware64-current' '--exclude=/*')
+
+# Mirror info for SlackwareARM cumulative tree.
+SLACKARMARCHIVE_TYPE="rsync"
+SLACKARMARCHIVE_LOCAL="slackwarearm/"
+SLACKARMARCHIVE_DESTDIR="cumulative/"
+SLACKARMARCHIVE_OPTIONS=()
+SLACKARMARCHIVE_FILTER=('--exclude=source/' '--include=/slackwareaarch64-*' '--exclude=/*')
 
 #
 # lftp example
@@ -239,6 +254,7 @@ done
 
   # Other remotes.
   (( ${SYNC_ALL:-0} == 1 )) || [[ $(printf "%(%H)T") =~ (02|08|14|20) ]] && RSYNC_LIST+=('ALIEN')
+  (( ${SYNC_ALL:-0} == 1 )) || [[ $(printf "%(%H)T") =~ (02|08|14|20) ]] && RSYNC_LIST+=('BONSLACK')
   (( ${SYNC_ALL:-0} == 1 )) || [[ $(printf "%(%H)T") =~ (02|08|14|20) ]] && RSYNC_LIST+=('CURRENTISO')
   (( ${SYNC_ALL:-0} == 1 )) || [[ $(printf "%(%H)T") =~ (02|08|14|20) ]] && RSYNC_LIST+=('KTOWN')
   (( ${SYNC_ALL:-0} == 1 )) || [[ $(printf "%(%H)T") =~ (02|08|14|20) ]] && RSYNC_LIST+=('LIVESLAK')
@@ -251,7 +267,8 @@ done
 
   # Locals.
   (( ${SYNC_ALL:-0} == 1 )) || [[ $(printf "%(%H)T") =~ .* ]] && RSYNC_LIST+=('SLACKARCHIVE')
-  (( ${SYNC_ALL:-0} == 1 )) || [[ $(printf "%(%H)T") =~ (02|08|14|20) ]] && RSYNC_LIST+=('MULTILIBARCHIVE')
+  (( ${SYNC_ALL:-0} == 1 )) || [[ $(printf "%(%H)T") =~ .* ]] && RSYNC_LIST+=('SLACKARMARCHIVE')
+  (( ${SYNC_ALL:-0} == 1 )) || [[ $(printf "%(%H)T") =~ .* ]] && RSYNC_LIST+=('MULTILIBARCHIVE')
 }
 (( ${#LFTP_LIST[@]} == 0 )) && {
   # Nothing to sync here currently.
@@ -260,10 +277,10 @@ done
 
 # Mirror a new Slackware version.
 [[ -v NEW_SLACK_VERSION ]] && {
-  "$DEBUG" "DEBUG: Checking for new Slackware release at $(date)." >&2
+  "${DEBUG:-:}" "DEBUG: Checking for new Slackware release at $(date)." >&2
   #export NEW_SLACK_VERSION
   /opt/bin/mirror-new-slackware-release "$NEW_SLACK_VERSION"
-  "$DEBUG" "DEBUG: Finished checking for new release at $(date)." >&2
+  "${DEBUG:-:}" "DEBUG: Finished checking for new release at $(date)." >&2
 }
 
 # Exit now if there's nothing to sync.
@@ -313,7 +330,7 @@ for RUN in $(seq -s' ' 1 "$MAX_RUNS"); do
     fi
 
     echo "INFO: Begining processing of '$VAR' (run $RUN) at $(printf "%(%c)T")." >&2
-    "$DEBUG" "DEBUG: Command line: RSYNC_PASSWORD=\"XXX\" $RSYNC_COMMAND ${!RSYNC_OPTS} ${RSYNC_VERBOSE[*]} ${!OPTIONS} ${!FILTER} $SRC ${!DESTDIR}" >&2
+    "${DEBUG:-:}" "DEBUG: Command line: RSYNC_PASSWORD=\"XXX\" $RSYNC_COMMAND ${!RSYNC_OPTS} ${RSYNC_VERBOSE[*]} ${!OPTIONS} ${!FILTER} $SRC ${!DESTDIR}" >&2
     while :; do
       RSYNC_PASSWORD="${!PASSWORD}" "$RSYNC_COMMAND" "${!RSYNC_OPTS}" "${RSYNC_VERBOSE[@]}" "${!OPTIONS}" "${!FILTER}" "$SRC" "${!DESTDIR}" >&2
       ERR_CODE="$?"
@@ -365,7 +382,7 @@ for RUN in $(seq -s' ' 1 "$MAX_RUNS"); do
     fi
 
     echo "INFO: Begining processing of '$VAR' (run $RUN) at $(printf "%(%c)T")." >&2
-    "$DEBUG" "DEBUG: Command line: $LFTP_COMMAND -c \"$LFTP_OPTIONS ${LFTP_VERBOSE[*]} ${!FILTER} ${!MIRROR}${!MODULE} ${!DESTDIR}\"" >&2
+    "${DEBUG:-:}" "DEBUG: Command line: $LFTP_COMMAND -c \"$LFTP_OPTIONS ${LFTP_VERBOSE[*]} ${!FILTER} ${!MIRROR}${!MODULE} ${!DESTDIR}\"" >&2
     while :; do
       "$LFTP_COMMAND" -c "$LFTP_OPTIONS ${LFTP_VERBOSE[*]} ${!FILTER} ${!MIRROR}/${!MODULE} ${!DESTDIR}" >&2
       ERR_CODE="$?"

mirror-wrapper

@@ -8,15 +8,16 @@ MIRRORING_USER='SLACKWAREUKINT\mirroring'
 DEPOSITORY_GROUP='SLACKWAREUKINT\depository'
 LOGSDIR="/var/log/duplication/mirroring/$(printf "%(%Y/%m)T")"
 LOGFILE="$(printf "%(%Y%m%d-%H%M%S)T")-$$"
-MIN_LOGFILE_SIZE="845"		# Used to prevent unnecessary emails - only messages over this size are sent.
+MIN_LOGFILE_SIZE="1210"		# Used to prevent unnecessary emails - only messages over this size are sent.
 # Where from/to to send emails.  Comment for no emailing.
 EMAIL_FROM="\"Server: ${HOSTNAME%%.*}\" <noreply@slackware.uk.net>"
 EMAIL_TO=("Systems' Administrator <sysadmin@slackware.uk>")
 
-# Functions
+# Functions.
 notify() {
   [[ -n "$EMAIL_FROM" ]] && (( "${#EMAIL_TO[@]}" != 0 )) && {
-    printf "%s: %s\\n%s: %s\\n%s:\\n%s\\n" "Exit code" "$ERR" "Logfile" "$LOGSDIR/$LOGFILE.xz" "Output" "$(cat "$LOGSDIR/$LOGFILE" | sed -re 's/^/  /g')" | mail -r "$EMAIL_FROM" -s "Mirroring $1" "${EMAIL_TO[@]}" >/dev/null 2>&1 || {
+    # shellcheck disable=SC2094
+    printf "%s: %s\\n%s: %s\\n%s:\\n%s\\n" "Exit code" "$ERR" "Logfile" "$LOGSDIR/$LOGFILE.xz" "Output" "$(sed -re 's/^/  /g' <"$LOGSDIR/$LOGFILE")" | mail -r "$EMAIL_FROM" -s "Mirroring $1" "${EMAIL_TO[@]}" >>"$LOGSDIR/$LOGFILE" 2>&1 || {
       printf "%s: %s\\n" "${0##*/}" "mail command failed" >&2
       return 1
     }
@@ -31,7 +32,7 @@ umask 027
 # Only run for the configured mirroring user.
 [[ "$(whoami)" != "$MIRRORING_USER" ]] && {
   printf "%s: %s\\n" "${0##*/}" "must be run by the '$MIRRORING_USER' user" >&2
-  printf "%s: %s: %s\\n" "${0##*/}" "to run manually:" "su '$MIRRORING_USER' $0 $@" >&2
+  printf "%s: %s: %s\\n" "${0##*/}" "to run manually:" "su '$MIRRORING_USER' -c \"$0 $*\"" >&2
   exit 1
 }
 
@@ -63,45 +64,64 @@ if [[ "$FLOCK" != "$0" ]]; then
   fi
 fi
 
-# Drop the caches as sometimes old user/groups are cached.
+ERR=0
-sudo /opt/sbin/drop-caches
 
 # Make sure there's no errant ownerships.
-printf "%s: %s\\n" "INFO" "Scanning depository directory for correct ownerships/permissions" | tee "$LOGSDIR/$LOGFILE"
+[[ $(printf "%(%H)T") =~ (02|08|14|20) ]] && {
-GLOBIGNORE="$DEPOSITORY/sbosrcarch"
+  printf "%s: %s\\n" "INFO" "Scanning depository directory for correct ownerships/permissions" | tee "$LOGSDIR/$LOGFILE"
-WRONG="$(find "$DEPOSITORY/"* \( \
+
-    \( \( -type b -o -type c -o -type p -o -type s \) -a \! -path "$DEPOSITORY/slackware/slackware-3.0/disk1/live/dev/*" \) -o \
+  # Drop the caches as sometimes old user/groups are cached.
-    \( -type l -a \! -perm 777 \) -o \
+  sudo /opt/sbin/drop-caches
-    \( -type d -a \! -perm 755 -a \! -perm 2755 \) -o \
+
-    \( -type f -a \! -perm 644 -a \! -perm  755 \) -o \
+  # Scan files.
-    \( \! -user "$MIRRORING_USER" \) -o \
+  GLOBIGNORE="$DEPOSITORY/sbosrcarch"
-    \( \! -group "$DEPOSITORY_GROUP" \) \
+  WRONG="$(find "$DEPOSITORY/"* -not -name .rsync-tmp -a \( \
-    \) -a -printf "%u:%g\t%M\t%p\n")"
+      \( \( -type b -o -type c -o -type p -o -type s \) -a -not -path "$DEPOSITORY/slackware/slackware-3.0/disk1/live/dev/*" \) -o \
-if [[ -n "$WRONG" ]]; then
+      \( -type l -a -not -perm 777 \) -o \
-  printf "%s: %s\\n" "ERROR" "This sync has been aborted!" | tee -a "$LOGSDIR/$LOGFILE"
+      \( -type d -a -not -perm 755 -a -not -perm 2755 \) -o \
-  printf "%s: %s\\n" "WARNING" "The following paths have erronious ownerships/permissions:" | tee -a "$LOGSDIR/$LOGFILE"
+      \( -type f -a -not -perm 644 -a -not -perm  755 \) -o \
-  printf "%s\\n" "$WRONG" | sed -re 's/^/  /g' | tee -a "$LOGSDIR/$LOGFILE"
+      \( -not -user "$MIRRORING_USER" \) -o \
-  ERR=-1
+      \( -not -group "$DEPOSITORY_GROUP" \) \
-else
+      \) -a -printf "%u:%g\t%M\t%p\n")"
-  printf "%s: %s\\n" "INFO" "No ownership/permission issues found" | tee -a "$LOGSDIR/$LOGFILE"
+
-  # Do the mirroring work.
+  # Report findings.
+  if [[ -n "$WRONG" ]]; then
+    printf "%s: %s\\n" "ERROR" "This sync has been aborted!" | tee -a "$LOGSDIR/$LOGFILE"
+    printf "%s: %s\\n" "ERROR" "The following paths have erronious ownerships/permissions:" | tee -a "$LOGSDIR/$LOGFILE"
+    printf "%s\\n" "$WRONG" | sed -re 's/^/  /g' | tee -a "$LOGSDIR/$LOGFILE"
+    ERR=-1
+  else
+    printf "%s: %s\\n" "INFO" "No ownership/permission issues found" | tee -a "$LOGSDIR/$LOGFILE"
+  fi
+}
+
+# Do the mirroring work.
+(( ERR == 0 )) && {
   umask 022
   export MIRRORING_USER
-  printf "%s: %s: %s\\n" "INFO" "Beginning sync command" "sg \"$DEPOSITORY_GROUP\" -c \"$COMMAND $@\"" | tee -a "$LOGSDIR/$LOGFILE"
+  printf "%s: %s: %s\\n" "INFO" "Beginning sync command" "sg \"$DEPOSITORY_GROUP\" -c \"$COMMAND $*\"" | tee -a "$LOGSDIR/$LOGFILE"
-  sg "$DEPOSITORY_GROUP" -c "$COMMAND $@" 2>&1 | tee -a "$LOGSDIR/$LOGFILE"
+  sg "$DEPOSITORY_GROUP" -c "$COMMAND $* 2>&1" | tee -a "$LOGSDIR/$LOGFILE"
   ERR="$?"
-fi
+}
 
 # Tell the sysadmin what went on.
-if (( "$ERR" == 0 )); then
+if (( ERR == 0 )); then
   # Only email a mirroring report if it's not the hourly Slackware tree only run.
   # The size of the log file determines if it gets emailed.
-  (( $(stat --printf="%s" "$LOGSDIR/$LOGFILE") > MIN_LOGFILE_SIZE )) && notify "report"
+  (( $(stat --printf="%s" "$LOGSDIR/$LOGFILE") > MIN_LOGFILE_SIZE )) && {
-  ERR="$?"
+    notify "report" || {
+      [[ -x /opt/bin/pushover-client ]] && /opt/bin/pushover-client "mirroring" -p -1 -s "notify() failure" -m "Check logfile: $LOGSDIR/$LOGFILE"
+    }
+  }
 else
-  # Mirroring failed, tell the admin.
-  notify "failure" || [[ -x /opt/bin/pushover-client ]] && /opt/bin/pushover-client "mirroring" -p -1 -s "Mirroring failure" -m "Check log in email"
   ERR="1"
+
+  # Mirroring failed, tell the admin.
+  if notify "failure"; then
+    [[ -x /opt/bin/pushover-client ]] && /opt/bin/pushover-client "mirroring" -p -1 -s "Mirroring failure" -m "Check email / logfile: $LOGSDIR/$LOGFILE"
+  else
+    [[ -x /opt/bin/pushover-client ]] && /opt/bin/pushover-client "mirroring" -p -1 -s "notify() failure" -m "Check logfile: $LOGSDIR/$LOGFILE"
+  fi
 fi
 
 # Compress the log to save some space.

sbosrcarch-wrapper

@@ -16,7 +16,8 @@ EMAIL_CC=("Urchlay <urchlay@slackware.uk>")
 # Functions
 notify() {
   [[ -n "$EMAIL_FROM" ]] && (( "${#EMAIL_TO[@]}" != 0 )) && {
-    printf "%s: %s\\n%s: %s\\n%s:\\n%s\\n" "Exit code" "$ERR" "Logfile" "$LOGSDIR/$LOGFILE.xz" "Output" "$(cat "$LOGSDIR/$LOGFILE" | sed -re 's/^/  /g')" | mail -r "$EMAIL_FROM" -s "SboSrcArch $1" "$(for i in $(seq -s ' ' 0 $(( ${#EMAIL_CC[@]} -1 ))); do echo ${EMAIL_CC[$i]:+-c "${EMAIL_CC[$i]}"}; done)" "${EMAIL_TO[@]}" >/dev/null 2>&1 || {
+    # shellcheck disable=SC2094
+    printf "%s: %s\\n%s: %s\\n%s:\\n%s\\n" "Exit code" "$ERR" "Logfile" "$LOGSDIR/$LOGFILE.xz" "Output" "$(sed -re 's/^/  /g' <"$LOGSDIR/$LOGFILE")" | mail -r "$EMAIL_FROM" -s "SboSrcArch $1" "$(for i in $(seq -s ' ' 0 $(( ${#EMAIL_CC[@]} -1 ))); do echo ${EMAIL_CC[$i]:+-c "${EMAIL_CC[$i]}"}; done)" "${EMAIL_TO[@]}" >>"$LOGSDIR/$LOGFILE"  2>&1 || {
       printf "%s: %s\\n" "${0##*/}" "mail command failed" >&2
       return 1
     }
@@ -31,7 +32,7 @@ umask 027
 # Only run for the configured sbosrcarch user.
 [[ "$(whoami)" != "$SBOSRCARCH_USER" ]] && {
   printf "%s: %s\\n" "${0##*/}" "must be run by the '$SBOSRCARCH_USER' user" >&2
-  printf "%s: %s: %s\\n" "${0##*/}" "to run manually:" "su '$SBOSRCARCH_USER' $0 $@" >&2
+  printf "%s: %s: %s\\n" "${0##*/}" "to run manually:" "su \"$SBOSRCARCH_USER\" -c \"$0 $*\"" >&2
   exit 1
 }
 
@@ -63,45 +64,66 @@ if [[ "$FLOCK" != "$0" ]]; then
   fi
 fi
 
-# Drop the caches as sometimes old user/groups are cached.
+ERR=0
-sudo /opt/sbin/drop-caches
 
 # Make sure there's no errant ownerships.
-printf "%s: %s\\n" "INFO" "Scanning sbosrcarch directory for correct ownerships/permissions" | tee "$LOGSDIR/$LOGFILE"
+[[ $(printf "%(%H)T") == "04" ]] && {
-WRONG="$(find "$DEPOSITORY/" \( \
+  printf "%s: %s\\n" "INFO" "Scanning sbosrcarch directory for correct ownerships/permissions" | tee "$LOGSDIR/$LOGFILE"
-    \( -type b -o -type c -o -type p -o -type s \) -o \
+
-    \( -type l -a \! -perm 777 \) -o \
+  # Drop the caches as sometimes old user/groups are cached.
-    \( -type d -a \! -perm 755 -a \! -perm 2755 \) -o \
+  sudo /opt/sbin/drop-caches
-    \( -type f -a \! -perm 644 -a \! -perm  755 \) -o \
+
-    \( \! -user "$SBOSRCARCH_USER" \) -o \
+  # Scan files.
-    \( \! -group "$DEPOSITORY_GROUP" \) \
+  WRONG="$(find "$DEPOSITORY/" \( \
-    \) -a -printf "%u:%g\t%M\t%p\n")"
+      \( -type b -o -type c -o -type p -o -type s \) -o \
-if [[ -n "$WRONG" ]]; then
+      \( -type l -a \! -perm 777 \) -o \
-  printf "%s: %s\\n" "ERROR" "This run has been aborted!" | tee -a "$LOGSDIR/$LOGFILE"
+      \( -type d -a \! -perm 755 -a \! -perm 2755 \) -o \
-  printf "%s: %s\\n" "WARNING" "The following paths have erronious ownerships/permissions:" | tee -a "$LOGSDIR/$LOGFILE"
+      \( -type f -a \! -perm 644 -a \! -perm  755 \) -o \
-  printf "%s\\n" "$WRONG" | sed -re 's/^/  /g' | tee -a "$LOGSDIR/$LOGFILE"
+      \( \! -user "$SBOSRCARCH_USER" \) -o \
-  ERR=-1
+      \( \! -group "$DEPOSITORY_GROUP" \) \
-else
+      \) -a -printf "%u:%g\t%M\t%p\n")"
-  printf "%s: %s\\n" "INFO" "No ownership/permission issues found" | tee -a "$LOGSDIR/$LOGFILE"
+
-  # Do the sbosrcarch work.
+  # Report findings.
+  if [[ -n "$WRONG" ]]; then
+    printf "%s: %s\\n" "ERROR" "This run has been aborted!" | tee -a "$LOGSDIR/$LOGFILE"
+    printf "%s: %s\\n" "ERROR" "The following paths have erronious ownerships/permissions:" | tee -a "$LOGSDIR/$LOGFILE"
+    printf "%s\\n" "$WRONG" | sed -re 's/^/  /g' | tee -a "$LOGSDIR/$LOGFILE"
+    ERR=-1
+  else
+    printf "%s: %s\\n" "INFO" "No ownership/permission issues found" | tee -a "$LOGSDIR/$LOGFILE"
+  fi
+}
+
+# Do the sbosrcarch work.
+(( ERR == 0 )) && {
   umask 022
-  printf "%s: %s: %s\\n" "INFO" "Beginning sbosrcarch run" "sg \"$DEPOSITORY_GROUP\" -c \"$COMMAND ${1:-update}${2:+ $2}${3:+ $3}\"" | tee -a "$LOGSDIR/$LOGFILE"
+
-  TMP_OUTPUT="$(sg "$DEPOSITORY_GROUP" -c "$COMMAND ${1:-update}${2:+ $2}${3:+ $3} 2>&1")"
+  printf "%s: %s: %s\\n" "INFO" "Beginning sbosrcarch run" "sg \"$DEPOSITORY_GROUP\" -c \"$COMMAND ${*:-update}\"" | tee -a "$LOGSDIR/$LOGFILE"
+  TMP_OUTPUT="$(sg "$DEPOSITORY_GROUP" -c "$COMMAND ${*:-update} 2>&1")"
   ERR="$?"
-  sg "$DEPOSITORY_GROUP" -c "$COMMAND status 2>&1" | tee -a "$LOGSDIR/$LOGFILE"
+
+  [[ "$*" != *status* ]] && {
+    sg "$DEPOSITORY_GROUP" -c "$COMMAND status 2>&1" | tee -a "$LOGSDIR/$LOGFILE"
+    (( ERR += PIPESTATUS[0] ))
+  }
+
   printf "\\n%s\\n" "$TMP_OUTPUT" | tee -a "$LOGSDIR/$LOGFILE"
-  (( ERR += $? ))
+}
-fi
 
 # Tell the sysadmin what went on.
 if (( "$ERR" == 0 )); then
   # Send a report.
-  notify "report"
+  notify "report" || {
-  ERR="$?"
+    [[ -x /opt/bin/pushover-client ]] && /opt/bin/pushover-client "mirroring" -p -1 -s "notify() failure" -m "Check logfile: $LOGSDIR/$LOGFILE.xz"
+  }
 else
-  # sbosrcarch failed, tell the admin.
-  notify "failure" || [[ -x /opt/bin/pushover-client ]] && /opt/bin/pushover-client "mirroring" -p -1 -s "SBoSrcArch failure" -m "Check log in email"
   ERR="1"
+  # sbosrcarch failed, tell the admin.
+  if notify "failure"; then
+    [[ -x /opt/bin/pushover-client ]] && /opt/bin/pushover-client "mirroring" -p -1 -s "SBoSrcArch failure" -m "Check email / logfile: $LOGSDIR/$LOGFILE.xz"
+  else
+    [[ -x /opt/bin/pushover-client ]] && /opt/bin/pushover-client "mirroring" -p -1 -s "notify() failure" -m "Check logfile: $LOGSDIR/$LOGFILE.xz"
+  fi
 fi
 
 # Compress the log to save some space.