diff --git a/mirror b/mirror index 0be86cd..2159951 100755 --- a/mirror +++ b/mirror @@ -40,14 +40,6 @@ ALIEN_DESTDIR="people/alien/" ALIEN_OPTIONS=() ALIEN_FILTER=() -# Mirror info for bonslack -BONSLACK_TYPE="rsync" -BONSLACK_MIRROR="bonslack.org" -BONSLACK_MODULE="bonslack/" -BONSLACK_DESTDIR="bonslack/" -BONSLACK_OPTIONS=() -BONSLACK_FILTER=('--exclude=/.status.html*' '--exclude=/.template' '--exclude=/HEADER.html' '--exclude=status.html*' '--exclude=etc/shadow') - # Mirror info for people/alien-current-iso CURRENTISO_TYPE="rsync" CURRENTISO_MIRROR="bear.alienbase.nl" @@ -122,7 +114,7 @@ SLACKTREE_FILTER=('--exclude=/*-iso' '--exclude=/slackware-pre*' '--exclude=/sla # Mirror info for Zenwalk ZENWALK_TYPE="rsync" -ZENWALK_MIRROR="ssh.zenwalk.org" +ZENWALK_MIRROR="download.zenwalk.org" ZENWALK_MODULE="zenwalk/" ZENWALK_DESTDIR="zenwalk/" ZENWALK_OPTIONS=() @@ -143,14 +135,7 @@ SLACKARCHIVE_TYPE="rsync" SLACKARCHIVE_LOCAL="slackware/" SLACKARCHIVE_DESTDIR="cumulative/" SLACKARCHIVE_OPTIONS=() -SLACKARCHIVE_FILTER=('--exclude=source/' '--include=/slackware-1[5-9].*' '--include=/slackware-[2-9][0-9].*' '--include=/slackware64-1[5-9].*' '--include=/slackware64-[2-9][0-9].*' '--include=/slackware-current' '--include=/slackware64-current' '--exclude=/*') - -# Mirror info for SlackwareARM cumulative tree. -SLACKARMARCHIVE_TYPE="rsync" -SLACKARMARCHIVE_LOCAL="slackwarearm/" -SLACKARMARCHIVE_DESTDIR="cumulative/" -SLACKARMARCHIVE_OPTIONS=() -SLACKARMARCHIVE_FILTER=('--exclude=source/' '--include=/slackwareaarch64-*' '--exclude=/*') +SLACKARCHIVE_FILTER=('--exclude=source/' '--include=/slackware-15.0' '--include=/slackware64-15.0' '--include=/slackware-current' '--include=/slackware64-current' '--exclude=/*') # # lftp example @@ -254,7 +239,6 @@ done # Other remotes. (( ${SYNC_ALL:-0} == 1 )) || [[ $(printf "%(%H)T") =~ (02|08|14|20) ]] && RSYNC_LIST+=('ALIEN') - (( ${SYNC_ALL:-0} == 1 )) || [[ $(printf "%(%H)T") =~ (02|08|14|20) ]] && RSYNC_LIST+=('BONSLACK') (( ${SYNC_ALL:-0} == 1 )) || [[ $(printf "%(%H)T") =~ (02|08|14|20) ]] && RSYNC_LIST+=('CURRENTISO') (( ${SYNC_ALL:-0} == 1 )) || [[ $(printf "%(%H)T") =~ (02|08|14|20) ]] && RSYNC_LIST+=('KTOWN') (( ${SYNC_ALL:-0} == 1 )) || [[ $(printf "%(%H)T") =~ (02|08|14|20) ]] && RSYNC_LIST+=('LIVESLAK') @@ -267,8 +251,7 @@ done # Locals. (( ${SYNC_ALL:-0} == 1 )) || [[ $(printf "%(%H)T") =~ .* ]] && RSYNC_LIST+=('SLACKARCHIVE') - (( ${SYNC_ALL:-0} == 1 )) || [[ $(printf "%(%H)T") =~ .* ]] && RSYNC_LIST+=('SLACKARMARCHIVE') - (( ${SYNC_ALL:-0} == 1 )) || [[ $(printf "%(%H)T") =~ .* ]] && RSYNC_LIST+=('MULTILIBARCHIVE') + (( ${SYNC_ALL:-0} == 1 )) || [[ $(printf "%(%H)T") =~ (02|08|14|20) ]] && RSYNC_LIST+=('MULTILIBARCHIVE') } (( ${#LFTP_LIST[@]} == 0 )) && { # Nothing to sync here currently. @@ -277,10 +260,10 @@ done # Mirror a new Slackware version. [[ -v NEW_SLACK_VERSION ]] && { - "${DEBUG:-:}" "DEBUG: Checking for new Slackware release at $(date)." >&2 + "$DEBUG" "DEBUG: Checking for new Slackware release at $(date)." >&2 #export NEW_SLACK_VERSION /opt/bin/mirror-new-slackware-release "$NEW_SLACK_VERSION" - "${DEBUG:-:}" "DEBUG: Finished checking for new release at $(date)." >&2 + "$DEBUG" "DEBUG: Finished checking for new release at $(date)." >&2 } # Exit now if there's nothing to sync. @@ -330,7 +313,7 @@ for RUN in $(seq -s' ' 1 "$MAX_RUNS"); do fi echo "INFO: Begining processing of '$VAR' (run $RUN) at $(printf "%(%c)T")." >&2 - "${DEBUG:-:}" "DEBUG: Command line: RSYNC_PASSWORD=\"XXX\" $RSYNC_COMMAND ${!RSYNC_OPTS} ${RSYNC_VERBOSE[*]} ${!OPTIONS} ${!FILTER} $SRC ${!DESTDIR}" >&2 + "$DEBUG" "DEBUG: Command line: RSYNC_PASSWORD=\"XXX\" $RSYNC_COMMAND ${!RSYNC_OPTS} ${RSYNC_VERBOSE[*]} ${!OPTIONS} ${!FILTER} $SRC ${!DESTDIR}" >&2 while :; do RSYNC_PASSWORD="${!PASSWORD}" "$RSYNC_COMMAND" "${!RSYNC_OPTS}" "${RSYNC_VERBOSE[@]}" "${!OPTIONS}" "${!FILTER}" "$SRC" "${!DESTDIR}" >&2 ERR_CODE="$?" @@ -382,7 +365,7 @@ for RUN in $(seq -s' ' 1 "$MAX_RUNS"); do fi echo "INFO: Begining processing of '$VAR' (run $RUN) at $(printf "%(%c)T")." >&2 - "${DEBUG:-:}" "DEBUG: Command line: $LFTP_COMMAND -c \"$LFTP_OPTIONS ${LFTP_VERBOSE[*]} ${!FILTER} ${!MIRROR}${!MODULE} ${!DESTDIR}\"" >&2 + "$DEBUG" "DEBUG: Command line: $LFTP_COMMAND -c \"$LFTP_OPTIONS ${LFTP_VERBOSE[*]} ${!FILTER} ${!MIRROR}${!MODULE} ${!DESTDIR}\"" >&2 while :; do "$LFTP_COMMAND" -c "$LFTP_OPTIONS ${LFTP_VERBOSE[*]} ${!FILTER} ${!MIRROR}/${!MODULE} ${!DESTDIR}" >&2 ERR_CODE="$?" diff --git a/mirror-wrapper b/mirror-wrapper index 20dcc4e..f8ed6e0 100755 --- a/mirror-wrapper +++ b/mirror-wrapper @@ -8,16 +8,15 @@ MIRRORING_USER='SLACKWAREUKINT\mirroring' DEPOSITORY_GROUP='SLACKWAREUKINT\depository' LOGSDIR="/var/log/duplication/mirroring/$(printf "%(%Y/%m)T")" LOGFILE="$(printf "%(%Y%m%d-%H%M%S)T")-$$" -MIN_LOGFILE_SIZE="1210" # Used to prevent unnecessary emails - only messages over this size are sent. +MIN_LOGFILE_SIZE="845" # Used to prevent unnecessary emails - only messages over this size are sent. # Where from/to to send emails. Comment for no emailing. EMAIL_FROM="\"Server: ${HOSTNAME%%.*}\" " EMAIL_TO=("Systems' Administrator ") -# Functions. +# Functions notify() { [[ -n "$EMAIL_FROM" ]] && (( "${#EMAIL_TO[@]}" != 0 )) && { - # shellcheck disable=SC2094 - printf "%s: %s\\n%s: %s\\n%s:\\n%s\\n" "Exit code" "$ERR" "Logfile" "$LOGSDIR/$LOGFILE.xz" "Output" "$(sed -re 's/^/ /g' <"$LOGSDIR/$LOGFILE")" | mail -r "$EMAIL_FROM" -s "Mirroring $1" "${EMAIL_TO[@]}" >>"$LOGSDIR/$LOGFILE" 2>&1 || { + printf "%s: %s\\n%s: %s\\n%s:\\n%s\\n" "Exit code" "$ERR" "Logfile" "$LOGSDIR/$LOGFILE.xz" "Output" "$(cat "$LOGSDIR/$LOGFILE" | sed -re 's/^/ /g')" | mail -r "$EMAIL_FROM" -s "Mirroring $1" "${EMAIL_TO[@]}" >/dev/null 2>&1 || { printf "%s: %s\\n" "${0##*/}" "mail command failed" >&2 return 1 } @@ -32,7 +31,7 @@ umask 027 # Only run for the configured mirroring user. [[ "$(whoami)" != "$MIRRORING_USER" ]] && { printf "%s: %s\\n" "${0##*/}" "must be run by the '$MIRRORING_USER' user" >&2 - printf "%s: %s: %s\\n" "${0##*/}" "to run manually:" "su '$MIRRORING_USER' -c \"$0 $*\"" >&2 + printf "%s: %s: %s\\n" "${0##*/}" "to run manually:" "su '$MIRRORING_USER' $0 $@" >&2 exit 1 } @@ -64,64 +63,45 @@ if [[ "$FLOCK" != "$0" ]]; then fi fi -ERR=0 +# Drop the caches as sometimes old user/groups are cached. +sudo /opt/sbin/drop-caches # Make sure there's no errant ownerships. -[[ $(printf "%(%H)T") =~ (02|08|14|20) ]] && { - printf "%s: %s\\n" "INFO" "Scanning depository directory for correct ownerships/permissions" | tee "$LOGSDIR/$LOGFILE" - - # Drop the caches as sometimes old user/groups are cached. - sudo /opt/sbin/drop-caches - - # Scan files. - GLOBIGNORE="$DEPOSITORY/sbosrcarch" - WRONG="$(find "$DEPOSITORY/"* -not -name .rsync-tmp -a \( \ - \( \( -type b -o -type c -o -type p -o -type s \) -a -not -path "$DEPOSITORY/slackware/slackware-3.0/disk1/live/dev/*" \) -o \ - \( -type l -a -not -perm 777 \) -o \ - \( -type d -a -not -perm 755 -a -not -perm 2755 \) -o \ - \( -type f -a -not -perm 644 -a -not -perm 755 \) -o \ - \( -not -user "$MIRRORING_USER" \) -o \ - \( -not -group "$DEPOSITORY_GROUP" \) \ - \) -a -printf "%u:%g\t%M\t%p\n")" - - # Report findings. - if [[ -n "$WRONG" ]]; then - printf "%s: %s\\n" "ERROR" "This sync has been aborted!" | tee -a "$LOGSDIR/$LOGFILE" - printf "%s: %s\\n" "ERROR" "The following paths have erronious ownerships/permissions:" | tee -a "$LOGSDIR/$LOGFILE" - printf "%s\\n" "$WRONG" | sed -re 's/^/ /g' | tee -a "$LOGSDIR/$LOGFILE" - ERR=-1 - else - printf "%s: %s\\n" "INFO" "No ownership/permission issues found" | tee -a "$LOGSDIR/$LOGFILE" - fi -} - -# Do the mirroring work. -(( ERR == 0 )) && { +printf "%s: %s\\n" "INFO" "Scanning depository directory for correct ownerships/permissions" | tee "$LOGSDIR/$LOGFILE" +GLOBIGNORE="$DEPOSITORY/sbosrcarch" +WRONG="$(find "$DEPOSITORY/"* \( \ + \( \( -type b -o -type c -o -type p -o -type s \) -a \! -path "$DEPOSITORY/slackware/slackware-3.0/disk1/live/dev/*" \) -o \ + \( -type l -a \! -perm 777 \) -o \ + \( -type d -a \! -perm 755 -a \! -perm 2755 \) -o \ + \( -type f -a \! -perm 644 -a \! -perm 755 \) -o \ + \( \! -user "$MIRRORING_USER" \) -o \ + \( \! -group "$DEPOSITORY_GROUP" \) \ + \) -a -printf "%u:%g\t%M\t%p\n")" +if [[ -n "$WRONG" ]]; then + printf "%s: %s\\n" "ERROR" "This sync has been aborted!" | tee -a "$LOGSDIR/$LOGFILE" + printf "%s: %s\\n" "WARNING" "The following paths have erronious ownerships/permissions:" | tee -a "$LOGSDIR/$LOGFILE" + printf "%s\\n" "$WRONG" | sed -re 's/^/ /g' | tee -a "$LOGSDIR/$LOGFILE" + ERR=-1 +else + printf "%s: %s\\n" "INFO" "No ownership/permission issues found" | tee -a "$LOGSDIR/$LOGFILE" + # Do the mirroring work. umask 022 export MIRRORING_USER - printf "%s: %s: %s\\n" "INFO" "Beginning sync command" "sg \"$DEPOSITORY_GROUP\" -c \"$COMMAND $*\"" | tee -a "$LOGSDIR/$LOGFILE" - sg "$DEPOSITORY_GROUP" -c "$COMMAND $* 2>&1" | tee -a "$LOGSDIR/$LOGFILE" + printf "%s: %s: %s\\n" "INFO" "Beginning sync command" "sg \"$DEPOSITORY_GROUP\" -c \"$COMMAND $@\"" | tee -a "$LOGSDIR/$LOGFILE" + sg "$DEPOSITORY_GROUP" -c "$COMMAND $@" 2>&1 | tee -a "$LOGSDIR/$LOGFILE" ERR="$?" -} +fi # Tell the sysadmin what went on. -if (( ERR == 0 )); then +if (( "$ERR" == 0 )); then # Only email a mirroring report if it's not the hourly Slackware tree only run. # The size of the log file determines if it gets emailed. - (( $(stat --printf="%s" "$LOGSDIR/$LOGFILE") > MIN_LOGFILE_SIZE )) && { - notify "report" || { - [[ -x /opt/bin/pushover-client ]] && /opt/bin/pushover-client "mirroring" -p -1 -s "notify() failure" -m "Check logfile: $LOGSDIR/$LOGFILE" - } - } + (( $(stat --printf="%s" "$LOGSDIR/$LOGFILE") > MIN_LOGFILE_SIZE )) && notify "report" + ERR="$?" else - ERR="1" - # Mirroring failed, tell the admin. - if notify "failure"; then - [[ -x /opt/bin/pushover-client ]] && /opt/bin/pushover-client "mirroring" -p -1 -s "Mirroring failure" -m "Check email / logfile: $LOGSDIR/$LOGFILE" - else - [[ -x /opt/bin/pushover-client ]] && /opt/bin/pushover-client "mirroring" -p -1 -s "notify() failure" -m "Check logfile: $LOGSDIR/$LOGFILE" - fi + notify "failure" || [[ -x /opt/bin/pushover-client ]] && /opt/bin/pushover-client "mirroring" -p -1 -s "Mirroring failure" -m "Check log in email" + ERR="1" fi # Compress the log to save some space. diff --git a/sbosrcarch-wrapper b/sbosrcarch-wrapper index 8f58c7f..ea52ffa 100755 --- a/sbosrcarch-wrapper +++ b/sbosrcarch-wrapper @@ -16,8 +16,7 @@ EMAIL_CC=("Urchlay ") # Functions notify() { [[ -n "$EMAIL_FROM" ]] && (( "${#EMAIL_TO[@]}" != 0 )) && { - # shellcheck disable=SC2094 - printf "%s: %s\\n%s: %s\\n%s:\\n%s\\n" "Exit code" "$ERR" "Logfile" "$LOGSDIR/$LOGFILE.xz" "Output" "$(sed -re 's/^/ /g' <"$LOGSDIR/$LOGFILE")" | mail -r "$EMAIL_FROM" -s "SboSrcArch $1" "$(for i in $(seq -s ' ' 0 $(( ${#EMAIL_CC[@]} -1 ))); do echo ${EMAIL_CC[$i]:+-c "${EMAIL_CC[$i]}"}; done)" "${EMAIL_TO[@]}" >>"$LOGSDIR/$LOGFILE" 2>&1 || { + printf "%s: %s\\n%s: %s\\n%s:\\n%s\\n" "Exit code" "$ERR" "Logfile" "$LOGSDIR/$LOGFILE.xz" "Output" "$(cat "$LOGSDIR/$LOGFILE" | sed -re 's/^/ /g')" | mail -r "$EMAIL_FROM" -s "SboSrcArch $1" "$(for i in $(seq -s ' ' 0 $(( ${#EMAIL_CC[@]} -1 ))); do echo ${EMAIL_CC[$i]:+-c "${EMAIL_CC[$i]}"}; done)" "${EMAIL_TO[@]}" >/dev/null 2>&1 || { printf "%s: %s\\n" "${0##*/}" "mail command failed" >&2 return 1 } @@ -32,7 +31,7 @@ umask 027 # Only run for the configured sbosrcarch user. [[ "$(whoami)" != "$SBOSRCARCH_USER" ]] && { printf "%s: %s\\n" "${0##*/}" "must be run by the '$SBOSRCARCH_USER' user" >&2 - printf "%s: %s: %s\\n" "${0##*/}" "to run manually:" "su \"$SBOSRCARCH_USER\" -c \"$0 $*\"" >&2 + printf "%s: %s: %s\\n" "${0##*/}" "to run manually:" "su '$SBOSRCARCH_USER' $0 $@" >&2 exit 1 } @@ -64,66 +63,45 @@ if [[ "$FLOCK" != "$0" ]]; then fi fi -ERR=0 +# Drop the caches as sometimes old user/groups are cached. +sudo /opt/sbin/drop-caches # Make sure there's no errant ownerships. -[[ $(printf "%(%H)T") == "04" ]] && { - printf "%s: %s\\n" "INFO" "Scanning sbosrcarch directory for correct ownerships/permissions" | tee "$LOGSDIR/$LOGFILE" - - # Drop the caches as sometimes old user/groups are cached. - sudo /opt/sbin/drop-caches - - # Scan files. - WRONG="$(find "$DEPOSITORY/" \( \ - \( -type b -o -type c -o -type p -o -type s \) -o \ - \( -type l -a \! -perm 777 \) -o \ - \( -type d -a \! -perm 755 -a \! -perm 2755 \) -o \ - \( -type f -a \! -perm 644 -a \! -perm 755 \) -o \ - \( \! -user "$SBOSRCARCH_USER" \) -o \ - \( \! -group "$DEPOSITORY_GROUP" \) \ - \) -a -printf "%u:%g\t%M\t%p\n")" - - # Report findings. - if [[ -n "$WRONG" ]]; then - printf "%s: %s\\n" "ERROR" "This run has been aborted!" | tee -a "$LOGSDIR/$LOGFILE" - printf "%s: %s\\n" "ERROR" "The following paths have erronious ownerships/permissions:" | tee -a "$LOGSDIR/$LOGFILE" - printf "%s\\n" "$WRONG" | sed -re 's/^/ /g' | tee -a "$LOGSDIR/$LOGFILE" - ERR=-1 - else - printf "%s: %s\\n" "INFO" "No ownership/permission issues found" | tee -a "$LOGSDIR/$LOGFILE" - fi -} - -# Do the sbosrcarch work. -(( ERR == 0 )) && { +printf "%s: %s\\n" "INFO" "Scanning sbosrcarch directory for correct ownerships/permissions" | tee "$LOGSDIR/$LOGFILE" +WRONG="$(find "$DEPOSITORY/" \( \ + \( -type b -o -type c -o -type p -o -type s \) -o \ + \( -type l -a \! -perm 777 \) -o \ + \( -type d -a \! -perm 755 -a \! -perm 2755 \) -o \ + \( -type f -a \! -perm 644 -a \! -perm 755 \) -o \ + \( \! -user "$SBOSRCARCH_USER" \) -o \ + \( \! -group "$DEPOSITORY_GROUP" \) \ + \) -a -printf "%u:%g\t%M\t%p\n")" +if [[ -n "$WRONG" ]]; then + printf "%s: %s\\n" "ERROR" "This run has been aborted!" | tee -a "$LOGSDIR/$LOGFILE" + printf "%s: %s\\n" "WARNING" "The following paths have erronious ownerships/permissions:" | tee -a "$LOGSDIR/$LOGFILE" + printf "%s\\n" "$WRONG" | sed -re 's/^/ /g' | tee -a "$LOGSDIR/$LOGFILE" + ERR=-1 +else + printf "%s: %s\\n" "INFO" "No ownership/permission issues found" | tee -a "$LOGSDIR/$LOGFILE" + # Do the sbosrcarch work. umask 022 - - printf "%s: %s: %s\\n" "INFO" "Beginning sbosrcarch run" "sg \"$DEPOSITORY_GROUP\" -c \"$COMMAND ${*:-update}\"" | tee -a "$LOGSDIR/$LOGFILE" - TMP_OUTPUT="$(sg "$DEPOSITORY_GROUP" -c "$COMMAND ${*:-update} 2>&1")" + printf "%s: %s: %s\\n" "INFO" "Beginning sbosrcarch run" "sg \"$DEPOSITORY_GROUP\" -c \"$COMMAND ${1:-update}${2:+ $2}${3:+ $3}\"" | tee -a "$LOGSDIR/$LOGFILE" + TMP_OUTPUT="$(sg "$DEPOSITORY_GROUP" -c "$COMMAND ${1:-update}${2:+ $2}${3:+ $3} 2>&1")" ERR="$?" - - [[ "$*" != *status* ]] && { - sg "$DEPOSITORY_GROUP" -c "$COMMAND status 2>&1" | tee -a "$LOGSDIR/$LOGFILE" - (( ERR += PIPESTATUS[0] )) - } - + sg "$DEPOSITORY_GROUP" -c "$COMMAND status 2>&1" | tee -a "$LOGSDIR/$LOGFILE" printf "\\n%s\\n" "$TMP_OUTPUT" | tee -a "$LOGSDIR/$LOGFILE" -} + (( ERR += $? )) +fi # Tell the sysadmin what went on. if (( "$ERR" == 0 )); then # Send a report. - notify "report" || { - [[ -x /opt/bin/pushover-client ]] && /opt/bin/pushover-client "mirroring" -p -1 -s "notify() failure" -m "Check logfile: $LOGSDIR/$LOGFILE.xz" - } + notify "report" + ERR="$?" else - ERR="1" # sbosrcarch failed, tell the admin. - if notify "failure"; then - [[ -x /opt/bin/pushover-client ]] && /opt/bin/pushover-client "mirroring" -p -1 -s "SBoSrcArch failure" -m "Check email / logfile: $LOGSDIR/$LOGFILE.xz" - else - [[ -x /opt/bin/pushover-client ]] && /opt/bin/pushover-client "mirroring" -p -1 -s "notify() failure" -m "Check logfile: $LOGSDIR/$LOGFILE.xz" - fi + notify "failure" || [[ -x /opt/bin/pushover-client ]] && /opt/bin/pushover-client "mirroring" -p -1 -s "SBoSrcArch failure" -m "Check log in email" + ERR="1" fi # Compress the log to save some space.