diff --git a/rest_pruner.sh b/rest_pruner.sh
index 491c108d9a4959857820504c7ca33e0b1b852fa4..095490b488e9fda60d10c832e30306b0b266fa82 100755
--- a/rest_pruner.sh
+++ b/rest_pruner.sh
@@ -5,16 +5,30 @@
#
# ----------------------------------------------------------------------
# 2024-02-01 v0.1 <axel.hahn@unibe.ch> first lines
+# 2024-02-02 v0.2 <axel.hahn@unibe.ch> add: timer, skip file, skin N days, limit process time, stats
# ======================================================================
cd "$( dirname $0 )" || exit
-_version=0.1
+_version=0.2
logdir=_last_prune
prune_basedir=
prune_params=
+prune_skipdays=7
+
+# stop pruning more repositories when running longer N seconds
+prune_timeout=7200
+
+# flag file inside repository to mark an archived backup
+sSkipfile=inactive.txt
+
+bOptAll=0
bOptDebug=0
+bOptStats=0
+bOptForce=0
+
+typeset -i iTimerStart; iTimerStart=$( date +%s )
typeset -i iCountDirs=0
typeset -i iCountMatch=0
@@ -36,18 +50,27 @@ cfgfile=rest_pruner.cfg || exit 1
function _showHelp(){
local _self; _self=$( basename $0 )
cat <<EOH
-
Pruner for restic rest server with append only option.
This script prunes all repositories on server side.
The config file [$cfgfile] contains <USER>:<RESTIC_PASSWORD>
If a directory matches ${prune_basedir}/<USER> then it will be pruned.
+Institute for Medical Education * University of Bern
+GNU GPL 3.0
+
+
SYNTAX:
$_self [OPTIONS] [FILTER]
+
OPTIONS:
-h, --help show help and exit.
+ -a, --all process all repositories, default: rest only + archives
+ -d, --debug show more infos
+ -f, --force force pruning; do not wait $prune_skipdays days
+ -s, --status show pruning status with last prunes
+
PARAMETERS:
FILTER regex to filter directory list in
@@ -56,13 +79,41 @@ PARAMETERS:
EXAMPLES:
$_self
Start pruning of all matching repositories
+
$_self mail
Prune servers that match "mail",
eg. my-mailhub.example.com
+ $_self --force mail
+ Prune servers that match "mail",
+ eg. my-mailhub.example.com
+
+ $_self --status
+ Show statistics with last prunes, if it is archive or
+ running or on error.
+ It shows repositories with passwords and archives.
+ Other repositories are not visible until you
+ specify -d.
+
EOH
}
+# get age of a file in sec
+# https://www.axel-hahn.de/blog/2020/03/09/bash-snippet-alter-einer-datei/
+# param string filename to test
+#
+function _getFileAge(){
+ echo $(($(date +%s) - $(date +%s -r "$1")))
+}
+
+# get execution time in seconds
+# global integer iTimerStart starting time in sec
+function _getTimer(){
+ local _iTimer
+ _iTimer=$( date +%s )
+ echo $(( _iTimer - iTimerStart ))
+}
+
# start prune of a given repository
# global string cfgfile name of the config file
# global integer rcAll sum of all prune exit status
@@ -78,62 +129,154 @@ function _prune(){
iCountDirs+=1
if [ -n "$mypw" ]; then
- local _user=$( stat -c "%U" "$_dir" )
- echo "----- $_dir"
- iCountMatch+=1
- bDoRun=1
-
- if ps -ef | grep "restic forget.*${_dir}" | grep -v "grep" | grep . ; then
- echo "SKIP: a process is still running..."
- bDoRun=0
- else
- rm -f "${logfile}.running"
- echo "TODO: check age of last prune run."
- fi
-
- if [ "$bDoRun" -eq "1" ]; then
- echo "Starting prune as user $_user ..."
- su - $_user - /bin/bash -c "
- echo START $( date ) $_dir
- export RESTIC_PASSWORD=$mypw
- restic forget -r $_dir $prune_params 2>&1
- " | tee "${logfile}.running"
- rc=${PIPESTATUS[0]}
- rcAll+=$rc
- echo END $( date ) exitcode $rc | tee -a "${logfile}.running"
- if [ "$rc" -eq "0" ]; then
- mv "${logfile}.running" "${logfile}"
- iCountPrune+=1
- else
- iCountPruneError+=1
- mv "${logfile}.running" "${logfile}.error"
+ iCountMatch+=1
+ bDoRun=1
+
+ if [ -f "$_dir/${sSkipfile}" ]; then
+ echo "SKIP: $_dir - inactive backup"
+ bDoRun=0
+ elif ps -eo command | grep "restic forget.*${_dir}" | grep -v "grep" | grep . ; then
+ echo "SKIP: $_dir - a process is still running..."
+ bDoRun=0
+ else
+ rm -f "${logfile}.running"
+
+ if [ "$bOptForce" = "0" ] && [ -f "${logfile}" ]; then
+ local _iAge; _iAge=$( _getFileAge "${logfile}" )
+ local _iDays; typeset -i _iDays=$(( _iAge/60/60/24 ))
+ if [ $_iDays -le $prune_skipdays ]; then
+ echo "SKIP: $_dir - Last run $_iDays days ($_iAge sec) ago - a prune will be started after $prune_skipdays days."
+ bDoRun=0
fi
fi
+ fi
+
+ if [ "$bDoRun" -eq "1" ]; then
+ local _user; _user=$( stat -c "%U" "$_dir" )
+ echo ">>>>> $( _getTimer ) >>>>> $_dir"
+ echo "Starting prune as user $_user ..."
+ su - $_user - /bin/bash -c "
+ echo START $( date ) $_dir
+ export RESTIC_PASSWORD=$mypw
+ restic forget -r $_dir $prune_params 2>&1
+ " | tee "${logfile}.running"
+ rc=${PIPESTATUS[0]}
+ rcAll+=$rc
+ echo END $( date ) exitcode $rc | tee -a "${logfile}.running"
+ if [ "$rc" -eq "0" ]; then
+ mv "${logfile}.running" "${logfile}"
+ iCountPrune+=1
+ else
+ iCountPruneError+=1
+ mv "${logfile}.running" "${logfile}.error"
+ fi
+ fi
+ else
+ test "$bOptAll" -eq "1" && echo "SKIP: $_dir - no password for this repo"
fi
}
+
+# start prune of a given repository
+# global string cfgfile name of the config file
+# global integer rcAll sum of all prune exit status
+#
+# param string directory to prune
+function _status(){
+
+ local _processes;
+ _processes="$( ps -eo command )"
+
+ echo "Prune status:"
+ for _dir in $( find ${prune_basedir} -maxdepth 1 -type d | grep -E "$filter")
+ do
+ local mybase; mybase=$( basename "${_dir}" )
+ local mypw; mypw=$( grep "^${mybase}:" "${cfgfile}" | cut -f2 -d ':')
+ local logfile="${logdir}/${mybase}.log"
+
+
+ local _flagPw="."
+ local _flagArchive="."
+ local _flagRunning="."
+ local _flagError="."
+ local _sLastPrune="?"
+
+ iCountDirs+=1
+
+ test -n "$mypw" && _flagPw="Y"
+ test -f "$_dir/${sSkipfile}" && _flagArchive="A"
+ test -f "${logfile}.error" && _flagError="E"
+ grep "restic forget.*${_dir}" <<< "$_processes" | grep -v "grep" | grep . >/dev/null && _flagRunning="R"
+
+ if [ -f "${logfile}" ]; then
+ local _iAge; _iAge=$( _getFileAge "${logfile}" )
+ local _iDays; typeset -i _iDays=$(( _iAge/60/60/24 ))
+ _sLastPrune="$_iDays days ago"
+ fi
+
+ if [ -n "$mypw" ] || [ "$_flagArchive" = "A" ] || [ "$bOptAll" -eq "1" ]; then
+ iCountMatch+=1
+ printf "%1s %1s %1s %1s %-12s %s\n" "$_flagPw" "$_flagArchive" "$_flagError" "$_flagRunning" "$_sLastPrune" "$_dir"
+ fi
+
+ done
+ if [ "$iCountMatch" -gt "0" ]; then
+ echo
+ echo ": : : : : :"
+ echo ": : : : : +-- repository dir"
+ echo ": : : : +-- last successful prune"
+ echo ": : : +-- is it currently running?"
+ echo ": : +-- Last prune on error?"
+ echo ": +-- Repository is archived?"
+ echo "+-- has password?"
+ else
+ echo "INFO: No (matching) Repository was found."
+ fi
+ echo
+}
# ----------------------------------------------------------------------
# MAIN
# ----------------------------------------------------------------------
-echo
-echo "========== RESTIC REST PRUNER v${_version} =========="
-echo
+cat << ENDHEADER
+------------------------------------------------------------------------------
+ --------========###| RESTIC REST PRUNER :: v${_version} |###=======--------
+------------------------------------------------------------------------------
-# ----- check parameters
+ENDHEADER
+# ----- check parameters
+allparams="$*"
while [[ "$#" -gt 0 ]]; do case $1 in
-h|--help) _showHelp; exit 0;;
+ -a|--all) bOptAll=1; shift;;
-d|--debug) bOptDebug=1; shift;;
- -p|--path) if ! grep ":{$2}:" <<< ":{$PATH}:" >/dev/null; then
- PATH="$2:$PATH";
- fi
- shift; shift;;
- -t|--target) export DOCKER_HOST="$2"; shift; shift;;
- *) echo "ERROR: Unknown parameter: $1"; _showHelp; exit 2;
+ -f|--force) bOptForce=1; shift;;
+ -s|--status) bOptStats=1; shift;;
+ *) if grep "^-" <<< "$1" >/dev/null ; then
+ echo; echo "ERROR: Unknown parameter: $1"; echo; _showHelp; exit 2
+ fi
+ break;
+ ;;
esac; done
+test "$bOptDebug" -eq "1" && cat << ENDDEBUGINFO
+----- 8< ----- DEBUG ------------------------------------------------ 8< -----
+
+Config:
+$( ls -l inc_config.sh && echo )
+$( grep "prune_.*=" inc_config.sh | grep -v "#")
+
+$( ls -l ${cfgfile} 2>&1 && echo)
+
+Parameters: $allparams
+
+----- 8< ----- /DEBUG ----------------------------------------------- 8< -----
+
+ENDDEBUGINFO
+
+
# ----- verify needed settings
if [ ! -f "$cfgfile" ]; then
@@ -192,18 +335,32 @@ fi
test -d "${logdir}" || mkdir -p "${logdir}"
filter=${1:-.}
-for mydir in $( find ${prune_basedir} -maxdepth 1 -type d | grep -E "$filter")
+
+test "$bOptStats" -eq "1" && _status "${mydir}"
+
+test "$bOptStats" -eq "0" && for mydir in $( find ${prune_basedir} -maxdepth 1 -type d | grep -E "$filter")
do
+ _iTimer=$( _getTimer )
+ if [ "$_iTimer" -gt "$prune_timeout" ]; then
+ echo "ABORT: Processing time reached $_iTimer sec - limit is $prune_timeout sec."
+ echo "Stopping before processing ${mydir}."
+ echo
+ break;
+ fi
_prune "${mydir}"
done
-echo
-echo "Status:"
-echo "Directories: $iCountDirs"
-echo "Matching : $iCountMatch"
-echo "Pruned : $iCountPrune"
-echo "Prune erors: $iCountPruneError"
-echo
+test "$bOptStats" -eq "0" && cat <<ENDSTATS
+
+Statistics:
+
+Elapsed time : $( _getTimer ) sec
+Directories : $iCountDirs
+Matching : $iCountMatch
+Pruned : $iCountPrune
+Prune erors : $iCountPruneError
+------------------------------------------------------------------------------
+ENDSTATS
echo "done - exitcode $rcAll"
exit $rcAll