Skip to content
Snippets Groups Projects
Commit 13cfbee3 authored by Hahn Axel (hahn)'s avatar Hahn Axel (hahn)
Browse files

prune add: timer, skip file, skin N days, limit process time, stats

parent d28cada0
No related branches found
No related tags found
1 merge request!4prune add: timer, skip file, skin N days, limit process time, stats
Hide whitespace changes
Inline Side-by-side
Showing
with 206 additions and 49 deletions
rest_pruner.sh
+ 206
− 49
View file @ 13cfbee3
...@@ -5,16 +5,30 @@ ...@@ -5,16 +5,30 @@
# #
# ---------------------------------------------------------------------- # ----------------------------------------------------------------------
# 2024-02-01 v0.1 <axel.hahn@unibe.ch> first lines # 2024-02-01 v0.1 <axel.hahn@unibe.ch> first lines
# 2024-02-02 v0.2 <axel.hahn@unibe.ch> add: timer, skip file, skin N days, limit process time, stats
# ====================================================================== # ======================================================================
cd "$( dirname $0 )" || exit cd "$( dirname $0 )" || exit
_version=0.1 _version=0.2
logdir=_last_prune logdir=_last_prune
prune_basedir= prune_basedir=
prune_params= prune_params=
prune_skipdays=7
# stop pruning more repositories when running longer N seconds
prune_timeout=7200
# flag file inside repository to mark an archived backup
sSkipfile=inactive.txt
bOptAll=0
bOptDebug=0 bOptDebug=0
bOptStats=0
bOptForce=0
typeset -i iTimerStart; iTimerStart=$( date +%s )
typeset -i iCountDirs=0 typeset -i iCountDirs=0
typeset -i iCountMatch=0 typeset -i iCountMatch=0
...@@ -36,18 +50,27 @@ cfgfile=rest_pruner.cfg || exit 1 ...@@ -36,18 +50,27 @@ cfgfile=rest_pruner.cfg || exit 1
function _showHelp(){ function _showHelp(){
local _self; _self=$( basename $0 ) local _self; _self=$( basename $0 )
cat <<EOH cat <<EOH
Pruner for restic rest server with append only option. Pruner for restic rest server with append only option.
This script prunes all repositories on server side. This script prunes all repositories on server side.
The config file [$cfgfile] contains <USER>:<RESTIC_PASSWORD> The config file [$cfgfile] contains <USER>:<RESTIC_PASSWORD>
If a directory matches ${prune_basedir}/<USER> then it will be pruned. If a directory matches ${prune_basedir}/<USER> then it will be pruned.
Institute for Medical Education * University of Bern
GNU GPL 3.0
SYNTAX: SYNTAX:
$_self [OPTIONS] [FILTER] $_self [OPTIONS] [FILTER]
OPTIONS: OPTIONS:
-h, --help show help and exit. -h, --help show help and exit.
-a, --all process all repositories, default: rest only + archives
-d, --debug show more infos
-f, --force force pruning; do not wait $prune_skipdays days
-s, --status show pruning status with last prunes
PARAMETERS: PARAMETERS:
FILTER regex to filter directory list in FILTER regex to filter directory list in
...@@ -56,13 +79,41 @@ PARAMETERS: ...@@ -56,13 +79,41 @@ PARAMETERS:
EXAMPLES: EXAMPLES:
$_self $_self
Start pruning of all matching repositories Start pruning of all matching repositories
$_self mail $_self mail
Prune servers that match "mail", Prune servers that match "mail",
eg. my-mailhub.example.com eg. my-mailhub.example.com
$_self --force mail
Prune servers that match "mail",
eg. my-mailhub.example.com
$_self --status
Show statistics with last prunes, if it is archive or
running or on error.
It shows repositories with passwords and archives.
Other repositories are not visible until you
specify -d.
EOH EOH
} }
# get age of a file in sec
# https://www.axel-hahn.de/blog/2020/03/09/bash-snippet-alter-einer-datei/
# param string filename to test
#
function _getFileAge(){
echo $(($(date +%s) - $(date +%s -r "$1")))
}
# get execution time in seconds
# global integer iTimerStart starting time in sec
function _getTimer(){
local _iTimer
_iTimer=$( date +%s )
echo $(( _iTimer - iTimerStart ))
}
# start prune of a given repository # start prune of a given repository
# global string cfgfile name of the config file # global string cfgfile name of the config file
# global integer rcAll sum of all prune exit status # global integer rcAll sum of all prune exit status
...@@ -78,62 +129,154 @@ function _prune(){ ...@@ -78,62 +129,154 @@ function _prune(){
iCountDirs+=1 iCountDirs+=1
if [ -n "$mypw" ]; then if [ -n "$mypw" ]; then
local _user=$( stat -c "%U" "$_dir" )
echo "----- $_dir"
iCountMatch+=1 iCountMatch+=1
bDoRun=1 bDoRun=1
if ps -ef | grep "restic forget.*${_dir}" | grep -v "grep" | grep . ; then if [ -f "$_dir/${sSkipfile}" ]; then
echo "SKIP: a process is still running..." echo "SKIP: $_dir - inactive backup"
bDoRun=0 bDoRun=0
else elif ps -eo command | grep "restic forget.*${_dir}" | grep -v "grep" | grep . ; then
rm -f "${logfile}.running" echo "SKIP: $_dir - a process is still running..."
echo "TODO: check age of last prune run." bDoRun=0
fi else
rm -f "${logfile}.running"
if [ "$bDoRun" -eq "1" ]; then
echo "Starting prune as user $_user ..." if [ "$bOptForce" = "0" ] && [ -f "${logfile}" ]; then
su - $_user - /bin/bash -c " local _iAge; _iAge=$( _getFileAge "${logfile}" )
echo START $( date ) $_dir local _iDays; typeset -i _iDays=$(( _iAge/60/60/24 ))
export RESTIC_PASSWORD=$mypw if [ $_iDays -le $prune_skipdays ]; then
restic forget -r $_dir $prune_params 2>&1 echo "SKIP: $_dir - Last run $_iDays days ($_iAge sec) ago - a prune will be started after $prune_skipdays days."
" | tee "${logfile}.running" bDoRun=0
rc=${PIPESTATUS[0]}
rcAll+=$rc
echo END $( date ) exitcode $rc | tee -a "${logfile}.running"
if [ "$rc" -eq "0" ]; then
mv "${logfile}.running" "${logfile}"
iCountPrune+=1
else
iCountPruneError+=1
mv "${logfile}.running" "${logfile}.error"
fi fi
fi fi
fi
if [ "$bDoRun" -eq "1" ]; then
local _user; _user=$( stat -c "%U" "$_dir" )
echo ">>>>> $( _getTimer ) >>>>> $_dir"
echo "Starting prune as user $_user ..."
su - $_user - /bin/bash -c "
echo START $( date ) $_dir
export RESTIC_PASSWORD=$mypw
restic forget -r $_dir $prune_params 2>&1
" | tee "${logfile}.running"
rc=${PIPESTATUS[0]}
rcAll+=$rc
echo END $( date ) exitcode $rc | tee -a "${logfile}.running"
if [ "$rc" -eq "0" ]; then
mv "${logfile}.running" "${logfile}"
iCountPrune+=1
else
iCountPruneError+=1
mv "${logfile}.running" "${logfile}.error"
fi
fi
else
test "$bOptAll" -eq "1" && echo "SKIP: $_dir - no password for this repo"
fi fi
} }
# start prune of a given repository
# global string cfgfile name of the config file
# global integer rcAll sum of all prune exit status
#
# param string directory to prune
function _status(){
local _processes;
_processes="$( ps -eo command )"
echo "Prune status:"
for _dir in $( find ${prune_basedir} -maxdepth 1 -type d | grep -E "$filter")
do
local mybase; mybase=$( basename "${_dir}" )
local mypw; mypw=$( grep "^${mybase}:" "${cfgfile}" | cut -f2 -d ':')
local logfile="${logdir}/${mybase}.log"
local _flagPw="."
local _flagArchive="."
local _flagRunning="."
local _flagError="."
local _sLastPrune="?"
iCountDirs+=1
test -n "$mypw" && _flagPw="Y"
test -f "$_dir/${sSkipfile}" && _flagArchive="A"
test -f "${logfile}.error" && _flagError="E"
grep "restic forget.*${_dir}" <<< "$_processes" | grep -v "grep" | grep . >/dev/null && _flagRunning="R"
if [ -f "${logfile}" ]; then
local _iAge; _iAge=$( _getFileAge "${logfile}" )
local _iDays; typeset -i _iDays=$(( _iAge/60/60/24 ))
_sLastPrune="$_iDays days ago"
fi
if [ -n "$mypw" ] || [ "$_flagArchive" = "A" ] || [ "$bOptAll" -eq "1" ]; then
iCountMatch+=1
printf "%1s %1s %1s %1s %-12s %s\n" "$_flagPw" "$_flagArchive" "$_flagError" "$_flagRunning" "$_sLastPrune" "$_dir"
fi
done
if [ "$iCountMatch" -gt "0" ]; then
echo
echo ": : : : : :"
echo ": : : : : +-- repository dir"
echo ": : : : +-- last successful prune"
echo ": : : +-- is it currently running?"
echo ": : +-- Last prune on error?"
echo ": +-- Repository is archived?"
echo "+-- has password?"
else
echo "INFO: No (matching) Repository was found."
fi
echo
}
# ---------------------------------------------------------------------- # ----------------------------------------------------------------------
# MAIN # MAIN
# ---------------------------------------------------------------------- # ----------------------------------------------------------------------
echo cat << ENDHEADER
echo "========== RESTIC REST PRUNER v${_version} ==========" ------------------------------------------------------------------------------
echo --------========###| RESTIC REST PRUNER :: v${_version} |###=======--------
------------------------------------------------------------------------------
# ----- check parameters ENDHEADER
# ----- check parameters
allparams="$*"
while [[ "$#" -gt 0 ]]; do case $1 in while [[ "$#" -gt 0 ]]; do case $1 in
-h|--help) _showHelp; exit 0;; -h|--help) _showHelp; exit 0;;
-a|--all) bOptAll=1; shift;;
-d|--debug) bOptDebug=1; shift;; -d|--debug) bOptDebug=1; shift;;
-p|--path) if ! grep ":{$2}:" <<< ":{$PATH}:" >/dev/null; then -f|--force) bOptForce=1; shift;;
PATH="$2:$PATH"; -s|--status) bOptStats=1; shift;;
fi *) if grep "^-" <<< "$1" >/dev/null ; then
shift; shift;; echo; echo "ERROR: Unknown parameter: $1"; echo; _showHelp; exit 2
-t|--target) export DOCKER_HOST="$2"; shift; shift;; fi
*) echo "ERROR: Unknown parameter: $1"; _showHelp; exit 2; break;
;;
esac; done esac; done
test "$bOptDebug" -eq "1" && cat << ENDDEBUGINFO
----- 8< ----- DEBUG ------------------------------------------------ 8< -----
Config:
$( ls -l inc_config.sh && echo )
$( grep "prune_.*=" inc_config.sh | grep -v "#")
$( ls -l ${cfgfile} 2>&1 && echo)
Parameters: $allparams
----- 8< ----- /DEBUG ----------------------------------------------- 8< -----
ENDDEBUGINFO
# ----- verify needed settings # ----- verify needed settings
if [ ! -f "$cfgfile" ]; then if [ ! -f "$cfgfile" ]; then
...@@ -192,18 +335,32 @@ fi ...@@ -192,18 +335,32 @@ fi
test -d "${logdir}" || mkdir -p "${logdir}" test -d "${logdir}" || mkdir -p "${logdir}"
filter=${1:-.} filter=${1:-.}
for mydir in $( find ${prune_basedir} -maxdepth 1 -type d | grep -E "$filter")
test "$bOptStats" -eq "1" && _status "${mydir}"
test "$bOptStats" -eq "0" && for mydir in $( find ${prune_basedir} -maxdepth 1 -type d | grep -E "$filter")
do do
_iTimer=$( _getTimer )
if [ "$_iTimer" -gt "$prune_timeout" ]; then
echo "ABORT: Processing time reached $_iTimer sec - limit is $prune_timeout sec."
echo "Stopping before processing ${mydir}."
echo
break;
fi
_prune "${mydir}" _prune "${mydir}"
done done
echo test "$bOptStats" -eq "0" && cat <<ENDSTATS
echo "Status:"
echo "Directories: $iCountDirs" Statistics:
echo "Matching : $iCountMatch"
echo "Pruned : $iCountPrune" Elapsed time : $( _getTimer ) sec
echo "Prune erors: $iCountPruneError" Directories : $iCountDirs
echo Matching : $iCountMatch
Pruned : $iCountPrune
Prune erors : $iCountPruneError
------------------------------------------------------------------------------
ENDSTATS
echo "done - exitcode $rcAll" echo "done - exitcode $rcAll"
exit $rcAll exit $rcAll
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment