Skip to content
Snippets Groups Projects

backup status: check age of restic backups when not using ssh

Merged backup status: check age of restic backups when not using ssh
update_backupstatus_for_restic_rest into master
Merged Hahn Axel (hahn) requested to merge update_backupstatus_for_restic_rest into master
Compare
and
1 file
+ 130
− 63
Compare changes
  • Side-by-side
  • Inline
storage_helper.sh
+ 130
− 63
@@ -33,6 +33,7 @@
# 2021-01-22 hahn show inactive backups in status page; colored lines by status
# 2021-11-12 hahn backup status: add handling for backup exitcode
# 2022-07-06 hahn backup status: show clear message on not started/ deleted backups
# 2024-03-22 hahn backup status: check age of restic backups when not using ssh
# ======================================================================
# ----------------------------------------------------------------------
@@ -124,7 +125,7 @@
# get content of all logfiles
function _getAllLogs(){
zcat $sLogdir/*.gz ; cat $sLogfile
ls $sLogdir/*.gz >/dev/null 2>&1 && ( zcat $sLogdir/*.gz ; cat $sLogfile )
}
# get a list of all servers in the log
@@ -184,7 +185,7 @@
test -f "$sTouchfile" && echo 1
test -f "$sTouchfile" || echo 0
}
# check if a backup for current server is running
# check if a backup for current server is inactive
# it returns 0 for no and 1 for yes
# global string sBackupClient FQDN of current server
# global string sBackupBasedir backup directory
@@ -197,6 +198,14 @@
fi
}
# get age of a file in sec
# param string filename to test
#
function _getFileAge(){
echo $(($(date +%s) - $(date +%s -r "$1")))
}
# ------------------------------------------------------------
# slot handling
# ------------------------------------------------------------
@@ -292,69 +301,127 @@
function PUBLIC_backupstatus(){
typeset -i local ierrors=0
local tbl="%s %-45s | %-20s | %-20s | %8s | %3s | %7s | %s \n"
local ierrors; typeset -i ierrors=0
local tbl
_showConnectionCount
echo
echo "This table shows the time and duration [s] of the last backup for each server."
echo
printf "$tbl" " " "server" "start" "end" "duration" "rc" "age [h]" "size"
echo "-----------------------------------------------------------------------------------------------------------------------------"
for myserver in $( _getLoggedServers )
do
# get data
setBackupclient $myserver
typeset -i local istart="$( _getLastBackupstart )"
typeset -i local iend="$( _getLastBackupend )"
typeset -i local bIsRunning=$( _isRunning )
typeset -i local bIsInactive=$( _isInactive )
typeset -i local iLastStatus=$( _getLastBackupstatus )
local size=$( _getBackupsize )
local tstart=$( date +"%Y-%m-%d %H:%M:%S" -d @${istart} )
local tend=$( date +"%Y-%m-%d %H:%M:%S" -d @${iend} )
typeset -i local iduration=${iend}-${istart}
typeset -i local iage=$( date +%s )-${iend}
typeset -i local iagehours=$iage/60/60
local sduration=$iduration
test $iduration -lt 0 && sduration="RUNNING"
sStatusRun="$(color ok)."
test $iend -eq 0 && sStatusRun="$(color warn)?"
test $bIsRunning -eq 1 && sStatusRun="$(color active)R"
test $bIsInactive -eq 1 && sStatusRun="$(color disabled)D"
# check values
if [ $iLastStatus -gt 0 ]; then
ierrors=$ierrors+1
sStatusRun="$(color error)E"
else
if [ $iend -gt 0 -a $iagehours -gt $iMaxbackupAge -a $bIsInactive -eq 0 ]; then
ierrors=$ierrors+1
sStatusRun="$(color error)E"
fi
fi
if [ $iend -eq 0 ]; then
tstart="-"
tend="-"
sduration=""
iagehours=""
fi
# output
printf "$tbl" "$sStatusRun" "$myserver" "${tstart}" "${tend}" "${sduration}" "$iLastStatus" "${iagehours}" "$size"
done
color reset
echo
echo "Legend"
echo ". OK | ? not started | R running | D disabled | E error"
echo
echo "total : $( _getLoggedServers | wc -l ) servers"
echo "errors: $ierrors"
# _showConnectionCount
echo
if ! _getLoggedServers | grep -qc . ; then
iKeepBackups=180
local iAgeH; typeset -i iAgeH
local iAgeD; typeset -i iAgeD
local sAge
local mydir
local bIsInactive; typeset -i bIsInactive
local iCounter; typeset -i iCounter=0
echo "Age of Restic backup repositories"
for myresticsnapshotdir in $( ls -1trd ${sBackupBasedir}/*/*/snapshots )
do
iCounter=$iCounter+1
mydir=$( dirname "${myresticsnapshotdir}")
sBackupClient=$( basename ${mydir} )
iAgeH=$( _getFileAge "${myresticsnapshotdir}" )/60/60
iAgeD=iAgeH/24
bIsInactive=$( _isInactive )
sMore=
sStatus="$(color ok)."
test $bIsInactive -eq 1 && sStatus="$(color disabled)D"
sFile=${mydir}/inactive.txt
if [ "$bIsInactive" -eq "1" ]; then
typeset -i iAge=`_getFileAge "$sFile"`
typeset -i iDays=$iKeepBackups-$iAge/60/60/24
sMore=" - INFO: keeping it $iKeepBackups d .. $iDays d left."
if [ "$iDays" -lt 0 ]; then
sMore=" - DELETE: overdued $iDays d."
sStatus="$(color error)E"
ierrors=$ierrors+1
fi
elif [ $iAgeH -ge 24 ]; then
sStatus="$(color error)E"
ierrors=$ierrors+1
fi
sAge="$iAgeH h"
if [ $iAgeH -gt 96 ]; then
sAge="$iAgeD d"
fi
printf "%2s %7s %s\n" "$sStatus" "$sAge" "${mydir}$sMore"
done
color reset
echo
echo "Legend"
echo ". OK | D disabled | E error"
echo
echo "total : $iCounter servers"
echo "errors: $ierrors"
echo
else
tbl="%s %-45s | %-20s | %-20s | %8s | %3s | %7s | %s \n"
echo "This table shows the time and duration [s] of the last backup for each server."
echo
printf "$tbl" " " "server" "start" "end" "duration" "rc" "age [h]" "size"
echo "-----------------------------------------------------------------------------------------------------------------------------"
for myserver in $( _getLoggedServers )
do
# get data
setBackupclient $myserver
typeset -i local istart="$( _getLastBackupstart )"
typeset -i local iend="$( _getLastBackupend )"
typeset -i local bIsRunning=$( _isRunning )
typeset -i local bIsInactive=$( _isInactive )
typeset -i local iLastStatus=$( _getLastBackupstatus )
local size=$( _getBackupsize )
local tstart=$( date +"%Y-%m-%d %H:%M:%S" -d @${istart} )
local tend=$( date +"%Y-%m-%d %H:%M:%S" -d @${iend} )
typeset -i local iduration=${iend}-${istart}
typeset -i local iage=$( date +%s )-${iend}
typeset -i local iagehours=$iage/60/60
local sduration=$iduration
test $iduration -lt 0 && sduration="RUNNING"
sStatusRun="$(color ok)."
test $iend -eq 0 && sStatusRun="$(color warn)?"
test $bIsRunning -eq 1 && sStatusRun="$(color active)R"
test $bIsInactive -eq 1 && sStatusRun="$(color disabled)D"
# check values
if [ $iLastStatus -gt 0 ]; then
ierrors=$ierrors+1
sStatusRun="$(color error)E"
else
if [ $iend -gt 0 -a $iagehours -gt $iMaxbackupAge -a $bIsInactive -eq 0 ]; then
ierrors=$ierrors+1
sStatusRun="$(color error)E"
fi
fi
if [ $iend -eq 0 ]; then
tstart="-"
tend="-"
sduration=""
iagehours=""
fi
# output
printf "$tbl" "$sStatusRun" "$myserver" "${tstart}" "${tend}" "${sduration}" "$iLastStatus" "${iagehours}" "$size"
done
color reset
echo
echo "Legend"
echo ". OK | ? not started | R running | D disabled | E error"
echo
echo "total : $( _getLoggedServers | wc -l ) servers"
echo "errors: $ierrors"
echo
fi
echo "rc=$ierrors"
exit $ierrors
}