Skip to content
Snippets Groups Projects

backup status: check age of restic backups when not using ssh

Merged backup status: check age of restic backups when not using ssh
update_backupstatus_for_restic_rest into master
Merged Hahn Axel (hahn) requested to merge update_backupstatus_for_restic_rest into master
Compare
and
1 file
+ 130
− 63
Compare changes
  • Side-by-side
  • Inline
storage_helper.sh
+ 130
− 63
@@ -33,6 +33,7 @@
@@ -33,6 +33,7 @@
# 2021-01-22 hahn show inactive backups in status page; colored lines by status
# 2021-01-22 hahn show inactive backups in status page; colored lines by status
# 2021-11-12 hahn backup status: add handling for backup exitcode
# 2021-11-12 hahn backup status: add handling for backup exitcode
# 2022-07-06 hahn backup status: show clear message on not started/ deleted backups
# 2022-07-06 hahn backup status: show clear message on not started/ deleted backups
 
# 2024-03-22 hahn backup status: check age of restic backups when not using ssh
# ======================================================================
# ======================================================================
# ----------------------------------------------------------------------
# ----------------------------------------------------------------------
@@ -124,7 +125,7 @@
@@ -124,7 +125,7 @@
# get content of all logfiles
# get content of all logfiles
function _getAllLogs(){
function _getAllLogs(){
zcat $sLogdir/*.gz ; cat $sLogfile
ls $sLogdir/*.gz >/dev/null 2>&1 && ( zcat $sLogdir/*.gz ; cat $sLogfile )
}
}
# get a list of all servers in the log
# get a list of all servers in the log
@@ -184,7 +185,7 @@
@@ -184,7 +185,7 @@
test -f "$sTouchfile" && echo 1
test -f "$sTouchfile" && echo 1
test -f "$sTouchfile" || echo 0
test -f "$sTouchfile" || echo 0
}
}
# check if a backup for current server is running
# check if a backup for current server is inactive
# it returns 0 for no and 1 for yes
# it returns 0 for no and 1 for yes
# global string sBackupClient FQDN of current server
# global string sBackupClient FQDN of current server
# global string sBackupBasedir backup directory
# global string sBackupBasedir backup directory
@@ -197,6 +198,14 @@
@@ -197,6 +198,14 @@
fi
fi
}
}
 
# get age of a file in sec
 
# param string filename to test
 
#
 
function _getFileAge(){
 
echo $(($(date +%s) - $(date +%s -r "$1")))
 
}
 
 
# ------------------------------------------------------------
# ------------------------------------------------------------
# slot handling
# slot handling
# ------------------------------------------------------------
# ------------------------------------------------------------
@@ -292,69 +301,127 @@
@@ -292,69 +301,127 @@
function PUBLIC_backupstatus(){
function PUBLIC_backupstatus(){
typeset -i local ierrors=0
local ierrors; typeset -i ierrors=0
local tbl="%s %-45s | %-20s | %-20s | %8s | %3s | %7s | %s \n"
local tbl
_showConnectionCount
# _showConnectionCount
echo
echo "This table shows the time and duration [s] of the last backup for each server."
echo
printf "$tbl" " " "server" "start" "end" "duration" "rc" "age [h]" "size"
echo "-----------------------------------------------------------------------------------------------------------------------------"
for myserver in $( _getLoggedServers )
do
# get data
setBackupclient $myserver
typeset -i local istart="$( _getLastBackupstart )"
typeset -i local iend="$( _getLastBackupend )"
typeset -i local bIsRunning=$( _isRunning )
typeset -i local bIsInactive=$( _isInactive )
typeset -i local iLastStatus=$( _getLastBackupstatus )
local size=$( _getBackupsize )
local tstart=$( date +"%Y-%m-%d %H:%M:%S" -d @${istart} )
local tend=$( date +"%Y-%m-%d %H:%M:%S" -d @${iend} )
typeset -i local iduration=${iend}-${istart}
typeset -i local iage=$( date +%s )-${iend}
typeset -i local iagehours=$iage/60/60
local sduration=$iduration
test $iduration -lt 0 && sduration="RUNNING"
sStatusRun="$(color ok)."
test $iend -eq 0 && sStatusRun="$(color warn)?"
test $bIsRunning -eq 1 && sStatusRun="$(color active)R"
test $bIsInactive -eq 1 && sStatusRun="$(color disabled)D"
# check values
if [ $iLastStatus -gt 0 ]; then
ierrors=$ierrors+1
sStatusRun="$(color error)E"
else
if [ $iend -gt 0 -a $iagehours -gt $iMaxbackupAge -a $bIsInactive -eq 0 ]; then
ierrors=$ierrors+1
sStatusRun="$(color error)E"
fi
fi
if [ $iend -eq 0 ]; then
tstart="-"
tend="-"
sduration=""
iagehours=""
fi
# output
printf "$tbl" "$sStatusRun" "$myserver" "${tstart}" "${tend}" "${sduration}" "$iLastStatus" "${iagehours}" "$size"
done
color reset
echo
echo "Legend"
echo ". OK | ? not started | R running | D disabled | E error"
echo
echo "total : $( _getLoggedServers | wc -l ) servers"
echo "errors: $ierrors"
echo
echo
 
if ! _getLoggedServers | grep -qc . ; then
 
iKeepBackups=180
 
local iAgeH; typeset -i iAgeH
 
local iAgeD; typeset -i iAgeD
 
local sAge
 
local mydir
 
local bIsInactive; typeset -i bIsInactive
 
local iCounter; typeset -i iCounter=0
 
echo "Age of Restic backup repositories"
 
for myresticsnapshotdir in $( ls -1trd ${sBackupBasedir}/*/*/snapshots )
 
do
 
iCounter=$iCounter+1
 
mydir=$( dirname "${myresticsnapshotdir}")
 
sBackupClient=$( basename ${mydir} )
 
iAgeH=$( _getFileAge "${myresticsnapshotdir}" )/60/60
 
iAgeD=iAgeH/24
 
bIsInactive=$( _isInactive )
 
 
sMore=
 
sStatus="$(color ok)."
 
 
test $bIsInactive -eq 1 && sStatus="$(color disabled)D"
 
sFile=${mydir}/inactive.txt
 
 
if [ "$bIsInactive" -eq "1" ]; then
 
typeset -i iAge=`_getFileAge "$sFile"`
 
typeset -i iDays=$iKeepBackups-$iAge/60/60/24
 
sMore=" - INFO: keeping it $iKeepBackups d .. $iDays d left."
 
if [ "$iDays" -lt 0 ]; then
 
sMore=" - DELETE: overdued $iDays d."
 
sStatus="$(color error)E"
 
ierrors=$ierrors+1
 
fi
 
elif [ $iAgeH -ge 24 ]; then
 
sStatus="$(color error)E"
 
ierrors=$ierrors+1
 
 
fi
 
 
sAge="$iAgeH h"
 
if [ $iAgeH -gt 96 ]; then
 
sAge="$iAgeD d"
 
fi
 
 
printf "%2s %7s %s\n" "$sStatus" "$sAge" "${mydir}$sMore"
 
done
 
color reset
 
echo
 
echo "Legend"
 
echo ". OK | D disabled | E error"
 
echo
 
echo "total : $iCounter servers"
 
echo "errors: $ierrors"
 
echo
 
 
else
 
tbl="%s %-45s | %-20s | %-20s | %8s | %3s | %7s | %s \n"
 
echo "This table shows the time and duration [s] of the last backup for each server."
 
echo
 
printf "$tbl" " " "server" "start" "end" "duration" "rc" "age [h]" "size"
 
echo "-----------------------------------------------------------------------------------------------------------------------------"
 
for myserver in $( _getLoggedServers )
 
do
 
# get data
 
setBackupclient $myserver
 
typeset -i local istart="$( _getLastBackupstart )"
 
typeset -i local iend="$( _getLastBackupend )"
 
typeset -i local bIsRunning=$( _isRunning )
 
typeset -i local bIsInactive=$( _isInactive )
 
typeset -i local iLastStatus=$( _getLastBackupstatus )
 
local size=$( _getBackupsize )
 
 
local tstart=$( date +"%Y-%m-%d %H:%M:%S" -d @${istart} )
 
local tend=$( date +"%Y-%m-%d %H:%M:%S" -d @${iend} )
 
typeset -i local iduration=${iend}-${istart}
 
typeset -i local iage=$( date +%s )-${iend}
 
typeset -i local iagehours=$iage/60/60
 
 
local sduration=$iduration
 
test $iduration -lt 0 && sduration="RUNNING"
 
 
sStatusRun="$(color ok)."
 
test $iend -eq 0 && sStatusRun="$(color warn)?"
 
test $bIsRunning -eq 1 && sStatusRun="$(color active)R"
 
test $bIsInactive -eq 1 && sStatusRun="$(color disabled)D"
 
 
# check values
 
if [ $iLastStatus -gt 0 ]; then
 
ierrors=$ierrors+1
 
sStatusRun="$(color error)E"
 
else
 
if [ $iend -gt 0 -a $iagehours -gt $iMaxbackupAge -a $bIsInactive -eq 0 ]; then
 
ierrors=$ierrors+1
 
sStatusRun="$(color error)E"
 
fi
 
fi
 
 
if [ $iend -eq 0 ]; then
 
tstart="-"
 
tend="-"
 
sduration=""
 
iagehours=""
 
fi
 
 
# output
 
printf "$tbl" "$sStatusRun" "$myserver" "${tstart}" "${tend}" "${sduration}" "$iLastStatus" "${iagehours}" "$size"
 
done
 
color reset
 
echo
 
echo "Legend"
 
echo ". OK | ? not started | R running | D disabled | E error"
 
echo
 
echo "total : $( _getLoggedServers | wc -l ) servers"
 
echo "errors: $ierrors"
 
echo
 
fi
echo "rc=$ierrors"
echo "rc=$ierrors"
exit $ierrors
exit $ierrors
}
}