diff --git a/storage_helper.sh b/storage_helper.sh index 0170d9c2ab7690a2b2ecaae1f2d7956a1c3f910a..604270f43c60d7c0336d615f4443256266aa4826 100755 --- a/storage_helper.sh +++ b/storage_helper.sh @@ -33,6 +33,7 @@ # 2021-01-22 hahn show inactive backups in status page; colored lines by status # 2021-11-12 hahn backup status: add handling for backup exitcode # 2022-07-06 hahn backup status: show clear message on not started/ deleted backups +# 2024-03-22 hahn backup status: check age of restic backups when not using ssh # ====================================================================== # ---------------------------------------------------------------------- @@ -124,7 +125,7 @@ # get content of all logfiles function _getAllLogs(){ - zcat $sLogdir/*.gz ; cat $sLogfile + ls $sLogdir/*.gz >/dev/null 2>&1 && ( zcat $sLogdir/*.gz ; cat $sLogfile ) } # get a list of all servers in the log @@ -184,7 +185,7 @@ test -f "$sTouchfile" && echo 1 test -f "$sTouchfile" || echo 0 } - # check if a backup for current server is running + # check if a backup for current server is inactive # it returns 0 for no and 1 for yes # global string sBackupClient FQDN of current server # global string sBackupBasedir backup directory @@ -197,6 +198,14 @@ fi } + # get age of a file in sec + # param string filename to test + # + function _getFileAge(){ + echo $(($(date +%s) - $(date +%s -r "$1"))) + } + + # ------------------------------------------------------------ # slot handling # ------------------------------------------------------------ @@ -292,69 +301,127 @@ function PUBLIC_backupstatus(){ - typeset -i local ierrors=0 - local tbl="%s %-45s | %-20s | %-20s | %8s | %3s | %7s | %s \n" + local ierrors; typeset -i ierrors=0 + local tbl - _showConnectionCount - echo - echo "This table shows the time and duration [s] of the last backup for each server." - echo - printf "$tbl" " " "server" "start" "end" "duration" "rc" "age [h]" "size" - echo "-----------------------------------------------------------------------------------------------------------------------------" - for myserver in $( _getLoggedServers ) - do - # get data - setBackupclient $myserver - typeset -i local istart="$( _getLastBackupstart )" - typeset -i local iend="$( _getLastBackupend )" - typeset -i local bIsRunning=$( _isRunning ) - typeset -i local bIsInactive=$( _isInactive ) - typeset -i local iLastStatus=$( _getLastBackupstatus ) - local size=$( _getBackupsize ) - - local tstart=$( date +"%Y-%m-%d %H:%M:%S" -d @${istart} ) - local tend=$( date +"%Y-%m-%d %H:%M:%S" -d @${iend} ) - typeset -i local iduration=${iend}-${istart} - typeset -i local iage=$( date +%s )-${iend} - typeset -i local iagehours=$iage/60/60 - - local sduration=$iduration - test $iduration -lt 0 && sduration="RUNNING" - - sStatusRun="$(color ok)." - test $iend -eq 0 && sStatusRun="$(color warn)?" - test $bIsRunning -eq 1 && sStatusRun="$(color active)R" - test $bIsInactive -eq 1 && sStatusRun="$(color disabled)D" - - # check values - if [ $iLastStatus -gt 0 ]; then - ierrors=$ierrors+1 - sStatusRun="$(color error)E" - else - if [ $iend -gt 0 -a $iagehours -gt $iMaxbackupAge -a $bIsInactive -eq 0 ]; then - ierrors=$ierrors+1 - sStatusRun="$(color error)E" - fi - fi - - if [ $iend -eq 0 ]; then - tstart="-" - tend="-" - sduration="" - iagehours="" - fi - - # output - printf "$tbl" "$sStatusRun" "$myserver" "${tstart}" "${tend}" "${sduration}" "$iLastStatus" "${iagehours}" "$size" - done - color reset - echo - echo "Legend" - echo ". OK | ? not started | R running | D disabled | E error" - echo - echo "total : $( _getLoggedServers | wc -l ) servers" - echo "errors: $ierrors" + # _showConnectionCount echo + if ! _getLoggedServers | grep -qc . ; then + iKeepBackups=180 + local iAgeH; typeset -i iAgeH + local iAgeD; typeset -i iAgeD + local sAge + local mydir + local bIsInactive; typeset -i bIsInactive + local iCounter; typeset -i iCounter=0 + echo "Age of Restic backup repositories" + for myresticsnapshotdir in $( ls -1trd ${sBackupBasedir}/*/*/snapshots ) + do + iCounter=$iCounter+1 + mydir=$( dirname "${myresticsnapshotdir}") + sBackupClient=$( basename ${mydir} ) + iAgeH=$( _getFileAge "${myresticsnapshotdir}" )/60/60 + iAgeD=iAgeH/24 + bIsInactive=$( _isInactive ) + + sMore= + sStatus="$(color ok)." + + test $bIsInactive -eq 1 && sStatus="$(color disabled)D" + sFile=${mydir}/inactive.txt + + if [ "$bIsInactive" -eq "1" ]; then + typeset -i iAge=`_getFileAge "$sFile"` + typeset -i iDays=$iKeepBackups-$iAge/60/60/24 + sMore=" - INFO: keeping it $iKeepBackups d .. $iDays d left." + if [ "$iDays" -lt 0 ]; then + sMore=" - DELETE: overdued $iDays d." + sStatus="$(color error)E" + ierrors=$ierrors+1 + fi + elif [ $iAgeH -ge 24 ]; then + sStatus="$(color error)E" + ierrors=$ierrors+1 + + fi + + sAge="$iAgeH h" + if [ $iAgeH -gt 96 ]; then + sAge="$iAgeD d" + fi + + printf "%2s %7s %s\n" "$sStatus" "$sAge" "${mydir}$sMore" + done + color reset + echo + echo "Legend" + echo ". OK | D disabled | E error" + echo + echo "total : $iCounter servers" + echo "errors: $ierrors" + echo + + else + tbl="%s %-45s | %-20s | %-20s | %8s | %3s | %7s | %s \n" + echo "This table shows the time and duration [s] of the last backup for each server." + echo + printf "$tbl" " " "server" "start" "end" "duration" "rc" "age [h]" "size" + echo "-----------------------------------------------------------------------------------------------------------------------------" + for myserver in $( _getLoggedServers ) + do + # get data + setBackupclient $myserver + typeset -i local istart="$( _getLastBackupstart )" + typeset -i local iend="$( _getLastBackupend )" + typeset -i local bIsRunning=$( _isRunning ) + typeset -i local bIsInactive=$( _isInactive ) + typeset -i local iLastStatus=$( _getLastBackupstatus ) + local size=$( _getBackupsize ) + + local tstart=$( date +"%Y-%m-%d %H:%M:%S" -d @${istart} ) + local tend=$( date +"%Y-%m-%d %H:%M:%S" -d @${iend} ) + typeset -i local iduration=${iend}-${istart} + typeset -i local iage=$( date +%s )-${iend} + typeset -i local iagehours=$iage/60/60 + + local sduration=$iduration + test $iduration -lt 0 && sduration="RUNNING" + + sStatusRun="$(color ok)." + test $iend -eq 0 && sStatusRun="$(color warn)?" + test $bIsRunning -eq 1 && sStatusRun="$(color active)R" + test $bIsInactive -eq 1 && sStatusRun="$(color disabled)D" + + # check values + if [ $iLastStatus -gt 0 ]; then + ierrors=$ierrors+1 + sStatusRun="$(color error)E" + else + if [ $iend -gt 0 -a $iagehours -gt $iMaxbackupAge -a $bIsInactive -eq 0 ]; then + ierrors=$ierrors+1 + sStatusRun="$(color error)E" + fi + fi + + if [ $iend -eq 0 ]; then + tstart="-" + tend="-" + sduration="" + iagehours="" + fi + + # output + printf "$tbl" "$sStatusRun" "$myserver" "${tstart}" "${tend}" "${sduration}" "$iLastStatus" "${iagehours}" "$size" + done + color reset + echo + echo "Legend" + echo ". OK | ? not started | R running | D disabled | E error" + echo + echo "total : $( _getLoggedServers | wc -l ) servers" + echo "errors: $ierrors" + echo + fi echo "rc=$ierrors" exit $ierrors }