check_clientbackup.sh

#!/bin/bash
# ==============================================================================
#
# check status of last performed backup
#
# This script checks the last logfile and scans for errors
# An error is a returncode <> 0 or a too old logfile
#
# ------------------------------------------------------------------------------
#
# ah - Axel Hahn <axel.hahn@iml.unibe.ch>
# ds - Daniel Schueler <daniel.schueler@iml.unibe.ch>
#
# 2016-12-09  ah,ds  v1.0
# 2022-01-19  ah     v1.1  fixes with shellcheck
# 2022-02-09  ah     v1.2  update after changes in logging
# 2022-10-07  ah     v1.3  unescape regex with space to prevent "grep: warning: stray \ before white space"
# 2022-10-20  ah     v1.4  show last backup, prune and verify
# ==============================================================================

. $(dirname $0)/jobhelper.sh

# ------------------------------------------------------------------------------
# CONFIG
# ------------------------------------------------------------------------------

typeset -i iOK=0
typeset -i iError

# limit when to warn if no backup was started
typeset -i iMaxAgeInHours=24

typeset -i iAge
typeset -i iAgeH
typeset -i iAgeD

logdir=$(dirname "$0")/logs

flagShowChangedFiles=0

# ------------------------------------------------------------------------------
# FUNCTIONS
# ------------------------------------------------------------------------------

# get age of a given file and fill global vars
#   iAge
#   iAgeH
#   iAgeD
# param  string  filename
function getAge(){
    typeset -i local tsfile=0
    test -r "$1" && tsfile=$( date +%s -r "$1" )

    iAge=$( date +%s )-$tsfile
    iAgeH=$iAge/60/60
    iAgeD=$iAge/60/60/24
}

# show age of a file hr like
function showAge(){
  getAge "$1"
  if [ $iAgeD -gt 10000 ]; then
    echo "NEVER"
  else
    test $iAgeD -gt 0 && echo -n "$iAgeD days"
    test $iAgeD -eq 0 && test $iAgeH -gt 0 && echo -n "$iAgeH h"
    test $iAgeD -eq 0 && test $iAgeH -eq 0 && echo -n "$iAge s"
    echo " ago"
  fi
}

# filter color in given output
# To use it set a pipe in front:
# <command> | filterColor
function filterColor(){
  sed -r "s/\x1B\[([0-9]{1,3}(;[0-9]{1,2})?)?[mGK]//g"
}

# ------------------------------------------------------------------------------
# MAIN
# ------------------------------------------------------------------------------

sShort="Client Backup -"

ls "$logdir"/*.log* >/dev/null
if [ $? -ne 0 ]; then
  sShort="$sShort logs were not found. Backup was never executed"
  iError=1
else

  backuplogfile=$(ls -1t "$logdir"/inc*.log "$logdir"/full*.log "$logdir"/auto*.log 2>/dev/null | head -1)
  transferlogfile=$(ls -1t "$logdir"/transfer*.log | head -1)

  logfile="${backuplogfile} ${transferlogfile}"
  lastlogfile="$(ls -1t $logfile | head -1)"

    echo LAST LOGS: "$logfile"
    echo
    # echo ">>> RETURNCODES"
    # grep "final.*rc=" $logfile

    # --- Monitoring Statusline
    if [ -f "$(dirname $0)"/transfer.sh ]; then
      sShort="$sShort WITH"
    else
      sShort="$sShort NO"
    fi
    sShort="$sShort transfer -"

    # --- count returncodes
    iOK=$(grep "final.*rc=0$" $logfile | wc -l )
    iError=$(grep "final.*rc=" $logfile | grep -v "rc=0$" | wc -l )
    echo "OK: $iOK ... Errors: $iError"
    
    # show errors:
    grep "final.*rc=" $logfile | grep -v "rc=0"

    # --- rc=0 must be here
    if [ $iOK -eq 0 ]; then
      echo "ERROR: no OK message was found. Something is messed up :-/"
      iError+=1
    fi
    echo

    # --- check age
    echo ">>> AGE"
    getAge "$lastlogfile"
    echo "age: $iAge sec ... $iAgeH h"
    sShort="$sShort last backup log: $(basename "$lastlogfile") ($iAgeH h ago) "
    if [ $iAgeH -gt $iMaxAgeInHours ]; then
      echo "Error: the last backup is older than $iMaxAgeInHours hours"
      sShort="$sShort ERROR: backup is older than $iMaxAgeInHours hours "
      iError+=1
    else
      echo "OK: last backup is younger $iMaxAgeInHours hours"
      sShort="$sShort OK: backup is younger than $iMaxAgeInHours hours "
      iOK+=1
    fi
    echo

    # --- changes (incremental backups only) and backup status infos
    if [ $flagShowChangedFiles -ne 0 ]; then
      sSearch="Args:|ElapsedTime|SourceFiles|SourceFileSize|RawDeltaSize"
      if echo "$logfile" | grep -E "(inc|auto)" >/dev/null; then
        sSearch="[ADM] |${sSearch}"
      fi
      echo ">>> Changes:"
      grep -E "^(${sSearch})" $logfile | cut -f 2- -d  ':'
      echo
    fi

    echo ">>> Summary of database backup actions:"
    cat $logfile | grep "__[A-Z][A-Z]*__" | grep '__DB__' | filterColor
    echo

    echo ">>> Summary of transfer actions:"
    cat $logfile | grep "__[A-Z][A-Z]*__" | grep -E '__(BACKUP|PRUNE|REPO|VERIFY)__' | filterColor
    echo

    sShort="$sShort - OK: $iOK ... Errors: $iError"

fi

echo ">>> Backup repository:"
echo "__LAST__backup " $(showAge "$logdir/last_backup")
echo "__LAST__prune  " $(showAge "$logdir/last_prune")
echo "__LAST__verify " $(showAge "$logdir/last_verify")
echo
echo "MONITORINFO: $sShort"
echo "STATUS $0 - final returncode rc=$iError"
exit "$iError"

# ------------------------------------------------------------------------------