#!/bin/bash
# ==============================================================================
#
# check status of last performed backup
#
# This script checks the last logfile and scans for errors
# An error is a returncode <> 0 or a too old logfile
#
# ------------------------------------------------------------------------------
#
# ah - Axel Hahn <axel.hahn@iml.unibe.ch>
# ds - Daniel Schueler <daniel.schueler@iml.unibe.ch>
#
# 2016-12-09  ah,ds  v1.0
# 2022-01-19  ah     v1.1  fixes with shellcheck
# 2022-02-09  ah     v1.2  update after changes in logging
# 2022-10-07  ah     v1.3  unescape regex with space to prevent "grep: warning: stray \ before white space"
# 2022-10-20  ah     v1.4  show last backup, prune and verify
# 2023-09-01  ah     v1.5  add counters for new/ changed/ deleted fileobjects (Restic only)
# 2023-09-04  ah     v1.6  fix perfdata dirs-del=
# 2024-08-28  ah     v1.7  update counter new/ changed/ deleted fileobjects without restic snapshot comparison
# ==============================================================================

. $(dirname $0)/includes/jobhelper.sh

# ------------------------------------------------------------------------------
# CONFIG
# ------------------------------------------------------------------------------

typeset -i iOK=0
typeset -i iError

# limit when to warn if no backup was started
typeset -i iMaxAgeInHours=24

typeset -i iAge
typeset -i iAgeH
typeset -i iAgeD

logdir=$(dirname "$0")/logs

flagShowChangedFiles=0

# ------------------------------------------------------------------------------
# FUNCTIONS
# ------------------------------------------------------------------------------

# get age of a given file and fill global vars
#   iAge
#   iAgeH
#   iAgeD
# param  string  filename
function getAge(){
    typeset -i local tsfile=0
    test -r "$1" && tsfile=$( date +%s -r "$1" )

    iAge=$( date +%s )-$tsfile
    iAgeH=$iAge/60/60
    iAgeD=$iAge/60/60/24
}

# show age of a file hr like
function showAge(){
  getAge "$1"
  if [ $iAgeD -gt 10000 ]; then
    echo "NEVER"
  else
    test $iAgeD -gt 0 && echo -n "$iAgeD days"
    test $iAgeD -eq 0 && test $iAgeH -gt 0 && echo -n "$iAgeH h"
    test $iAgeD -eq 0 && test $iAgeH -eq 0 && echo -n "$iAge s"
    echo " ago"
  fi
}

# filter color in given output
# To use it set a pipe in front:
# <command> | filterColor
function filterColor(){
  sed -r "s/\x1B\[([0-9]{1,3}(;[0-9]{1,2})?)?[mGK]//g"
}

# ------------------------------------------------------------------------------
# MAIN
# ------------------------------------------------------------------------------

sShort="Client Backup -"

ls "$logdir"/*.log* >/dev/null
if [ $? -ne 0 ]; then
  sShort="$sShort logs were not found. Backup was never executed"
  iError=1
else

  backuplogfile=$(ls -1t "$logdir"/inc*.log "$logdir"/full*.log "$logdir"/auto*.log 2>/dev/null | head -1)
  transferlogfile=$(ls -1t "$logdir"/transfer*.log | head -1)

  logfile="${backuplogfile} ${transferlogfile}"
  lastlogfile="$(ls -1t $logfile | head -1)"

    echo LAST LOGS: "$logfile"
    echo
    # echo ">>> RETURNCODES"
    # grep "final.*rc=" $logfile

    # --- Monitoring Statusline
    if [ -f "$(dirname $0)"/transfer.sh ]; then
      sShort="$sShort WITH"
    else
      sShort="$sShort NO"
    fi
    sShort="$sShort transfer -"

    # --- count returncodes
    iOK=$(grep "final.*rc=0$" $logfile | wc -l )
    iError=$(grep "final.*rc=" $logfile | grep -v "rc=0$" | wc -l )
    echo "OK: $iOK ... Errors: $iError"
    
    # show errors:
    grep "final.*rc=" $logfile | grep -v "rc=0"

    # --- rc=0 must be here
    if [ $iOK -eq 0 ]; then
      echo "ERROR: no OK message was found. Something is messed up :-/"
      iError+=1
    fi
    echo

    # --- check age
    echo ">>> AGE"
    getAge "$lastlogfile"
    echo "age: $iAge sec ... $iAgeH h"
    sShort="$sShort last backup log: $(basename "$lastlogfile") ($iAgeH h ago) "
    if [ $iAgeH -gt $iMaxAgeInHours ]; then
      echo "Error: the last backup is older than $iMaxAgeInHours hours"
      sShort="$sShort ERROR: backup is older than $iMaxAgeInHours hours "
      iError+=1
    else
      echo "OK: last backup is younger $iMaxAgeInHours hours"
      sShort="$sShort OK: backup is younger than $iMaxAgeInHours hours "
      iOK+=1
    fi
    echo

    # --- changes (incremental backups only) and backup status infos
    if [ $flagShowChangedFiles -ne 0 ]; then
      sSearch="Args:|ElapsedTime|SourceFiles|SourceFileSize|RawDeltaSize"
      if echo "$logfile" | grep -E "(inc|auto)" >/dev/null; then
        sSearch="[+-M] |${sSearch}"
      fi
      echo ">>> Changes:"
      grep -E "^(${sSearch})" $logfile | cut -f 2- -d  ':'
      echo
    fi

    if cat $logfile | grep restic >/dev/null; then

      echo ">>> Counters:"

      # This is faster andd doesn't need comparison of snapshots in output
      # and works with initial backups too
      typeset -i iModFDelete=0
      typeset -i iModDDelete=0
      iModFNew=$(    cat $logfile | grep "^Files:.*unmodified" | awk '{sum+=$2;} END {print sum;}' )
      iModFChange=$( cat $logfile | grep "^Files:.*unmodified" | awk '{sum+=$4;} END {print sum;}' )
      iModFNone=$(   cat $logfile | grep "^Files:.*unmodified" | awk '{sum+=$6;} END {print sum;}' )
      iModFDelete=$( cat $logfile | grep "^Files:.*removed"    | awk '{sum+=$4;} END {print sum;}' )

      iModDNew=$(    cat $logfile | grep "^Dirs:.*unmodified"  | awk '{sum+=$2;} END {print sum;}' )
      iModDChange=$( cat $logfile | grep "^Dirs:.*unmodified"  | awk '{sum+=$4;} END {print sum;}' )
      iModDNone=$(   cat $logfile | grep "^Dirs:.*unmodified"  | awk '{sum+=$6;} END {print sum;}' )
      iModDDelete=$( cat $logfile | grep "^Dirs:.*removed"     | awk '{sum+=$4;} END {print sum;}' )

      # old way:
      # iModFNew=$(    cat $logfile | grep -c "^+    /.*[^/]$" )
      # iModDNew=$(    cat $logfile | grep -c "^+    /.*/$" )
      # iModFChange=$( cat $logfile | grep -c "^M    /.*[^/]$" )
      # iModDChange=$( cat $logfile | grep -c "^M    /.*/$" )
      # iModFDelete=$( cat $logfile | grep -c "^-    /.*[^/]$" )
      # iModDDelete=$( cat $logfile | grep -c "^-    /.*/$" )

      printf "__COUNTERS__New:       %10s files %10s dirs\n" $iModFNew    $iModDNew
      printf "__COUNTERS__Changed:   %10s files %10s dirs\n" $iModFChange $iModDChange
      printf "__COUNTERS__Deleted:   %10s files %10s dirs\n" $iModFDelete $iModDDelete
      printf "__COUNTERS__Unchanged: %10s files %10s dirs\n" $iModFNone   $iModDNone
      echo "__PERFDATA__ |files-new=$iModFNew;;;0 files-mod=$iModFChange;;;0 files-del=$iModFDelete dirs-new=$iModDNew;;;0 dirs-mod=$iModDChange;;;0 dirs-del=$iModDDelete;;;0"
      echo
    fi

    echo ">>> Summary of database backup actions:"
    cat $logfile | grep '__DB__' | filterColor | cut -f 1 -d ' ' | sort -u | while read -r dbprefix
    do
      ( grep "$dbprefix SKIP" $logfile || grep "$dbprefix " $logfile ) | cut -f 2- -d ':'
    done
    echo

    echo ">>> Summary of transfer actions:"
    cat $logfile | grep -E '__(BACKUP|PRUNE|REPO|VERIFY)__' | filterColor
    echo

    sShort="$sShort - OK: $iOK ... Errors: $iError"

fi

echo ">>> Backup repository:"
echo "__LAST__backup " $(showAge "$logdir/last_backup")
echo "__LAST__prune  " $(showAge "$logdir/last_prune")
echo "__LAST__verify " $(showAge "$logdir/last_verify")
echo
echo "MONITORINFO: $sShort"
echo "STATUS $0 - final returncode rc=$iError"
exit "$iError"

# ------------------------------------------------------------------------------