#!/bin/bash
# ======================================================================
#
# IML BACKUP STORAGE HELPER
#
# ----------------------------------------------------------------------
#
# SYNTAX:
#   storage_helper.sh PARAMETERS
#
# PARAMETERS:
#
#   register [hostname]
#     init backup for hostname; backup client must interrupt/ repeat
#     return code is <> 0
#
#   unregister [hostname] [statuscode]
#     send info that a backup was done to delete the  connection flag
#     for the given host.
#
#   status
#     show current connections
#
#   usage
#     show diskusage infos
#
# ----------------------------------------------------------------------
# 2019-11-01  hahn  write more clear log messages
# 2021-06-xx  hahn  added setactive + setinactive
# 2021-06-16  hahn  added backupstatus
# 2021-06-18  hahn  store backup status in status files ... do not grep frpom logs
# 2021-06-21  hahn  use single status file; added backup size
# 2021-01-22  hahn  show inactive backups in status page; colored lines by status
# 2021-11-12  hahn  backup status: add handling for backup exitcode
# 2022-07-06  hahn  backup status: show clear message on not started/ deleted backups
# 2024-03-22  hahn  backup status: check age of restic backups when not using ssh
# ======================================================================

# ----------------------------------------------------------------------
# CONFIG
# ----------------------------------------------------------------------


    . `dirname $0`/inc_config.sh

    # set in setBackupclient
    sBackupClient=dummy

    # value is set in _showConnectionCount
    iConnections=0

    # max age of last backup in h
    iMaxbackupAge=36


# ----------------------------------------------------------------------
# FUNCTIONS
# ----------------------------------------------------------------------


  # ------------------------------------------------------------
  # color text
  # ------------------------------------------------------------
    function color(){
      sColorcode=""
      case $1 in
        "reset") sColorcode="0"
                 ;;
        "head")  sColorcode="33" # yellow
                 ;;
        "cmd")   sColorcode="94" # light blue
                 ;;
        "input") sColorcode="92" # green
                 ;;
        "ok") sColorcode="92" # green
                 ;;
        "error") sColorcode="91" # red
                 ;;
        "warn")  sColorcode="33" # yellow
                 ;;
        "active") sColorcode="94" # light blue
                 ;;
        "disabled") sColorcode="90" # gray
                 ;;
      esac
      if [ ! -z ${sColorcode} ]; then
        echo -ne "\e[${sColorcode}m"
      fi
    }

    function _checkDir(){
      typeset -i iCount

      mydir=$1

      if [ -d $mydir ]; then
        iCount=`find $mydir -type f -cmin -720 | wc -l `
        if [ $iCount -eq 0 ]; then
          color error
        fi
        echo -n "$iCount        "
        echo -n "`du -hs $mydir #| awk ' { print $1 } '`        "

        if [ $iCount -eq 0 ]; then
          echo -n "<<< WARNING: No changed files detected !!! rc=1"
          color reset
          iErrors=$iErrors+1
        fi
        echo

      else
        echo SKIP: $mydir is not a directory
      fi
    }

  # ------------------------------------------------------------
  # logging
  # ------------------------------------------------------------
    # add a line into logfile
    # param  string  message (date will be added as prefix)
    function _addLog(){
        # echo `date` $* >>${sLogDir}${sLogfile}
        echo `date "+%Y-%m-%d %H:%M:%S"` $* >>${sLogfile}
    }

    # get content of all logfiles
    function _getAllLogs(){
        ls $sLogdir/*.gz >/dev/null 2>&1 && ( zcat $sLogdir/*.gz ; cat $sLogfile )
    }

    # get a list of all servers in the log
    function _getLoggedServers(){
        _getAllLogs | cut -f 2 -d '[' | cut -f 1 -d ']' | sort -u
    }

    # get timestamp of last backup start of a given server
    # global  string  sBackupClient  FQDN of current server
    function _getLastBackupstart(){
        #test -r "${sStatusDir}/${sBackupClient}_start" && stat -c %Y "${sStatusDir}/${sBackupClient}_start"
        #echo $tsBackupStart
        #local _srv=$1
        #_getAllLogs | sort | grep "\[$_srv\]" | grep ACCEPTED | tail -1 | cut -f -2 -d " "

        local tsBackupStart=
        test -r "$sStatusDir/${sBackupClient}" && . "$sStatusDir/${sBackupClient}"
        echo $tsBackupStart
    }
    
    # get timestamp of last backup end of a given server
    # global  string  sBackupClient  FQDN of current server
    function _getLastBackupend(){
        # test -r "${sStatusDir}/${sBackupClient}_end" && stat -c %Y "${sStatusDir}/${sBackupClient}_end"
        # local _srv=$1
        # _getAllLogs | sort | grep "\[$_srv\]" | grep REMOVED | tail -1 | cut -f -2 -d " "
        local tsBackupEnd=
        test -r "$sStatusDir/${sBackupClient}" && . "$sStatusDir/${sBackupClient}"
        echo $tsBackupEnd
    }
    
    # get status code of last backup from status file
    # global  string  sBackupClient  FQDN of current server
    function _getLastBackupstatus(){
        if [ -r "${sStatusDir}/${sBackupClient}" ]; then
          local backupRc=
          . "${sStatusDir}/${sBackupClient}"
          echo $backupRc
        else
          echo "-2"
        fi
    }
    # get size of backup data from status file
    # global  string  sBackupClient  FQDN of current server
    function _getBackupsize(){
        local size=
        test -r "$sStatusDir/${sBackupClient}" && . "$sStatusDir/${sBackupClient}"
        test -z "$size" && echo "[ no metadata: not started yet or deleted ]"
        echo $size
    }

    # check if a backup for current server is running
    # it returns 0 for no and 1 for yes
    # global  string  sBackupClient  FQDN of current server
    function _isRunning(){
        sTouchfile=`getConnFilename $sBackupClient`
        test -f "$sTouchfile" && echo 1
        test -f "$sTouchfile" || echo 0
    } 
    # check if a backup for current server is inactive
    # it returns 0 for no and 1 for yes
    # global  string  sBackupClient  FQDN of current server
    # global  string  sBackupBasedir backup directory
    function _isInactive(){
        ls ${sBackupBasedir}/*/$sBackupClient/inactive.txt >/dev/null 2>/dev/null
        if [ $? -eq 0 ]; then
            echo 1
        else
            echo 0
        fi
    }

    # get age of a file in sec
    # param  string  filename to test
    #
    function _getFileAge(){
        echo $(($(date +%s) - $(date +%s -r "$1")))
    }


  # ------------------------------------------------------------
  # slot handling
  # ------------------------------------------------------------

    # show current connction count
    function _showConnectionCount(){
        iConnections=`getConnectionCount`
        sMax=$iMaxConnections
        if [ $iMaxConnections -eq 0 ]; then
          sMax="0 [unlimited]"
        fi
        echo STATUS: $iConnections existing connection\(s\). Allowed maximum is $sMax.
    }

    
    # helper: get filename for backup client connection
    function getConnFilename(){
        # sBackupClient=$1
        echo $sConncectionDir/$sBackupClient
    }

    # allow a slot for a new connection
    # see PUBLIC_register
    function addConnection(){
        sTouchfile=`getConnFilename $sBackupClient`
        if [ -f "$sTouchfile" ]; then
            echo INFO: server [$sBackupClient] has a slot already
            _addLog "IGNORE request for a slot - [$sBackupClient] has a slot already"
        else
            touch "$sTouchfile"
            echo "client='$sBackupClient'" > "$sTouchfile"
            echo "# $( date """+%Y-%m-%d %H:%M:%S""" )" >> "$sTouchfile"
            echo "tsBackupStart=$( date +%s )" >> "$sTouchfile"
            _addLog "REQUEST a slot for server [$sBackupClient]"
        fi
    }

    # remove slot of a connection
    # see PUBLIC_unregister
    # param  int  backup status
    function freeConnection(){
        local _status=$1
        test -z "$1" && _status=-1
        sTouchfile=`getConnFilename $sBackupClient`
        if [ -f "$sTouchfile" ]; then
            echo "# $( date """+%Y-%m-%d %H:%M:%S""" )" >> "$sTouchfile"
            echo "tsBackupEnd=$( date +%s )" >> "$sTouchfile"
            echo "backupRc=$_status" >> "$sTouchfile"
            echo "size='$( du -hs ${sBackupBasedir}/*/$sBackupClient 2>/dev/null )'" >> "$sTouchfile"
            mv "$sTouchfile" "$sStatusDir/${sBackupClient}"
        else
            echo INFO: server [$sBackupClient] had no reserved slot
            _addLog "IGNORE freeing the connection - [$sBackupClient] had no reserved slot"
        fi
    }


    # count of conection slots
    function getConnectionCount(){
        # who | grep "^${sBackupUser}" | wc -l
        find $sConncectionDir -type f | wc -l
    }

    # helper function: set internal var for hostname of backup client
    function setBackupclient(){
        sBackupClient=`echo $1 | sed s"#[^a-zA-Z\.0-9\-]#_#g"`
        if [ -z $sBackupClient ]; then
            echo ERROR: no backup client was given
            exit 1
        fi
        # read last backup status
        typeset -i tsBackupStart=0
        typeset -i tsBackupEnd=0
        typeset -i backupRc=-2
        # test -r "$sStatusDir/${sBackupClient}" && . "$sStatusDir/${sBackupClient}"
    }

    # list repositories ... used in setactive/ setinactive
    function _listRepodirs(){
        for mydir in $( ls -1d ${sBackupBasedir}/*/* | grep -vE '(_active|somethinglsetodisable)' )
        do
                echo -n "$mydir "
                test -f "$mydir/inactive.txt" && ( color error ; echo -n "(inactive)" )
                test -f "$mydir/inactive.txt" || ( color ok    ; echo -n "(active)"   )
                color reset
                echo
        done
    }

# ----------------------------------------------------------------------
# PUBLIC FUNCTIONS
# ----------------------------------------------------------------------


    function PUBLIC_backupstatus(){
        local ierrors; typeset -i ierrors=0
        local tbl

        # _showConnectionCount
        echo
        if ! _getLoggedServers | grep -qc . ; then
            iKeepBackups=180
            local iAgeH; typeset -i iAgeH
            local iAgeD; typeset -i iAgeD
            local sAge
            local mydir
            local bIsInactive; typeset -i bIsInactive
            local iCounter; typeset -i iCounter=0
            echo "Age of Restic backup repositories"
            for myresticsnapshotdir in $( ls -1trd ${sBackupBasedir}/*/*/snapshots )
            do
                iCounter=$iCounter+1
                mydir=$( dirname "${myresticsnapshotdir}")
                sBackupClient=$( basename ${mydir} )
                iAgeH=$( _getFileAge "${myresticsnapshotdir}" )/60/60
                iAgeD=iAgeH/24
                bIsInactive=$( _isInactive )

                sMore=
                sStatus="$(color ok)."

                test $bIsInactive -eq 1  && sStatus="$(color disabled)D"
                sFile=${mydir}/inactive.txt

                if [ "$bIsInactive" -eq "1" ]; then
                     typeset -i iAge=`_getFileAge "$sFile"`
                     typeset -i iDays=$iKeepBackups-$iAge/60/60/24
                     sMore=" - INFO: keeping it $iKeepBackups d .. $iDays d left."
                     if [ "$iDays" -lt 0 ]; then
                        sMore=" - DELETE: overdued $iDays d."
                        sStatus="$(color error)E"
                        ierrors=$ierrors+1
                     fi
                elif [ $iAgeH -ge 24 ]; then
                    sStatus="$(color error)E"
                    ierrors=$ierrors+1

                fi

                sAge="$iAgeH h"
                if [ $iAgeH -gt 96 ]; then
                    sAge="$iAgeD d"
                fi

                printf "%2s  %7s  %s\n" "$sStatus" "$sAge" "${mydir}$sMore"
            done
            color reset
            echo
            echo "Legend"
            echo ". OK | D disabled | E error"
            echo
            echo "total : $iCounter servers"
            echo "errors: $ierrors"
            echo

        else
            tbl="%s %-45s | %-20s | %-20s | %8s | %3s | %7s | %s \n"
            echo "This table shows the time and duration [s] of the last backup for each server."
            echo
            printf "$tbl" " " "server" "start" "end" "duration" "rc" "age [h]" "size"
            echo "-----------------------------------------------------------------------------------------------------------------------------"
            for myserver in $( _getLoggedServers )
            do
                # get data
                setBackupclient $myserver
                typeset -i local istart="$( _getLastBackupstart )"
                typeset -i local iend="$( _getLastBackupend )"
                typeset -i local bIsRunning=$( _isRunning )
                typeset -i local bIsInactive=$( _isInactive )
                typeset -i local iLastStatus=$( _getLastBackupstatus )
                local size=$( _getBackupsize )

                local tstart=$( date +"%Y-%m-%d %H:%M:%S" -d @${istart} )
                local tend=$( date +"%Y-%m-%d %H:%M:%S" -d @${iend} )
                typeset -i local iduration=${iend}-${istart}
                typeset -i local iage=$( date +%s )-${iend}
                typeset -i local iagehours=$iage/60/60

                local sduration=$iduration
                test $iduration -lt 0 && sduration="RUNNING"

                sStatusRun="$(color ok)."
                test $iend        -eq 0  && sStatusRun="$(color warn)?"
                test $bIsRunning  -eq 1  && sStatusRun="$(color active)R"
                test $bIsInactive -eq 1  && sStatusRun="$(color disabled)D"

                # check values
                if [ $iLastStatus -gt 0 ]; then
                    ierrors=$ierrors+1
                    sStatusRun="$(color error)E"
                else
                if [ $iend -gt 0 -a $iagehours -gt $iMaxbackupAge -a $bIsInactive -eq 0 ]; then
                    ierrors=$ierrors+1
                    sStatusRun="$(color error)E"
                fi
                fi

                if [ $iend -eq 0 ]; then
                    tstart="-"
                    tend="-"
                    sduration=""
                    iagehours=""
                fi

                # output
                printf "$tbl" "$sStatusRun" "$myserver" "${tstart}" "${tend}" "${sduration}" "$iLastStatus" "${iagehours}" "$size"
            done
            color reset
            echo
            echo "Legend"
            echo ". OK | ? not started | R running | D disabled | E error"
            echo
            echo "total : $( _getLoggedServers | wc -l ) servers"
            echo "errors: $ierrors"
            echo
        fi
        echo "rc=$ierrors"
        exit $ierrors
    }

    function PUBLIC_setactive(){
        echo --- list of inactive backup repositories
        _listRepodirs | grep '\(inactive\)'
        echo
        echo -n "Repo to activate again >"
        read inputdir
        if [ -f "$inputdir/inactive.txt" ]; then
                cat "$inputdir/inactive.txt"
                echo
                echo Removing $inputdir/inactive.txt
                rm -f "$inputdir/inactive.txt"
                echo If it was a mistake run $( basename $0 ) setinactive 
        else
                echo SKIP [$inputdir]
        fi
        echo
    }

    function PUBLIC_setinactive(){
        echo --- list of backup repositories
        _listRepodirs
        echo
        echo -n "Repo to deactivate >"
        read inputdir
        if [ -d "$inputdir" -a ! -f "$inputdir/inactive.txt" ]; then
                echo -n "Optional short message >"
                read mymessage
                echo "$( date ) | $mymessage" > "$inputdir/inactive.txt"
                ls -l "$inputdir/inactive.txt"
        else
                echo SKIP [$inputdir]
        fi
        echo
    }

    # return backup directory for a client
    function PUBLIC_getBackupdir(){
        setBackupclient $1
        echo $sBackupBasedir/$sBackupClient
        exit 0
    }

    # register a hostname to start backups
    # param  string  FQDN
    function PUBLIC_register(){
        setBackupclient $1
        echo "REGISTER $1"
        echo

        addConnection
        # iConnections=`getConnectionCount`
        _showConnectionCount >/dev/null

        if [ $iConnections -le $iMaxConnections -o $iMaxConnections -eq 0 ]; then
            sStatus="OK"
            iExit=0
            _addLog "ACCEPTED [$sBackupClient] got a slot."
        else
            sStatus="FAILED"
            iExit=1

            freeConnection
            _addLog "REJECTED maximum of $iMaxConnections connections exceeded - request of server [$sBackupClient] failed."
        fi

        _showConnectionCount
        echo $sStatus - rc=$iExit
        exit $iExit

    }

    # unregister a backup slot
    # param  string  FQDN
    # param  int     optional: return code of the backup run
    function PUBLIC_unregister(){
        typeset -i local _status=$2
        setBackupclient $1
        echo "UNREGISTER $1"
        echo
        freeConnection $_status
        _addLog "REMOVED slot for [$sBackupClient]"
        _showConnectionCount
        exit 0
    }

    # show used slots
    function PUBLIC_status(){
        _showConnectionCount
        echo
        if [ $iConnections -gt 0 ]; then
            find $sConncectionDir -type f -exec ls -l {} \; | nl
            echo
        fi
        exit 0
    }

    function PUBLIC_usage(){
        typeset -i iErrors=0

        # can be "full"
        sDetail=$1

        echo "DISK USAGE:"
        echo
        echo "time                   : `date`"
        echo "Backups are located in : $sBackupBasedir"
        echo "dirs                   : `find ${sBackupBasedir} -type d | wc -l`"
        echo "files                  : `find ${sBackupBasedir} -type f | wc -l`"
        echo "volume size            : `df -h ${sBackupBasedir} | tail -1 | awk '{ print $2 }'`"
        echo "backup size            : `du -hs ${sBackupBasedir} | cut -f 1 `"
        echo "free                   : `df -h ${sBackupBasedir} | tail -1 | awk '{ print $4 }'`"
        echo
        echo sizes:
        # ls -l "${sBackupBasedir}" | grep "^d" | fgrep -v "`basename $sConncectionDir`"
        du -hs ${sBackupBasedir}*
        echo

        echo "changed files last 12 hours and used diskspace:"
        echo
        _checkDir ${sBackupBasedir}
        echo
        if [ ! -z ${sDetail} ]; then
          sFilter=${sDetail}
          if [ "${sDetail}" = "full" ]; then
            sFilter=
          fi

          echo "... and by server and backup sets:"
          echo

          for myserver in `ls -1 ${sBackupBasedir} | fgrep -v "_active" | grep "$sFilter"`
          do
            color head
            echo ----- $myserver
            color reset
            echo
            _checkDir ${sBackupBasedir}/$myserver
            echo
            for subdir in `ls -1 ${sBackupBasedir}/$myserver`
            do
              _checkDir ${sBackupBasedir}/$myserver/$subdir
            done
            echo
          done
        fi

        # --- show result
        echo -n "Status: "
        if [ $iErrors -eq 0 ]; then
          echo "OK"
        else
          color error
          echo "$iErrors Errors detected."
        fi
        color reset
        echo rc=$iErrors
        exit $iErrors
    }

# ----------------------------------------------------------------------
# MAIN
# ----------------------------------------------------------------------


    if [ ! -d "$sConncectionDir" ]; then
        mkdir -p "$sConncectionDir" || exit 1
    fi
    if [ ! -d "$sStatusDir" ]; then
        mkdir -p "$sStatusDir" || exit 1
    fi

    color head
    echo
    echo "---------- :: STORAGE :: `hostname` :: ----------"
    echo
    color reset

    sFunction="PUBLIC_$1"
    fgrep "${sFunction}()" $0 >/dev/null


    if [ $? -ne 0 ]; then
        echo SYNTAX:
        echo "`basename $0` [function]"
        echo
        echo "  backupstatus"
        echo "    show all servers and their backup times"
        echo
        echo "  register [hostname]"
        echo "    add a slot for a backup client"
        echo
        echo "  unregister [hostname] [statuscode]"
        echo "    remove slot for a backup client."
        echo "    The statuscode is an integer of the return code."
        echo
        echo "  setactive"
        echo "    Reactivate an inctave backup repo."
        echo
        echo "  setinactive"
        echo "    Mark a backup repo as inactive."
        echo "    This prevents errors of missing backup data"
        echo "    if a host is obsolete and was shut down."
        echo
        echo "  status"
        echo "    show current reserved slots"
        echo
        echo "  usage [full|[filter to hostname]] - DEPRECATED"
        echo "    show used diskspace"
        echo "    you can set a string to show additionally details if these hosts"
        echo "    if you add \"full\" then details of all hosts will be shown"
        echo
        #echo SCAN... all functions are:
        #grep "function\ PUBLIC_" $0 | cut -f 2 -d "_" | cut -f 1 -d "("
    else
        shift 1
        $sFunction $*
    fi

# ----------------------------------------------------------------------