#!/bin/bash
# ================================================================================
#
# TRANSFER LOCAL DATA TO BACKUP STORAGE
#
# SYNTAX:
#   transfer.sh       - incremental backup
#   transfer.sh full  - full backup
#   transfer.sh dumps - transfer local dumps only
#   transfer.sh prune - cleanup backup data
#   transfer.sh help  - show help
#
# --------------------------------------------------------------------------------
# ah - Axel Hahn <axel.hahn@iml.unibe.ch>
# ds - Daniel Schueler <daniel.schueler@iml.unibe.ch>
#
# 2016-11-10  ah,ds  v1.0
# 2016-12-19  ah,ds  v1.1  added parameter "dumps"
# 2017-02-16  ah,ds  v1.2  added support for storage slots
# 2017-10-11  ah,ds  v1.3  added support for duplicity param --ssh-backend
# 2017-10-17  ah,ds  v1.4  remove PIPESTATUS for Debian8 compatibility
# 2017-11-17  ah,ds  v1.5  check pid of lockfile in process list if process still runs
# 2018-06-19  ah,ds  v1.6  replace --exclude with --exclude regexp in custom dirs
# 2019-06-05  ah,ds  v1.7  add custom cache dir
# 2019-09-09  ah,ds  v1.8  add testfile on target
# 2019-10-30  ah,ds  v1.9  for rsync targets: create remote target dir with ssh command
# 2020-01-21  ah,ds  v1.10 show colored OK or FAILED at the end of output
# 2020-02-25  ah,ds, v1.11 fix test -z with non existing vars; show final backup status
# 2021-01-29  ah,ds, v1.12 abort on empty passphrase
# 2021-05-19  ah,ds, v2.0  plugin driven to support multiple backup tools (duplicity + restic)
# 2021-12-02  ah     v2.1  added parameter "prune" to cleanup only
# 2022-02-10  ah     v2.2  update logging (removing tee)
# 2022-10-01  ah     v2.3  customize prune and verify action
# 2022-10-04  ah     v2.4  prune and verify are non directory based
# 2022-10-07  ah     v2.5  unescape regex with space to prevent "grep: warning: stray \ before white space"
# 2022-10-20  ah     v2.6  move hook 20-before-transfer (after init of the backup tool)
# 2022-10-21  ah     v2.7  shell fixes; 
# 2022-11-04  ah     v2.8  rename hooks
# 2022-11-07  ah     v2.9  run brefore-transfer hook after registration
# 2023-02-22  ah     v2.10 fix touch of last_backup
# 2024-02-02  ah     v2.11 support "never" for prune-after, verify-after
# ================================================================================


# --------------------------------------------------------------------------------
# CONFIG
# --------------------------------------------------------------------------------

  # . `dirname $0`/inc_config.sh

  . $(dirname $0)/includes/jobhelper.sh || exit 1
  . $(dirname $0)/includes/inc_bash.sh  || exit 1

  typeset -i rc=0
  typeset -i doBackup=1
  typeset -i doPrune
  typeset -i doVerify
  typeset -i doValue

  if [ ! -r "${DIRFILE}" -o ! -r "${STORAGEFILE}" ]; then
    echo "SKIP backup of local files - one of the files is not readable (no error): ${DIRFILE} | ${STORAGEFILE}"
    exit 0
  fi

  STORAGE_BIN=$(_j_getvar "${STORAGEFILE}" "bin")
  STORAGE_BASEDIR=$(_j_getvar "${STORAGEFILE}" "storage")
  STORAGE_TESTFILE=$(_j_getvar "${STORAGEFILE}" "storage-file")
  PASSPHRASE=$(_j_getvar "${STORAGEFILE}" "passphrase")

  STORAGE_REGISTER=$(_j_getvar "${STORAGEFILE}" "storage-register")
  typeset -i TIMER_TRANSFER_START
  TIMER_TRANSFER_START=$(date +%s)

  # check
  if [ -z "$STORAGE_BIN" ]; then
    STORAGE_BIN=restic
  fi
  CFGPREFIX=${STORAGE_BIN}_

  if [ -z "$STORAGE_BASEDIR" ]; then
    color error
    echo ERROR: missing config for backup target.
    echo There must be an entry "storage = " in ${STORAGEFILE}
    color reset
    exit 1
  fi
  if [ -n "$STORAGE_TESTFILE" -a ! -f "$STORAGE_TESTFILE" ]; then
    color error
    echo "ERROR: missing testfile $STORAGE_TESTFILE on backup target."
    echo "The Backup target disk / NAS is not mounted."
    color reset
    exit 1
  fi
  # support old value
  if [ -z "${PASSPHRASE}" ]; then
    echo "WARNING: The value gnupg-passphrase in ${STORAGEFILE} is deprecated. Replace it with passphrase=..."
    PASSPHRASE=$(_j_getvar "${STORAGEFILE}" "gnupg-passphrase")
  fi

  if [ -z "${PASSPHRASE}" ]; then
    echo "ERROR: no value passphrase was set in ${STORAGEFILE} to encrypt backup data."
    echo "Aborting."
    exit 1
  fi
  
  # For duplicity only
  # METHOD incremental is default; full backup will be triggered with 
  # first param "full"
  METHOD=

  ACTION=backup

  transferlog="${DIR_LOGS}/transfer-$(date +%Y%m%d-%H%M%S).log"
  lockfile="${DIR_LOGS}/transfer.running"
  lastbackupfile="${DIR_LOGS}/last_backup"
  lastprunefile="${DIR_LOGS}/last_prune"
  lastverifyfile="${DIR_LOGS}/last_verify"
  rcfile=/tmp/transfer-rc.$$.tmp


# --------------------------------------------------------------------------------
# FUNCTIONS
# --------------------------------------------------------------------------------

# get age of a file in sec
# https://www.axel-hahn.de/blog/2020/03/09/bash-snippet-alter-einer-datei/
# param  string  filename to test
# return integer
function _getFileAge(){
    echo $(($(date +%s) - $(date +%s -r "$1")))
}

# set default behaviur based on limit (from config) and its last
# execution (read from touchfile)
# Function is enabled if no touchfile was found or it is older than
# given limit
#
# It is used to set flags for prune and verify
# param  string  name of action; one of prune|verify
# param  string  filename of touchfile of last execution
function setAction(){
  local action=$1
  local myfile=$2
  local doforce=$3

  local iLimit
  iLimit=$(_j_getvar ${STORAGEFILE} "${action}-after")
  if [ "$iLimit" = "never" ]; then
    echo "Info: $action is set to [never] and is disabled."
    doValue=0
    return
  fi

  if [ ! -f "${myfile}" ]; then
    echo "Info: $action is ENABLED - no last $action detected"
    doValue=1
  else
    if [ "$doforce" -eq "1" ]; then
      echo "Force $action"
      doValue=1
    else
      typeset -i iLastDone
      iLastDone=$( _getFileAge "${myfile}" )/60
      typeset -i iLastDoneD=iLastDone/60/24
      echo "Info: Last $action was $iLastDone min ago ($iLastDoneD days). Limit is $iLimit days."
      if [ $iLastDoneD -ge $iLimit ]; then 
        echo "Info: $action is ENABLED - last $action is outdated"
        doValue=1
      else
        echo "Info: $action is not needed yet."
        doValue=0
      fi
    fi
  fi
}

# --------------------------------------------------------------------------------
# MAIN
# --------------------------------------------------------------------------------

  if [ "$1" = "help" -o "$1" = "-h" -o "$1" = "-?" ]; then
    echo "HELP:

  Transfer local files to a backup target.

    target       $( echo ${STORAGE_BASEDIR} | sed 's#:[^:]*@#:**********@#' )
    backup tool  $STORAGE_BIN

  PARAMETERS:
    transfer.sh        - incremental backup
    transfer.sh full   - full backup (Duplicity only)
    transfer.sh dumps  - transfer local dumps only
    transfer.sh prune  - cleanup backup data only (no backup)
    transfer.sh verify - verify backup data (no backup)
    transfer.sh help   - show this help (works with -h and -? too)
    "
    exit 0
  fi

  # set defaults for prune and verify
  echo ">>> Detect default behaviour:"
  setAction "prune"  "$lastprunefile"  0;  doPrune=$doValue
  setAction "verify" "$lastverifyfile" 0; doVerify=$doValue
  echo

  echo ">>> Check parameters"
  if [ "$1" = "prune" ]; then
    echo "Info: Forcing prune only by parameter."
    ACTION=$1
    doBackup=0
    setAction "prune"  "$lastprunefile" 1;  doPrune=$doValue
    doVerify=0
    transferlog="${DIR_LOGS}/prune-$(date +%Y%m%d-%H%M%S).log"
  fi
  if [ "$1" = "verify" ]; then
    echo "Info: Forcing verify only by parameter."
    ACTION=$1
    doBackup=0
    doPrune=0
    doVerify=1
    transferlog="${DIR_LOGS}/verify-$(date +%Y%m%d-%H%M%S).log"
  fi

  if [ "${doBackup}${doPrune}${doVerify}" = "000" ]; then
    echo "Nothing to do. Aborting."
    exit 0
  fi

  exec 1> >( tee -a "$transferlog" ) 2>&1
  echo "INFO: Start logging into $transferlog"  

  h1 "$( date )  TRANSFER LOCAL DATA TO STORAGE"

  echo "TOOL     : $STORAGE_BIN"
  echo "ACTION   : $ACTION"
  echo "TARGET   : ${STORAGE_BASEDIR}" | sed 's#:[^:]*@#:**********@#'
  echo

  if [ "$ACTION" = "backup" ]; then
    echo "METHOD   : $METHOD"
    echo "REGISTER : ${STORAGE_REGISTER}"
    echo "PRUNE    : $doPrune"
    echo "VERIFY   : $doVerify"
    echo
  fi

  . $(dirname $0)/plugins/transfer/$STORAGE_BIN.sh || exit 1

# --------------------------------------------------------------------------------
# ----- Check requirements

  t_checkRequirements || exit 1

  test -z "$STORAGE_REGISTER" || . $(dirname $0)/plugins/register/$STORAGE_REGISTER.sh || exit 1

  echo Check locking of a running transfer
  if [ -f "${lockfile}" ]; then
    color error
    echo A lock file for a running transfer was found
    cat "${lockfile}"
    color reset
    echo

    # 1659 - check process id inside the lock file
    # detect pid from lockfile and search for this process
    lockpid=$(cat "${lockfile}" | cut -f 2 -d "-" | cut -f 4 -d " " | grep "[0-9]")
    if [ -z "$lockpid" ]; then
      color error
      echo "ERROR: pid was not fetched from lock file. Check the transfer processes manually, please."
      color reset
      exit 1
    fi
    echo "transfer processes with pid or ppid ${lockpid}:"
    color cmd
    ps -ef | grep "$lockpid" | grep "transfer"
    rccheck=$?
    color reset
    if [ $rccheck -eq 0 ]; then
      color error
      echo "ERROR: The transfer with pid $lockpid seems to be still active. Aborting."
      color reset
      exit 1
    fi
    color ok
    echo "OK, the transfer seems not to be active anymore. I IGNORE the lock and continue..."
    color reset
  fi

  echo Creating a lock file ...
  echo "transfer started $( date )  - process id $$" > "${lockfile}"
  if [ $? -ne 0 ]; then
    color error
    echo "ABORT - unable to create transfer lock"
    color reset
    exit 2
  fi

# --------------------------------------------------------------------------------
# ----- BACKUP VARS

  # parameters for all 
  t_setVars || exit 1
  export ARGS_DEFAULT
  ARGS_DEFAULT="$( t_getParamDefault $1 $2 )"


  if [ "$1" = "dumps" ]; then
    sDirs2Backup="$(_j_getvar ${JOBFILE} dir-localdumps)"
  else
    sDirs2Backup="$(j_getDirs2Backup)"
  fi


  sParamExclude=
  for sItem in $(_j_getvar "${DIRFILE}" exclude)
  do
    sParamExclude="$sParamExclude $( t_getParamExlude $sItem )"
  done

  
  sFileSshPrivkey=$(_j_getvar ${STORAGEFILE} "ssh-privatekey")
  if [ -n "$sFileSshPrivkey" ]; then
    ARGS_DEFAULT="${ARGS_DEFAULT} $( t_getParamSshKey $sFileSshPrivkey )"
  fi


  # task#3046 - add custom cache dir
  sCacheDir=$(_j_getvar "${STORAGEFILE}" "${CFGPREFIX}cachedir")
  if [ -n "$sCacheDir" ]; then
    ARGS_DEFAULT="${ARGS_DEFAULT} $( t_getParamCacheDir $sCacheDir )"
  fi


# --------------------------------------------------------------------------------
# ----- PRE transfer


  h2 "$( date )  Wait for a free slot"
  if [ -z "$STORAGE_REGISTER" ]; then
    echo "SKIP"
  else
    iExit=1
    until [ $iExit -eq 0 ]; do
      registerBackupSlot "$FQDN"
      iExit=$?
      if [ $iExit -ne 0 ]; then
          statusBackupSlot
          iRnd=$(($RANDOM%30+30))
          echo "I wait a bit ... random time ... $iRnd sec ..."
          sleep $iRnd
      fi
    done
  fi

  _j_runHooks "300-before-transfer"

  h2 "$( date )  PRE transfer tasks"
  t_backupDoPreTasks
  echo

# --------------------------------------------------------------------------------
# ----- START BACKUPS
  (
    if [ "$ACTION" = "backup" ]; then
      for mydir in $sDirs2Backup
      do

        # remove ending slash ... otherwise duplicity will fail
        mydir=$(echo "$mydir" | sed 's#\/$##g')

        if [ -d "$mydir" ]; then

          BACKUP_DIR=$mydir

            h2 "$( date )  STORE $BACKUP_DIR"

            # --- build parameters
            sSafeName=$(j_getSafename "$BACKUP_DIR")
            sTarget="$( t_backupDirGetTarget $BACKUP_DIR )"

            ARGS_BACKUP="${sParamExclude} $( t_getParamBackup )"


            # detect custom backup sets and add its includes and excludes
            backupid=$(j_getSetnameOfPath "$BACKUP_DIR")
            sSpaceReplace="___SPACE___"


            if [ ! -z $backupid ]; then
              for sItem in $(_j_getvar ${DIRFILE} "${backupid}\-\-include" | sed "s# #${sSpaceReplace}#g")
              do
                ARGS_BACKUP="${ARGS_BACKUP} $( t_getParamInlude $sItem)"
              done
              for sItem in $(_j_getvar ${DIRFILE} "${backupid}\-\-exclude" | sed "s# #${sSpaceReplace}#g")
              do
                ARGS_BACKUP="${ARGS_BACKUP} $( t_getParamExlude $sItem)"
              done
            fi


            # --- pre task
            h3 "$( date )  PRE backup task for ${BACKUP_DIR}"
            t_backupDirDoPreTasks
            # sCmdPre="$( t_backupDirDoPreTasks )"


            # --- backup
            h3 "$( date )  Backup ${BACKUP_DIR}"
            if [ $doBackup -eq 0 ]; then
              echo "SKIP backup"
            else

              _j_runHooks "310-before-folder-transfer"

              sCmd="$( t_backupDirGetCmdBackup )"
              echo "what:    ${BACKUP_DIR}"
              echo "target:  ${sTarget}" | sed 's#:[^:]*@#:**********@#'
              echo "command: $sCmd"
              echo
              color cmd
              $sCmd
              fetchrc
              color reset
              echo

              t_rcCheckBackup $myrc "${BACKUP_DIR}"
              # test $myrc -ne 0 && j_notify "Dir ${BACKUP_DIR}" "Backup for ${BACKUP_DIR} failed with rc=$myrc. See log for details: $JOB_LOGFILE" 1

              _j_runHooks "320-after-folder-transfer" "$myrc"

            fi
            echo


            # --- post action
            h3 "$( date )  POST backup task for ${BACKUP_DIR}"
            t_backupDirDoPostTasks
            echo

        else
          color warning
          echo "DIR SKIP $mydir ... does not exist (no error)"
          color reset
        fi 
        echo
      done
      test $rc -eq 0 && touch "${lastbackupfile}"
    else
      echo "SKIP backup of dirs"
    fi
 
    echo $rc > ${rcfile}
    exit $rc

  ) 

  # rc=${PIPESTATUS[0]}
  rc=$(cat ${rcfile})

# --------------------------------------------------------------------------------
# --- transfer POST tasks

  h2 "$( date )  POST transfer tasks"

  # --- prune
  if [ $doPrune -eq 0 ]; then
    echo "SKIP prune"
  else
    h3 "$( date )  PRUNE repository data"
    if t_backupDoPrune; then
      touch "${lastprunefile}"
    else
      rc+=1
      j_notify "Prune" "Pruning old data in the repostitory failed." 1
    fi
    ls -l "${lastprunefile}"
    echo
  fi
  echo

  # --- verify
  if [ $doVerify -eq 0 ]; then
    echo "SKIP verify"
  else
    h3 "$( date )  VERIFY repository data"
    if t_backupDoVerify; then
      touch "${lastverifyfile}"
    else
      rc+=1
      j_notify "Verify" "Verify of repository data failed." 1
    fi
    ls -l "${lastverifyfile}"
    echo
  fi
  echo

  # --- unlock
  t_backupDoPostTasks

  rm -f "${lockfile}" "${rcfile}"
  echo "Local lock file was removed."

  h2 "$( date )  Unregister used slot"
  if [ -z "$STORAGE_REGISTER" ]; then
    echo "SKIP"
  else
    unregisterBackupSlot "$FQDN" $rc
  fi

  h2 "$( date )  Backup finished"
  echo "STATUS $0 exit with final returncode rc=$rc"
  echo
  if [ $rc -eq 0 ]; then
    color ok
    echo Backup OK
  else
    color error
    echo Backup FAILED :-/
  fi
  color reset

  _j_runHooks "400-post-backup" "$rc"
  echo
  typeset -i TIMER_TRANSFER
  TIMER_TRANSFER=$(date +%s)-$TIMER_TRANSFER_START
  echo "$( date )  $ACTION DONE in $TIMER_TRANSFER sec"

  ls -l "$transferlog"
  exit $rc

# --------------------------------------------------------------------------------