#!/bin/bash
# ================================================================================
#
# TRANSFER LOCAL DATA TO BACKUP STORAGE
#
# SYNTAX:
#   transfer.sh       - incremental backup
#   transfer.sh full  - full backup
#   transfer.sh dumps - transfer local dumps only
#
# --------------------------------------------------------------------------------
# ah - Axel Hahn <axel.hahn@iml.unibe.ch>
# ds - Daniel Schueler <daniel.schueler@iml.unibe.ch>
#
# 2016-11-10  ah,ds  v1.0
# 2016-12-19  ah,ds  v1.1  added parameter "dumps"
# 2017-02-16  ah,ds  v1.2  added support for storage slots
# 2017-10-11  ah,ds  v1.3  added support for duplicity param --ssh-backend
# 2017-10-17  ah,ds  v1.4  remove PIPESTATUS for Debian8 compatibility
# 2017-11-17  ah,ds  v1.5  check pid of lockfile in process list if process still runs
# 2018-06-19  ah,ds  v1.6  replace --exclude with --exclude regexp in custom dirs
# 2019-06-05  ah,ds  v1.7  add custom cache dir
# 2019-09-09  ah,ds  v1.8  add testfile on target
# 2019-10-30  ah,ds  v1.9  for rsync targets: create remote target dir with ssh command
# 2020-01-21  ah,ds  v1.10 show colored OK or FAILED at the end of output
# 2020-02-25  ah,ds, v1.11 fix test -z with non existing vars; show final backup status
# 2021-01-29  ah,ds, v1.12 abort on empty passphrase
# 2021-05-19  ah,ds, v2.0  plugin driven to support multiple backup tools (duplicity + restic)
# ================================================================================


# --------------------------------------------------------------------------------
# CONFIG
# --------------------------------------------------------------------------------

  # . `dirname $0`/inc_config.sh

  . `dirname $0`/jobhelper.sh
  . `dirname $0`/inc_bash.sh

  typeset -i rc=0  
  if [ ! -r "${DIRFILE}" -o ! -r "${STORAGEFILE}" ]; then
    echo "SKIP backup of local files - one of the files is not readable (no error): ${DIRFILE} | ${STORAGEFILE}"
    exit 0
  fi

  STORAGE_BIN=`_j_getvar ${STORAGEFILE} "bin"`
  STORAGE_BASEDIR=`_j_getvar ${STORAGEFILE} "storage"`
  STORAGE_TESTFILE=`_j_getvar ${STORAGEFILE} "storage-file"`
  PASSPHRASE=`_j_getvar ${STORAGEFILE} "passphrase"`

  STORAGE_REGISTER=`_j_getvar ${STORAGEFILE} "storage-register"`
  typeset -i TIMER_TRANSFER_START=`date +%s`

  # check
  if [ -z "$STORAGE_BIN" ]; then
    # STORAGE_BIN=restic
    STORAGE_BIN=duplicity
  fi
  CFGPREFIX=${STORAGE_BIN}_

  if [ -z "$STORAGE_BASEDIR" ]; then
    color error
    echo ERROR: missing config for backup target.
    echo There must be an entry "storage = " in ${STORAGEFILE}
    color reset
    exit 1
  fi
  if [ ! -z "$STORAGE_TESTFILE" -a ! -f "$STORAGE_TESTFILE" ]; then
    color error
    echo ERROR: missing testfile $STORAGE_TESTFILE on backup target.
    echo The Backup target disk / NAS is not mounted.
    color reset
    exit 1
  fi
  # support old value
  if [ -z "${PASSPHRASE}" ]; then
    echo "WARNING: The value gnupg-passphrase in ${STORAGEFILE} is deprecated. Replace it with passphrase=..."
    PASSPHRASE=`_j_getvar ${STORAGEFILE} "gnupg-passphrase"`
  fi

  if [ -z "${PASSPHRASE}" ]; then
    echo "ERROR: no value passphrase was set in ${STORAGEFILE} to encrypt backup data."
    echo "Aborting."
    exit 1
  fi
  
  # For duplicity only
  # METHOD incremental is default; full backup will be triggered with 
  # first param "full"
  METHOD=


  transferlog="${DIR_LOGS}/transfer-`date +%Y%m%d`.log"
  lockfile="${DIR_LOGS}/transfer.running"
  rcfile=/tmp/transfer-rc.$$.tmp

# --------------------------------------------------------------------------------
# FUNCTIONS
# --------------------------------------------------------------------------------


# --------------------------------------------------------------------------------
# MAIN
# --------------------------------------------------------------------------------


  
  h1 `date` TRANSFER LOCAL DATA TO STORAGE                      | tee -a $transferlog
  
  echo "METHOD   : $METHOD"                                     | tee -a $transferlog
  echo "TARGET   : ${STORAGE_BASEDIR}"                          | tee -a $transferlog
  echo "REGISTER : ${STORAGE_REGISTER}"                         | tee -a $transferlog
  echo "TOOL     : $STORAGE_BIN"                                | tee -a $transferlog
  echo                                                          | tee -a $transferlog

  . `dirname $0`/plugins/transfer/$STORAGE_BIN.sh || exit 1
  test -z "$STORAGE_REGISTER" || . `dirname $0`/plugins/register/$STORAGE_REGISTER.sh || exit 1

# --------------------------------------------------------------------------------
# ----- Check requirements

  t_checkRequirements || exit 1

  echo Check locking of a running transfer
  if [ -f "${lockfile}" ]; then
    color error
    echo A lock file for a running transfer was found
    cat "${lockfile}"
    color reset
    echo

    # 1659 - check process id inside the lock file
    # detect pid from lockfile and search for this process
    lockpid=`cat "${lockfile}" | cut -f 2 -d "-" | cut -f 4 -d " " | grep "[0-9]"`
    if [ -z $lockpid ]; then
      color error
      echo ERROR: pid was not fetched from lock file. Check the transfer processes manually, please.
      color reset
      exit 1
    fi
    echo transfer processes with pid or ppid ${lockpid}:
    color cmd
    ps -ef | grep $lockpid | grep "transfer"
    rccheck=$?
    color reset
    if [ $rccheck -eq 0 ]; then
      color error
      echo ERROR: The transfer with pid $lockpid seems to be still active. Aborting.
      color reset
      exit 1
    fi
    color ok
    echo OK, the transfer seems not to be active anymore. I IGNORE the lock and continue...
    color reset
  fi

  echo Creating a lock file ...
  echo "transfer started `date` - process id $$" > "${lockfile}"
  if [ $? -ne 0 ]; then
    color error
    echo ABORT - unable to create transfer lock
    color reset
    exit 2
  fi

# --------------------------------------------------------------------------------
# ----- BACKUP VARS

  # parameters for all 
  t_setVars || exit 1
  ARGS_DEFAULT="$( t_getParamDefault $1 $2 )"


  if [ "$1" = "dumps" ]; then
    sDirs2Backup="`_j_getvar ${JOBFILE} dir-localdumps`"
  else
    sDirs2Backup="`j_getDirs2Backup`"
  fi


  sParamExclude=
  for sItem in `_j_getvar ${DIRFILE} exclude`
  do
    sParamExclude="$sParamExclude $( t_getParamExlude $sItem )"
  done

  
  sFileSshPrivkey=`_j_getvar ${STORAGEFILE} "ssh-privatekey"`
  if [ ! -z $sFileSshPrivkey ]; then
    ARGS_DEFAULT="${ARGS_DEFAULT} $( t_getParamSshKey $sFileSshPrivkey )"
  fi


  # task#3046 - add custom cache dir
  sCacheDir=`_j_getvar ${STORAGEFILE} "${CFGPREFIX}cachedir"`
  if [ ! -z $sCacheDir ]; then
    ARGS_DEFAULT="${ARGS_DEFAULT} $( t_getParamCacheDir $sCacheDir )"
  fi


# --------------------------------------------------------------------------------
# ----- PRE transfer

  h2 "`date` Wait for a free slot"
  if [ -z "$STORAGE_REGISTER" ]; then
    echo "SKIP"
  else
    iExit=1
    until [ $iExit -eq 0 ]; do
      registerBackupSlot `hostname -f`
      iExit=$?
      if [ $iExit -ne 0 ]; then
          statusBackupSlot
          iRnd=$(($RANDOM%30+30))
          echo "I wait a bit ... random time ... $iRnd sec ..."
          sleep $iRnd
      fi
    done
  fi | tee -a $transferlog


  h2 "`date` PRE transfer tasks"
  t_backupDoPreTasks
  echo

# --------------------------------------------------------------------------------
# ----- START BACKUPS
  (
    for mydir in $sDirs2Backup
    do

      # remove ending slash ... otherwise duplicity will fail
      mydir=`echo $mydir | sed 's#\/$##g'`

      if [ -d "$mydir" ]; then

        h2 "`date` STORE $mydir" 

        # --- build parameters
        sSafeName=`j_getSafename "$mydir"`
        sTarget="$( t_backupDirGetTarget $mydir )"

        ARGS_BACKUP="${sParamExclude} $( t_getParamBackup )"


        # detect custom backup sets and add its includes and excludes
        backupid=`j_getSetnameOfPath "$mydir"`
        sSpaceReplace="___SPACE___"


        if [ ! -z $backupid ]; then
          for sItem in `_j_getvar ${DIRFILE} "${backupid}\-\-include" | sed "s#\ #${sSpaceReplace}#g"`
          do
            ARGS_BACKUP="${ARGS_BACKUP} $( t_getParamInlude $sItem)"
          done
          for sItem in `_j_getvar ${DIRFILE} "${backupid}\-\-exclude" | sed "s#\ #${sSpaceReplace}#g"`
          do
            ARGS_BACKUP="${ARGS_BACKUP} $( t_getParamExlude $sItem)"
          done
        fi


        # --- pre task
        h3 "`date` PRE backup task for ${mydir}"
        t_backupDirDoPreTasks
        # sCmdPre="$( t_backupDirDoPreTasks )"


        # --- backup
        h3 "`date` Backup ${mydir}"

        sCmd="$( t_backupDirGetCmdBackup )"
        echo "what:    ${mydir}"
        echo "target:  ${sTarget}"
        echo "command: $sCmd"
        echo
        color cmd
        $sCmd
        fetchrc
        color reset
        echo

        t_rcCheckBackup $myrc

        if [ $myrc -ne 0 ]; then
          color error
          echo DIR ERROR ${mydir} rc=$myrc during file transfer

        else
          color ok
          echo DIR OK ${mydir} was successful.
        fi
        color reset
        echo


        # --- post action
        h3 "`date` POST backup task for ${mydir}"
        t_backupDirDoPostTasks
        echo
        echo

      else
        color warning
        echo "DIR SKIP $mydir ... does not exist (no error)"
        color reset
      fi 
      echo
    done
 
    echo $rc > ${rcfile}
    exit $rc

  ) | tee -a $transferlog 

  # rc=${PIPESTATUS[0]}
  rc=`cat ${rcfile}`

# --------------------------------------------------------------------------------
# --- transfer POST tasks

  h2 "`date` POST transfer tasks"
  t_backupDoPostTasks

  rm -f "${lockfile}" "${rcfile}"
  echo "Local lock file was removed."

  h2 "`date` Unregister used slot"
  if [ -z "$STORAGE_REGISTER" ]; then
    echo "SKIP"
  else
    unregisterBackupSlot `hostname -f` $rc
  fi | tee -a $transferlog

  h2 "`date` Backup finished"
  echo STATUS $0 exit with final returncode rc=$rc                                    | tee -a $transferlog
  echo                                                                                | tee -a $transferlog
  if [ $rc -eq 0 ]; then
    color ok
    echo Backup OK                                                                    | tee -a $transferlog
  else
    color error
    echo Backup FAILED :-/                                                            | tee -a $transferlog
  fi
  color reset
  echo                                                                                | tee -a $transferlog
  typeset -i TIMER_TRANSFER=`date +%s`-$TIMER_TRANSFER_START
  echo `date` TRANSFER DONE in $TIMER_TRANSFER sec                                    | tee -a $transferlog

  ls -l $transferlog                                                                  
  exit $rc

# --------------------------------------------------------------------------------
