transfer.sh

#!/bin/bash
# ================================================================================
#
# TRANSFER LOCAL DATA TO BACKUP STORAGE
#
# SYNTAX:
#   transfer.sh       - incremental backup
#   transfer.sh full  - full backup
#   transfer.sh dumps - transfer local dumps only
#   transfer.sh prune - cleanup backup data
#   transfer.sh help  - show help
#
# --------------------------------------------------------------------------------
# ah - Axel Hahn <axel.hahn@iml.unibe.ch>
# ds - Daniel Schueler <daniel.schueler@iml.unibe.ch>
#
# 2016-11-10  ah,ds  v1.0
# 2016-12-19  ah,ds  v1.1  added parameter "dumps"
# 2017-02-16  ah,ds  v1.2  added support for storage slots
# 2017-10-11  ah,ds  v1.3  added support for duplicity param --ssh-backend
# 2017-10-17  ah,ds  v1.4  remove PIPESTATUS for Debian8 compatibility
# 2017-11-17  ah,ds  v1.5  check pid of lockfile in process list if process still runs
# 2018-06-19  ah,ds  v1.6  replace --exclude with --exclude regexp in custom dirs
# 2019-06-05  ah,ds  v1.7  add custom cache dir
# 2019-09-09  ah,ds  v1.8  add testfile on target
# 2019-10-30  ah,ds  v1.9  for rsync targets: create remote target dir with ssh command
# 2020-01-21  ah,ds  v1.10 show colored OK or FAILED at the end of output
# 2020-02-25  ah,ds, v1.11 fix test -z with non existing vars; show final backup status
# 2021-01-29  ah,ds, v1.12 abort on empty passphrase
# 2021-05-19  ah,ds, v2.0  plugin driven to support multiple backup tools (duplicity + restic)
# 2021-12-02  ah     v2.1  added parameter "prune" to cleanup only
# 2022-02-10  ah     v2.2  update logging (removing tee)
# 2022-10-01  ah     v2.3  customize prune and verify action
# 2022-10-04  ah     v2.4  prune and verify are non directory based
# ================================================================================


# --------------------------------------------------------------------------------
# CONFIG
# --------------------------------------------------------------------------------

  # . `dirname $0`/inc_config.sh

  . `dirname $0`/jobhelper.sh
  . `dirname $0`/inc_bash.sh

  typeset -i rc=0
  typeset -i doBackup=1
  typeset -i doPrune
  typeset -i doVerify
  typeset -i doValue

  if [ ! -r "${DIRFILE}" -o ! -r "${STORAGEFILE}" ]; then
    echo "SKIP backup of local files - one of the files is not readable (no error): ${DIRFILE} | ${STORAGEFILE}"
    exit 0
  fi

  STORAGE_BIN=`_j_getvar ${STORAGEFILE} "bin"`
  STORAGE_BASEDIR=`_j_getvar ${STORAGEFILE} "storage"`
  STORAGE_TESTFILE=`_j_getvar ${STORAGEFILE} "storage-file"`
  PASSPHRASE=`_j_getvar ${STORAGEFILE} "passphrase"`

  STORAGE_REGISTER=`_j_getvar ${STORAGEFILE} "storage-register"`
  typeset -i TIMER_TRANSFER_START=`date +%s`

  # check
  if [ -z "$STORAGE_BIN" ]; then
    # STORAGE_BIN=restic
    STORAGE_BIN=duplicity
  fi
  CFGPREFIX=${STORAGE_BIN}_

  if [ -z "$STORAGE_BASEDIR" ]; then
    color error
    echo ERROR: missing config for backup target.
    echo There must be an entry "storage = " in ${STORAGEFILE}
    color reset
    exit 1
  fi
  if [ ! -z "$STORAGE_TESTFILE" -a ! -f "$STORAGE_TESTFILE" ]; then
    color error
    echo ERROR: missing testfile $STORAGE_TESTFILE on backup target.
    echo The Backup target disk / NAS is not mounted.
    color reset
    exit 1
  fi
  # support old value
  if [ -z "${PASSPHRASE}" ]; then
    echo "WARNING: The value gnupg-passphrase in ${STORAGEFILE} is deprecated. Replace it with passphrase=..."
    PASSPHRASE=`_j_getvar ${STORAGEFILE} "gnupg-passphrase"`
  fi

  if [ -z "${PASSPHRASE}" ]; then
    echo "ERROR: no value passphrase was set in ${STORAGEFILE} to encrypt backup data."
    echo "Aborting."
    exit 1
  fi
  
  # For duplicity only
  # METHOD incremental is default; full backup will be triggered with 
  # first param "full"
  METHOD=

  ACTION=backup

  transferlog="${DIR_LOGS}/transfer-$(date +%Y%m%d-%H%M%S).log"
  lockfile="${DIR_LOGS}/transfer.running"
  lastbackupfile="${DIR_LOGS}/last_backup"
  lastprunefile="${DIR_LOGS}/last_prune"
  lastverifyfile="${DIR_LOGS}/last_verify"
  rcfile=/tmp/transfer-rc.$$.tmp


# --------------------------------------------------------------------------------
# FUNCTIONS
# --------------------------------------------------------------------------------

# get age of a file in sec
# https://www.axel-hahn.de/blog/2020/03/09/bash-snippet-alter-einer-datei/
# param  string  filename to test
# return integer
function _getFileAge(){
    echo $(($(date +%s) - $(date +%s -r "$1")))
}

# set default behaviur based on limit (from config) and its last
# execution (read from touchfile)
# Function is enabled if no touchfile was found or it is older than
# given limit
#
# It is used to set flags for prune and verify
# param  string  name of action; one of prune|verify
# param  string  filename of touchfile of last execution
function setAction(){
  local action=$1
  local myfile=$2

  typeset -i local iLimit=$(_j_getvar ${STORAGEFILE} "$action-after")

  if [ ! -f "${myfile}" ]; then
    echo "Info: $action is ENABLED - no last $action detected"
    doValue=1
  else
    typeset -i iLastDone=$( _getFileAge "${myfile}" )/60
    typeset -i iLastDoneD=iLastDone/60/24
    echo "Info: Last $action was $iLastDone min ago ($iLastDoneD days). Limit is $iLimit days."
    if [ $iLastDoneD -ge $iLimit ]; then 
      echo "Info: $action is ENABLED - last $action is outdated"
      doValue=1
    else
      echo "Info: $action is not needed yet."
      doValue=0
    fi
  fi
}

# --------------------------------------------------------------------------------
# MAIN
# --------------------------------------------------------------------------------

  if [ "$1" = "help" -o "$1" = "-h" -o "$1" = "-?" ]; then
    echo "HELP:

  Transfer local files to a backup target.

    target       $( echo ${STORAGE_BASEDIR} | sed 's#:[^:]*@#:**********@#' )
    backup tool  $STORAGE_BIN

  PARAMETERS:
    transfer.sh        - incremental backup
    transfer.sh full   - full backup (Duplicity only)
    transfer.sh dumps  - transfer local dumps only
    transfer.sh prune  - cleanup backup data only (no backup)
    transfer.sh verify - verify backup data (no backup)
    transfer.sh help   - show this help (works with -h and -? too)
    "
    exit 0
  fi

  # set defaults for prune and verify
  echo ">>> Detect default behaviour:"
  setAction "prune"  $lastprunefile;  doPrune=$doValue
  setAction "verify" $lastverifyfile; doVerify=$doValue
  echo

  echo ">>> Check parameters"
  if [ "$1" = "prune" ]; then
    echo "Info: Forcing prune only by parameter."
    ACTION=$1
    doBackup=0
    doPrune=1
    doVerify=0
    transferlog="${DIR_LOGS}/prune-$(date +%Y%m%d-%H%M%S).log"
  fi
  if [ "$1" = "verify" ]; then
    echo "Info: Forcing verify only by parameter."
    ACTION=$1
    doBackup=0
    doPrune=0
    doVerify=1
    transferlog="${DIR_LOGS}/verify-$(date +%Y%m%d-%H%M%S).log"
  fi
  exec 1> >( tee -a "$transferlog" ) 2>&1
  echo "INFO: Start logging into $transferlog"  

  h1 `date` TRANSFER LOCAL DATA TO STORAGE

  echo "TOOL     : $STORAGE_BIN"
  echo "ACTION   : $ACTION"
  echo "TARGET   : ${STORAGE_BASEDIR}" | sed 's#:[^:]*@#:**********@#'
  echo

  if [ "$ACTION" = "backup" ]; then
    echo "METHOD   : $METHOD"
    echo "REGISTER : ${STORAGE_REGISTER}"
    echo "PRUNE    : $doPrune"
    echo "VERIFY   : $doVerify"
    echo
  fi

  . `dirname $0`/plugins/transfer/$STORAGE_BIN.sh || exit 1
  test -z "$STORAGE_REGISTER" || . `dirname $0`/plugins/register/$STORAGE_REGISTER.sh || exit 1

# --------------------------------------------------------------------------------
# ----- Check requirements

  t_checkRequirements || exit 1

  echo Check locking of a running transfer
  if [ -f "${lockfile}" ]; then
    color error
    echo A lock file for a running transfer was found
    cat "${lockfile}"
    color reset
    echo

    # 1659 - check process id inside the lock file
    # detect pid from lockfile and search for this process
    lockpid=`cat "${lockfile}" | cut -f 2 -d "-" | cut -f 4 -d " " | grep "[0-9]"`
    if [ -z $lockpid ]; then
      color error
      echo ERROR: pid was not fetched from lock file. Check the transfer processes manually, please.
      color reset
      exit 1
    fi
    echo transfer processes with pid or ppid ${lockpid}:
    color cmd
    ps -ef | grep $lockpid | grep "transfer"
    rccheck=$?
    color reset
    if [ $rccheck -eq 0 ]; then
      color error
      echo ERROR: The transfer with pid $lockpid seems to be still active. Aborting.
      color reset
      exit 1
    fi
    color ok
    echo OK, the transfer seems not to be active anymore. I IGNORE the lock and continue...
    color reset
  fi

  echo Creating a lock file ...
  echo "transfer started `date` - process id $$" > "${lockfile}"
  if [ $? -ne 0 ]; then
    color error
    echo ABORT - unable to create transfer lock
    color reset
    exit 2
  fi

# --------------------------------------------------------------------------------
# ----- BACKUP VARS

  # parameters for all 
  t_setVars || exit 1
  ARGS_DEFAULT="$( t_getParamDefault $1 $2 )"


  if [ "$1" = "dumps" ]; then
    sDirs2Backup="`_j_getvar ${JOBFILE} dir-localdumps`"
  else
    sDirs2Backup="`j_getDirs2Backup`"
  fi


  sParamExclude=
  for sItem in `_j_getvar ${DIRFILE} exclude`
  do
    sParamExclude="$sParamExclude $( t_getParamExlude $sItem )"
  done

  
  sFileSshPrivkey=`_j_getvar ${STORAGEFILE} "ssh-privatekey"`
  if [ ! -z $sFileSshPrivkey ]; then
    ARGS_DEFAULT="${ARGS_DEFAULT} $( t_getParamSshKey $sFileSshPrivkey )"
  fi


  # task#3046 - add custom cache dir
  sCacheDir=`_j_getvar ${STORAGEFILE} "${CFGPREFIX}cachedir"`
  if [ ! -z $sCacheDir ]; then
    ARGS_DEFAULT="${ARGS_DEFAULT} $( t_getParamCacheDir $sCacheDir )"
  fi


# --------------------------------------------------------------------------------
# ----- PRE transfer

  h2 "`date` Wait for a free slot"
  if [ -z "$STORAGE_REGISTER" ]; then
    echo "SKIP"
  else
    iExit=1
    until [ $iExit -eq 0 ]; do
      registerBackupSlot `hostname -f`
      iExit=$?
      if [ $iExit -ne 0 ]; then
          statusBackupSlot
          iRnd=$(($RANDOM%30+30))
          echo "I wait a bit ... random time ... $iRnd sec ..."
          sleep $iRnd
      fi
    done
  fi


  h2 "`date` PRE transfer tasks"
  t_backupDoPreTasks
  echo

# --------------------------------------------------------------------------------
# ----- START BACKUPS
  (
    for mydir in $sDirs2Backup
    do

      # remove ending slash ... otherwise duplicity will fail
      mydir=`echo $mydir | sed 's#\/$##g'`

      if [ -d "$mydir" ]; then

        BACKUP_DIR=$mydir

        if [ "$ACTION" = "backup" ]; then
          
          h2 "`date` STORE $BACKUP_DIR"

          # --- build parameters
          sSafeName=`j_getSafename "$BACKUP_DIR"`
          sTarget="$( t_backupDirGetTarget $BACKUP_DIR )"

          ARGS_BACKUP="${sParamExclude} $( t_getParamBackup )"


          # detect custom backup sets and add its includes and excludes
          backupid=`j_getSetnameOfPath "$BACKUP_DIR"`
          sSpaceReplace="___SPACE___"


          if [ ! -z $backupid ]; then
            for sItem in `_j_getvar ${DIRFILE} "${backupid}\-\-include" | sed "s#\ #${sSpaceReplace}#g"`
            do
              ARGS_BACKUP="${ARGS_BACKUP} $( t_getParamInlude $sItem)"
            done
            for sItem in `_j_getvar ${DIRFILE} "${backupid}\-\-exclude" | sed "s#\ #${sSpaceReplace}#g"`
            do
              ARGS_BACKUP="${ARGS_BACKUP} $( t_getParamExlude $sItem)"
            done
          fi


          # --- pre task
          h3 "`date` PRE backup task for ${BACKUP_DIR}"
          t_backupDirDoPreTasks
          # sCmdPre="$( t_backupDirDoPreTasks )"


          # --- backup
          h3 "`date` Backup ${BACKUP_DIR}"
          if [ $doBackup -eq 0 ]; then
            echo "SKIP backup"
          else
            sCmd="$( t_backupDirGetCmdBackup )"
            echo "what:    ${BACKUP_DIR}"
            echo "target:  ${sTarget}"
            echo "command: $sCmd"
            echo
            color cmd
            $sCmd
            fetchrc
            color reset
            echo

            t_rcCheckBackup $myrc "${BACKUP_DIR}"

          fi
          echo


          # --- post action
          h3 "`date` POST backup task for ${BACKUP_DIR}"
          t_backupDirDoPostTasks
          echo

          touch ${lastbackupfile}
        fi

        # --- prune
        if [ $doPrune -eq 0 ]; then
          echo "SKIP forget data"
        else
          h3 "`date` FORGET data of ${BACKUP_DIR}"
          t_backupDoForget
          echo
        fi
        echo

      else
        color warning
        echo "DIR SKIP $mydir ... does not exist (no error)"
        color reset
      fi 
      echo
    done

    # --- prune
    if [ $doPrune -eq 0 ]; then
      echo "SKIP prune"
    else
      h3 "`date` PRUNE repository data"
      t_backupDoPrune
      touch ${lastprunefile}
      echo
    fi
    echo

    # --- verify
    if [ $doVerify -eq 0 ]; then
      echo "SKIP verify"
    else
      h3 "`date` VERIFY repository data"
      t_backupDoVerify
      touch ${lastverifyfile}
      echo
    fi
    echo
 
    echo $rc > ${rcfile}
    exit $rc

  ) 

  # rc=${PIPESTATUS[0]}
  rc=`cat ${rcfile}`

# --------------------------------------------------------------------------------
# --- transfer POST tasks

  h2 "`date` POST transfer tasks"
  t_backupDoPostTasks

  rm -f "${lockfile}" "${rcfile}"
  echo "Local lock file was removed."

  h2 "`date` Unregister used slot"
  if [ -z "$STORAGE_REGISTER" ]; then
    echo "SKIP"
  else
    unregisterBackupSlot `hostname -f` $rc
  fi

  h2 "`date` Backup finished"
  echo STATUS $0 exit with final returncode rc=$rc
  echo
  if [ $rc -eq 0 ]; then
    color ok
    echo Backup OK
  else
    color error
    echo Backup FAILED :-/
  fi
  color reset
  echo
  typeset -i TIMER_TRANSFER=`date +%s`-$TIMER_TRANSFER_START
  echo `date` $ACTION DONE in $TIMER_TRANSFER sec

  ls -l $transferlog                                                                  
  exit $rc

# --------------------------------------------------------------------------------