keep old transfer.sh for a moment

transfer_legacy.sh

+#!/bin/bash
+# ================================================================================
+#
+# TRANSFER LOCAL DATA TO BACKUP STORAGE
+#
+# SYNTAX:
+#   transfer.sh       - incremental backup
+#   transfer.sh full  - full backup
+#   transfer.sh dumps - transfer local dumps only
+#
+# duplicity see
+# - man pages http://duplicity.nongnu.org/duplicity.1.html
+# - example http://duplicity.nongnu.org/contrib/jwfull
+# - example http://duplicity.nongnu.org/contrib/jwincr
+#
+# this script was build to support ssh backend only
+#
+# --------------------------------------------------------------------------------
+# ah - Axel Hahn <axel.hahn@iml.unibe.ch>
+# ds - Daniel Schueler <daniel.schueler@iml.unibe.ch>
+#
+# 2016-11-10  ah,ds  v1.0
+# 2016-12-19  ah,ds  v1.1  added parameter "dumps"
+# 2017-02-16  ah,ds  v1.2  added support for storage slots
+# 2017-10-11  ah,ds  v1.3  added support for duplicity param --ssh-backend
+# 2017-10-17  ah,ds  v1.4  remove PIPESTATUS for Debian8 compatibility
+# 2017-11-17  ah,ds  v1.5  check pid of lockfile in process list if process still runs
+# 2018-06-19  ah,ds  v1.6  replace --exclude with --exclude regexp in custom dirs
+# 2019-06-05  ah,ds  v1.7  add custom cache dir
+# 2019-09-09  ah,ds  v1.8  add testfile on target
+# 2019-10-30  ah,ds  v1.9  for rsync targets: create remote target dir with ssh command
+# 2020-01-21  ah,ds  v1.10 show colored OK or FAILED at the end of output
+# 2020-02-25  ah,ds, v1.11 fix test -z with non existing vars; show final backup status
+# 2021-01-29  ah,ds, v1.12 abort on empty passphrase
+# ================================================================================
+
+
+# --------------------------------------------------------------------------------
+# CONFIG
+# --------------------------------------------------------------------------------
+
+  # . `dirname $0`/inc_config.sh
+
+  . `dirname $0`/jobhelper.sh
+  . `dirname $0`/inc_bash.sh
+
+  typeset -i rc=0
+
+  STORAGE_BASEDIR=`_j_getvar ${STORAGEFILE} "storage"`
+  STORAGE_TESTFILE=`_j_getvar ${STORAGEFILE} "storage-file"`
+  STORAGE_KEEP=`_j_getvar ${STORAGEFILE} "keep"`
+  STORAGE_VOLSIZE=`_j_getvar ${STORAGEFILE} "volsize"`
+
+  # check
+  if [ -z "$STORAGE_BASEDIR" ]; then
+    color error
+    echo ERROR: missing config for backup target.
+    echo There must be an entry storage in ${STORAGEFILE}
+    color reset
+    exit 1
+  fi
+  if [ ! -z "$STORAGE_TESTFILE" -a ! -f "$STORAGE_TESTFILE" ]; then
+    color error
+    echo ERROR: missing testfile $STORAGE_TESTFILE on backup target.
+    echo The Backup target disk / NAS is not mounted.
+    color reset
+    exit 1
+  fi
+  
+  if [ -z "$STORAGE_KEEP" ]; then
+    color error
+    echo ERROR: missing config for backup target.
+    echo There must be an entry keep in ${STORAGEFILE}
+    color reset
+    exit 1
+  fi
+
+  # METHOD incremental is default; full backup will be triggered with 
+  # first param "full"
+  METHOD=
+
+
+  transferlog="${DIR_LOGS}/transfer-`date +%Y%m%d`.log"
+  lockfile="${DIR_LOGS}/transfer.running"
+  rcfile=/tmp/transfer-rc.$$.tmp
+
+# --------------------------------------------------------------------------------
+# FUNCTIONS
+# --------------------------------------------------------------------------------
+
+
+# --------------------------------------------------------------------------------
+# MAIN
+# --------------------------------------------------------------------------------
+
+
+  
+  h1 `date` TRANSFER LOCAL DATA TO STORAGE                                            | tee -a $transferlog
+  
+  echo METHOD: $METHOD                                                                | tee -a $transferlog
+  echo TARGET: ${STORAGE_BASEDIR}                                                     | tee -a $transferlog
+  echo                                                                                | tee -a $transferlog
+
+# ----- Check requirements
+
+  j_requireUser "root"
+  j_requireBinary "duplicity"
+
+  echo Check locking of a running transfer
+  if [ -f "${lockfile}" ]; then
+    color error
+    echo A lock file for a running transfer was found
+    cat "${lockfile}"
+    color reset
+    echo
+
+    # 1659 - check process id inside the lock file
+    # detect pid from lockfile and search for this process
+    lockpid=`cat "${lockfile}" | cut -f 2 -d "-" | cut -f 4 -d " " | grep "[0-9]"`
+    if [ -z $lockpid ]; then
+      color error
+      echo ERROR: pid was not fetched from lock file. Check the transfer processes manually, please.
+      color reset
+      exit 1
+    fi
+    echo transfer processes with pid or ppid ${lockpid}:
+    color cmd
+    ps -ef | grep $lockpid | grep "transfer"
+    rccheck=$?
+    color reset
+    if [ $rccheck -eq 0 ]; then
+      color error
+      echo ERROR: The transfer with pid $lockpid seems to be still active. Aborting.
+      color reset
+      exit 1
+    fi
+    color ok
+    echo OK, the transfer seems not to be active anymore. I IGNORE the lock and continue...
+    color reset
+  fi
+
+  echo Creating a lock file ...
+  echo "transfer started `date` - process id $$" > "${lockfile}"
+  if [ $? -ne 0 ]; then
+    color error
+    echo ABORT - unable to create transfer lock
+    color reset
+    exit 2
+  fi
+
+# ----- BACKUP VARS
+
+  # parameters for all 
+  sParams=
+
+  if [ "$1" = "dumps" ]; then
+    sDirs2Backup="`_j_getvar ${DIRFILE} dir-localdumps`"
+  else
+    sDirs2Backup="`j_getDirs2Backup`"
+  fi
+
+  if [ "$1" = "full" ]; then
+    METHOD="full"
+  fi
+  if [ "$1" = "auto" ]; then
+    METHOD="--full-if-older-than $2"
+  fi
+
+
+  PASSPHRASE=`_j_getvar ${STORAGEFILE} "gnupg-passphrase"`
+  if [ -z "${PASSPHRASE}" ]; then
+    echo "ERROR: no value gnupg-passphrase was set in ${STORAGEFILE} to encrypt backup data."
+    echo "Aborting."
+    exit 1
+  fi
+
+  sParamExclude=
+  # build exclude param list for duplicity
+  #
+  # REMARK: the excludes must fit the include definition
+  #         On non matching excludes duplicity stops
+  # 
+  for sItem in `_j_getvar ${DIRFILE} exclude`
+  do
+    sParamExclude="$sParamExclude --exclude-regexp """$sItem""" "
+  done
+  # 
+  # sExcludefile="${DIR_JOBS}/transfer-exclude.txt"
+  # 
+  # if [ -f ${sExcludefile} ]; then
+  #   sParamExclude="$sParamExclude --exclude-filelist """${sExcludefile}""" "
+  # fi
+
+  export PASSPHRASE
+
+  # task#1623 - fallback ssh backend for Debian 8
+  sSshBackend=`_j_getvar ${STORAGEFILE} "ssh-backend"`
+  if [ ! -z $sSshBackend ]; then
+    sParams="${sParams} --ssh-backend $sSshBackend"
+  fi
+
+  sFileSshPrivkey=`_j_getvar ${STORAGEFILE} "ssh-privatekey"`
+  if [ ! -z $sFileSshPrivkey ]; then
+    sParams="${sParams} --ssh-options="""-oIdentityFile=${sFileSshPrivkey}"""  "
+  fi
+
+  # task#3046 - add custom cache dir
+  sCacheDir=`_j_getvar ${STORAGEFILE} "cachedir"`
+  if [ ! -z $sCacheDir ]; then
+    sParams="${sParams} --archive-dir=$sCacheDir"
+    if [ ! -d $sCacheDir ]; then
+      mkdir -p $sCacheDir
+      chmod 750 $sCacheDir
+    fi
+  fi
+
+  
+# ----- Wait for a free slot
+
+  j_transferStart | tee -a $transferlog
+
+
+# ----- START BACKUPS
+
+  (
+    for mydir in $sDirs2Backup
+    do
+
+      # remove ending slash ... otherwise duplicity will fail
+      mydir=`echo $mydir | sed 's#\/$##g'`
+
+      if [ -d "$mydir" ]; then
+
+        h2 "`date` STORE $mydir" 
+
+        # --- build parameters for duplicity
+        sSafeName=`j_getSafename "$mydir"`
+        sTarget=`j_getFullTarget "$mydir"`
+
+        sBackupParams=
+        sBackupParams="${sBackupParams} ${sParams} "
+        sBackupParams="${sBackupParams} ${sParamExclude} "
+        if [ ! -z $STORAGE_VOLSIZE ]; then
+          sBackupParams="${sBackupParams} --volsize ${STORAGE_VOLSIZE} "
+        fi
+
+        # verbosity level to fetch changed files from log
+        sBackupParams="${sBackupParams} -v8"
+
+        # add asynchronous upload
+        sBackupParams="${sBackupParams} --asynchronous-upload"
+
+        # ---------- START --------------------
+        # detect custom backup sets and add its includes and excludes
+        backupid=`j_getSetnameOfPath "$mydir"`
+        sSpaceReplace="___SPACE___"
+
+        if [ ! -z $backupid ]; then
+          for param in include exclude
+          do
+            for sItem in `_j_getvar ${DIRFILE} "${backupid}\-\-${param}" | sed "s#\ #${sSpaceReplace}#g"`
+            do
+              sBackupParams="${sBackupParams} --${param}-regexp """$sItem""" "
+            done
+          done
+          sBackupParams=`echo ${sBackupParams} | sed "s#${sSpaceReplace}# #g"`
+
+        fi
+        # ---------- ENDE --------------------
+
+        # --- for rsync only: create remote directory
+        echo ${sTarget} | fgrep "rsync://"  >/dev/null
+        if [ $? -eq 0 ]; then
+          # sshTarget=`echo ${sTarget} | sed "s#rsync://#scp://#"`
+          # echo Creating remote directory with fetching collection-status on $sshTarget
+          # color cmd
+          # duplicity collection-status ${sParams} ${sshTarget}
+          # color reset
+          sshTarget=`echo ${sTarget} | cut -f 3 -d '/'`
+          RemoteDir=`echo ${sTarget} | cut -f 4- -d '/'`
+          cmd="ssh"
+          if [ ! -z ${sFileSshPrivkey} ]; then
+            cmd="${cmd} -i ${sFileSshPrivkey}"
+          fi
+          cmd="${cmd} ${sshTarget} mkdir -p ${RemoteDir} 2>/dev/null ; ls -ld ${RemoteDir} "
+          echo Creating remote directory first ...
+          color cmd
+          $cmd
+          color reset    
+        fi
+    
+        # --- backup
+        echo backup to target: ${sTarget}
+        echo duplicity ${METHOD} ${sBackupParams} ${mydir} ${sTarget}
+        color cmd
+        duplicity ${METHOD} ${sBackupParams} ${mydir} ${sTarget}
+        fetchrc
+        color reset
+        echo
+
+        if [ $myrc -ne 0 ]; then
+          color error
+          echo DIR ERROR ${mydir} rc=$myrc during file transfer
+          case $myrc in
+            23) echo A lock file was found. Maybe this server was rebooted while performing a backup.
+                echo If so delete the file lockfile.lock named in the output and start $0 again.
+                ;;
+            31) echo Maybe you it is a problem with the gpg-agent.conf
+                ls -l ~/.gnupg/gpg-agent.conf && cat ~/.gnupg/gpg-agent.conf
+                ;;
+          esac
+
+        else
+          color ok
+          echo DIR OK ${mydir} was successful.
+        fi
+        color reset
+        echo
+
+        # --- cleanup on remote target
+        h2 "`date` Cleanup old backup data"
+        echo duplicity remove-older-than $STORAGE_KEEP --force ${sParams} ${sTarget}
+        color cmd
+        duplicity remove-older-than $STORAGE_KEEP --force ${sParams} ${sTarget}
+        fetchrc
+        color reset
+
+        echo
+        echo
+
+      else
+        color error
+        echo "DIR SKIP $mydir ... does not exist (no error)"
+        color reset
+      fi 
+      echo
+    done
+ 
+    echo $rc > ${rcfile}
+    exit $rc
+
+  ) | tee -a $transferlog 
+
+  # rc=${PIPESTATUS[0]}
+  rc=`cat ${rcfile}`
+
+  rm -f "${lockfile}" "${rcfile}"
+  j_transferEnd
+
+  echo STATUS $0 exit with final returncode rc=$rc                                    | tee -a $transferlog
+  echo                                                                                | tee -a $transferlog
+  if [ $rc -eq 0 ]; then
+    color ok
+    echo Backup OK                                                                    | tee -a $transferlog
+  else
+    color error
+    echo Backup FAILED :-/                                                            | tee -a $transferlog
+  fi
+  color reset
+  echo                                                                                | tee -a $transferlog
+  echo `date` TRANSFER DONE                                                           | tee -a $transferlog
+
+  ls -l $transferlog                                                                  
+  exit $rc
+
+# --------------------------------------------------------------------------------