#!/bin/bash # ================================================================================ # # TRANSFER LOCAL DATA TO BACKUP STORAGE # # SYNTAX: # transfer.sh - incremental backup # transfer.sh full - full backup # transfer.sh dumps - transfer local dumps only # # -------------------------------------------------------------------------------- # ah - Axel Hahn <axel.hahn@iml.unibe.ch> # ds - Daniel Schueler <daniel.schueler@iml.unibe.ch> # # 2016-11-10 ah,ds v1.0 # 2016-12-19 ah,ds v1.1 added parameter "dumps" # 2017-02-16 ah,ds v1.2 added support for storage slots # 2017-10-11 ah,ds v1.3 added support for duplicity param --ssh-backend # 2017-10-17 ah,ds v1.4 remove PIPESTATUS for Debian8 compatibility # 2017-11-17 ah,ds v1.5 check pid of lockfile in process list if process still runs # 2018-06-19 ah,ds v1.6 replace --exclude with --exclude regexp in custom dirs # 2019-06-05 ah,ds v1.7 add custom cache dir # 2019-09-09 ah,ds v1.8 add testfile on target # 2019-10-30 ah,ds v1.9 for rsync targets: create remote target dir with ssh command # 2020-01-21 ah,ds v1.10 show colored OK or FAILED at the end of output # 2020-02-25 ah,ds, v1.11 fix test -z with non existing vars; show final backup status # 2021-01-29 ah,ds, v1.12 abort on empty passphrase # 2021-05-19 ah,ds, v2.0 plugin driven to support multiple backup tools (duplicity + restic) # ================================================================================ # -------------------------------------------------------------------------------- # CONFIG # -------------------------------------------------------------------------------- # . `dirname $0`/inc_config.sh . `dirname $0`/jobhelper.sh . `dirname $0`/inc_bash.sh typeset -i rc=0 if [ ! -r "${DIRFILE}" -o ! -r "${STORAGEFILE}" ]; then echo "SKIP backup of local files - one of the files is not readable (no error): ${DIRFILE} | ${STORAGEFILE}" exit 0 fi STORAGE_BIN=`_j_getvar ${STORAGEFILE} "bin"` STORAGE_BASEDIR=`_j_getvar ${STORAGEFILE} "storage"` STORAGE_TESTFILE=`_j_getvar ${STORAGEFILE} "storage-file"` PASSPHRASE=`_j_getvar ${STORAGEFILE} "passphrase"` STORAGE_REGISTER=`_j_getvar ${STORAGEFILE} "storage-register"` typeset -i TIMER_TRANSFER_START=`date +%s` # check if [ -z "$STORAGE_BIN" ]; then # STORAGE_BIN=restic STORAGE_BIN=duplicity fi CFGPREFIX=${STORAGE_BIN}_ if [ -z "$STORAGE_BASEDIR" ]; then color error echo ERROR: missing config for backup target. echo There must be an entry "storage = " in ${STORAGEFILE} color reset exit 1 fi if [ ! -z "$STORAGE_TESTFILE" -a ! -f "$STORAGE_TESTFILE" ]; then color error echo ERROR: missing testfile $STORAGE_TESTFILE on backup target. echo The Backup target disk / NAS is not mounted. color reset exit 1 fi # support old value if [ -z "${PASSPHRASE}" ]; then echo "WARNING: The value gnupg-passphrase in ${STORAGEFILE} is deprecated. Replace it with passphrase=..." PASSPHRASE=`_j_getvar ${STORAGEFILE} "gnupg-passphrase"` fi if [ -z "${PASSPHRASE}" ]; then echo "ERROR: no value passphrase was set in ${STORAGEFILE} to encrypt backup data." echo "Aborting." exit 1 fi # For duplicity only # METHOD incremental is default; full backup will be triggered with # first param "full" METHOD= transferlog="${DIR_LOGS}/transfer-`date +%Y%m%d`.log" lockfile="${DIR_LOGS}/transfer.running" rcfile=/tmp/transfer-rc.$$.tmp # -------------------------------------------------------------------------------- # FUNCTIONS # -------------------------------------------------------------------------------- # -------------------------------------------------------------------------------- # MAIN # -------------------------------------------------------------------------------- h1 `date` TRANSFER LOCAL DATA TO STORAGE | tee -a $transferlog echo "METHOD : $METHOD" | tee -a $transferlog echo "TARGET : ${STORAGE_BASEDIR}" | tee -a $transferlog echo "REGISTER : ${STORAGE_REGISTER}" | tee -a $transferlog echo "TOOL : $STORAGE_BIN" | tee -a $transferlog echo | tee -a $transferlog . `dirname $0`/plugins/transfer/$STORAGE_BIN.sh || exit 1 test -z "$STORAGE_REGISTER" || . `dirname $0`/plugins/register/$STORAGE_REGISTER.sh || exit 1 # -------------------------------------------------------------------------------- # ----- Check requirements t_checkRequirements || exit 1 echo Check locking of a running transfer if [ -f "${lockfile}" ]; then color error echo A lock file for a running transfer was found cat "${lockfile}" color reset echo # 1659 - check process id inside the lock file # detect pid from lockfile and search for this process lockpid=`cat "${lockfile}" | cut -f 2 -d "-" | cut -f 4 -d " " | grep "[0-9]"` if [ -z $lockpid ]; then color error echo ERROR: pid was not fetched from lock file. Check the transfer processes manually, please. color reset exit 1 fi echo transfer processes with pid or ppid ${lockpid}: color cmd ps -ef | grep $lockpid | grep "transfer" rccheck=$? color reset if [ $rccheck -eq 0 ]; then color error echo ERROR: The transfer with pid $lockpid seems to be still active. Aborting. color reset exit 1 fi color ok echo OK, the transfer seems not to be active anymore. I IGNORE the lock and continue... color reset fi echo Creating a lock file ... echo "transfer started `date` - process id $$" > "${lockfile}" if [ $? -ne 0 ]; then color error echo ABORT - unable to create transfer lock color reset exit 2 fi # -------------------------------------------------------------------------------- # ----- BACKUP VARS # parameters for all t_setVars || exit 1 ARGS_DEFAULT="$( t_getParamDefault $1 $2 )" if [ "$1" = "dumps" ]; then sDirs2Backup="`_j_getvar ${JOBFILE} dir-localdumps`" else sDirs2Backup="`j_getDirs2Backup`" fi sParamExclude= for sItem in `_j_getvar ${DIRFILE} exclude` do sParamExclude="$sParamExclude $( t_getParamExlude $sItem )" done sFileSshPrivkey=`_j_getvar ${STORAGEFILE} "ssh-privatekey"` if [ ! -z $sFileSshPrivkey ]; then ARGS_DEFAULT="${ARGS_DEFAULT} $( t_getParamSshKey $sFileSshPrivkey )" fi # task#3046 - add custom cache dir sCacheDir=`_j_getvar ${STORAGEFILE} "${CFGPREFIX}cachedir"` if [ ! -z $sCacheDir ]; then ARGS_DEFAULT="${ARGS_DEFAULT} $( t_getParamCacheDir $sCacheDir )" fi # -------------------------------------------------------------------------------- # ----- PRE transfer h2 "`date` Wait for a free slot" if [ -z "$STORAGE_REGISTER" ]; then echo "SKIP" else iExit=1 until [ $iExit -eq 0 ]; do registerBackupSlot `hostname -f` iExit=$? if [ $iExit -ne 0 ]; then statusBackupSlot iRnd=$(($RANDOM%30+30)) echo "I wait a bit ... random time ... $iRnd sec ..." sleep $iRnd fi done fi | tee -a $transferlog h2 "`date` PRE transfer tasks" t_backupDoPreTasks echo # -------------------------------------------------------------------------------- # ----- START BACKUPS ( for mydir in $sDirs2Backup do # remove ending slash ... otherwise duplicity will fail mydir=`echo $mydir | sed 's#\/$##g'` if [ -d "$mydir" ]; then h2 "`date` STORE $mydir" # --- build parameters sSafeName=`j_getSafename "$mydir"` sTarget="$( t_backupDirGetTarget $mydir )" ARGS_BACKUP="${sParamExclude} $( t_getParamBackup )" # detect custom backup sets and add its includes and excludes backupid=`j_getSetnameOfPath "$mydir"` sSpaceReplace="___SPACE___" if [ ! -z $backupid ]; then for sItem in `_j_getvar ${DIRFILE} "${backupid}\-\-include" | sed "s#\ #${sSpaceReplace}#g"` do ARGS_BACKUP="${ARGS_BACKUP} $( t_getParamInlude $sItem)" done for sItem in `_j_getvar ${DIRFILE} "${backupid}\-\-exclude" | sed "s#\ #${sSpaceReplace}#g"` do ARGS_BACKUP="${ARGS_BACKUP} $( t_getParamExlude $sItem)" done fi # --- pre task h3 "`date` PRE backup task for ${mydir}" t_backupDirDoPreTasks # sCmdPre="$( t_backupDirDoPreTasks )" # --- backup h3 "`date` Backup ${mydir}" sCmd="$( t_backupDirGetCmdBackup )" echo "what: ${mydir}" echo "target: ${sTarget}" echo "command: $sCmd" echo color cmd $sCmd fetchrc color reset echo t_rcCheckBackup $myrc if [ $myrc -ne 0 ]; then color error echo DIR ERROR ${mydir} rc=$myrc during file transfer else color ok echo DIR OK ${mydir} was successful. fi color reset echo # --- post action h3 "`date` POST backup task for ${mydir}" t_backupDirDoPostTasks echo echo else color warning echo "DIR SKIP $mydir ... does not exist (no error)" color reset fi echo done echo $rc > ${rcfile} exit $rc ) | tee -a $transferlog # rc=${PIPESTATUS[0]} rc=`cat ${rcfile}` # -------------------------------------------------------------------------------- # --- transfer POST tasks h2 "`date` POST transfer tasks" t_backupDoPostTasks rm -f "${lockfile}" "${rcfile}" echo "Local lock file was removed." h2 "`date` Unregister used slot" if [ -z "$STORAGE_REGISTER" ]; then echo "SKIP" else unregisterBackupSlot `hostname -f` $rc fi | tee -a $transferlog h2 "`date` Backup finished" echo STATUS $0 exit with final returncode rc=$rc | tee -a $transferlog echo | tee -a $transferlog if [ $rc -eq 0 ]; then color ok echo Backup OK | tee -a $transferlog else color error echo Backup FAILED :-/ | tee -a $transferlog fi color reset echo | tee -a $transferlog typeset -i TIMER_TRANSFER=`date +%s`-$TIMER_TRANSFER_START echo `date` TRANSFER DONE in $TIMER_TRANSFER sec | tee -a $transferlog ls -l $transferlog exit $rc # --------------------------------------------------------------------------------