Select Git revision
check_file_age
-
Hahn Axel (hahn) authoredHahn Axel (hahn) authored
transfer.sh 10.77 KiB
#!/bin/bash
# ================================================================================
#
# TRANSFER LOCAL DATA TO BACKUP STORAGE
#
# SYNTAX:
# transfer.sh - incremental backup
# transfer.sh full - full backup
# transfer.sh dumps - transfer local dumps only
#
# --------------------------------------------------------------------------------
# ah - Axel Hahn <axel.hahn@iml.unibe.ch>
# ds - Daniel Schueler <daniel.schueler@iml.unibe.ch>
#
# 2016-11-10 ah,ds v1.0
# 2016-12-19 ah,ds v1.1 added parameter "dumps"
# 2017-02-16 ah,ds v1.2 added support for storage slots
# 2017-10-11 ah,ds v1.3 added support for duplicity param --ssh-backend
# 2017-10-17 ah,ds v1.4 remove PIPESTATUS for Debian8 compatibility
# 2017-11-17 ah,ds v1.5 check pid of lockfile in process list if process still runs
# 2018-06-19 ah,ds v1.6 replace --exclude with --exclude regexp in custom dirs
# 2019-06-05 ah,ds v1.7 add custom cache dir
# 2019-09-09 ah,ds v1.8 add testfile on target
# 2019-10-30 ah,ds v1.9 for rsync targets: create remote target dir with ssh command
# 2020-01-21 ah,ds v1.10 show colored OK or FAILED at the end of output
# 2020-02-25 ah,ds, v1.11 fix test -z with non existing vars; show final backup status
# 2021-01-29 ah,ds, v1.12 abort on empty passphrase
# 2021-05-19 ah,ds, v2.0 plugin driven to support multiple backup tools (duplicity + restic)
# ================================================================================
# --------------------------------------------------------------------------------
# CONFIG
# --------------------------------------------------------------------------------
# . `dirname $0`/inc_config.sh
. `dirname $0`/jobhelper.sh
. `dirname $0`/inc_bash.sh
typeset -i rc=0
if [ ! -r "${DIRFILE}" -o ! -r "${STORAGEFILE}" ]; then
echo "SKIP backup of local files - one of the files is not readable (no error): ${DIRFILE} | ${STORAGEFILE}"
exit 0
fi
STORAGE_BIN=`_j_getvar ${STORAGEFILE} "bin"`
STORAGE_BASEDIR=`_j_getvar ${STORAGEFILE} "storage"`
STORAGE_TESTFILE=`_j_getvar ${STORAGEFILE} "storage-file"`
PASSPHRASE=`_j_getvar ${STORAGEFILE} "passphrase"`
STORAGE_REGISTER=`_j_getvar ${STORAGEFILE} "storage-register"`
typeset -i TIMER_TRANSFER_START=`date +%s`
# check
if [ -z "$STORAGE_BIN" ]; then
# STORAGE_BIN=restic
STORAGE_BIN=duplicity
fi
CFGPREFIX=${STORAGE_BIN}_
if [ -z "$STORAGE_BASEDIR" ]; then
color error
echo ERROR: missing config for backup target.
echo There must be an entry "storage = " in ${STORAGEFILE}
color reset
exit 1
fi
if [ ! -z "$STORAGE_TESTFILE" -a ! -f "$STORAGE_TESTFILE" ]; then
color error
echo ERROR: missing testfile $STORAGE_TESTFILE on backup target.
echo The Backup target disk / NAS is not mounted.
color reset
exit 1
fi
# support old value
if [ -z "${PASSPHRASE}" ]; then
echo "WARNING: The value gnupg-passphrase in ${STORAGEFILE} is deprecated. Replace it with passphrase=..."
PASSPHRASE=`_j_getvar ${STORAGEFILE} "gnupg-passphrase"`
fi
if [ -z "${PASSPHRASE}" ]; then
echo "ERROR: no value passphrase was set in ${STORAGEFILE} to encrypt backup data."
echo "Aborting."
exit 1
fi
# For duplicity only
# METHOD incremental is default; full backup will be triggered with
# first param "full"
METHOD=
transferlog="${DIR_LOGS}/transfer-`date +%Y%m%d`.log"
lockfile="${DIR_LOGS}/transfer.running"
rcfile=/tmp/transfer-rc.$$.tmp
# --------------------------------------------------------------------------------
# FUNCTIONS
# --------------------------------------------------------------------------------
# --------------------------------------------------------------------------------
# MAIN
# --------------------------------------------------------------------------------
h1 `date` TRANSFER LOCAL DATA TO STORAGE | tee -a $transferlog
echo "METHOD : $METHOD" | tee -a $transferlog
echo "TARGET : ${STORAGE_BASEDIR}" | tee -a $transferlog
echo "REGISTER : ${STORAGE_REGISTER}" | tee -a $transferlog
echo "TOOL : $STORAGE_BIN" | tee -a $transferlog
echo | tee -a $transferlog
. `dirname $0`/plugins/transfer/$STORAGE_BIN.sh || exit 1
test -z "$STORAGE_REGISTER" || . `dirname $0`/plugins/register/$STORAGE_REGISTER.sh || exit 1
# --------------------------------------------------------------------------------
# ----- Check requirements
t_checkRequirements || exit 1
echo Check locking of a running transfer
if [ -f "${lockfile}" ]; then
color error
echo A lock file for a running transfer was found
cat "${lockfile}"
color reset
echo
# 1659 - check process id inside the lock file
# detect pid from lockfile and search for this process
lockpid=`cat "${lockfile}" | cut -f 2 -d "-" | cut -f 4 -d " " | grep "[0-9]"`
if [ -z $lockpid ]; then
color error
echo ERROR: pid was not fetched from lock file. Check the transfer processes manually, please.
color reset
exit 1
fi
echo transfer processes with pid or ppid ${lockpid}:
color cmd
ps -ef | grep $lockpid | grep "transfer"
rccheck=$?
color reset
if [ $rccheck -eq 0 ]; then
color error
echo ERROR: The transfer with pid $lockpid seems to be still active. Aborting.
color reset
exit 1
fi
color ok
echo OK, the transfer seems not to be active anymore. I IGNORE the lock and continue...
color reset
fi
echo Creating a lock file ...
echo "transfer started `date` - process id $$" > "${lockfile}"
if [ $? -ne 0 ]; then
color error
echo ABORT - unable to create transfer lock
color reset
exit 2
fi
# --------------------------------------------------------------------------------
# ----- BACKUP VARS
# parameters for all
t_setVars || exit 1
ARGS_DEFAULT="$( t_getParamDefault $1 $2 )"
if [ "$1" = "dumps" ]; then
sDirs2Backup="`_j_getvar ${JOBFILE} dir-localdumps`"
else
sDirs2Backup="`j_getDirs2Backup`"
fi
sParamExclude=
for sItem in `_j_getvar ${DIRFILE} exclude`
do
sParamExclude="$sParamExclude $( t_getParamExlude $sItem )"
done
sFileSshPrivkey=`_j_getvar ${STORAGEFILE} "ssh-privatekey"`
if [ ! -z $sFileSshPrivkey ]; then
ARGS_DEFAULT="${ARGS_DEFAULT} $( t_getParamSshKey $sFileSshPrivkey )"
fi
# task#3046 - add custom cache dir
sCacheDir=`_j_getvar ${STORAGEFILE} "${CFGPREFIX}cachedir"`
if [ ! -z $sCacheDir ]; then
ARGS_DEFAULT="${ARGS_DEFAULT} $( t_getParamCacheDir $sCacheDir )"
fi
# --------------------------------------------------------------------------------
# ----- PRE transfer
h2 "`date` Wait for a free slot"
if [ -z "$STORAGE_REGISTER" ]; then
echo "SKIP"
else
iExit=1
until [ $iExit -eq 0 ]; do
registerBackupSlot `hostname -f`
iExit=$?
if [ $iExit -ne 0 ]; then
statusBackupSlot
iRnd=$(($RANDOM%30+30))
echo "I wait a bit ... random time ... $iRnd sec ..."
sleep $iRnd
fi
done
fi | tee -a $transferlog
h2 "`date` PRE transfer tasks"
t_backupDoPreTasks
echo
# --------------------------------------------------------------------------------
# ----- START BACKUPS
(
for mydir in $sDirs2Backup
do
# remove ending slash ... otherwise duplicity will fail
mydir=`echo $mydir | sed 's#\/$##g'`
if [ -d "$mydir" ]; then
h2 "`date` STORE $mydir"
# --- build parameters
sSafeName=`j_getSafename "$mydir"`
sTarget="$( t_backupDirGetTarget $mydir )"
ARGS_BACKUP="${sParamExclude} $( t_getParamBackup )"
# detect custom backup sets and add its includes and excludes
backupid=`j_getSetnameOfPath "$mydir"`
sSpaceReplace="___SPACE___"
if [ ! -z $backupid ]; then
for sItem in `_j_getvar ${DIRFILE} "${backupid}\-\-include" | sed "s#\ #${sSpaceReplace}#g"`
do
ARGS_BACKUP="${ARGS_BACKUP} $( t_getParamInlude $sItem)"
done
for sItem in `_j_getvar ${DIRFILE} "${backupid}\-\-exclude" | sed "s#\ #${sSpaceReplace}#g"`
do
ARGS_BACKUP="${ARGS_BACKUP} $( t_getParamExlude $sItem)"
done
fi
# --- pre task
h3 "`date` PRE backup task for ${mydir}"
t_backupDirDoPreTasks
# sCmdPre="$( t_backupDirDoPreTasks )"
# --- backup
h3 "`date` Backup ${mydir}"
sCmd="$( t_backupDirGetCmdBackup )"
echo "what: ${mydir}"
echo "target: ${sTarget}"
echo "command: $sCmd"
echo
color cmd
$sCmd
fetchrc
color reset
echo
t_rcCheckBackup $myrc
if [ $myrc -ne 0 ]; then
color error
echo DIR ERROR ${mydir} rc=$myrc during file transfer
else
color ok
echo DIR OK ${mydir} was successful.
fi
color reset
echo
# --- post action
h3 "`date` POST backup task for ${mydir}"
t_backupDirDoPostTasks
echo
echo
else
color warning
echo "DIR SKIP $mydir ... does not exist (no error)"
color reset
fi
echo
done
echo $rc > ${rcfile}
exit $rc
) | tee -a $transferlog
# rc=${PIPESTATUS[0]}
rc=`cat ${rcfile}`
# --------------------------------------------------------------------------------
# --- transfer POST tasks
h2 "`date` POST transfer tasks"
t_backupDoPostTasks
rm -f "${lockfile}" "${rcfile}"
echo "Local lock file was removed."
h2 "`date` Unregister used slot"
if [ -z "$STORAGE_REGISTER" ]; then
echo "SKIP"
else
unregisterBackupSlot `hostname -f` $rc
fi | tee -a $transferlog
h2 "`date` Backup finished"
echo STATUS $0 exit with final returncode rc=$rc | tee -a $transferlog
echo | tee -a $transferlog
if [ $rc -eq 0 ]; then
color ok
echo Backup OK | tee -a $transferlog
else
color error
echo Backup FAILED :-/ | tee -a $transferlog
fi
color reset
echo | tee -a $transferlog
typeset -i TIMER_TRANSFER=`date +%s`-$TIMER_TRANSFER_START
echo `date` TRANSFER DONE in $TIMER_TRANSFER sec | tee -a $transferlog
ls -l $transferlog
exit $rc
# --------------------------------------------------------------------------------