From ee7359e73593fc3b3f720d8031adcda35c833596 Mon Sep 17 00:00:00 2001
From: "Hahn Axel (hahn)" <axel.hahn@iml.unibe.ch>
Date: Tue, 25 May 2021 13:01:30 +0200
Subject: [PATCH] keep old transfer.sh for a moment
---
transfer_legacy.sh | 366 +++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 366 insertions(+)
create mode 100755 transfer_legacy.sh
diff --git a/transfer_legacy.sh b/transfer_legacy.sh
new file mode 100755
index 0000000..132aaaa
--- /dev/null
+++ b/transfer_legacy.sh
@@ -0,0 +1,366 @@
+#!/bin/bash
+# ================================================================================
+#
+# TRANSFER LOCAL DATA TO BACKUP STORAGE
+#
+# SYNTAX:
+# transfer.sh - incremental backup
+# transfer.sh full - full backup
+# transfer.sh dumps - transfer local dumps only
+#
+# duplicity see
+# - man pages http://duplicity.nongnu.org/duplicity.1.html
+# - example http://duplicity.nongnu.org/contrib/jwfull
+# - example http://duplicity.nongnu.org/contrib/jwincr
+#
+# this script was build to support ssh backend only
+#
+# --------------------------------------------------------------------------------
+# ah - Axel Hahn <axel.hahn@iml.unibe.ch>
+# ds - Daniel Schueler <daniel.schueler@iml.unibe.ch>
+#
+# 2016-11-10 ah,ds v1.0
+# 2016-12-19 ah,ds v1.1 added parameter "dumps"
+# 2017-02-16 ah,ds v1.2 added support for storage slots
+# 2017-10-11 ah,ds v1.3 added support for duplicity param --ssh-backend
+# 2017-10-17 ah,ds v1.4 remove PIPESTATUS for Debian8 compatibility
+# 2017-11-17 ah,ds v1.5 check pid of lockfile in process list if process still runs
+# 2018-06-19 ah,ds v1.6 replace --exclude with --exclude regexp in custom dirs
+# 2019-06-05 ah,ds v1.7 add custom cache dir
+# 2019-09-09 ah,ds v1.8 add testfile on target
+# 2019-10-30 ah,ds v1.9 for rsync targets: create remote target dir with ssh command
+# 2020-01-21 ah,ds v1.10 show colored OK or FAILED at the end of output
+# 2020-02-25 ah,ds, v1.11 fix test -z with non existing vars; show final backup status
+# 2021-01-29 ah,ds, v1.12 abort on empty passphrase
+# ================================================================================
+
+
+# --------------------------------------------------------------------------------
+# CONFIG
+# --------------------------------------------------------------------------------
+
+ # . `dirname $0`/inc_config.sh
+
+ . `dirname $0`/jobhelper.sh
+ . `dirname $0`/inc_bash.sh
+
+ typeset -i rc=0
+
+ STORAGE_BASEDIR=`_j_getvar ${STORAGEFILE} "storage"`
+ STORAGE_TESTFILE=`_j_getvar ${STORAGEFILE} "storage-file"`
+ STORAGE_KEEP=`_j_getvar ${STORAGEFILE} "keep"`
+ STORAGE_VOLSIZE=`_j_getvar ${STORAGEFILE} "volsize"`
+
+ # check
+ if [ -z "$STORAGE_BASEDIR" ]; then
+ color error
+ echo ERROR: missing config for backup target.
+ echo There must be an entry storage in ${STORAGEFILE}
+ color reset
+ exit 1
+ fi
+ if [ ! -z "$STORAGE_TESTFILE" -a ! -f "$STORAGE_TESTFILE" ]; then
+ color error
+ echo ERROR: missing testfile $STORAGE_TESTFILE on backup target.
+ echo The Backup target disk / NAS is not mounted.
+ color reset
+ exit 1
+ fi
+
+ if [ -z "$STORAGE_KEEP" ]; then
+ color error
+ echo ERROR: missing config for backup target.
+ echo There must be an entry keep in ${STORAGEFILE}
+ color reset
+ exit 1
+ fi
+
+ # METHOD incremental is default; full backup will be triggered with
+ # first param "full"
+ METHOD=
+
+
+ transferlog="${DIR_LOGS}/transfer-`date +%Y%m%d`.log"
+ lockfile="${DIR_LOGS}/transfer.running"
+ rcfile=/tmp/transfer-rc.$$.tmp
+
+# --------------------------------------------------------------------------------
+# FUNCTIONS
+# --------------------------------------------------------------------------------
+
+
+# --------------------------------------------------------------------------------
+# MAIN
+# --------------------------------------------------------------------------------
+
+
+
+ h1 `date` TRANSFER LOCAL DATA TO STORAGE | tee -a $transferlog
+
+ echo METHOD: $METHOD | tee -a $transferlog
+ echo TARGET: ${STORAGE_BASEDIR} | tee -a $transferlog
+ echo | tee -a $transferlog
+
+# ----- Check requirements
+
+ j_requireUser "root"
+ j_requireBinary "duplicity"
+
+ echo Check locking of a running transfer
+ if [ -f "${lockfile}" ]; then
+ color error
+ echo A lock file for a running transfer was found
+ cat "${lockfile}"
+ color reset
+ echo
+
+ # 1659 - check process id inside the lock file
+ # detect pid from lockfile and search for this process
+ lockpid=`cat "${lockfile}" | cut -f 2 -d "-" | cut -f 4 -d " " | grep "[0-9]"`
+ if [ -z $lockpid ]; then
+ color error
+ echo ERROR: pid was not fetched from lock file. Check the transfer processes manually, please.
+ color reset
+ exit 1
+ fi
+ echo transfer processes with pid or ppid ${lockpid}:
+ color cmd
+ ps -ef | grep $lockpid | grep "transfer"
+ rccheck=$?
+ color reset
+ if [ $rccheck -eq 0 ]; then
+ color error
+ echo ERROR: The transfer with pid $lockpid seems to be still active. Aborting.
+ color reset
+ exit 1
+ fi
+ color ok
+ echo OK, the transfer seems not to be active anymore. I IGNORE the lock and continue...
+ color reset
+ fi
+
+ echo Creating a lock file ...
+ echo "transfer started `date` - process id $$" > "${lockfile}"
+ if [ $? -ne 0 ]; then
+ color error
+ echo ABORT - unable to create transfer lock
+ color reset
+ exit 2
+ fi
+
+# ----- BACKUP VARS
+
+ # parameters for all
+ sParams=
+
+ if [ "$1" = "dumps" ]; then
+ sDirs2Backup="`_j_getvar ${DIRFILE} dir-localdumps`"
+ else
+ sDirs2Backup="`j_getDirs2Backup`"
+ fi
+
+ if [ "$1" = "full" ]; then
+ METHOD="full"
+ fi
+ if [ "$1" = "auto" ]; then
+ METHOD="--full-if-older-than $2"
+ fi
+
+
+ PASSPHRASE=`_j_getvar ${STORAGEFILE} "gnupg-passphrase"`
+ if [ -z "${PASSPHRASE}" ]; then
+ echo "ERROR: no value gnupg-passphrase was set in ${STORAGEFILE} to encrypt backup data."
+ echo "Aborting."
+ exit 1
+ fi
+
+ sParamExclude=
+ # build exclude param list for duplicity
+ #
+ # REMARK: the excludes must fit the include definition
+ # On non matching excludes duplicity stops
+ #
+ for sItem in `_j_getvar ${DIRFILE} exclude`
+ do
+ sParamExclude="$sParamExclude --exclude-regexp """$sItem""" "
+ done
+ #
+ # sExcludefile="${DIR_JOBS}/transfer-exclude.txt"
+ #
+ # if [ -f ${sExcludefile} ]; then
+ # sParamExclude="$sParamExclude --exclude-filelist """${sExcludefile}""" "
+ # fi
+
+ export PASSPHRASE
+
+ # task#1623 - fallback ssh backend for Debian 8
+ sSshBackend=`_j_getvar ${STORAGEFILE} "ssh-backend"`
+ if [ ! -z $sSshBackend ]; then
+ sParams="${sParams} --ssh-backend $sSshBackend"
+ fi
+
+ sFileSshPrivkey=`_j_getvar ${STORAGEFILE} "ssh-privatekey"`
+ if [ ! -z $sFileSshPrivkey ]; then
+ sParams="${sParams} --ssh-options="""-oIdentityFile=${sFileSshPrivkey}""" "
+ fi
+
+ # task#3046 - add custom cache dir
+ sCacheDir=`_j_getvar ${STORAGEFILE} "cachedir"`
+ if [ ! -z $sCacheDir ]; then
+ sParams="${sParams} --archive-dir=$sCacheDir"
+ if [ ! -d $sCacheDir ]; then
+ mkdir -p $sCacheDir
+ chmod 750 $sCacheDir
+ fi
+ fi
+
+
+# ----- Wait for a free slot
+
+ j_transferStart | tee -a $transferlog
+
+
+# ----- START BACKUPS
+
+ (
+ for mydir in $sDirs2Backup
+ do
+
+ # remove ending slash ... otherwise duplicity will fail
+ mydir=`echo $mydir | sed 's#\/$##g'`
+
+ if [ -d "$mydir" ]; then
+
+ h2 "`date` STORE $mydir"
+
+ # --- build parameters for duplicity
+ sSafeName=`j_getSafename "$mydir"`
+ sTarget=`j_getFullTarget "$mydir"`
+
+ sBackupParams=
+ sBackupParams="${sBackupParams} ${sParams} "
+ sBackupParams="${sBackupParams} ${sParamExclude} "
+ if [ ! -z $STORAGE_VOLSIZE ]; then
+ sBackupParams="${sBackupParams} --volsize ${STORAGE_VOLSIZE} "
+ fi
+
+ # verbosity level to fetch changed files from log
+ sBackupParams="${sBackupParams} -v8"
+
+ # add asynchronous upload
+ sBackupParams="${sBackupParams} --asynchronous-upload"
+
+ # ---------- START --------------------
+ # detect custom backup sets and add its includes and excludes
+ backupid=`j_getSetnameOfPath "$mydir"`
+ sSpaceReplace="___SPACE___"
+
+ if [ ! -z $backupid ]; then
+ for param in include exclude
+ do
+ for sItem in `_j_getvar ${DIRFILE} "${backupid}\-\-${param}" | sed "s#\ #${sSpaceReplace}#g"`
+ do
+ sBackupParams="${sBackupParams} --${param}-regexp """$sItem""" "
+ done
+ done
+ sBackupParams=`echo ${sBackupParams} | sed "s#${sSpaceReplace}# #g"`
+
+ fi
+ # ---------- ENDE --------------------
+
+ # --- for rsync only: create remote directory
+ echo ${sTarget} | fgrep "rsync://" >/dev/null
+ if [ $? -eq 0 ]; then
+ # sshTarget=`echo ${sTarget} | sed "s#rsync://#scp://#"`
+ # echo Creating remote directory with fetching collection-status on $sshTarget
+ # color cmd
+ # duplicity collection-status ${sParams} ${sshTarget}
+ # color reset
+ sshTarget=`echo ${sTarget} | cut -f 3 -d '/'`
+ RemoteDir=`echo ${sTarget} | cut -f 4- -d '/'`
+ cmd="ssh"
+ if [ ! -z ${sFileSshPrivkey} ]; then
+ cmd="${cmd} -i ${sFileSshPrivkey}"
+ fi
+ cmd="${cmd} ${sshTarget} mkdir -p ${RemoteDir} 2>/dev/null ; ls -ld ${RemoteDir} "
+ echo Creating remote directory first ...
+ color cmd
+ $cmd
+ color reset
+ fi
+
+ # --- backup
+ echo backup to target: ${sTarget}
+ echo duplicity ${METHOD} ${sBackupParams} ${mydir} ${sTarget}
+ color cmd
+ duplicity ${METHOD} ${sBackupParams} ${mydir} ${sTarget}
+ fetchrc
+ color reset
+ echo
+
+ if [ $myrc -ne 0 ]; then
+ color error
+ echo DIR ERROR ${mydir} rc=$myrc during file transfer
+ case $myrc in
+ 23) echo A lock file was found. Maybe this server was rebooted while performing a backup.
+ echo If so delete the file lockfile.lock named in the output and start $0 again.
+ ;;
+ 31) echo Maybe you it is a problem with the gpg-agent.conf
+ ls -l ~/.gnupg/gpg-agent.conf && cat ~/.gnupg/gpg-agent.conf
+ ;;
+ esac
+
+ else
+ color ok
+ echo DIR OK ${mydir} was successful.
+ fi
+ color reset
+ echo
+
+ # --- cleanup on remote target
+ h2 "`date` Cleanup old backup data"
+ echo duplicity remove-older-than $STORAGE_KEEP --force ${sParams} ${sTarget}
+ color cmd
+ duplicity remove-older-than $STORAGE_KEEP --force ${sParams} ${sTarget}
+ fetchrc
+ color reset
+
+ echo
+ echo
+
+ else
+ color error
+ echo "DIR SKIP $mydir ... does not exist (no error)"
+ color reset
+ fi
+ echo
+ done
+
+ echo $rc > ${rcfile}
+ exit $rc
+
+ ) | tee -a $transferlog
+
+ # rc=${PIPESTATUS[0]}
+ rc=`cat ${rcfile}`
+
+ rm -f "${lockfile}" "${rcfile}"
+ j_transferEnd
+
+ echo STATUS $0 exit with final returncode rc=$rc | tee -a $transferlog
+ echo | tee -a $transferlog
+ if [ $rc -eq 0 ]; then
+ color ok
+ echo Backup OK | tee -a $transferlog
+ else
+ color error
+ echo Backup FAILED :-/ | tee -a $transferlog
+ fi
+ color reset
+ echo | tee -a $transferlog
+ echo `date` TRANSFER DONE | tee -a $transferlog
+
+ ls -l $transferlog
+ exit $rc
+
+# --------------------------------------------------------------------------------
--
GitLab