From a704e62380e9a75b6491edf9794ecb285708e1de Mon Sep 17 00:00:00 2001
From: "Hahn Axel (hahn)" <axel.hahn@iml.unibe.ch>
Date: Tue, 25 May 2021 13:03:04 +0200
Subject: [PATCH] transfer with multiple backend tools
---
inc_bash.sh | 15 ++-
jobhelper.sh | 2 +-
plugins/transfer/_template.sh | 167 +++++++++++++++++++++++
plugins/transfer/duplicity.sh | 239 +++++++++++++++++++++++++++++++++
plugins/transfer/restic.sh | 240 ++++++++++++++++++++++++++++++++++
transfer.sh | 198 +++++++++++++---------------
6 files changed, 748 insertions(+), 113 deletions(-)
create mode 100644 plugins/transfer/_template.sh
create mode 100644 plugins/transfer/duplicity.sh
create mode 100644 plugins/transfer/restic.sh
diff --git a/inc_bash.sh b/inc_bash.sh
index 3564302..7920ff2 100755
--- a/inc_bash.sh
+++ b/inc_bash.sh
@@ -34,6 +34,8 @@ function fetchrc(){
;;
"ok") sColorcode="92" # green
;;
+ "warning") sColorcode="33" # yellow
+ ;;
"error") sColorcode="91" # red
;;
esac
@@ -49,7 +51,8 @@ function fetchrc(){
function h1(){
color head
echo
- echo "====== $* ======"
+ echo
+ echo "########## $* ##########"
echo
color reset
}
@@ -57,7 +60,15 @@ function fetchrc(){
function h2(){
color head
- echo "--- $*"
+ echo
+ echo "========== $* =========="
+ echo
+ color reset
+ }
+
+ function h3(){
+ color head
+ echo "----- $*"
color reset
}
diff --git a/jobhelper.sh b/jobhelper.sh
index 52c1d6c..497b5ba 100755
--- a/jobhelper.sh
+++ b/jobhelper.sh
@@ -425,7 +425,7 @@ function _j_storagehelper(){
# rsync://[backupuser]@[storage]//[targetdir]/
sProtocol=`j_getFullTarget "" | cut -f 1 -d ":"`
- if [ ${sProtocol} = "rsync" -o ${sProtocol} = "scp" ]; then
+ if [ ${sProtocol} = "rsync" -o ${sProtocol} = "scp" -o ${sProtocol} = "sftp" ]; then
sSshTarget=`j_getFullTarget "" | cut -f 3 -d "/"`
# echo target: $sSshTarget
diff --git a/plugins/transfer/_template.sh b/plugins/transfer/_template.sh
new file mode 100644
index 0000000..5eee997
--- /dev/null
+++ b/plugins/transfer/_template.sh
@@ -0,0 +1,167 @@
+#!/bin/bash
+# ================================================================================
+#
+# TRANSFER :: PLUGIN - TEMPLATE
+#
+# this script will be included in ../../transfer.sh
+#
+# --------------------------------------------------------------------------------
+# ah - Axel Hahn <axel.hahn@iml.unibe.ch>
+# 2021-05-19 ah v0.0 INIT ... WIP
+# ================================================================================
+
+# --------------------------------------------------------------------------------
+# ENV VARS
+# export variables that are needed by the backup tool
+# --------------------------------------------------------------------------------
+
+# export PASSPHRASE
+
+# --------------------------------------------------------------------------------
+# INIT
+# --------------------------------------------------------------------------------
+
+ function t_checkRequirements(){
+ echo "DEBUG: function t_checkRequirements"
+ j_requireUser "root"
+ # j_requireBinary "duplicity"
+ }
+
+ # set variables
+ function t_setVars(){
+ echo "DEBUG: t_setVars"
+ }
+
+# --------------------------------------------------------------------------------
+# GENERATE PARAMS :: ALL DIRS
+# --------------------------------------------------------------------------------
+
+ # return a string with default params
+ # param string param1 of transfer.sh; one of full|inc|auto
+ # param string param2 of transfer.sh; for auto: date i.e. 3M for 3 monthes
+ function t_getParamDefault(){
+ echo "DEBUG: function t_getParamDefault $*"
+
+ # local _method=
+
+ # --- method
+ # test "$1" = "full" && _method="full"
+ # test "$1" = "auto" && _method="--full-if-older-than $2"
+ # echo -n "$_method"
+
+ }
+
+ # return a cli parameter for a single exlude directory
+ # param string cache directory for local index files
+ function t_getParamCacheDir(){
+ if [ ! -z "$1" ]; then
+ local sCacheDir="$1"
+ if [ ! -d $sCacheDir ]; then
+ mkdir -p $sCacheDir
+ chmod 750 $sCacheDir
+ fi
+ echo --archive-dir=$sCacheDir
+ fi
+ }
+
+ # return a cli parameter for a single exlude directory
+ # param string exlude pattern
+ function t_getParamExlude(){
+ # test -z "$1" || echo --exclude-regexp """$*"""
+ }
+ # return a cli parameter for a single exlude directory
+ # param string exlude pattern
+ function t_getParamInlude(){
+ # test -z "$1" || echo --include-regexp """$*"""
+ }
+
+ # return a cli parameter to use an ssh keyfile
+ # param string filename if ssh private key file
+ function t_getParamSshKey(){
+ # test -z "$1" || echo --ssh-options="""-oIdentityFile=$1"""
+ }
+
+# --------------------------------------------------------------------------------
+# ACTIONS :: TRANSFER
+# --------------------------------------------------------------------------------
+ # pre backup actions
+ # uses global vars from ../../transfer.sh
+ function t_cmdPre(){
+ # echo TODO PRE actions before starting transfer
+ }
+
+
+ # post backup actions
+ # uses global vars from ../../transfer.sh
+ function t_cmdPost(){
+ # echo TODO POST actions after all transfers
+ }
+
+# --------------------------------------------------------------------------------
+# GENERATE PARAMS :: SINGLE DIR
+# --------------------------------------------------------------------------------
+
+ # get target url/ directory
+ # param string directory to backup
+ function t_sd_getTarget(){
+ # directory based target
+ j_getFullTarget "$1"
+ # for host based backup target - remove param:
+ # j_getFullTarget ""
+ }
+
+ function t_sd_getCmdBackup(){
+ # echo duplicity ${sBackupParams} ${mydir} ${sTarget}
+ }
+
+ # pre backup actions
+ # uses global vars from ../../transfer.sh
+ function t_sd_cmdPre(){
+ }
+
+
+ # post backup actions
+ # uses global vars from ../../transfer.sh
+ function t_sd_cmdPost(){
+ }
+
+# --------------------------------------------------------------------------------
+# VERIFY RETURNCODES
+# --------------------------------------------------------------------------------
+
+ # init repository
+ function t_rcCheckInit(){
+ case $1 in
+ 0) color ok; echo "OK - the repository was created." ;;
+ *) color error; echo "Verify output above - returncode of init was $1" ;;
+ esac
+ color reset
+ }
+ # backup files
+ function t_rcCheckBackup(){
+ case $1 in
+ 0) color ok; echo "OK" ;;
+ *) color error; echo "Backup error - returncode was $1" ;;
+ esac
+ color reset
+ }
+
+ # repoitory cleanup
+ function t_rcCheckCleanup(){
+ case $1 in
+ 0) color ok; echo "OK" ;;
+ *) color error; echo "Cleanup error - returncode was $1" ;;
+ esac
+ color reset
+ }
+
+ # restore files
+ function t_rcCheckRestore(){
+ case $1 in
+ 0) color ok; echo "OK" ;;
+ *) color error; echo "Restore error - returncode was $1" ;;
+ esac
+ color reset
+ }
+
+# --------------------------------------------------------------------------------
diff --git a/plugins/transfer/duplicity.sh b/plugins/transfer/duplicity.sh
new file mode 100644
index 0000000..b398da6
--- /dev/null
+++ b/plugins/transfer/duplicity.sh
@@ -0,0 +1,239 @@
+#!/bin/bash
+# ================================================================================
+#
+# TRANSFER :: PLUGIN - DUPLICITY
+#
+# this script will be included in ../../transfer.sh
+#
+# --------------------------------------------------------------------------------
+# ah - Axel Hahn <axel.hahn@iml.unibe.ch>
+# 2021-05-19 ah v0.0 INIT ... WIP
+# ================================================================================
+
+# --------------------------------------------------------------------------------
+# ENV VARS
+# export variables that are needed by the backup tool
+# --------------------------------------------------------------------------------
+
+export PASSPHRASE
+
+# --------------------------------------------------------------------------------
+# INIT
+# --------------------------------------------------------------------------------
+
+ # check requirements
+ function t_checkRequirements(){
+ echo "DEBUG: function t_checkRequirements"
+
+ j_requireUser "root"
+ j_requireBinary "duplicity"
+ }
+
+ # set variables
+ function t_setVars(){
+ echo "DEBUG: t_setVars"
+
+ }
+
+# --------------------------------------------------------------------------------
+# GENERATE PARAMS :: ALL DIRS
+# --------------------------------------------------------------------------------
+
+ # return a string with default params
+ # param string param1 of transfer.sh; one of full|inc|auto
+ # param string param2 of transfer.sh; for auto: date i.e. 3M for 3 monthes
+ function t_getParamDefault(){
+
+ local _method=
+ local _volsize=
+
+ # --- method
+ test "$1" = "full" && _method="full"
+ test "$1" = "auto" && _method="--full-if-older-than $2"
+ echo -n "$_method"
+
+ # --- backend
+ # task#1623 - fallback ssh backend for Debian 8
+ sSshBackend=`_j_getvar ${STORAGEFILE} "ssh-backend"`
+ if [ ! -z $sSshBackend ]; then
+ echo -n " --ssh-backend $sSshBackend"
+ fi
+
+ # --- volume size
+ _volsize=`_j_getvar ${STORAGEFILE} "duplicity_volsize"`
+ test -z "$_volsize" && _volsize=`_j_getvar ${STORAGEFILE} "volsize"`
+
+ if [ ! -z $_volsize ]; then
+ echo -n " --volsize ${_volsize}"
+ fi
+
+ # --- verbosity level to fetch changed files from log
+ echo -n " -v8"
+
+ # --- add asynchronous upload
+ echo -n " --asynchronous-upload"
+
+ }
+ # return a string with backup default params
+ function t_getParamBackup(){
+ echo -n ''
+ }
+ # return a cli parameter for a single exlude directory
+ # param string cache directory for local index files
+ function t_getParamCacheDir(){
+ if [ ! -z "$1" ]; then
+ local sCacheDir="$1"
+ if [ ! -d $sCacheDir ]; then
+ mkdir -p $sCacheDir
+ chmod 750 $sCacheDir
+ fi
+ echo --archive-dir=$sCacheDir
+ fi
+ }
+
+ # return a cli parameter for a single exlude directory
+ # param string exlude pattern
+ function t_getParamExlude(){
+ test -z "$1" || echo --exclude-regexp "'"$*"'"
+ }
+ # return a cli parameter for a single exlude directory
+ # param string exlude pattern
+ function t_getParamInlude(){
+ test -z "$1" || echo --include-regexp "'"$*"'"
+ }
+
+ # return a cli parameter to use an ssh keyfile
+ # param string filename if ssh private key file
+ function t_getParamSshKey(){
+ test -z "$1" || echo --ssh-options="'"-oIdentityFile=$1"'"
+ }
+
+# --------------------------------------------------------------------------------
+# ACTIONS :: TRANSFER
+# --------------------------------------------------------------------------------
+ # pre backup actions
+ # uses global vars from ../../transfer.sh
+ function t_cmdPre(){
+ echo No PRE actions before starting transfer
+ }
+
+
+ # post backup actions
+ # uses global vars from ../../transfer.sh
+ function t_cmdPost(){
+ echo No POST actions after all transfers
+ }
+
+# --------------------------------------------------------------------------------
+# GENERATE PARAMS :: SINGLE DIR
+# --------------------------------------------------------------------------------
+
+ # get target url/ directory
+ # param string directory to backup
+ function t_sd_getTarget(){
+ # duplicity has a directory based target
+ j_getFullTarget "$1"
+ }
+
+ # get string with complete backup command
+ function t_sd_getCmdBackup(){
+ echo duplicity ${sBackupParams} ${mydir} ${sTarget}
+ }
+
+ # pre backup tasks
+ # uses global vars from ../../transfer.sh
+ function t_sd_cmdPre(){
+ # --- for rsync only: create remote directory
+ echo ${sTarget} | fgrep "rsync://" >/dev/null
+ if [ $? -eq 0 ]; then
+ # sshTarget=`echo ${sTarget} | sed "s#rsync://#scp://#"`
+ # echo Creating remote directory with fetching collection-status on $sshTarget
+ # color cmd
+ # duplicity collection-status ${sParams} ${sshTarget}
+ # color reset
+ sshTarget=`echo ${sTarget} | cut -f 3 -d '/'`
+ RemoteDir=`echo ${sTarget} | cut -f 4- -d '/'`
+ cmd="ssh"
+ if [ ! -z ${sFileSshPrivkey} ]; then
+ cmd="${cmd} -i ${sFileSshPrivkey}"
+ fi
+ cmd="${cmd} ${sshTarget} mkdir -p ${RemoteDir} 2>/dev/null ; ls -ld ${RemoteDir} "
+ echo Creating remote directory first ...
+ color cmd
+ $cmd
+ color reset
+ fi
+ }
+
+ # post backup tasks
+ # uses global vars from ../../transfer.sh
+ function t_sd_cmdPost(){
+ local STORAGE_KEEP=`_j_getvar ${STORAGEFILE} "duplicity_keep"`
+ test -z "$STORAGE_KEEP" && STORAGE_KEEP=`_j_getvar ${STORAGEFILE} "keep"`
+
+ if [ -z "$STORAGE_KEEP" ]; then
+ color error
+ echo ERROR: missing config for backup target.
+ echo There must be an entry keep in ${STORAGEFILE}
+ color reset
+ exit 1
+ fi
+
+ cmd="duplicity remove-older-than $STORAGE_KEEP --force ${sParams} ${sTarget}"
+ echo $cmd
+ color cmd
+ $cmd
+ fetchrc
+ color reset
+
+ t_rcCheckCleanup $myrc
+
+ }
+# --------------------------------------------------------------------------------
+# VERIFY RETURNCODES
+# --------------------------------------------------------------------------------
+
+ # init repository
+ function t_rcCheckInit(){
+ case $1 in
+ 0) color ok; echo "OK - the repository was created." ;;
+ *) color error; echo "Verify output above - returncode of init was $1" ;;
+ esac
+ color reset
+ }
+ # backup files
+ function t_rcCheckBackup(){
+ case $1 in
+ 0) color ok; echo "OK" ;;
+ 23) color error
+ echo A lock file was found. Maybe this server was rebooted while performing a backup.
+ echo If so delete the file lockfile.lock named in the output and start $0 again.
+ ;;
+ 31) color error
+ echo Maybe you it is a problem with the gpg-agent.conf
+ ls -l ~/.gnupg/gpg-agent.conf && cat ~/.gnupg/gpg-agent.conf
+ ;;
+ *) color error; echo "Backup error - returncode was $1" ;;
+ esac
+ color reset
+ }
+
+ # repoitory cleanup
+ function t_rcCheckCleanup(){
+ case $1 in
+ 0) color ok; echo "OK" ;;
+ *) color error; echo "Cleanup error - returncode was $1" ;;
+ esac
+ color reset
+ }
+
+ # restore files
+ function t_rcCheckRestore(){
+ case $1 in
+ 0) color ok; echo "OK" ;;
+ *) color error; echo "Restore error - returncode was $1" ;;
+ esac
+ color reset
+ }
+
+# --------------------------------------------------------------------------------
diff --git a/plugins/transfer/restic.sh b/plugins/transfer/restic.sh
new file mode 100644
index 0000000..1c7d8fe
--- /dev/null
+++ b/plugins/transfer/restic.sh
@@ -0,0 +1,240 @@
+#!/bin/bash
+# ================================================================================
+#
+# TRANSFER :: PLUGIN - TEMPLATE
+#
+# this script will be included in ../../transfer.sh
+#
+# --------------------------------------------------------------------------------
+# ah - Axel Hahn <axel.hahn@iml.unibe.ch>
+# 2021-05-19 ah v0.0 INIT ... WIP
+# ================================================================================
+
+# --------------------------------------------------------------------------------
+# ENV VARS
+# export variables that are needed by the backup tool
+# --------------------------------------------------------------------------------
+
+# export PASSPHRASE
+
+# --------------------------------------------------------------------------------
+# INIT
+# --------------------------------------------------------------------------------
+
+
+ function t_checkRequirements(){
+ j_requireUser "root"
+ j_requireBinary "restic"
+ }
+
+ function t_setVars(){
+ export RESTIC_PASSWORD=$PASSPHRASE
+
+ # if we set RESTIC_REPOSITORY then "-r TARGET" is not
+ # needed in restic commands
+ export RESTIC_REPOSITORY=$( j_getFullTarget )
+
+ # WORKAROUND for bug while writing on a SMB target
+ export GODEBUG="asyncpreemptoff=1"
+ }
+
+# --------------------------------------------------------------------------------
+# GENERATE PARAMS :: ALL DIRS
+# --------------------------------------------------------------------------------
+
+ # return a string with default params
+ # param string param1 of transfer.sh; one of full|inc|auto
+ # param string param2 of transfer.sh; for auto: date i.e. 3M for 3 monthes
+ function t_getParamDefault(){
+ # verbose to see more details
+ echo -n --verbose=$( _j_getvar ${STORAGEFILE} "restic_verbose")
+
+ }
+ # return a string with backup default params
+ function t_getParamBackup(){
+ # tagging
+ echo -n --tag $( _j_getvar ${STORAGEFILE} "restic_tag")
+ }
+
+ # return a cli parameter for a single exlude directory
+ # param string cache directory for local index files
+ function t_getParamCacheDir(){
+ if [ ! -z "$1" ]; then
+ local sCacheDir="$1"
+ if [ ! -d $sCacheDir ]; then
+ mkdir -p $sCacheDir
+ chmod 750 $sCacheDir
+ fi
+ echo --cache-dir $sCacheDir
+ fi
+ }
+
+ # return a cli parameter for a single exlude directory
+ # param string exlude pattern
+ function t_getParamExlude(){
+ test -z "$1" || echo --exclude "'"$*"'"
+ }
+ # return a cli parameter for a single exlude directory
+ # param string exlude pattern
+ function t_getParamInlude(){
+ test -z "$1" || echo --include "'"$*"'"
+ }
+
+ # return a cli parameter to use an ssh keyfile
+ # param string filename if ssh private key file
+ function t_getParamSshKey(){
+ if [ ! -z "$1" ]; then
+ # scp://backup@storage.example.com//netshare/backup/one
+ # ^^^^^^^^^^^^^^^^^^^^^^^^^^
+ # sshtarget fetches user@host ... if protocol matches
+ echo $STORAGE_BASEDIR | grep -E "^(scp|sftp|rsync):" >/dev/null
+ if [ $? -eq 0 ]; then
+ local sshtarget=$( echo $STORAGE_BASEDIR | cut -f 3 -d '/' )
+ echo -o sftp.command="'"ssh -i ${1} ${sshtarget} -s sftp"'"
+ # echo -n "-o sftp.command=''"
+ fi
+ fi
+ }
+
+# --------------------------------------------------------------------------------
+# ACTIONS :: TRANSFER
+# --------------------------------------------------------------------------------
+ # pre backup actions
+ # uses global vars from ../../transfer.sh
+ function t_cmdPre(){
+
+ local _mycmd="restic init ${sParams}"
+ echo $_mycmd
+ color cmd
+ eval $_mycmd
+ local _myrc=$?
+ color reset
+
+ # detect return code ... do not abort on any error.
+ t_rcCheckInit $_myrc
+
+ }
+
+
+ # post backup actions
+ # uses global vars from ../../transfer.sh
+ function t_cmdPost(){
+
+ echo "--- CLEANUP local data:"
+ echo restic cache --cleanup
+ color cmd
+ restic cache --cleanup
+ local _myrc=$?
+ color reset
+ case $_myrc in
+ 0) color ok; echo "OK" ;;
+ *) color error; echo "Cleanup error - returncode was $_myrc" ;;
+ esac
+ color reset
+ echo
+
+ echo "--- UNLOCK ... just in case :-)"
+ echo restic unlock ${sParams}
+ eval restic unlock ${sParams}
+ echo
+
+ echo "--- PRUNE"
+ local _tag=$( _j_getvar ${STORAGEFILE} "restic_tag")
+
+ local _keep_h=$( _j_getvar ${STORAGEFILE} "restic_keep-hourly")
+ local _keep_d=$( _j_getvar ${STORAGEFILE} "restic_keep-daily")
+ local _keep_w=$( _j_getvar ${STORAGEFILE} "restic_keep-weekly")
+ local _keep_m=$( _j_getvar ${STORAGEFILE} "restic_keep-monthly")
+ local _keep_y=$( _j_getvar ${STORAGEFILE} "restic_keep-yearly")
+
+ local _mycmd="restic forget \
+ ${sParams} \
+ --tag $_tag \
+ --group-by "paths,tags" \
+ --keep-hourly $_keep_h \
+ --keep-daily $_keep_d \
+ --keep-weekly $_keep_w \
+ --keep-monthly $_keep_m \
+ --keep-yearly $_keep_y"
+ echo $_mycmd
+ color cmd
+ eval $_mycmd
+ local _myrc=$?
+ color reset
+
+ t_rcCheckCleanup $_myrc
+ echo
+ }
+
+# --------------------------------------------------------------------------------
+# ACTIONS :: SINGLE DIR
+# --------------------------------------------------------------------------------
+
+ # get target url/ directory
+ # param string directory to backup
+ function t_sd_getTarget(){
+ # directory based target
+ # j_getFullTarget "$1"
+ # for host based backup target - remove param:
+ j_getFullTarget ""
+ }
+
+ function t_sd_getCmdBackup(){
+ echo eval restic backup ${sBackupParams} ${mydir}
+ }
+
+ # pre backup actions
+ # uses global vars from ../../transfer.sh
+ function t_sd_cmdPre(){
+ echo "Nothing here."
+ }
+
+
+ # post backup actions
+ # uses global vars from ../../transfer.sh
+ function t_sd_cmdPost(){
+ echo "Nothing here."
+ }
+
+# --------------------------------------------------------------------------------
+# VERIFY RETURNCODES
+# --------------------------------------------------------------------------------
+
+ # init repository
+ function t_rcCheckInit(){
+ case $1 in
+ 0) color ok; echo "OK - the repository was created." ;;
+ 1) color warning; echo "You can ignore the error if the repository was initialized already." ;;
+ *) color error; echo "Verify output above - returncode of restic init was $1" ;;
+ esac
+ color reset
+ }
+ # backup files
+ function t_rcCheckBackup(){
+ case $1 in
+ 0) color ok; echo "OK" ;;
+ 1) color error; echo "Unable to connect with restic repository." ;;
+ *) color error; echo "Backup error - returncode was $1" ;;
+ esac
+ color reset
+ }
+
+ # repoitory cleanup
+ function t_rcCheckCleanup(){
+ case $1 in
+ 0) color ok; echo "OK" ;;
+ *) color error; echo "Cleanup error - returncode was $1" ;;
+ esac
+ color reset
+ }
+
+ # restore files
+ function t_rcCheckRestore(){
+ case $1 in
+ 0) color ok; echo "OK" ;;
+ *) color error; echo "Restore error - returncode was $1" ;;
+ esac
+ color reset
+ }
+
+# --------------------------------------------------------------------------------
diff --git a/transfer.sh b/transfer.sh
index 132aaaa..48e8cbc 100755
--- a/transfer.sh
+++ b/transfer.sh
@@ -32,6 +32,7 @@
# 2020-01-21 ah,ds v1.10 show colored OK or FAILED at the end of output
# 2020-02-25 ah,ds, v1.11 fix test -z with non existing vars; show final backup status
# 2021-01-29 ah,ds, v1.12 abort on empty passphrase
+# 2021-05-19 ah,ds, v2.0 plugin driven
# ================================================================================
@@ -46,12 +47,17 @@
typeset -i rc=0
+ STORAGE_BIN=`_j_getvar ${STORAGEFILE} "bin"`
STORAGE_BASEDIR=`_j_getvar ${STORAGEFILE} "storage"`
STORAGE_TESTFILE=`_j_getvar ${STORAGEFILE} "storage-file"`
- STORAGE_KEEP=`_j_getvar ${STORAGEFILE} "keep"`
- STORAGE_VOLSIZE=`_j_getvar ${STORAGEFILE} "volsize"`
+ PASSPHRASE=`_j_getvar ${STORAGEFILE} "passphrase"`
# check
+ if [ -z "$STORAGE_BIN" ]; then
+ STORAGE_BIN=restic
+ # STORAGE_BIN=duplicity
+ fi
+
if [ -z "$STORAGE_BASEDIR" ]; then
color error
echo ERROR: missing config for backup target.
@@ -66,15 +72,18 @@
color reset
exit 1
fi
-
- if [ -z "$STORAGE_KEEP" ]; then
- color error
- echo ERROR: missing config for backup target.
- echo There must be an entry keep in ${STORAGEFILE}
- color reset
- exit 1
+ # support old value
+ if [ -z "${PASSPHRASE}" ]; then
+ echo "WARNING: The value gnupg-passphrase in ${STORAGEFILE} is deprecated. Replace it with passphrase=..."
+ PASSPHRASE=`_j_getvar ${STORAGEFILE} "gnupg-passphrase"`
fi
+ if [ -z "${PASSPHRASE}" ]; then
+ echo "ERROR: no value passphrase was set in ${STORAGEFILE} to encrypt backup data."
+ echo "Aborting."
+ exit 1
+ fi
+
# METHOD incremental is default; full backup will be triggered with
# first param "full"
METHOD=
@@ -99,12 +108,16 @@
echo METHOD: $METHOD | tee -a $transferlog
echo TARGET: ${STORAGE_BASEDIR} | tee -a $transferlog
+ echo TOOL : $STORAGE_BIN | tee -a $transferlog
echo | tee -a $transferlog
+ . `dirname $0`/plugins/transfer/$STORAGE_BIN.sh || exit 1
+
+
+# --------------------------------------------------------------------------------
# ----- Check requirements
- j_requireUser "root"
- j_requireBinary "duplicity"
+ t_checkRequirements || exit 1
echo Check locking of a running transfer
if [ -f "${lockfile}" ]; then
@@ -148,10 +161,13 @@
exit 2
fi
+# --------------------------------------------------------------------------------
# ----- BACKUP VARS
# parameters for all
- sParams=
+ t_setVars || exit 1
+ sParams="$( t_getParamDefault $1 $2 )"
+
if [ "$1" = "dumps" ]; then
sDirs2Backup="`_j_getvar ${DIRFILE} dir-localdumps`"
@@ -159,31 +175,18 @@
sDirs2Backup="`j_getDirs2Backup`"
fi
- if [ "$1" = "full" ]; then
- METHOD="full"
- fi
- if [ "$1" = "auto" ]; then
- METHOD="--full-if-older-than $2"
- fi
-
- PASSPHRASE=`_j_getvar ${STORAGEFILE} "gnupg-passphrase"`
- if [ -z "${PASSPHRASE}" ]; then
- echo "ERROR: no value gnupg-passphrase was set in ${STORAGEFILE} to encrypt backup data."
- echo "Aborting."
- exit 1
- fi
sParamExclude=
- # build exclude param list for duplicity
- #
- # REMARK: the excludes must fit the include definition
- # On non matching excludes duplicity stops
- #
for sItem in `_j_getvar ${DIRFILE} exclude`
do
- sParamExclude="$sParamExclude --exclude-regexp """$sItem""" "
+ sParamExclude="$sParamExclude $( t_getParamExlude $sItem )"
done
+ echo DEBUG: sParamExclude = $sParamExclude
+
+
+
+
#
# sExcludefile="${DIR_JOBS}/transfer-exclude.txt"
#
@@ -191,37 +194,33 @@
# sParamExclude="$sParamExclude --exclude-filelist """${sExcludefile}""" "
# fi
- export PASSPHRASE
-
- # task#1623 - fallback ssh backend for Debian 8
- sSshBackend=`_j_getvar ${STORAGEFILE} "ssh-backend"`
- if [ ! -z $sSshBackend ]; then
- sParams="${sParams} --ssh-backend $sSshBackend"
- fi
+
sFileSshPrivkey=`_j_getvar ${STORAGEFILE} "ssh-privatekey"`
if [ ! -z $sFileSshPrivkey ]; then
- sParams="${sParams} --ssh-options="""-oIdentityFile=${sFileSshPrivkey}""" "
+ sParams="${sParams} $( t_getParamSshKey $sFileSshPrivkey )"
fi
+
# task#3046 - add custom cache dir
sCacheDir=`_j_getvar ${STORAGEFILE} "cachedir"`
if [ ! -z $sCacheDir ]; then
- sParams="${sParams} --archive-dir=$sCacheDir"
- if [ ! -d $sCacheDir ]; then
- mkdir -p $sCacheDir
- chmod 750 $sCacheDir
- fi
+ sParams="${sParams} $( t_getParamCacheDir $sCacheDir )"
fi
-
-# ----- Wait for a free slot
+# --------------------------------------------------------------------------------
+# ----- PRE transfer
+
+ h2 "`date` Wait for a free slot"
j_transferStart | tee -a $transferlog
+ h2 "`date` PRE transfer tasks"
+ t_cmdPre
-# ----- START BACKUPS
+# --------------------------------------------------------------------------------
+# ----- START BACKUPS
(
for mydir in $sDirs2Backup
do
@@ -233,82 +232,55 @@
h2 "`date` STORE $mydir"
- # --- build parameters for duplicity
+ # --- build parameters
sSafeName=`j_getSafename "$mydir"`
- sTarget=`j_getFullTarget "$mydir"`
-
- sBackupParams=
- sBackupParams="${sBackupParams} ${sParams} "
- sBackupParams="${sBackupParams} ${sParamExclude} "
- if [ ! -z $STORAGE_VOLSIZE ]; then
- sBackupParams="${sBackupParams} --volsize ${STORAGE_VOLSIZE} "
- fi
+ sTarget="$( t_sd_getTarget $mydir )"
- # verbosity level to fetch changed files from log
- sBackupParams="${sBackupParams} -v8"
+ sBackupParams="${sParams} ${sParamExclude} $( t_getParamBackup )"
- # add asynchronous upload
- sBackupParams="${sBackupParams} --asynchronous-upload"
- # ---------- START --------------------
# detect custom backup sets and add its includes and excludes
backupid=`j_getSetnameOfPath "$mydir"`
sSpaceReplace="___SPACE___"
+
if [ ! -z $backupid ]; then
- for param in include exclude
+ for sItem in `_j_getvar ${DIRFILE} "${backupid}\-\-include" | sed "s#\ #${sSpaceReplace}#g"`
do
- for sItem in `_j_getvar ${DIRFILE} "${backupid}\-\-${param}" | sed "s#\ #${sSpaceReplace}#g"`
- do
- sBackupParams="${sBackupParams} --${param}-regexp """$sItem""" "
- done
+ sBackupParams="${sBackupParams} $( t_getParamInlude $sItem)"
+ done
+ for sItem in `_j_getvar ${DIRFILE} "${backupid}\-\-exclude" | sed "s#\ #${sSpaceReplace}#g"`
+ do
+ sBackupParams="${sBackupParams} $( t_getParamExlude $sItem)"
done
- sBackupParams=`echo ${sBackupParams} | sed "s#${sSpaceReplace}# #g"`
-
- fi
- # ---------- ENDE --------------------
-
- # --- for rsync only: create remote directory
- echo ${sTarget} | fgrep "rsync://" >/dev/null
- if [ $? -eq 0 ]; then
- # sshTarget=`echo ${sTarget} | sed "s#rsync://#scp://#"`
- # echo Creating remote directory with fetching collection-status on $sshTarget
- # color cmd
- # duplicity collection-status ${sParams} ${sshTarget}
- # color reset
- sshTarget=`echo ${sTarget} | cut -f 3 -d '/'`
- RemoteDir=`echo ${sTarget} | cut -f 4- -d '/'`
- cmd="ssh"
- if [ ! -z ${sFileSshPrivkey} ]; then
- cmd="${cmd} -i ${sFileSshPrivkey}"
- fi
- cmd="${cmd} ${sshTarget} mkdir -p ${RemoteDir} 2>/dev/null ; ls -ld ${RemoteDir} "
- echo Creating remote directory first ...
- color cmd
- $cmd
- color reset
fi
-
+
+
+ # --- pre task
+ h3 "`date` PRE backup task for ${mydir}"
+ t_sd_cmdPre
+ # sCmdPre="$( t_sd_cmdPre )"
+
+
# --- backup
- echo backup to target: ${sTarget}
- echo duplicity ${METHOD} ${sBackupParams} ${mydir} ${sTarget}
+ h3 "`date` Backup ${mydir}"
+
+ sCmd="$( t_sd_getCmdBackup)"
+ echo "what: ${mydir}"
+ echo "target: ${sTarget}"
+ echo "command: $sCmd"
+ echo
color cmd
- duplicity ${METHOD} ${sBackupParams} ${mydir} ${sTarget}
+ $sCmd
fetchrc
color reset
echo
+ t_rcCheckBackup $myrc
+
if [ $myrc -ne 0 ]; then
color error
echo DIR ERROR ${mydir} rc=$myrc during file transfer
- case $myrc in
- 23) echo A lock file was found. Maybe this server was rebooted while performing a backup.
- echo If so delete the file lockfile.lock named in the output and start $0 again.
- ;;
- 31) echo Maybe you it is a problem with the gpg-agent.conf
- ls -l ~/.gnupg/gpg-agent.conf && cat ~/.gnupg/gpg-agent.conf
- ;;
- esac
else
color ok
@@ -317,19 +289,15 @@
color reset
echo
- # --- cleanup on remote target
- h2 "`date` Cleanup old backup data"
- echo duplicity remove-older-than $STORAGE_KEEP --force ${sParams} ${sTarget}
- color cmd
- duplicity remove-older-than $STORAGE_KEEP --force ${sParams} ${sTarget}
- fetchrc
- color reset
+ # --- post action
+ h3 "`date` POST backup task for ${mydir}"
+ t_sd_cmdPost
echo
echo
else
- color error
+ color warning
echo "DIR SKIP $mydir ... does not exist (no error)"
color reset
fi
@@ -344,7 +312,17 @@
# rc=${PIPESTATUS[0]}
rc=`cat ${rcfile}`
+# --------------------------------------------------------------------------------
+# --- transfer POST tasks
+
+ h2 "`date` POST transfer tasks"
+ t_cmdPost
+
+
rm -f "${lockfile}" "${rcfile}"
+ echo "Local lock file was removed."
+
+
j_transferEnd
echo STATUS $0 exit with final returncode rc=$rc | tee -a $transferlog
--
GitLab