Skip to content
Snippets Groups Projects
Commit a704e623 authored by Hahn Axel (hahn)'s avatar Hahn Axel (hahn)
Browse files

transfer with multiple backend tools

parent ee7359e7
No related branches found
No related tags found
1 merge request!1Version 2
Hide whitespace changes
Inline Side-by-side
inc_bash.sh
+ 13
− 2
View file @ a704e623
...@@ -34,6 +34,8 @@ function fetchrc(){ ...@@ -34,6 +34,8 @@ function fetchrc(){
;; ;;
"ok") sColorcode="92" # green "ok") sColorcode="92" # green
;; ;;
"warning") sColorcode="33" # yellow
;;
"error") sColorcode="91" # red "error") sColorcode="91" # red
;; ;;
esac esac
...@@ -49,7 +51,8 @@ function fetchrc(){ ...@@ -49,7 +51,8 @@ function fetchrc(){
function h1(){ function h1(){
color head color head
echo echo
echo "====== $* ======" echo
echo "########## $* ##########"
echo echo
color reset color reset
} }
...@@ -57,7 +60,15 @@ function fetchrc(){ ...@@ -57,7 +60,15 @@ function fetchrc(){
function h2(){ function h2(){
color head color head
echo "--- $*" echo
echo "========== $* =========="
echo
color reset
}
function h3(){
color head
echo "----- $*"
color reset color reset
} }
......
jobhelper.sh
+ 1
− 1
View file @ a704e623
...@@ -425,7 +425,7 @@ function _j_storagehelper(){ ...@@ -425,7 +425,7 @@ function _j_storagehelper(){
# rsync://[backupuser]@[storage]//[targetdir]/ # rsync://[backupuser]@[storage]//[targetdir]/
sProtocol=`j_getFullTarget "" | cut -f 1 -d ":"` sProtocol=`j_getFullTarget "" | cut -f 1 -d ":"`
if [ ${sProtocol} = "rsync" -o ${sProtocol} = "scp" ]; then if [ ${sProtocol} = "rsync" -o ${sProtocol} = "scp" -o ${sProtocol} = "sftp" ]; then
sSshTarget=`j_getFullTarget "" | cut -f 3 -d "/"` sSshTarget=`j_getFullTarget "" | cut -f 3 -d "/"`
# echo target: $sSshTarget # echo target: $sSshTarget
......
plugins/transfer/_template.sh 0 → 100644
+ 167
− 0
View file @ a704e623
#!/bin/bash
# ================================================================================
#
# TRANSFER :: PLUGIN - TEMPLATE
#
# this script will be included in ../../transfer.sh
#
# --------------------------------------------------------------------------------
# ah - Axel Hahn <axel.hahn@iml.unibe.ch>
# 2021-05-19 ah v0.0 INIT ... WIP
# ================================================================================
# --------------------------------------------------------------------------------
# ENV VARS
# export variables that are needed by the backup tool
# --------------------------------------------------------------------------------
# export PASSPHRASE
# --------------------------------------------------------------------------------
# INIT
# --------------------------------------------------------------------------------
function t_checkRequirements(){
echo "DEBUG: function t_checkRequirements"
j_requireUser "root"
# j_requireBinary "duplicity"
}
# set variables
function t_setVars(){
echo "DEBUG: t_setVars"
}
# --------------------------------------------------------------------------------
# GENERATE PARAMS :: ALL DIRS
# --------------------------------------------------------------------------------
# return a string with default params
# param string param1 of transfer.sh; one of full|inc|auto
# param string param2 of transfer.sh; for auto: date i.e. 3M for 3 monthes
function t_getParamDefault(){
echo "DEBUG: function t_getParamDefault $*"
# local _method=
# --- method
# test "$1" = "full" && _method="full"
# test "$1" = "auto" && _method="--full-if-older-than $2"
# echo -n "$_method"
}
# return a cli parameter for a single exlude directory
# param string cache directory for local index files
function t_getParamCacheDir(){
if [ ! -z "$1" ]; then
local sCacheDir="$1"
if [ ! -d $sCacheDir ]; then
mkdir -p $sCacheDir
chmod 750 $sCacheDir
fi
echo --archive-dir=$sCacheDir
fi
}
# return a cli parameter for a single exlude directory
# param string exlude pattern
function t_getParamExlude(){
# test -z "$1" || echo --exclude-regexp """$*"""
}
# return a cli parameter for a single exlude directory
# param string exlude pattern
function t_getParamInlude(){
# test -z "$1" || echo --include-regexp """$*"""
}
# return a cli parameter to use an ssh keyfile
# param string filename if ssh private key file
function t_getParamSshKey(){
# test -z "$1" || echo --ssh-options="""-oIdentityFile=$1"""
}
# --------------------------------------------------------------------------------
# ACTIONS :: TRANSFER
# --------------------------------------------------------------------------------
# pre backup actions
# uses global vars from ../../transfer.sh
function t_cmdPre(){
# echo TODO PRE actions before starting transfer
}
# post backup actions
# uses global vars from ../../transfer.sh
function t_cmdPost(){
# echo TODO POST actions after all transfers
}
# --------------------------------------------------------------------------------
# GENERATE PARAMS :: SINGLE DIR
# --------------------------------------------------------------------------------
# get target url/ directory
# param string directory to backup
function t_sd_getTarget(){
# directory based target
j_getFullTarget "$1"
# for host based backup target - remove param:
# j_getFullTarget ""
}
function t_sd_getCmdBackup(){
# echo duplicity ${sBackupParams} ${mydir} ${sTarget}
}
# pre backup actions
# uses global vars from ../../transfer.sh
function t_sd_cmdPre(){
}
# post backup actions
# uses global vars from ../../transfer.sh
function t_sd_cmdPost(){
}
# --------------------------------------------------------------------------------
# VERIFY RETURNCODES
# --------------------------------------------------------------------------------
# init repository
function t_rcCheckInit(){
case $1 in
0) color ok; echo "OK - the repository was created." ;;
*) color error; echo "Verify output above - returncode of init was $1" ;;
esac
color reset
}
# backup files
function t_rcCheckBackup(){
case $1 in
0) color ok; echo "OK" ;;
*) color error; echo "Backup error - returncode was $1" ;;
esac
color reset
}
# repoitory cleanup
function t_rcCheckCleanup(){
case $1 in
0) color ok; echo "OK" ;;
*) color error; echo "Cleanup error - returncode was $1" ;;
esac
color reset
}
# restore files
function t_rcCheckRestore(){
case $1 in
0) color ok; echo "OK" ;;
*) color error; echo "Restore error - returncode was $1" ;;
esac
color reset
}
# --------------------------------------------------------------------------------
plugins/transfer/duplicity.sh 0 → 100644
+ 239
− 0
View file @ a704e623
#!/bin/bash
# ================================================================================
#
# TRANSFER :: PLUGIN - DUPLICITY
#
# this script will be included in ../../transfer.sh
#
# --------------------------------------------------------------------------------
# ah - Axel Hahn <axel.hahn@iml.unibe.ch>
# 2021-05-19 ah v0.0 INIT ... WIP
# ================================================================================
# --------------------------------------------------------------------------------
# ENV VARS
# export variables that are needed by the backup tool
# --------------------------------------------------------------------------------
export PASSPHRASE
# --------------------------------------------------------------------------------
# INIT
# --------------------------------------------------------------------------------
# check requirements
function t_checkRequirements(){
echo "DEBUG: function t_checkRequirements"
j_requireUser "root"
j_requireBinary "duplicity"
}
# set variables
function t_setVars(){
echo "DEBUG: t_setVars"
}
# --------------------------------------------------------------------------------
# GENERATE PARAMS :: ALL DIRS
# --------------------------------------------------------------------------------
# return a string with default params
# param string param1 of transfer.sh; one of full|inc|auto
# param string param2 of transfer.sh; for auto: date i.e. 3M for 3 monthes
function t_getParamDefault(){
local _method=
local _volsize=
# --- method
test "$1" = "full" && _method="full"
test "$1" = "auto" && _method="--full-if-older-than $2"
echo -n "$_method"
# --- backend
# task#1623 - fallback ssh backend for Debian 8
sSshBackend=`_j_getvar ${STORAGEFILE} "ssh-backend"`
if [ ! -z $sSshBackend ]; then
echo -n " --ssh-backend $sSshBackend"
fi
# --- volume size
_volsize=`_j_getvar ${STORAGEFILE} "duplicity_volsize"`
test -z "$_volsize" && _volsize=`_j_getvar ${STORAGEFILE} "volsize"`
if [ ! -z $_volsize ]; then
echo -n " --volsize ${_volsize}"
fi
# --- verbosity level to fetch changed files from log
echo -n " -v8"
# --- add asynchronous upload
echo -n " --asynchronous-upload"
}
# return a string with backup default params
function t_getParamBackup(){
echo -n ''
}
# return a cli parameter for a single exlude directory
# param string cache directory for local index files
function t_getParamCacheDir(){
if [ ! -z "$1" ]; then
local sCacheDir="$1"
if [ ! -d $sCacheDir ]; then
mkdir -p $sCacheDir
chmod 750 $sCacheDir
fi
echo --archive-dir=$sCacheDir
fi
}
# return a cli parameter for a single exlude directory
# param string exlude pattern
function t_getParamExlude(){
test -z "$1" || echo --exclude-regexp "'"$*"'"
}
# return a cli parameter for a single exlude directory
# param string exlude pattern
function t_getParamInlude(){
test -z "$1" || echo --include-regexp "'"$*"'"
}
# return a cli parameter to use an ssh keyfile
# param string filename if ssh private key file
function t_getParamSshKey(){
test -z "$1" || echo --ssh-options="'"-oIdentityFile=$1"'"
}
# --------------------------------------------------------------------------------
# ACTIONS :: TRANSFER
# --------------------------------------------------------------------------------
# pre backup actions
# uses global vars from ../../transfer.sh
function t_cmdPre(){
echo No PRE actions before starting transfer
}
# post backup actions
# uses global vars from ../../transfer.sh
function t_cmdPost(){
echo No POST actions after all transfers
}
# --------------------------------------------------------------------------------
# GENERATE PARAMS :: SINGLE DIR
# --------------------------------------------------------------------------------
# get target url/ directory
# param string directory to backup
function t_sd_getTarget(){
# duplicity has a directory based target
j_getFullTarget "$1"
}
# get string with complete backup command
function t_sd_getCmdBackup(){
echo duplicity ${sBackupParams} ${mydir} ${sTarget}
}
# pre backup tasks
# uses global vars from ../../transfer.sh
function t_sd_cmdPre(){
# --- for rsync only: create remote directory
echo ${sTarget} | fgrep "rsync://" >/dev/null
if [ $? -eq 0 ]; then
# sshTarget=`echo ${sTarget} | sed "s#rsync://#scp://#"`
# echo Creating remote directory with fetching collection-status on $sshTarget
# color cmd
# duplicity collection-status ${sParams} ${sshTarget}
# color reset
sshTarget=`echo ${sTarget} | cut -f 3 -d '/'`
RemoteDir=`echo ${sTarget} | cut -f 4- -d '/'`
cmd="ssh"
if [ ! -z ${sFileSshPrivkey} ]; then
cmd="${cmd} -i ${sFileSshPrivkey}"
fi
cmd="${cmd} ${sshTarget} mkdir -p ${RemoteDir} 2>/dev/null ; ls -ld ${RemoteDir} "
echo Creating remote directory first ...
color cmd
$cmd
color reset
fi
}
# post backup tasks
# uses global vars from ../../transfer.sh
function t_sd_cmdPost(){
local STORAGE_KEEP=`_j_getvar ${STORAGEFILE} "duplicity_keep"`
test -z "$STORAGE_KEEP" && STORAGE_KEEP=`_j_getvar ${STORAGEFILE} "keep"`
if [ -z "$STORAGE_KEEP" ]; then
color error
echo ERROR: missing config for backup target.
echo There must be an entry keep in ${STORAGEFILE}
color reset
exit 1
fi
cmd="duplicity remove-older-than $STORAGE_KEEP --force ${sParams} ${sTarget}"
echo $cmd
color cmd
$cmd
fetchrc
color reset
t_rcCheckCleanup $myrc
}
# --------------------------------------------------------------------------------
# VERIFY RETURNCODES
# --------------------------------------------------------------------------------
# init repository
function t_rcCheckInit(){
case $1 in
0) color ok; echo "OK - the repository was created." ;;
*) color error; echo "Verify output above - returncode of init was $1" ;;
esac
color reset
}
# backup files
function t_rcCheckBackup(){
case $1 in
0) color ok; echo "OK" ;;
23) color error
echo A lock file was found. Maybe this server was rebooted while performing a backup.
echo If so delete the file lockfile.lock named in the output and start $0 again.
;;
31) color error
echo Maybe you it is a problem with the gpg-agent.conf
ls -l ~/.gnupg/gpg-agent.conf && cat ~/.gnupg/gpg-agent.conf
;;
*) color error; echo "Backup error - returncode was $1" ;;
esac
color reset
}
# repoitory cleanup
function t_rcCheckCleanup(){
case $1 in
0) color ok; echo "OK" ;;
*) color error; echo "Cleanup error - returncode was $1" ;;
esac
color reset
}
# restore files
function t_rcCheckRestore(){
case $1 in
0) color ok; echo "OK" ;;
*) color error; echo "Restore error - returncode was $1" ;;
esac
color reset
}
# --------------------------------------------------------------------------------
plugins/transfer/restic.sh 0 → 100644
+ 240
− 0
View file @ a704e623
#!/bin/bash
# ================================================================================
#
# TRANSFER :: PLUGIN - TEMPLATE
#
# this script will be included in ../../transfer.sh
#
# --------------------------------------------------------------------------------
# ah - Axel Hahn <axel.hahn@iml.unibe.ch>
# 2021-05-19 ah v0.0 INIT ... WIP
# ================================================================================
# --------------------------------------------------------------------------------
# ENV VARS
# export variables that are needed by the backup tool
# --------------------------------------------------------------------------------
# export PASSPHRASE
# --------------------------------------------------------------------------------
# INIT
# --------------------------------------------------------------------------------
function t_checkRequirements(){
j_requireUser "root"
j_requireBinary "restic"
}
function t_setVars(){
export RESTIC_PASSWORD=$PASSPHRASE
# if we set RESTIC_REPOSITORY then "-r TARGET" is not
# needed in restic commands
export RESTIC_REPOSITORY=$( j_getFullTarget )
# WORKAROUND for bug while writing on a SMB target
export GODEBUG="asyncpreemptoff=1"
}
# --------------------------------------------------------------------------------
# GENERATE PARAMS :: ALL DIRS
# --------------------------------------------------------------------------------
# return a string with default params
# param string param1 of transfer.sh; one of full|inc|auto
# param string param2 of transfer.sh; for auto: date i.e. 3M for 3 monthes
function t_getParamDefault(){
# verbose to see more details
echo -n --verbose=$( _j_getvar ${STORAGEFILE} "restic_verbose")
}
# return a string with backup default params
function t_getParamBackup(){
# tagging
echo -n --tag $( _j_getvar ${STORAGEFILE} "restic_tag")
}
# return a cli parameter for a single exlude directory
# param string cache directory for local index files
function t_getParamCacheDir(){
if [ ! -z "$1" ]; then
local sCacheDir="$1"
if [ ! -d $sCacheDir ]; then
mkdir -p $sCacheDir
chmod 750 $sCacheDir
fi
echo --cache-dir $sCacheDir
fi
}
# return a cli parameter for a single exlude directory
# param string exlude pattern
function t_getParamExlude(){
test -z "$1" || echo --exclude "'"$*"'"
}
# return a cli parameter for a single exlude directory
# param string exlude pattern
function t_getParamInlude(){
test -z "$1" || echo --include "'"$*"'"
}
# return a cli parameter to use an ssh keyfile
# param string filename if ssh private key file
function t_getParamSshKey(){
if [ ! -z "$1" ]; then
# scp://backup@storage.example.com//netshare/backup/one
# ^^^^^^^^^^^^^^^^^^^^^^^^^^
# sshtarget fetches user@host ... if protocol matches
echo $STORAGE_BASEDIR | grep -E "^(scp|sftp|rsync):" >/dev/null
if [ $? -eq 0 ]; then
local sshtarget=$( echo $STORAGE_BASEDIR | cut -f 3 -d '/' )
echo -o sftp.command="'"ssh -i ${1} ${sshtarget} -s sftp"'"
# echo -n "-o sftp.command=''"
fi
fi
}
# --------------------------------------------------------------------------------
# ACTIONS :: TRANSFER
# --------------------------------------------------------------------------------
# pre backup actions
# uses global vars from ../../transfer.sh
function t_cmdPre(){
local _mycmd="restic init ${sParams}"
echo $_mycmd
color cmd
eval $_mycmd
local _myrc=$?
color reset
# detect return code ... do not abort on any error.
t_rcCheckInit $_myrc
}
# post backup actions
# uses global vars from ../../transfer.sh
function t_cmdPost(){
echo "--- CLEANUP local data:"
echo restic cache --cleanup
color cmd
restic cache --cleanup
local _myrc=$?
color reset
case $_myrc in
0) color ok; echo "OK" ;;
*) color error; echo "Cleanup error - returncode was $_myrc" ;;
esac
color reset
echo
echo "--- UNLOCK ... just in case :-)"
echo restic unlock ${sParams}
eval restic unlock ${sParams}
echo
echo "--- PRUNE"
local _tag=$( _j_getvar ${STORAGEFILE} "restic_tag")
local _keep_h=$( _j_getvar ${STORAGEFILE} "restic_keep-hourly")
local _keep_d=$( _j_getvar ${STORAGEFILE} "restic_keep-daily")
local _keep_w=$( _j_getvar ${STORAGEFILE} "restic_keep-weekly")
local _keep_m=$( _j_getvar ${STORAGEFILE} "restic_keep-monthly")
local _keep_y=$( _j_getvar ${STORAGEFILE} "restic_keep-yearly")
local _mycmd="restic forget \
${sParams} \
--tag $_tag \
--group-by "paths,tags" \
--keep-hourly $_keep_h \
--keep-daily $_keep_d \
--keep-weekly $_keep_w \
--keep-monthly $_keep_m \
--keep-yearly $_keep_y"
echo $_mycmd
color cmd
eval $_mycmd
local _myrc=$?
color reset
t_rcCheckCleanup $_myrc
echo
}
# --------------------------------------------------------------------------------
# ACTIONS :: SINGLE DIR
# --------------------------------------------------------------------------------
# get target url/ directory
# param string directory to backup
function t_sd_getTarget(){
# directory based target
# j_getFullTarget "$1"
# for host based backup target - remove param:
j_getFullTarget ""
}
function t_sd_getCmdBackup(){
echo eval restic backup ${sBackupParams} ${mydir}
}
# pre backup actions
# uses global vars from ../../transfer.sh
function t_sd_cmdPre(){
echo "Nothing here."
}
# post backup actions
# uses global vars from ../../transfer.sh
function t_sd_cmdPost(){
echo "Nothing here."
}
# --------------------------------------------------------------------------------
# VERIFY RETURNCODES
# --------------------------------------------------------------------------------
# init repository
function t_rcCheckInit(){
case $1 in
0) color ok; echo "OK - the repository was created." ;;
1) color warning; echo "You can ignore the error if the repository was initialized already." ;;
*) color error; echo "Verify output above - returncode of restic init was $1" ;;
esac
color reset
}
# backup files
function t_rcCheckBackup(){
case $1 in
0) color ok; echo "OK" ;;
1) color error; echo "Unable to connect with restic repository." ;;
*) color error; echo "Backup error - returncode was $1" ;;
esac
color reset
}
# repoitory cleanup
function t_rcCheckCleanup(){
case $1 in
0) color ok; echo "OK" ;;
*) color error; echo "Cleanup error - returncode was $1" ;;
esac
color reset
}
# restore files
function t_rcCheckRestore(){
case $1 in
0) color ok; echo "OK" ;;
*) color error; echo "Restore error - returncode was $1" ;;
esac
color reset
}
# --------------------------------------------------------------------------------
transfer.sh
+ 88
− 110
View file @ a704e623
...@@ -32,6 +32,7 @@ ...@@ -32,6 +32,7 @@
# 2020-01-21 ah,ds v1.10 show colored OK or FAILED at the end of output # 2020-01-21 ah,ds v1.10 show colored OK or FAILED at the end of output
# 2020-02-25 ah,ds, v1.11 fix test -z with non existing vars; show final backup status # 2020-02-25 ah,ds, v1.11 fix test -z with non existing vars; show final backup status
# 2021-01-29 ah,ds, v1.12 abort on empty passphrase # 2021-01-29 ah,ds, v1.12 abort on empty passphrase
# 2021-05-19 ah,ds, v2.0 plugin driven
# ================================================================================ # ================================================================================
...@@ -46,12 +47,17 @@ ...@@ -46,12 +47,17 @@
typeset -i rc=0 typeset -i rc=0
STORAGE_BIN=`_j_getvar ${STORAGEFILE} "bin"`
STORAGE_BASEDIR=`_j_getvar ${STORAGEFILE} "storage"` STORAGE_BASEDIR=`_j_getvar ${STORAGEFILE} "storage"`
STORAGE_TESTFILE=`_j_getvar ${STORAGEFILE} "storage-file"` STORAGE_TESTFILE=`_j_getvar ${STORAGEFILE} "storage-file"`
STORAGE_KEEP=`_j_getvar ${STORAGEFILE} "keep"` PASSPHRASE=`_j_getvar ${STORAGEFILE} "passphrase"`
STORAGE_VOLSIZE=`_j_getvar ${STORAGEFILE} "volsize"`
# check # check
if [ -z "$STORAGE_BIN" ]; then
STORAGE_BIN=restic
# STORAGE_BIN=duplicity
fi
if [ -z "$STORAGE_BASEDIR" ]; then if [ -z "$STORAGE_BASEDIR" ]; then
color error color error
echo ERROR: missing config for backup target. echo ERROR: missing config for backup target.
...@@ -66,15 +72,18 @@ ...@@ -66,15 +72,18 @@
color reset color reset
exit 1 exit 1
fi fi
# support old value
if [ -z "$STORAGE_KEEP" ]; then if [ -z "${PASSPHRASE}" ]; then
color error echo "WARNING: The value gnupg-passphrase in ${STORAGEFILE} is deprecated. Replace it with passphrase=..."
echo ERROR: missing config for backup target. PASSPHRASE=`_j_getvar ${STORAGEFILE} "gnupg-passphrase"`
echo There must be an entry keep in ${STORAGEFILE}
color reset
exit 1
fi fi
if [ -z "${PASSPHRASE}" ]; then
echo "ERROR: no value passphrase was set in ${STORAGEFILE} to encrypt backup data."
echo "Aborting."
exit 1
fi
# METHOD incremental is default; full backup will be triggered with # METHOD incremental is default; full backup will be triggered with
# first param "full" # first param "full"
METHOD= METHOD=
...@@ -99,12 +108,16 @@ ...@@ -99,12 +108,16 @@
echo METHOD: $METHOD | tee -a $transferlog echo METHOD: $METHOD | tee -a $transferlog
echo TARGET: ${STORAGE_BASEDIR} | tee -a $transferlog echo TARGET: ${STORAGE_BASEDIR} | tee -a $transferlog
echo TOOL : $STORAGE_BIN | tee -a $transferlog
echo | tee -a $transferlog echo | tee -a $transferlog
. `dirname $0`/plugins/transfer/$STORAGE_BIN.sh || exit 1
# --------------------------------------------------------------------------------
# ----- Check requirements # ----- Check requirements
j_requireUser "root" t_checkRequirements || exit 1
j_requireBinary "duplicity"
echo Check locking of a running transfer echo Check locking of a running transfer
if [ -f "${lockfile}" ]; then if [ -f "${lockfile}" ]; then
...@@ -148,10 +161,13 @@ ...@@ -148,10 +161,13 @@
exit 2 exit 2
fi fi
# --------------------------------------------------------------------------------
# ----- BACKUP VARS # ----- BACKUP VARS
# parameters for all # parameters for all
sParams= t_setVars || exit 1
sParams="$( t_getParamDefault $1 $2 )"
if [ "$1" = "dumps" ]; then if [ "$1" = "dumps" ]; then
sDirs2Backup="`_j_getvar ${DIRFILE} dir-localdumps`" sDirs2Backup="`_j_getvar ${DIRFILE} dir-localdumps`"
...@@ -159,31 +175,18 @@ ...@@ -159,31 +175,18 @@
sDirs2Backup="`j_getDirs2Backup`" sDirs2Backup="`j_getDirs2Backup`"
fi fi
if [ "$1" = "full" ]; then
METHOD="full"
fi
if [ "$1" = "auto" ]; then
METHOD="--full-if-older-than $2"
fi
PASSPHRASE=`_j_getvar ${STORAGEFILE} "gnupg-passphrase"`
if [ -z "${PASSPHRASE}" ]; then
echo "ERROR: no value gnupg-passphrase was set in ${STORAGEFILE} to encrypt backup data."
echo "Aborting."
exit 1
fi
sParamExclude= sParamExclude=
# build exclude param list for duplicity
#
# REMARK: the excludes must fit the include definition
# On non matching excludes duplicity stops
#
for sItem in `_j_getvar ${DIRFILE} exclude` for sItem in `_j_getvar ${DIRFILE} exclude`
do do
sParamExclude="$sParamExclude --exclude-regexp """$sItem""" " sParamExclude="$sParamExclude $( t_getParamExlude $sItem )"
done done
echo DEBUG: sParamExclude = $sParamExclude
# #
# sExcludefile="${DIR_JOBS}/transfer-exclude.txt" # sExcludefile="${DIR_JOBS}/transfer-exclude.txt"
# #
...@@ -191,37 +194,33 @@ ...@@ -191,37 +194,33 @@
# sParamExclude="$sParamExclude --exclude-filelist """${sExcludefile}""" " # sParamExclude="$sParamExclude --exclude-filelist """${sExcludefile}""" "
# fi # fi
export PASSPHRASE
# task#1623 - fallback ssh backend for Debian 8
sSshBackend=`_j_getvar ${STORAGEFILE} "ssh-backend"`
if [ ! -z $sSshBackend ]; then
sParams="${sParams} --ssh-backend $sSshBackend"
fi
sFileSshPrivkey=`_j_getvar ${STORAGEFILE} "ssh-privatekey"` sFileSshPrivkey=`_j_getvar ${STORAGEFILE} "ssh-privatekey"`
if [ ! -z $sFileSshPrivkey ]; then if [ ! -z $sFileSshPrivkey ]; then
sParams="${sParams} --ssh-options="""-oIdentityFile=${sFileSshPrivkey}""" " sParams="${sParams} $( t_getParamSshKey $sFileSshPrivkey )"
fi fi
# task#3046 - add custom cache dir # task#3046 - add custom cache dir
sCacheDir=`_j_getvar ${STORAGEFILE} "cachedir"` sCacheDir=`_j_getvar ${STORAGEFILE} "cachedir"`
if [ ! -z $sCacheDir ]; then if [ ! -z $sCacheDir ]; then
sParams="${sParams} --archive-dir=$sCacheDir" sParams="${sParams} $( t_getParamCacheDir $sCacheDir )"
if [ ! -d $sCacheDir ]; then
mkdir -p $sCacheDir
chmod 750 $sCacheDir
fi
fi fi
# ----- Wait for a free slot
# --------------------------------------------------------------------------------
# ----- PRE transfer
h2 "`date` Wait for a free slot"
j_transferStart | tee -a $transferlog j_transferStart | tee -a $transferlog
h2 "`date` PRE transfer tasks"
t_cmdPre
# ----- START BACKUPS
# --------------------------------------------------------------------------------
# ----- START BACKUPS
( (
for mydir in $sDirs2Backup for mydir in $sDirs2Backup
do do
...@@ -233,82 +232,55 @@ ...@@ -233,82 +232,55 @@
h2 "`date` STORE $mydir" h2 "`date` STORE $mydir"
# --- build parameters for duplicity # --- build parameters
sSafeName=`j_getSafename "$mydir"` sSafeName=`j_getSafename "$mydir"`
sTarget=`j_getFullTarget "$mydir"` sTarget="$( t_sd_getTarget $mydir )"
sBackupParams=
sBackupParams="${sBackupParams} ${sParams} "
sBackupParams="${sBackupParams} ${sParamExclude} "
if [ ! -z $STORAGE_VOLSIZE ]; then
sBackupParams="${sBackupParams} --volsize ${STORAGE_VOLSIZE} "
fi
# verbosity level to fetch changed files from log sBackupParams="${sParams} ${sParamExclude} $( t_getParamBackup )"
sBackupParams="${sBackupParams} -v8"
# add asynchronous upload
sBackupParams="${sBackupParams} --asynchronous-upload"
# ---------- START --------------------
# detect custom backup sets and add its includes and excludes # detect custom backup sets and add its includes and excludes
backupid=`j_getSetnameOfPath "$mydir"` backupid=`j_getSetnameOfPath "$mydir"`
sSpaceReplace="___SPACE___" sSpaceReplace="___SPACE___"
if [ ! -z $backupid ]; then if [ ! -z $backupid ]; then
for param in include exclude for sItem in `_j_getvar ${DIRFILE} "${backupid}\-\-include" | sed "s#\ #${sSpaceReplace}#g"`
do do
for sItem in `_j_getvar ${DIRFILE} "${backupid}\-\-${param}" | sed "s#\ #${sSpaceReplace}#g"` sBackupParams="${sBackupParams} $( t_getParamInlude $sItem)"
do done
sBackupParams="${sBackupParams} --${param}-regexp """$sItem""" " for sItem in `_j_getvar ${DIRFILE} "${backupid}\-\-exclude" | sed "s#\ #${sSpaceReplace}#g"`
done do
sBackupParams="${sBackupParams} $( t_getParamExlude $sItem)"
done done
sBackupParams=`echo ${sBackupParams} | sed "s#${sSpaceReplace}# #g"`
fi
# ---------- ENDE --------------------
# --- for rsync only: create remote directory
echo ${sTarget} | fgrep "rsync://" >/dev/null
if [ $? -eq 0 ]; then
# sshTarget=`echo ${sTarget} | sed "s#rsync://#scp://#"`
# echo Creating remote directory with fetching collection-status on $sshTarget
# color cmd
# duplicity collection-status ${sParams} ${sshTarget}
# color reset
sshTarget=`echo ${sTarget} | cut -f 3 -d '/'`
RemoteDir=`echo ${sTarget} | cut -f 4- -d '/'`
cmd="ssh"
if [ ! -z ${sFileSshPrivkey} ]; then
cmd="${cmd} -i ${sFileSshPrivkey}"
fi
cmd="${cmd} ${sshTarget} mkdir -p ${RemoteDir} 2>/dev/null ; ls -ld ${RemoteDir} "
echo Creating remote directory first ...
color cmd
$cmd
color reset
fi fi
# --- pre task
h3 "`date` PRE backup task for ${mydir}"
t_sd_cmdPre
# sCmdPre="$( t_sd_cmdPre )"
# --- backup # --- backup
echo backup to target: ${sTarget} h3 "`date` Backup ${mydir}"
echo duplicity ${METHOD} ${sBackupParams} ${mydir} ${sTarget}
sCmd="$( t_sd_getCmdBackup)"
echo "what: ${mydir}"
echo "target: ${sTarget}"
echo "command: $sCmd"
echo
color cmd color cmd
duplicity ${METHOD} ${sBackupParams} ${mydir} ${sTarget} $sCmd
fetchrc fetchrc
color reset color reset
echo echo
t_rcCheckBackup $myrc
if [ $myrc -ne 0 ]; then if [ $myrc -ne 0 ]; then
color error color error
echo DIR ERROR ${mydir} rc=$myrc during file transfer echo DIR ERROR ${mydir} rc=$myrc during file transfer
case $myrc in
23) echo A lock file was found. Maybe this server was rebooted while performing a backup.
echo If so delete the file lockfile.lock named in the output and start $0 again.
;;
31) echo Maybe you it is a problem with the gpg-agent.conf
ls -l ~/.gnupg/gpg-agent.conf && cat ~/.gnupg/gpg-agent.conf
;;
esac
else else
color ok color ok
...@@ -317,19 +289,15 @@ ...@@ -317,19 +289,15 @@
color reset color reset
echo echo
# --- cleanup on remote target
h2 "`date` Cleanup old backup data"
echo duplicity remove-older-than $STORAGE_KEEP --force ${sParams} ${sTarget}
color cmd
duplicity remove-older-than $STORAGE_KEEP --force ${sParams} ${sTarget}
fetchrc
color reset
# --- post action
h3 "`date` POST backup task for ${mydir}"
t_sd_cmdPost
echo echo
echo echo
else else
color error color warning
echo "DIR SKIP $mydir ... does not exist (no error)" echo "DIR SKIP $mydir ... does not exist (no error)"
color reset color reset
fi fi
...@@ -344,7 +312,17 @@ ...@@ -344,7 +312,17 @@
# rc=${PIPESTATUS[0]} # rc=${PIPESTATUS[0]}
rc=`cat ${rcfile}` rc=`cat ${rcfile}`
# --------------------------------------------------------------------------------
# --- transfer POST tasks
h2 "`date` POST transfer tasks"
t_cmdPost
rm -f "${lockfile}" "${rcfile}" rm -f "${lockfile}" "${rcfile}"
echo "Local lock file was removed."
j_transferEnd j_transferEnd
echo STATUS $0 exit with final returncode rc=$rc | tee -a $transferlog echo STATUS $0 exit with final returncode rc=$rc | tee -a $transferlog
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment