Skip to content
Snippets Groups Projects
Commit 484384e1 authored by Hahn Axel (hahn)'s avatar Hahn Axel (hahn)
Browse files

Merge branch '5814-migrate-repo-to-v2' into 'master' 

5814 migrate repo to v2

See merge request !87
parents eab77b6f d9305959
No related branches found
No related tags found
1 merge request!875814 migrate repo to v2
Show whitespace changes
Inline Side-by-side
plugins/transfer/restic.sh
+ 32
− 41
View file @ 484384e1
......@@ -14,6 +14,7 @@
# 2022-03-07 ah v0.4 add verify in post task
# 2022-05-10 ah v0.5 fix handling with nocache flag (use globally as default param - not in backup only)
# 2022-05-16 ah v0.6 added restic prune
# 2022-10-21 ah v0.7 simplify restic exec commands in _restic; remove --prune in check
# ================================================================================
# --------------------------------------------------------------------------------
......@@ -119,6 +120,19 @@
fi
}
# execute a restic command
# param string options and subcommand
function _restic(){
local _mycmd="restic $* ${ARGS_DEFAULT}"
echo "$_mycmd"
sleep 3
color cmd
eval "$_mycmd"
local _myrc=$?
color reset
return $_myrc
}
# --------------------------------------------------------------------------------
# BACKUP ACTIONS :: TRANSFER
# --------------------------------------------------------------------------------
......@@ -127,30 +141,22 @@
# function t_backupDoPreTasks(){
function t_backupDoPreTasks(){
if eval restic snapshots ${ARGS_DEFAULT} >/dev/null 2>&1
# if eval restic snapshots ${ARGS_DEFAULT} >/dev/null 2>&1
if _restic list keys >/dev/null 2>&1
then
echo "__REPO__ OK, Backup repository already exists."
echo "--- UNLOCK ... just in case :-)"
eval restic unlock ${ARGS_DEFAULT}
color reset
_restic unlock
echo
else
echo "Backup repository needs to be created."
local _mycmd="restic init ${ARGS_DEFAULT}"
echo $_mycmd
color cmd
eval $_mycmd
_restic init
local _myrc=$?
color reset
# detect return code ... and abort on any error.
t_rcCheckInit $_myrc
fi
echo restic unlock ${ARGS_DEFAULT}
color cmd
eval restic unlock ${ARGS_DEFAULT}
color reset
echo
}
......@@ -161,26 +167,23 @@
# --------------------
echo "--- UNLOCK ... just in case :-)"
echo restic unlock ${ARGS_DEFAULT}
color cmd
eval restic unlock ${ARGS_DEFAULT}
color reset
_restic unlock
echo
}
# prune old data
# uses global vars from ../../transfer.sh
# return exitcode of restic prune
function t_backupDoPrune(){
# --------------------
echo "--- FORGET (in all pathes of repository)"
local _tag=$( _j_getvar ${STORAGEFILE} "${CFGPREFIX}tag")
local _mycmd="restic forget \
local _mycmd="forget \
${ARGS_DEFAULT} \
--tag $_tag \
--group-by paths,tags \
--prune \
--cleanup-cache"
local _keep
......@@ -192,49 +195,39 @@
fi
done
echo $_mycmd
sleep 3
color cmd
eval $_mycmd
local _myrc=$?
color reset
_restic "$_mycmd"
_myrc=$?
t_rcCheckCleanup $_myrc
echo
# --------------------
echo "--- PRUNE (whole repository)"
_mycmd="restic prune ${ARGS_DEFAULT}"
echo $_mycmd
sleep 3
color cmd
eval $_mycmd
local _myrc=$?
color reset
_restic prune
_myrc=$?
t_rcCheckPrune $_myrc
_j_runHooks "26-after-prune" "$_myrc"
echo
return $_myrc
}
# verify backup data
# uses global vars from ../../transfer.sh
# return exitcode of restic check
function t_backupDoVerify(){
# --------------------
echo "--- VERIFY (whole repository)"
# param --read-data takes a long time. Maybe use an extra job with it.
# _mycmd="time restic check ${ARGS_DEFAULT} --with-cache --read-data"
_mycmd="restic check ${ARGS_DEFAULT} --with-cache"
echo $_mycmd
sleep 3
color cmd
eval $_mycmd
local _myrc=$?
color reset
_restic check --with-cache
_myrc=$?
t_rcCheckVerify $_myrc
_j_runHooks "28-after-verify" "$_myrc"
echo
return $_myrc
}
# --------------------------------------------------------------------------------
......@@ -280,9 +273,7 @@
if [ "${_snapshotLast}" = "${_snapshotNow}" ]; then
echo "This was the initial (full) Backup"
else
color cmd
eval restic diff ${ARGS_DEFAULT} "${_snapshotLast}" "${_snapshotNow}"
color reset
_restic diff "${_snapshotLast}" "${_snapshotNow}"
fi
echo
......
transfer.sh
+ 94
− 78
View file @ 484384e1
......@@ -34,6 +34,7 @@
# 2022-10-04 ah v2.4 prune and verify are non directory based
# 2022-10-07 ah v2.5 unescape regex with space to prevent "grep: warning: stray \ before white space"
# 2022-10-20 ah v2.6 move hook 20-before-transfer (after init of the backup tool)
# 2022-10-21 ah v2.7 shell fixes;
# ================================================================================
......@@ -43,8 +44,8 @@
# . `dirname $0`/inc_config.sh
. `dirname $0`/jobhelper.sh
. `dirname $0`/inc_bash.sh
. $(dirname $0)/jobhelper.sh
. $(dirname $0)/inc_bash.sh
typeset -i rc=0
typeset -i doBackup=1
......@@ -57,13 +58,14 @@
exit 0
fi
STORAGE_BIN=`_j_getvar ${STORAGEFILE} "bin"`
STORAGE_BASEDIR=`_j_getvar ${STORAGEFILE} "storage"`
STORAGE_TESTFILE=`_j_getvar ${STORAGEFILE} "storage-file"`
PASSPHRASE=`_j_getvar ${STORAGEFILE} "passphrase"`
STORAGE_BIN=$(_j_getvar "${STORAGEFILE}" "bin")
STORAGE_BASEDIR=$(_j_getvar "${STORAGEFILE}" "storage")
STORAGE_TESTFILE=$(_j_getvar "${STORAGEFILE}" "storage-file")
PASSPHRASE=$(_j_getvar "${STORAGEFILE}" "passphrase")
STORAGE_REGISTER=`_j_getvar ${STORAGEFILE} "storage-register"`
typeset -i TIMER_TRANSFER_START=`date +%s`
STORAGE_REGISTER=$(_j_getvar "${STORAGEFILE}" "storage-register")
typeset -i TIMER_TRANSFER_START
TIMER_TRANSFER_START=$(date +%s)
# check
if [ -z "$STORAGE_BIN" ]; then
......@@ -78,17 +80,17 @@
color reset
exit 1
fi
if [ ! -z "$STORAGE_TESTFILE" -a ! -f "$STORAGE_TESTFILE" ]; then
if [ -n "$STORAGE_TESTFILE" -a ! -f "$STORAGE_TESTFILE" ]; then
color error
echo ERROR: missing testfile $STORAGE_TESTFILE on backup target.
echo The Backup target disk / NAS is not mounted.
echo "ERROR: missing testfile $STORAGE_TESTFILE on backup target."
echo "The Backup target disk / NAS is not mounted."
color reset
exit 1
fi
# support old value
if [ -z "${PASSPHRASE}" ]; then
echo "WARNING: The value gnupg-passphrase in ${STORAGEFILE} is deprecated. Replace it with passphrase=..."
PASSPHRASE=`_j_getvar ${STORAGEFILE} "gnupg-passphrase"`
PASSPHRASE=$(_j_getvar "${STORAGEFILE}" "gnupg-passphrase")
fi
if [ -z "${PASSPHRASE}" ]; then
......@@ -136,13 +138,15 @@ function setAction(){
local action=$1
local myfile=$2
typeset -i local iLimit=$(_j_getvar ${STORAGEFILE} "$action-after")
typeset -i local iLimit
iLimit=$(_j_getvar ${STORAGEFILE} "$action-after")
if [ ! -f "${myfile}" ]; then
echo "Info: $action is ENABLED - no last $action detected"
doValue=1
else
typeset -i iLastDone=$( _getFileAge "${myfile}" )/60
typeset -i iLastDone
iLastDone=$( _getFileAge "${myfile}" )/60
typeset -i iLastDoneD=iLastDone/60/24
echo "Info: Last $action was $iLastDone min ago ($iLastDoneD days). Limit is $iLimit days."
if [ $iLastDoneD -ge $iLimit ]; then
......@@ -180,8 +184,8 @@ function setAction(){
# set defaults for prune and verify
echo ">>> Detect default behaviour:"
setAction "prune" $lastprunefile; doPrune=$doValue
setAction "verify" $lastverifyfile; doVerify=$doValue
setAction "prune" "$lastprunefile"; doPrune=$doValue
setAction "verify" "$lastverifyfile"; doVerify=$doValue
echo
echo ">>> Check parameters"
......@@ -204,7 +208,7 @@ function setAction(){
exec 1> >( tee -a "$transferlog" ) 2>&1
echo "INFO: Start logging into $transferlog"
h1 `date` TRANSFER LOCAL DATA TO STORAGE
h1 "$( date ) TRANSFER LOCAL DATA TO STORAGE"
echo "TOOL : $STORAGE_BIN"
echo "ACTION : $ACTION"
......@@ -219,14 +223,14 @@ function setAction(){
echo
fi
. `dirname $0`/plugins/transfer/$STORAGE_BIN.sh || exit 1
. $(dirname $0)/plugins/transfer/$STORAGE_BIN.sh || exit 1
# --------------------------------------------------------------------------------
# ----- Check requirements
t_checkRequirements || exit 1
test -z "$STORAGE_REGISTER" || . `dirname $0`/plugins/register/$STORAGE_REGISTER.sh || exit 1
test -z "$STORAGE_REGISTER" || . $(dirname $0)/plugins/register/$STORAGE_REGISTER.sh || exit 1
echo Check locking of a running transfer
if [ -f "${lockfile}" ]; then
......@@ -238,34 +242,34 @@ function setAction(){
# 1659 - check process id inside the lock file
# detect pid from lockfile and search for this process
lockpid=`cat "${lockfile}" | cut -f 2 -d "-" | cut -f 4 -d " " | grep "[0-9]"`
if [ -z $lockpid ]; then
lockpid=$(cat "${lockfile}" | cut -f 2 -d "-" | cut -f 4 -d " " | grep "[0-9]")
if [ -z "$lockpid" ]; then
color error
echo ERROR: pid was not fetched from lock file. Check the transfer processes manually, please.
echo "ERROR: pid was not fetched from lock file. Check the transfer processes manually, please."
color reset
exit 1
fi
echo transfer processes with pid or ppid ${lockpid}:
echo "transfer processes with pid or ppid ${lockpid}:"
color cmd
ps -ef | grep $lockpid | grep "transfer"
ps -ef | grep "$lockpid" | grep "transfer"
rccheck=$?
color reset
if [ $rccheck -eq 0 ]; then
color error
echo ERROR: The transfer with pid $lockpid seems to be still active. Aborting.
echo "ERROR: The transfer with pid $lockpid seems to be still active. Aborting."
color reset
exit 1
fi
color ok
echo OK, the transfer seems not to be active anymore. I IGNORE the lock and continue...
echo "OK, the transfer seems not to be active anymore. I IGNORE the lock and continue..."
color reset
fi
echo Creating a lock file ...
echo "transfer started `date` - process id $$" > "${lockfile}"
echo "transfer started $( date ) - process id $$" > "${lockfile}"
if [ $? -ne 0 ]; then
color error
echo ABORT - unable to create transfer lock
echo "ABORT - unable to create transfer lock"
color reset
exit 2
fi
......@@ -275,32 +279,33 @@ function setAction(){
# parameters for all
t_setVars || exit 1
export ARGS_DEFAULT
ARGS_DEFAULT="$( t_getParamDefault $1 $2 )"
if [ "$1" = "dumps" ]; then
sDirs2Backup="`_j_getvar ${JOBFILE} dir-localdumps`"
sDirs2Backup="$(_j_getvar ${JOBFILE} dir-localdumps)"
else
sDirs2Backup="`j_getDirs2Backup`"
sDirs2Backup="$(j_getDirs2Backup)"
fi
sParamExclude=
for sItem in `_j_getvar ${DIRFILE} exclude`
for sItem in $(_j_getvar "${DIRFILE}" exclude)
do
sParamExclude="$sParamExclude $( t_getParamExlude $sItem )"
done
sFileSshPrivkey=`_j_getvar ${STORAGEFILE} "ssh-privatekey"`
if [ ! -z $sFileSshPrivkey ]; then
sFileSshPrivkey=$(_j_getvar ${STORAGEFILE} "ssh-privatekey")
if [ -n "$sFileSshPrivkey" ]; then
ARGS_DEFAULT="${ARGS_DEFAULT} $( t_getParamSshKey $sFileSshPrivkey )"
fi
# task#3046 - add custom cache dir
sCacheDir=`_j_getvar ${STORAGEFILE} "${CFGPREFIX}cachedir"`
if [ ! -z $sCacheDir ]; then
sCacheDir=$(_j_getvar "${STORAGEFILE}" "${CFGPREFIX}cachedir")
if [ -n "$sCacheDir" ]; then
ARGS_DEFAULT="${ARGS_DEFAULT} $( t_getParamCacheDir $sCacheDir )"
fi
......@@ -311,13 +316,13 @@ function setAction(){
_j_runHooks "20-before-transfer"
h2 "`date` Wait for a free slot"
h2 "$( date ) Wait for a free slot"
if [ -z "$STORAGE_REGISTER" ]; then
echo "SKIP"
else
iExit=1
until [ $iExit -eq 0 ]; do
registerBackupSlot `hostname -f`
registerBackupSlot "$(hostname -f)"
iExit=$?
if [ $iExit -ne 0 ]; then
statusBackupSlot
......@@ -329,7 +334,7 @@ function setAction(){
fi
h2 "`date` PRE transfer tasks"
h2 "$( date ) PRE transfer tasks"
t_backupDoPreTasks
echo
......@@ -341,32 +346,32 @@ function setAction(){
do
# remove ending slash ... otherwise duplicity will fail
mydir=`echo $mydir | sed 's#\/$##g'`
mydir=$(echo "$mydir" | sed 's#\/$##g')
if [ -d "$mydir" ]; then
BACKUP_DIR=$mydir
h2 "`date` STORE $BACKUP_DIR"
h2 "$( date ) STORE $BACKUP_DIR"
# --- build parameters
sSafeName=`j_getSafename "$BACKUP_DIR"`
sSafeName=$(j_getSafename "$BACKUP_DIR")
sTarget="$( t_backupDirGetTarget $BACKUP_DIR )"
ARGS_BACKUP="${sParamExclude} $( t_getParamBackup )"
# detect custom backup sets and add its includes and excludes
backupid=`j_getSetnameOfPath "$BACKUP_DIR"`
backupid=$(j_getSetnameOfPath "$BACKUP_DIR")
sSpaceReplace="___SPACE___"
if [ ! -z $backupid ]; then
for sItem in `_j_getvar ${DIRFILE} "${backupid}\-\-include" | sed "s# #${sSpaceReplace}#g"`
for sItem in $(_j_getvar ${DIRFILE} "${backupid}\-\-include" | sed "s# #${sSpaceReplace}#g")
do
ARGS_BACKUP="${ARGS_BACKUP} $( t_getParamInlude $sItem)"
done
for sItem in `_j_getvar ${DIRFILE} "${backupid}\-\-exclude" | sed "s# #${sSpaceReplace}#g"`
for sItem in $(_j_getvar ${DIRFILE} "${backupid}\-\-exclude" | sed "s# #${sSpaceReplace}#g")
do
ARGS_BACKUP="${ARGS_BACKUP} $( t_getParamExlude $sItem)"
done
......@@ -374,13 +379,13 @@ function setAction(){
# --- pre task
h3 "`date` PRE backup task for ${BACKUP_DIR}"
h3 "$( date ) PRE backup task for ${BACKUP_DIR}"
t_backupDirDoPreTasks
# sCmdPre="$( t_backupDirDoPreTasks )"
# --- backup
h3 "`date` Backup ${BACKUP_DIR}"
h3 "$( date ) Backup ${BACKUP_DIR}"
if [ $doBackup -eq 0 ]; then
echo "SKIP backup"
else
......@@ -406,7 +411,7 @@ function setAction(){
# --- post action
h3 "`date` POST backup task for ${BACKUP_DIR}"
h3 "$( date ) POST backup task for ${BACKUP_DIR}"
t_backupDirDoPostTasks
echo
......@@ -423,13 +428,30 @@ function setAction(){
echo "SKIP backup of dirs"
fi
echo $rc > ${rcfile}
exit $rc
)
# rc=${PIPESTATUS[0]}
rc=$(cat ${rcfile})
# --------------------------------------------------------------------------------
# --- transfer POST tasks
h2 "$( date ) POST transfer tasks"
# --- prune
if [ $doPrune -eq 0 ]; then
echo "SKIP prune"
else
h3 "`date` PRUNE repository data"
t_backupDoPrune
touch ${lastprunefile}
h3 "$( date ) PRUNE repository data"
if t_backupDoPrune; then
touch "${lastprunefile}"
else
rc+=1
fi
ls -l "${lastprunefile}"
echo
fi
echo
......@@ -438,39 +460,32 @@ function setAction(){
if [ $doVerify -eq 0 ]; then
echo "SKIP verify"
else
h3 "`date` VERIFY repository data"
t_backupDoVerify
touch ${lastverifyfile}
h3 "$( date ) VERIFY repository data"
if t_backupDoVerify; then
touch "${lastverifyfile}"
else
rc+=1
fi
ls -l "${lastverifyfile}"
echo
fi
echo
echo $rc > ${rcfile}
exit $rc
)
# rc=${PIPESTATUS[0]}
rc=`cat ${rcfile}`
# --------------------------------------------------------------------------------
# --- transfer POST tasks
h2 "`date` POST transfer tasks"
# --- unlock
t_backupDoPostTasks
rm -f "${lockfile}" "${rcfile}"
echo "Local lock file was removed."
h2 "`date` Unregister used slot"
h2 "$( date ) Unregister used slot"
if [ -z "$STORAGE_REGISTER" ]; then
echo "SKIP"
else
unregisterBackupSlot `hostname -f` $rc
unregisterBackupSlot "$(hostname -f)" $rc
fi
h2 "`date` Backup finished"
echo STATUS $0 exit with final returncode rc=$rc
h2 "$( date ) Backup finished"
echo "STATUS $0 exit with final returncode rc=$rc"
echo
if [ $rc -eq 0 ]; then
color ok
......@@ -483,10 +498,11 @@ function setAction(){
_j_runHooks "30-post-backup" "$rc"
echo
typeset -i TIMER_TRANSFER=`date +%s`-$TIMER_TRANSFER_START
echo `date` $ACTION DONE in $TIMER_TRANSFER sec
typeset -i TIMER_TRANSFER
TIMER_TRANSFER=$(date +%s)-$TIMER_TRANSFER_START
echo "$( date ) $ACTION DONE in $TIMER_TRANSFER sec"
ls -l $transferlog
ls -l "$transferlog"
exit $rc
# --------------------------------------------------------------------------------
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment