diff --git a/plugins/transfer/restic.sh b/plugins/transfer/restic.sh index 826ae8c097b3a48508b52ed8b12287ec8d4e0628..ba4d0cce82105efa8f4d0206cc95e11ff14c2156 100644 --- a/plugins/transfer/restic.sh +++ b/plugins/transfer/restic.sh @@ -14,6 +14,7 @@ # 2022-03-07 ah v0.4 add verify in post task # 2022-05-10 ah v0.5 fix handling with nocache flag (use globally as default param - not in backup only) # 2022-05-16 ah v0.6 added restic prune +# 2022-10-21 ah v0.7 simplify restic exec commands in _restic; remove --prune in check # ================================================================================ # -------------------------------------------------------------------------------- @@ -119,6 +120,19 @@ fi } + # execute a restic command + # param string options and subcommand + function _restic(){ + local _mycmd="restic $* ${ARGS_DEFAULT}" + echo "$_mycmd" + sleep 3 + color cmd + eval "$_mycmd" + local _myrc=$? + color reset + return $_myrc + } + # -------------------------------------------------------------------------------- # BACKUP ACTIONS :: TRANSFER # -------------------------------------------------------------------------------- @@ -127,30 +141,22 @@ # function t_backupDoPreTasks(){ function t_backupDoPreTasks(){ - if eval restic snapshots ${ARGS_DEFAULT} >/dev/null 2>&1 + # if eval restic snapshots ${ARGS_DEFAULT} >/dev/null 2>&1 + if _restic list keys >/dev/null 2>&1 then echo "__REPO__ OK, Backup repository already exists." echo "--- UNLOCK ... just in case :-)" - eval restic unlock ${ARGS_DEFAULT} - color reset + _restic unlock echo else echo "Backup repository needs to be created." - local _mycmd="restic init ${ARGS_DEFAULT}" - echo $_mycmd - color cmd - eval $_mycmd + _restic init local _myrc=$? - color reset # detect return code ... and abort on any error. t_rcCheckInit $_myrc fi - echo restic unlock ${ARGS_DEFAULT} - color cmd - eval restic unlock ${ARGS_DEFAULT} - color reset echo } @@ -161,26 +167,23 @@ # -------------------- echo "--- UNLOCK ... just in case :-)" - echo restic unlock ${ARGS_DEFAULT} - color cmd - eval restic unlock ${ARGS_DEFAULT} - color reset + _restic unlock echo } # prune old data # uses global vars from ../../transfer.sh + # return exitcode of restic prune function t_backupDoPrune(){ # -------------------- echo "--- FORGET (in all pathes of repository)" local _tag=$( _j_getvar ${STORAGEFILE} "${CFGPREFIX}tag") - local _mycmd="restic forget \ + local _mycmd="forget \ ${ARGS_DEFAULT} \ --tag $_tag \ --group-by paths,tags \ - --prune \ --cleanup-cache" local _keep @@ -192,49 +195,39 @@ fi done - echo $_mycmd - sleep 3 - color cmd - eval $_mycmd - local _myrc=$? - color reset + _restic "$_mycmd" + _myrc=$? t_rcCheckCleanup $_myrc echo # -------------------- echo "--- PRUNE (whole repository)" - _mycmd="restic prune ${ARGS_DEFAULT}" - echo $_mycmd - sleep 3 - color cmd - eval $_mycmd - local _myrc=$? - color reset + _restic prune + _myrc=$? t_rcCheckPrune $_myrc _j_runHooks "26-after-prune" "$_myrc" echo + return $_myrc } # verify backup data # uses global vars from ../../transfer.sh + # return exitcode of restic check function t_backupDoVerify(){ # -------------------- echo "--- VERIFY (whole repository)" # param --read-data takes a long time. Maybe use an extra job with it. # _mycmd="time restic check ${ARGS_DEFAULT} --with-cache --read-data" - _mycmd="restic check ${ARGS_DEFAULT} --with-cache" - echo $_mycmd - sleep 3 - color cmd - eval $_mycmd - local _myrc=$? - color reset + _restic check --with-cache + _myrc=$? t_rcCheckVerify $_myrc + _j_runHooks "28-after-verify" "$_myrc" echo + return $_myrc } # -------------------------------------------------------------------------------- @@ -280,9 +273,7 @@ if [ "${_snapshotLast}" = "${_snapshotNow}" ]; then echo "This was the initial (full) Backup" else - color cmd - eval restic diff ${ARGS_DEFAULT} "${_snapshotLast}" "${_snapshotNow}" - color reset + _restic diff "${_snapshotLast}" "${_snapshotNow}" fi echo diff --git a/transfer.sh b/transfer.sh index 33f4c58b19bb2a3926cf9c0c6c2548cf394e1cb4..ccdd6464d0b8c3d10b40dc377af3524fe5aa279c 100755 --- a/transfer.sh +++ b/transfer.sh @@ -34,6 +34,7 @@ # 2022-10-04 ah v2.4 prune and verify are non directory based # 2022-10-07 ah v2.5 unescape regex with space to prevent "grep: warning: stray \ before white space" # 2022-10-20 ah v2.6 move hook 20-before-transfer (after init of the backup tool) +# 2022-10-21 ah v2.7 shell fixes; # ================================================================================ @@ -43,8 +44,8 @@ # . `dirname $0`/inc_config.sh - . `dirname $0`/jobhelper.sh - . `dirname $0`/inc_bash.sh + . $(dirname $0)/jobhelper.sh + . $(dirname $0)/inc_bash.sh typeset -i rc=0 typeset -i doBackup=1 @@ -57,13 +58,14 @@ exit 0 fi - STORAGE_BIN=`_j_getvar ${STORAGEFILE} "bin"` - STORAGE_BASEDIR=`_j_getvar ${STORAGEFILE} "storage"` - STORAGE_TESTFILE=`_j_getvar ${STORAGEFILE} "storage-file"` - PASSPHRASE=`_j_getvar ${STORAGEFILE} "passphrase"` + STORAGE_BIN=$(_j_getvar "${STORAGEFILE}" "bin") + STORAGE_BASEDIR=$(_j_getvar "${STORAGEFILE}" "storage") + STORAGE_TESTFILE=$(_j_getvar "${STORAGEFILE}" "storage-file") + PASSPHRASE=$(_j_getvar "${STORAGEFILE}" "passphrase") - STORAGE_REGISTER=`_j_getvar ${STORAGEFILE} "storage-register"` - typeset -i TIMER_TRANSFER_START=`date +%s` + STORAGE_REGISTER=$(_j_getvar "${STORAGEFILE}" "storage-register") + typeset -i TIMER_TRANSFER_START + TIMER_TRANSFER_START=$(date +%s) # check if [ -z "$STORAGE_BIN" ]; then @@ -78,17 +80,17 @@ color reset exit 1 fi - if [ ! -z "$STORAGE_TESTFILE" -a ! -f "$STORAGE_TESTFILE" ]; then + if [ -n "$STORAGE_TESTFILE" -a ! -f "$STORAGE_TESTFILE" ]; then color error - echo ERROR: missing testfile $STORAGE_TESTFILE on backup target. - echo The Backup target disk / NAS is not mounted. + echo "ERROR: missing testfile $STORAGE_TESTFILE on backup target." + echo "The Backup target disk / NAS is not mounted." color reset exit 1 fi # support old value if [ -z "${PASSPHRASE}" ]; then echo "WARNING: The value gnupg-passphrase in ${STORAGEFILE} is deprecated. Replace it with passphrase=..." - PASSPHRASE=`_j_getvar ${STORAGEFILE} "gnupg-passphrase"` + PASSPHRASE=$(_j_getvar "${STORAGEFILE}" "gnupg-passphrase") fi if [ -z "${PASSPHRASE}" ]; then @@ -136,13 +138,15 @@ function setAction(){ local action=$1 local myfile=$2 - typeset -i local iLimit=$(_j_getvar ${STORAGEFILE} "$action-after") + typeset -i local iLimit + iLimit=$(_j_getvar ${STORAGEFILE} "$action-after") if [ ! -f "${myfile}" ]; then echo "Info: $action is ENABLED - no last $action detected" doValue=1 else - typeset -i iLastDone=$( _getFileAge "${myfile}" )/60 + typeset -i iLastDone + iLastDone=$( _getFileAge "${myfile}" )/60 typeset -i iLastDoneD=iLastDone/60/24 echo "Info: Last $action was $iLastDone min ago ($iLastDoneD days). Limit is $iLimit days." if [ $iLastDoneD -ge $iLimit ]; then @@ -180,8 +184,8 @@ function setAction(){ # set defaults for prune and verify echo ">>> Detect default behaviour:" - setAction "prune" $lastprunefile; doPrune=$doValue - setAction "verify" $lastverifyfile; doVerify=$doValue + setAction "prune" "$lastprunefile"; doPrune=$doValue + setAction "verify" "$lastverifyfile"; doVerify=$doValue echo echo ">>> Check parameters" @@ -204,7 +208,7 @@ function setAction(){ exec 1> >( tee -a "$transferlog" ) 2>&1 echo "INFO: Start logging into $transferlog" - h1 `date` TRANSFER LOCAL DATA TO STORAGE + h1 "$( date ) TRANSFER LOCAL DATA TO STORAGE" echo "TOOL : $STORAGE_BIN" echo "ACTION : $ACTION" @@ -219,14 +223,14 @@ function setAction(){ echo fi - . `dirname $0`/plugins/transfer/$STORAGE_BIN.sh || exit 1 + . $(dirname $0)/plugins/transfer/$STORAGE_BIN.sh || exit 1 # -------------------------------------------------------------------------------- # ----- Check requirements t_checkRequirements || exit 1 - test -z "$STORAGE_REGISTER" || . `dirname $0`/plugins/register/$STORAGE_REGISTER.sh || exit 1 + test -z "$STORAGE_REGISTER" || . $(dirname $0)/plugins/register/$STORAGE_REGISTER.sh || exit 1 echo Check locking of a running transfer if [ -f "${lockfile}" ]; then @@ -238,34 +242,34 @@ function setAction(){ # 1659 - check process id inside the lock file # detect pid from lockfile and search for this process - lockpid=`cat "${lockfile}" | cut -f 2 -d "-" | cut -f 4 -d " " | grep "[0-9]"` - if [ -z $lockpid ]; then + lockpid=$(cat "${lockfile}" | cut -f 2 -d "-" | cut -f 4 -d " " | grep "[0-9]") + if [ -z "$lockpid" ]; then color error - echo ERROR: pid was not fetched from lock file. Check the transfer processes manually, please. + echo "ERROR: pid was not fetched from lock file. Check the transfer processes manually, please." color reset exit 1 fi - echo transfer processes with pid or ppid ${lockpid}: + echo "transfer processes with pid or ppid ${lockpid}:" color cmd - ps -ef | grep $lockpid | grep "transfer" + ps -ef | grep "$lockpid" | grep "transfer" rccheck=$? color reset if [ $rccheck -eq 0 ]; then color error - echo ERROR: The transfer with pid $lockpid seems to be still active. Aborting. + echo "ERROR: The transfer with pid $lockpid seems to be still active. Aborting." color reset exit 1 fi color ok - echo OK, the transfer seems not to be active anymore. I IGNORE the lock and continue... + echo "OK, the transfer seems not to be active anymore. I IGNORE the lock and continue..." color reset fi echo Creating a lock file ... - echo "transfer started `date` - process id $$" > "${lockfile}" + echo "transfer started $( date ) - process id $$" > "${lockfile}" if [ $? -ne 0 ]; then color error - echo ABORT - unable to create transfer lock + echo "ABORT - unable to create transfer lock" color reset exit 2 fi @@ -275,32 +279,33 @@ function setAction(){ # parameters for all t_setVars || exit 1 + export ARGS_DEFAULT ARGS_DEFAULT="$( t_getParamDefault $1 $2 )" if [ "$1" = "dumps" ]; then - sDirs2Backup="`_j_getvar ${JOBFILE} dir-localdumps`" + sDirs2Backup="$(_j_getvar ${JOBFILE} dir-localdumps)" else - sDirs2Backup="`j_getDirs2Backup`" + sDirs2Backup="$(j_getDirs2Backup)" fi sParamExclude= - for sItem in `_j_getvar ${DIRFILE} exclude` + for sItem in $(_j_getvar "${DIRFILE}" exclude) do sParamExclude="$sParamExclude $( t_getParamExlude $sItem )" done - sFileSshPrivkey=`_j_getvar ${STORAGEFILE} "ssh-privatekey"` - if [ ! -z $sFileSshPrivkey ]; then + sFileSshPrivkey=$(_j_getvar ${STORAGEFILE} "ssh-privatekey") + if [ -n "$sFileSshPrivkey" ]; then ARGS_DEFAULT="${ARGS_DEFAULT} $( t_getParamSshKey $sFileSshPrivkey )" fi # task#3046 - add custom cache dir - sCacheDir=`_j_getvar ${STORAGEFILE} "${CFGPREFIX}cachedir"` - if [ ! -z $sCacheDir ]; then + sCacheDir=$(_j_getvar "${STORAGEFILE}" "${CFGPREFIX}cachedir") + if [ -n "$sCacheDir" ]; then ARGS_DEFAULT="${ARGS_DEFAULT} $( t_getParamCacheDir $sCacheDir )" fi @@ -311,13 +316,13 @@ function setAction(){ _j_runHooks "20-before-transfer" - h2 "`date` Wait for a free slot" + h2 "$( date ) Wait for a free slot" if [ -z "$STORAGE_REGISTER" ]; then echo "SKIP" else iExit=1 until [ $iExit -eq 0 ]; do - registerBackupSlot `hostname -f` + registerBackupSlot "$(hostname -f)" iExit=$? if [ $iExit -ne 0 ]; then statusBackupSlot @@ -329,7 +334,7 @@ function setAction(){ fi - h2 "`date` PRE transfer tasks" + h2 "$( date ) PRE transfer tasks" t_backupDoPreTasks echo @@ -341,32 +346,32 @@ function setAction(){ do # remove ending slash ... otherwise duplicity will fail - mydir=`echo $mydir | sed 's#\/$##g'` + mydir=$(echo "$mydir" | sed 's#\/$##g') if [ -d "$mydir" ]; then BACKUP_DIR=$mydir - h2 "`date` STORE $BACKUP_DIR" + h2 "$( date ) STORE $BACKUP_DIR" # --- build parameters - sSafeName=`j_getSafename "$BACKUP_DIR"` + sSafeName=$(j_getSafename "$BACKUP_DIR") sTarget="$( t_backupDirGetTarget $BACKUP_DIR )" ARGS_BACKUP="${sParamExclude} $( t_getParamBackup )" # detect custom backup sets and add its includes and excludes - backupid=`j_getSetnameOfPath "$BACKUP_DIR"` + backupid=$(j_getSetnameOfPath "$BACKUP_DIR") sSpaceReplace="___SPACE___" if [ ! -z $backupid ]; then - for sItem in `_j_getvar ${DIRFILE} "${backupid}\-\-include" | sed "s# #${sSpaceReplace}#g"` + for sItem in $(_j_getvar ${DIRFILE} "${backupid}\-\-include" | sed "s# #${sSpaceReplace}#g") do ARGS_BACKUP="${ARGS_BACKUP} $( t_getParamInlude $sItem)" done - for sItem in `_j_getvar ${DIRFILE} "${backupid}\-\-exclude" | sed "s# #${sSpaceReplace}#g"` + for sItem in $(_j_getvar ${DIRFILE} "${backupid}\-\-exclude" | sed "s# #${sSpaceReplace}#g") do ARGS_BACKUP="${ARGS_BACKUP} $( t_getParamExlude $sItem)" done @@ -374,13 +379,13 @@ function setAction(){ # --- pre task - h3 "`date` PRE backup task for ${BACKUP_DIR}" + h3 "$( date ) PRE backup task for ${BACKUP_DIR}" t_backupDirDoPreTasks # sCmdPre="$( t_backupDirDoPreTasks )" # --- backup - h3 "`date` Backup ${BACKUP_DIR}" + h3 "$( date ) Backup ${BACKUP_DIR}" if [ $doBackup -eq 0 ]; then echo "SKIP backup" else @@ -406,7 +411,7 @@ function setAction(){ # --- post action - h3 "`date` POST backup task for ${BACKUP_DIR}" + h3 "$( date ) POST backup task for ${BACKUP_DIR}" t_backupDirDoPostTasks echo @@ -422,28 +427,6 @@ function setAction(){ else echo "SKIP backup of dirs" fi - - # --- prune - if [ $doPrune -eq 0 ]; then - echo "SKIP prune" - else - h3 "`date` PRUNE repository data" - t_backupDoPrune - touch ${lastprunefile} - echo - fi - echo - - # --- verify - if [ $doVerify -eq 0 ]; then - echo "SKIP verify" - else - h3 "`date` VERIFY repository data" - t_backupDoVerify - touch ${lastverifyfile} - echo - fi - echo echo $rc > ${rcfile} exit $rc @@ -451,26 +434,58 @@ function setAction(){ ) # rc=${PIPESTATUS[0]} - rc=`cat ${rcfile}` + rc=$(cat ${rcfile}) # -------------------------------------------------------------------------------- # --- transfer POST tasks - h2 "`date` POST transfer tasks" + h2 "$( date ) POST transfer tasks" + + # --- prune + if [ $doPrune -eq 0 ]; then + echo "SKIP prune" + else + h3 "$( date ) PRUNE repository data" + if t_backupDoPrune; then + touch "${lastprunefile}" + else + rc+=1 + fi + ls -l "${lastprunefile}" + echo + fi + echo + + # --- verify + if [ $doVerify -eq 0 ]; then + echo "SKIP verify" + else + h3 "$( date ) VERIFY repository data" + if t_backupDoVerify; then + touch "${lastverifyfile}" + else + rc+=1 + fi + ls -l "${lastverifyfile}" + echo + fi + echo + + # --- unlock t_backupDoPostTasks rm -f "${lockfile}" "${rcfile}" echo "Local lock file was removed." - h2 "`date` Unregister used slot" + h2 "$( date ) Unregister used slot" if [ -z "$STORAGE_REGISTER" ]; then echo "SKIP" else - unregisterBackupSlot `hostname -f` $rc + unregisterBackupSlot "$(hostname -f)" $rc fi - h2 "`date` Backup finished" - echo STATUS $0 exit with final returncode rc=$rc + h2 "$( date ) Backup finished" + echo "STATUS $0 exit with final returncode rc=$rc" echo if [ $rc -eq 0 ]; then color ok @@ -483,10 +498,11 @@ function setAction(){ _j_runHooks "30-post-backup" "$rc" echo - typeset -i TIMER_TRANSFER=`date +%s`-$TIMER_TRANSFER_START - echo `date` $ACTION DONE in $TIMER_TRANSFER sec + typeset -i TIMER_TRANSFER + TIMER_TRANSFER=$(date +%s)-$TIMER_TRANSFER_START + echo "$( date ) $ACTION DONE in $TIMER_TRANSFER sec" - ls -l $transferlog + ls -l "$transferlog" exit $rc # --------------------------------------------------------------------------------