diff --git a/jobs/transfer.job.dist b/jobs/transfer.job.dist
index c4d6465ed84f5d04f1f15d3db43ef9fb3919225c..9c1fa30777e933f583e4f79ab17a8680eaa073f4 100644
--- a/jobs/transfer.job.dist
+++ b/jobs/transfer.job.dist
@@ -104,6 +104,14 @@ sambashares = 1
restore-path = /restore
+# ----------------------------------------------------------------------
+# automatic tasks on each backup run
+# ----------------------------------------------------------------------
+
+# number of days when to prune and verify backup.
+# set it to 0 to run prune and verify in each backup run.
+prune-after = 3
+verify-after = 7
# ----------------------------------------------------------------------
#
diff --git a/plugins/transfer/duplicity.sh b/plugins/transfer/duplicity.sh
index 2840527e784796d967fbad6752f8b570cdabdcfc..2b1788c16734b024aa1182ddeeea06118b93d3c7 100644
--- a/plugins/transfer/duplicity.sh
+++ b/plugins/transfer/duplicity.sh
@@ -172,7 +172,12 @@
color reset
exit 1
fi
+ }
+ # prune old data
+ # uses global vars from ../../transfer.sh
+ function t_backupDoPrune(){
+ echo "--- FORGET some data"
cmd="duplicity remove-older-than $STORAGE_KEEP --force ${ARGS_DEFAULT} ${STORAGE_TARGETPATH}"
echo $cmd
color cmd
@@ -181,8 +186,14 @@
color reset
t_rcCheckCleanup $myrc
-
}
+ # verify backup data
+ # uses global vars from ../../transfer.sh
+ function t_backupDoVerify(){
+ echo "--- VERIFY is not implemented yet for duplicity."
+ }
+
+
# --------------------------------------------------------------------------------
# RESTORE
# --------------------------------------------------------------------------------
diff --git a/plugins/transfer/restic.sh b/plugins/transfer/restic.sh
index cb6635c483f30434ab6faa16300f8a7515ddb291..663848b4f76611a05cdd2587e0ea4ee12d473a38 100644
--- a/plugins/transfer/restic.sh
+++ b/plugins/transfer/restic.sh
@@ -142,6 +142,11 @@
# detect return code ... and abort on any error.
t_rcCheckInit $_myrc
fi
+ echo restic unlock ${ARGS_DEFAULT}
+ color cmd
+ eval restic unlock ${ARGS_DEFAULT}
+ color reset
+ echo
}
@@ -157,6 +162,11 @@
color reset
echo
+ }
+
+ # prune old data
+ # uses global vars from ../../transfer.sh
+ function t_backupDoPrune(){
# --------------------
echo "--- FORGET some data"
local _tag=$( _j_getvar ${STORAGEFILE} "${CFGPREFIX}tag")
@@ -199,7 +209,10 @@
t_rcCheckPrune $_myrc
echo
-
+ }
+ # verify backup data
+ # uses global vars from ../../transfer.sh
+ function t_backupDoVerify(){
# --------------------
echo "--- VERIFY"
# param --read-data takes a long time. Maybe use an extra job with it.
@@ -214,7 +227,6 @@
t_rcCheckVerify $_myrc
echo
-
}
# --------------------------------------------------------------------------------
@@ -385,7 +397,7 @@
# repository cleanup
# param integer exitcode of command
function t_rcCheckCleanup(){
- echo -n "__PRUNE__ "
+ echo -n "__CLEANUP__ "
case $1 in
0) color ok; echo "OK" ;;
*) color error; echo "FAILED ${2} - Cleanup error - returncode was $1" ;;
diff --git a/prune.sh b/prune.sh
new file mode 100755
index 0000000000000000000000000000000000000000..8ea35cbb187a7e080b29312d5c222a1c03484010
--- /dev/null
+++ b/prune.sh
@@ -0,0 +1,2 @@
+#!/bin/bash
+$(dirname $0 )/transfer.sh prune
\ No newline at end of file
diff --git a/transfer.sh b/transfer.sh
index 216162ee2bc2768ab8e77ee323d850cd9e85889a..3cd45727788ec270d7227579cc0913dbe64c2f7b 100755
--- a/transfer.sh
+++ b/transfer.sh
@@ -30,6 +30,7 @@
# 2021-05-19 ah,ds, v2.0 plugin driven to support multiple backup tools (duplicity + restic)
# 2021-12-02 ah v2.1 added parameter "prune" to cleanup only
# 2022-02-10 ah v2.2 update logging (removing tee)
+# 2022-10-01 ah v2.3 customize prune and verify action
# ================================================================================
@@ -44,7 +45,9 @@
typeset -i rc=0
typeset -i doBackup=1
- typeset -i doPrune=1
+ typeset -i doPrune
+ typeset -i doVerify
+ typeset -i doValue
if [ ! -r "${DIRFILE}" -o ! -r "${STORAGEFILE}" ]; then
echo "SKIP backup of local files - one of the files is not readable (no error): ${DIRFILE} | ${STORAGEFILE}"
@@ -97,15 +100,58 @@
# first param "full"
METHOD=
+ ACTION=backup
transferlog="${DIR_LOGS}/transfer-$(date +%Y%m%d-%H%M%S).log"
lockfile="${DIR_LOGS}/transfer.running"
+ lastbackupfile="${DIR_LOGS}/last_backup"
+ lastprunefile="${DIR_LOGS}/last_prune"
+ lastverifyfile="${DIR_LOGS}/last_verify"
rcfile=/tmp/transfer-rc.$$.tmp
+
# --------------------------------------------------------------------------------
# FUNCTIONS
# --------------------------------------------------------------------------------
+# get age of a file in sec
+# https://www.axel-hahn.de/blog/2020/03/09/bash-snippet-alter-einer-datei/
+# param string filename to test
+# return integer
+function _getFileAge(){
+ echo $(($(date +%s) - $(date +%s -r "$1")))
+}
+
+# set default behaviur based on limit (from config) and its last
+# execution (read from touchfile)
+# Function is enabled if no touchfile was found or it is older than
+# given limit
+#
+# It is used to set flags for prune and verify
+# param string name of action; one of prune|verify
+# param string filename of touchfile of last execution
+function setAction(){
+ local action=$1
+ local myfile=$2
+
+ typeset -i local iLimit=$(_j_getvar ${STORAGEFILE} "$action-after")
+
+ if [ ! -f "${myfile}" ]; then
+ echo "Info: $action is ENABLED - no last $action detected"
+ doValue=1
+ else
+ typeset -i iLastDone=$( _getFileAge "${myfile}" )/60
+ typeset -i iLastDoneD=iLastDone/60/24
+ echo "Info: Last $action was $iLastDone min ago ($iLastDoneD days). Limit is $iLimit days."
+ if [ $iLastDoneD -ge $iLimit ]; then
+ echo "Info: $action is ENABLED - last $action is outdated"
+ doValue=1
+ else
+ echo "Info: $action is not needed yet."
+ doValue=0
+ fi
+ fi
+}
# --------------------------------------------------------------------------------
# MAIN
@@ -116,32 +162,61 @@
Transfer local files to a backup target.
- target ${STORAGE_BASEDIR}
+ target $( echo ${STORAGE_BASEDIR} | sed 's#:[^:]*@#:**********@#' )
backup tool $STORAGE_BIN
PARAMETERS:
- transfer.sh - incremental backup
- transfer.sh full - full backup
- transfer.sh dumps - transfer local dumps only
- transfer.sh prune - cleanup backup data only (no backup)
- transfer.sh help - show this help (works with -h and -? too)
+ transfer.sh - incremental backup
+ transfer.sh full - full backup (Duplicity only)
+ transfer.sh dumps - transfer local dumps only
+ transfer.sh prune - cleanup backup data only (no backup)
+ transfer.sh verify - verify backup data (no backup)
+ transfer.sh help - show this help (works with -h and -? too)
"
exit 0
fi
+
+ # set defaults for prune and verify
+ echo ">>> Detect default behaviuor:"
+ setAction "prune" $lastprunefile; doPrune=$doValue
+ setAction "verify" $lastverifyfile; doVerify=$doValue
+ echo
+
+ echo ">>> Check parameters"
if [ "$1" = "prune" ]; then
+ echo "Info: Forcing prune only by parameter."
+ ACTION=$1
doBackup=0
+ doPrune=1
+ doVerify=0
+ transferlog="${DIR_LOGS}/prune-$(date +%Y%m%d-%H%M%S).log"
+ fi
+ if [ "$1" = "verify" ]; then
+ echo "Info: Forcing verify only by parameter."
+ ACTION=$1
+ doBackup=0
+ doPrune=0
+ doVerify=1
+ transferlog="${DIR_LOGS}/verify-$(date +%Y%m%d-%H%M%S).log"
fi
exec 1> >( tee -a "$transferlog" ) 2>&1
- echo "INFO: Start logging into $transferlog"
+ echo "INFO: Start logging into $transferlog"
h1 `date` TRANSFER LOCAL DATA TO STORAGE
- echo "METHOD : $METHOD"
- echo "TARGET : ${STORAGE_BASEDIR}"
- echo "REGISTER : ${STORAGE_REGISTER}"
echo "TOOL : $STORAGE_BIN"
+ echo "ACTION : $ACTION"
+ echo "TARGET : ${STORAGE_BASEDIR}" | sed 's#:[^:]*@#:**********@#'
echo
+ if [ "$ACTION" = "backup" ]; then
+ echo "METHOD : $METHOD"
+ echo "REGISTER : ${STORAGE_REGISTER}"
+ echo "PRUNE : $doPrune"
+ echo "VERIFY : $doVerify"
+ echo
+ fi
+
. `dirname $0`/plugins/transfer/$STORAGE_BIN.sh || exit 1
test -z "$STORAGE_REGISTER" || . `dirname $0`/plugins/register/$STORAGE_REGISTER.sh || exit 1
@@ -264,66 +339,89 @@
if [ -d "$mydir" ]; then
BACKUP_DIR=$mydir
- h2 "`date` STORE $BACKUP_DIR"
- # --- build parameters
- sSafeName=`j_getSafename "$BACKUP_DIR"`
- sTarget="$( t_backupDirGetTarget $BACKUP_DIR )"
+ if [ "$ACTION" = "backup" ]; then
+
+ h2 "`date` STORE $BACKUP_DIR"
- ARGS_BACKUP="${sParamExclude} $( t_getParamBackup )"
+ # --- build parameters
+ sSafeName=`j_getSafename "$BACKUP_DIR"`
+ sTarget="$( t_backupDirGetTarget $BACKUP_DIR )"
+ ARGS_BACKUP="${sParamExclude} $( t_getParamBackup )"
- # detect custom backup sets and add its includes and excludes
- backupid=`j_getSetnameOfPath "$BACKUP_DIR"`
- sSpaceReplace="___SPACE___"
+ # detect custom backup sets and add its includes and excludes
+ backupid=`j_getSetnameOfPath "$BACKUP_DIR"`
+ sSpaceReplace="___SPACE___"
- if [ ! -z $backupid ]; then
- for sItem in `_j_getvar ${DIRFILE} "${backupid}\-\-include" | sed "s#\ #${sSpaceReplace}#g"`
- do
- ARGS_BACKUP="${ARGS_BACKUP} $( t_getParamInlude $sItem)"
- done
- for sItem in `_j_getvar ${DIRFILE} "${backupid}\-\-exclude" | sed "s#\ #${sSpaceReplace}#g"`
- do
- ARGS_BACKUP="${ARGS_BACKUP} $( t_getParamExlude $sItem)"
- done
- fi
+ if [ ! -z $backupid ]; then
+ for sItem in `_j_getvar ${DIRFILE} "${backupid}\-\-include" | sed "s#\ #${sSpaceReplace}#g"`
+ do
+ ARGS_BACKUP="${ARGS_BACKUP} $( t_getParamInlude $sItem)"
+ done
+ for sItem in `_j_getvar ${DIRFILE} "${backupid}\-\-exclude" | sed "s#\ #${sSpaceReplace}#g"`
+ do
+ ARGS_BACKUP="${ARGS_BACKUP} $( t_getParamExlude $sItem)"
+ done
+ fi
- # --- pre task
- h3 "`date` PRE backup task for ${BACKUP_DIR}"
- t_backupDirDoPreTasks
- # sCmdPre="$( t_backupDirDoPreTasks )"
+ # --- pre task
+ h3 "`date` PRE backup task for ${BACKUP_DIR}"
+ t_backupDirDoPreTasks
+ # sCmdPre="$( t_backupDirDoPreTasks )"
- # --- backup
- h3 "`date` Backup ${BACKUP_DIR}"
- if [ $doBackup -eq 0 ]; then
- echo "SKIP backup"
- else
- sCmd="$( t_backupDirGetCmdBackup )"
- echo "what: ${BACKUP_DIR}"
- echo "target: ${sTarget}"
- echo "command: $sCmd"
- echo
- color cmd
- $sCmd
- fetchrc
- color reset
+
+ # --- backup
+ h3 "`date` Backup ${BACKUP_DIR}"
+ if [ $doBackup -eq 0 ]; then
+ echo "SKIP backup"
+ else
+ sCmd="$( t_backupDirGetCmdBackup )"
+ echo "what: ${BACKUP_DIR}"
+ echo "target: ${sTarget}"
+ echo "command: $sCmd"
+ echo
+ color cmd
+ $sCmd
+ fetchrc
+ color reset
+ echo
+
+ t_rcCheckBackup $myrc "${BACKUP_DIR}"
+
+ fi
echo
- t_rcCheckBackup $myrc "${BACKUP_DIR}"
- fi
- echo
+ # --- post action
+ h3 "`date` POST backup task for ${BACKUP_DIR}"
+ t_backupDirDoPostTasks
+ echo
+ touch ${lastbackupfile}
+ fi
- # --- post action
- h3 "`date` POST backup task for ${BACKUP_DIR}"
+ # --- prune
if [ $doPrune -eq 0 ]; then
echo "SKIP prune"
else
- t_backupDirDoPostTasks
+ h3 "`date` PRUNE for ${BACKUP_DIR}"
+ t_backupDoPrune
+ touch ${lastprunefile}
+ echo
+ fi
+ echo
+
+ # --- verify
+ if [ $doVerify -eq 0 ]; then
+ echo "SKIP verify"
+ else
+ h3 "`date` VERIFY for ${BACKUP_DIR}"
+ t_backupDoVerify
+ touch ${lastverifyfile}
echo
fi
echo
@@ -373,7 +471,7 @@
color reset
echo
typeset -i TIMER_TRANSFER=`date +%s`-$TIMER_TRANSFER_START
- echo `date` TRANSFER DONE in $TIMER_TRANSFER sec
+ echo `date` $ACTION DONE in $TIMER_TRANSFER sec
ls -l $transferlog
exit $rc
diff --git a/verify.sh b/verify.sh
new file mode 100755
index 0000000000000000000000000000000000000000..954a5558fb6bd37ce7d2960460ddce5c6651a7af
--- /dev/null
+++ b/verify.sh
@@ -0,0 +1,2 @@
+#!/bin/bash
+$(dirname $0 )/transfer.sh verify
\ No newline at end of file