customize prune and verify

jobs/transfer.job.dist

@@ -104,6 +104,14 @@ sambashares = 1
 
 restore-path = /restore
 
+# ----------------------------------------------------------------------
+# automatic tasks on each backup run
+# ----------------------------------------------------------------------
+
+# number of days when to prune and verify backup.
+# set it to 0 to run prune and verify in each backup run.
+prune-after = 3
+verify-after = 7
 
 # ----------------------------------------------------------------------
 #

plugins/transfer/duplicity.sh

@@ -172,7 +172,12 @@
             color reset
             exit 1
         fi
+    }
 
+    # prune old data
+    # uses global vars from ../../transfer.sh
+    function t_backupDoPrune(){
+        echo "--- FORGET some data"
         cmd="duplicity remove-older-than $STORAGE_KEEP --force ${ARGS_DEFAULT} ${STORAGE_TARGETPATH}"
         echo $cmd
         color cmd
@@ -181,8 +186,14 @@
         color reset
         
         t_rcCheckCleanup $myrc
-
     }
+    # verify backup data
+    # uses global vars from ../../transfer.sh
+    function t_backupDoVerify(){
+        echo "--- VERIFY is not implemented yet for duplicity."
+    }
+
+
 # --------------------------------------------------------------------------------
 # RESTORE
 # --------------------------------------------------------------------------------

plugins/transfer/restic.sh

@@ -142,6 +142,11 @@
             # detect return code ... and abort on any error.
             t_rcCheckInit $_myrc
         fi
+        echo restic unlock ${ARGS_DEFAULT}
+        color cmd
+        eval restic unlock ${ARGS_DEFAULT}
+        color reset
+        echo
     }
 
 
@@ -157,6 +162,11 @@
         color reset
         echo
 
+    }
+
+    # prune old data
+    # uses global vars from ../../transfer.sh
+    function t_backupDoPrune(){
         # --------------------
         echo "--- FORGET some data"
         local _tag=$( _j_getvar ${STORAGEFILE} "${CFGPREFIX}tag")
@@ -199,7 +209,10 @@
 
         t_rcCheckPrune $_myrc
         echo
-
+    }
+    # verify backup data
+    # uses global vars from ../../transfer.sh
+    function t_backupDoVerify(){
         # --------------------
         echo "--- VERIFY"
         # param --read-data takes a long time. Maybe use an extra job with it.
@@ -214,7 +227,6 @@
 
         t_rcCheckVerify $_myrc
         echo
-
     }
 
 # --------------------------------------------------------------------------------
@@ -385,7 +397,7 @@
     # repository cleanup
     # param  integer  exitcode of command
     function t_rcCheckCleanup(){
-        echo -n "__PRUNE__ "
+        echo -n "__CLEANUP__ "
         case $1 in
             0) color ok;      echo "OK" ;;
             *) color error;   echo "FAILED ${2} - Cleanup error - returncode was $1" ;;

prune.sh

+#!/bin/bash
+$(dirname $0 )/transfer.sh prune
\ No newline at end of file

transfer.sh

@@ -30,6 +30,7 @@
 # 2021-05-19  ah,ds, v2.0  plugin driven to support multiple backup tools (duplicity + restic)
 # 2021-12-02  ah     v2.1  added parameter "prune" to cleanup only
 # 2022-02-10  ah     v2.2  update logging (removing tee)
+# 2022-10-01  ah     v2.3  customize prune and verify action
 # ================================================================================
 
 
@@ -44,7 +45,9 @@
 
   typeset -i rc=0
   typeset -i doBackup=1
-  typeset -i doPrune=1
+  typeset -i doPrune
+  typeset -i doVerify
+  typeset -i doValue
 
   if [ ! -r "${DIRFILE}" -o ! -r "${STORAGEFILE}" ]; then
     echo "SKIP backup of local files - one of the files is not readable (no error): ${DIRFILE} | ${STORAGEFILE}"
@@ -97,15 +100,58 @@
   # first param "full"
   METHOD=
 
+  ACTION=backup
 
   transferlog="${DIR_LOGS}/transfer-$(date +%Y%m%d-%H%M%S).log"
   lockfile="${DIR_LOGS}/transfer.running"
+  lastbackupfile="${DIR_LOGS}/last_backup"
+  lastprunefile="${DIR_LOGS}/last_prune"
+  lastverifyfile="${DIR_LOGS}/last_verify"
   rcfile=/tmp/transfer-rc.$$.tmp
 
+
 # --------------------------------------------------------------------------------
 # FUNCTIONS
 # --------------------------------------------------------------------------------
 
+# get age of a file in sec
+# https://www.axel-hahn.de/blog/2020/03/09/bash-snippet-alter-einer-datei/
+# param  string  filename to test
+# return integer
+function _getFileAge(){
+    echo $(($(date +%s) - $(date +%s -r "$1")))
+}
+
+# set default behaviur based on limit (from config) and its last
+# execution (read from touchfile)
+# Function is enabled if no touchfile was found or it is older than
+# given limit
+#
+# It is used to set flags for prune and verify
+# param  string  name of action; one of prune|verify
+# param  string  filename of touchfile of last execution
+function setAction(){
+  local action=$1
+  local myfile=$2
+
+  typeset -i local iLimit=$(_j_getvar ${STORAGEFILE} "$action-after")
+
+  if [ ! -f "${myfile}" ]; then
+    echo "Info: $action is ENABLED - no last $action detected"
+    doValue=1
+  else
+    typeset -i iLastDone=$( _getFileAge "${myfile}" )/60
+    typeset -i iLastDoneD=iLastDone/60/24
+    echo "Info: Last $action was $iLastDone min ago ($iLastDoneD days). Limit is $iLimit days."
+    if [ $iLastDoneD -ge $iLimit ]; then 
+      echo "Info: $action is ENABLED - last $action is outdated"
+      doValue=1
+    else
+      echo "Info: $action is not needed yet."
+      doValue=0
+    fi
+  fi
+}
 
 # --------------------------------------------------------------------------------
 # MAIN
@@ -116,32 +162,61 @@
 
   Transfer local files to a backup target.
 
-    target       ${STORAGE_BASEDIR}
+    target       $( echo ${STORAGE_BASEDIR} | sed 's#:[^:]*@#:**********@#' )
     backup tool  $STORAGE_BIN
 
   PARAMETERS:
-    transfer.sh       - incremental backup
-    transfer.sh full  - full backup
-    transfer.sh dumps - transfer local dumps only
-    transfer.sh prune - cleanup backup data only (no backup)
-    transfer.sh help  - show this help (works with -h and -? too)
+    transfer.sh        - incremental backup
+    transfer.sh full   - full backup (Duplicity only)
+    transfer.sh dumps  - transfer local dumps only
+    transfer.sh prune  - cleanup backup data only (no backup)
+    transfer.sh verify - verify backup data (no backup)
+    transfer.sh help   - show this help (works with -h and -? too)
     "
     exit 0
   fi
+
+  # set defaults for prune and verify
+  echo ">>> Detect default behaviuor:"
+  setAction "prune"  $lastprunefile;  doPrune=$doValue
+  setAction "verify" $lastverifyfile; doVerify=$doValue
+  echo
+
+  echo ">>> Check parameters"
   if [ "$1" = "prune" ]; then
+    echo "Info: Forcing prune only by parameter."
+    ACTION=$1
     doBackup=0
+    doPrune=1
+    doVerify=0
+    transferlog="${DIR_LOGS}/prune-$(date +%Y%m%d-%H%M%S).log"
+  fi
+  if [ "$1" = "verify" ]; then
+    echo "Info: Forcing verify only by parameter."
+    ACTION=$1
+    doBackup=0
+    doPrune=0
+    doVerify=1
+    transferlog="${DIR_LOGS}/verify-$(date +%Y%m%d-%H%M%S).log"
   fi
   exec 1> >( tee -a "$transferlog" ) 2>&1
-  echo "INFO: Start logging into $transferlog"
+  echo "INFO: Start logging into $transferlog"  
 
   h1 `date` TRANSFER LOCAL DATA TO STORAGE
 
-  echo "METHOD   : $METHOD"
-  echo "TARGET   : ${STORAGE_BASEDIR}"
-  echo "REGISTER : ${STORAGE_REGISTER}"
   echo "TOOL     : $STORAGE_BIN"
+  echo "ACTION   : $ACTION"
+  echo "TARGET   : ${STORAGE_BASEDIR}" | sed 's#:[^:]*@#:**********@#'
   echo
 
+  if [ "$ACTION" = "backup" ]; then
+    echo "METHOD   : $METHOD"
+    echo "REGISTER : ${STORAGE_REGISTER}"
+    echo "PRUNE    : $doPrune"
+    echo "VERIFY   : $doVerify"
+    echo
+  fi
+
   . `dirname $0`/plugins/transfer/$STORAGE_BIN.sh || exit 1
   test -z "$STORAGE_REGISTER" || . `dirname $0`/plugins/register/$STORAGE_REGISTER.sh || exit 1
 
@@ -264,66 +339,89 @@
       if [ -d "$mydir" ]; then
 
         BACKUP_DIR=$mydir
-        h2 "`date` STORE $BACKUP_DIR"
 
-        # --- build parameters
-        sSafeName=`j_getSafename "$BACKUP_DIR"`
-        sTarget="$( t_backupDirGetTarget $BACKUP_DIR )"
+        if [ "$ACTION" = "backup" ]; then
+          
+          h2 "`date` STORE $BACKUP_DIR"
 
-        ARGS_BACKUP="${sParamExclude} $( t_getParamBackup )"
+          # --- build parameters
+          sSafeName=`j_getSafename "$BACKUP_DIR"`
+          sTarget="$( t_backupDirGetTarget $BACKUP_DIR )"
 
+          ARGS_BACKUP="${sParamExclude} $( t_getParamBackup )"
 
-        # detect custom backup sets and add its includes and excludes
-        backupid=`j_getSetnameOfPath "$BACKUP_DIR"`
-        sSpaceReplace="___SPACE___"
 
+          # detect custom backup sets and add its includes and excludes
+          backupid=`j_getSetnameOfPath "$BACKUP_DIR"`
+          sSpaceReplace="___SPACE___"
 
-        if [ ! -z $backupid ]; then
-          for sItem in `_j_getvar ${DIRFILE} "${backupid}\-\-include" | sed "s#\ #${sSpaceReplace}#g"`
-          do
-            ARGS_BACKUP="${ARGS_BACKUP} $( t_getParamInlude $sItem)"
-          done
-          for sItem in `_j_getvar ${DIRFILE} "${backupid}\-\-exclude" | sed "s#\ #${sSpaceReplace}#g"`
-          do
-            ARGS_BACKUP="${ARGS_BACKUP} $( t_getParamExlude $sItem)"
-          done
-        fi
 
+          if [ ! -z $backupid ]; then
+            for sItem in `_j_getvar ${DIRFILE} "${backupid}\-\-include" | sed "s#\ #${sSpaceReplace}#g"`
+            do
+              ARGS_BACKUP="${ARGS_BACKUP} $( t_getParamInlude $sItem)"
+            done
+            for sItem in `_j_getvar ${DIRFILE} "${backupid}\-\-exclude" | sed "s#\ #${sSpaceReplace}#g"`
+            do
+              ARGS_BACKUP="${ARGS_BACKUP} $( t_getParamExlude $sItem)"
+            done
+          fi
 
-        # --- pre task
-        h3 "`date` PRE backup task for ${BACKUP_DIR}"
-        t_backupDirDoPreTasks
-        # sCmdPre="$( t_backupDirDoPreTasks )"
 
+          # --- pre task
+          h3 "`date` PRE backup task for ${BACKUP_DIR}"
+          t_backupDirDoPreTasks
+          # sCmdPre="$( t_backupDirDoPreTasks )"
 
-        # --- backup
-        h3 "`date` Backup ${BACKUP_DIR}"
-        if [ $doBackup -eq 0 ]; then
-          echo "SKIP backup"
-        else
-          sCmd="$( t_backupDirGetCmdBackup )"
-          echo "what:    ${BACKUP_DIR}"
-          echo "target:  ${sTarget}"
-          echo "command: $sCmd"
-          echo
-          color cmd
-          $sCmd
-          fetchrc
-          color reset
+
+          # --- backup
+          h3 "`date` Backup ${BACKUP_DIR}"
+          if [ $doBackup -eq 0 ]; then
+            echo "SKIP backup"
+          else
+            sCmd="$( t_backupDirGetCmdBackup )"
+            echo "what:    ${BACKUP_DIR}"
+            echo "target:  ${sTarget}"
+            echo "command: $sCmd"
+            echo
+            color cmd
+            $sCmd
+            fetchrc
+            color reset
+            echo
+
+            t_rcCheckBackup $myrc "${BACKUP_DIR}"
+
+          fi
           echo
 
-          t_rcCheckBackup $myrc "${BACKUP_DIR}"
 
-        fi
-        echo
+          # --- post action
+          h3 "`date` POST backup task for ${BACKUP_DIR}"
+          t_backupDirDoPostTasks
+          echo
 
+          touch ${lastbackupfile}
+        fi
 
-        # --- post action
-        h3 "`date` POST backup task for ${BACKUP_DIR}"
+        # --- prune
         if [ $doPrune -eq 0 ]; then
           echo "SKIP prune"
         else
-          t_backupDirDoPostTasks
+          h3 "`date` PRUNE for ${BACKUP_DIR}"
+          t_backupDoPrune
+          touch ${lastprunefile}
+          echo
+        fi
+        echo
+
+        # --- verify
+        if [ $doVerify -eq 0 ]; then
+          echo "SKIP verify"
+        else
+          h3 "`date` VERIFY for ${BACKUP_DIR}"
+          t_backupDoVerify
+          touch ${lastverifyfile}
           echo
         fi
         echo
@@ -373,7 +471,7 @@
   color reset
   echo
   typeset -i TIMER_TRANSFER=`date +%s`-$TIMER_TRANSFER_START
-  echo `date` TRANSFER DONE in $TIMER_TRANSFER sec
+  echo `date` $ACTION DONE in $TIMER_TRANSFER sec
 
   ls -l $transferlog                                                                  
   exit $rc

verify.sh

+#!/bin/bash
+$(dirname $0 )/transfer.sh verify
\ No newline at end of file