Merge branch 'prune_and_verify' into 'master'

Prune and verify

See merge request !75

docs/30_Configuration/50_File_transfer.job.md

@@ -107,6 +107,14 @@ sambashares = 1
 
 restore-path = /restore
 
+# ----------------------------------------------------------------------
+# automatic tasks on each backup run
+# ----------------------------------------------------------------------
+
+# number of days when to prune and verify backup.
+# set it to 0 to run prune and verify in each backup run.
+prune-after = 3
+verify-after = 7
 
 # ----------------------------------------------------------------------
 #
@@ -239,6 +247,15 @@ restore any data anymore.
 
 `passphrase = EnterYourSecretHere`
 
+## Prune-after
+
+You can define how often to prune old data.
+Set an integer for age in days here. If it is 0 then it will be executed on each backup run.
+
+`prune-after = 3`
+
+Remark: You can force the prune functionality by executing `transfer.sh prune`
+
 ## restic_*
 
 Variables that are specific for restic. They will be used when you set
@@ -334,3 +351,11 @@ Duplicity supports 2 modes.
 
 see also **bin**
 
+## Verify-after
+
+You can define how often to verify integrity of the backup repoitory data.
+Set an integer for age in days here. If it is 0 then it will be executed on each backup run.
+
+`verify-after = 3`
+
+Remark: You can force the verify functionality by executing `transfer.sh verify`
\ No newline at end of file

docs/40_Usage/10_Backup.md

@@ -38,4 +38,3 @@ OPTIONS:
     -i, --inc     force incrmenental backup
 
 ```
-

docs/40_Usage/30_Filetransfer.md

 ```
-# ./transfer.sh -?
+> ./transfer.sh -h
 
   ___ ___ ___ ___         _______            __
  |   |   Y   |   |       |   _   .---.-.----|  |--.--.--.-----.
@@ -20,14 +20,14 @@ HELP:
 
   Transfer local files to a backup target.
 
-    target       sftp://imlbackup@storage-connector.iml.unibe.ch//netshare/restic-backup
+    target       rest:https://backup:**********@mysynology.dsmynas.net:8000/linux-pc
     backup tool  restic
 
   PARAMETERS:
     transfer.sh        - incremental backup
-    transfer.sh full  - full backup
+    transfer.sh full   - full backup (Duplicity only)
     transfer.sh dumps  - transfer local dumps only
     transfer.sh prune  - cleanup backup data only (no backup)
+    transfer.sh verify - verify backup data (no backup)
     transfer.sh help   - show this help (works with -h and -? too)
-    
 ```

docs/40_Usage/40_Cronjob.md

+## Backup
+
 To start a daily backup at 23:03:
 
 ```shell
@@ -7,7 +9,8 @@ $ cat /etc/cron.d/client-backup
 
 ... or with using the cronwrapper
 
-see <https://github.com/axelhahn/cronwrapper>
+see 📗 <https://www.axel-hahn.de/docs/cronwrapper/>
+🧾 <https://github.com/axelhahn/cronwrapper>
 
 TTL is 1440 for 1440 min = 1d
 
@@ -15,3 +18,29 @@ TTL is 1440 for 1440 min = 1d
 $ cat /etc/cron.d/client-backup 
 3 23 * * * root /usr/local/bin/cronwrapper.sh 1440 /opt/imlbackup/client/backup.sh 'iml-backup'
 ```
+
+## Prune and verify
+
+By default prune and verify are executed on each backup run. This is a safe way. A very safe way.
+
+BUT: By doing this the backup with Restic is running for a few seconds and prune + verify take a much-much longer time. In many cases it is a good idea to split backup and prune + verify.
+
+**Example:**
+
+You want to start a daily (or hourly) backup. Pruning and verify shall run on Saturday and Sunday.
+
+In jobs/transfer.job set a limit greater 7 days:
+
+```txt
+prune-after = 14
+verify-after = 14
+```
+
+Then define your weekly yobs on Saturday and Sunday.
+
+```shell
+$ cat /etc/cron.d/client-backup 
+3 23 * * * root /usr/local/bin/cronwrapper.sh 1440 /opt/imlbackup/client/backup.sh 'iml-backup'
+7 18 * * 6 root /usr/local/bin/cronwrapper.sh 1440 /opt/imlbackup/client/prune.sh 'iml-backup-prune'
+7 18 * * 0 root /usr/local/bin/cronwrapper.sh 1440 /opt/imlbackup/client/verify.sh 'iml-backup-verify'
+```

docs/99_Glossary.md

+## Explaination of some words
+
+* **backup** = save current data "somewhare".<br>Store your backup outside the local system to have a copy you can access if the system is damaged.
+
+* **cronjob** = start a command in a defined cycle. ie. eve3ry hour, every day, once a week and so on
+
+* **database dump** = backup a database / scheme into a single file
+
+* **full backup** = backup of all data of the local system.
+
+* **incremental backup** = backup of data that have changed since the last backup.
+
+* **prune** = delete old backup sets and thin data in the backup repository. <br>As an example: You can make an hourly backup and define to keep all of them for a week, daily backups for 90 days and monthly backups starting from 1 to 36 monthes (=3 years). The prune actions removes data in the backup storage to thin out your backup data.
+  
+* **restore** = copy data back from a made backup to the local system.
+
+* **ttl** = time to live<br>Definition how long an information is valid before it needs to be updated again.
+
+* **verify** = check the integrity of backup data
\ No newline at end of file

jobs/transfer.job.dist

@@ -104,6 +104,14 @@ sambashares = 1
 
 restore-path = /restore
 
+# ----------------------------------------------------------------------
+# automatic tasks on each backup run
+# ----------------------------------------------------------------------
+
+# number of days when to prune and verify backup.
+# set it to 0 to run prune and verify in each backup run.
+prune-after = 3
+verify-after = 7
 
 # ----------------------------------------------------------------------
 #

plugins/transfer/duplicity.sh

@@ -172,7 +172,12 @@
             color reset
             exit 1
         fi
+    }
 
+    # prune old data
+    # uses global vars from ../../transfer.sh
+    function t_backupDoPrune(){
+        echo "--- FORGET some data"
         cmd="duplicity remove-older-than $STORAGE_KEEP --force ${ARGS_DEFAULT} ${STORAGE_TARGETPATH}"
         echo $cmd
         color cmd
@@ -181,8 +186,14 @@
         color reset
         
         t_rcCheckCleanup $myrc
-
     }
+    # verify backup data
+    # uses global vars from ../../transfer.sh
+    function t_backupDoVerify(){
+        echo "--- VERIFY is not implemented yet for duplicity."
+    }
+
+
 # --------------------------------------------------------------------------------
 # RESTORE
 # --------------------------------------------------------------------------------

plugins/transfer/restic.sh

@@ -142,6 +142,11 @@
             # detect return code ... and abort on any error.
             t_rcCheckInit $_myrc
         fi
+        echo restic unlock ${ARGS_DEFAULT}
+        color cmd
+        eval restic unlock ${ARGS_DEFAULT}
+        color reset
+        echo
     }
 
 
@@ -157,6 +162,11 @@
         color reset
         echo
 
+    }
+
+    # prune old data
+    # uses global vars from ../../transfer.sh
+    function t_backupDoPrune(){
         # --------------------
         echo "--- FORGET some data"
         local _tag=$( _j_getvar ${STORAGEFILE} "${CFGPREFIX}tag")
@@ -199,7 +209,10 @@
 
         t_rcCheckPrune $_myrc
         echo
-
+    }
+    # verify backup data
+    # uses global vars from ../../transfer.sh
+    function t_backupDoVerify(){
         # --------------------
         echo "--- VERIFY"
         # param --read-data takes a long time. Maybe use an extra job with it.
@@ -214,7 +227,6 @@
 
         t_rcCheckVerify $_myrc
         echo
-
     }
 
 # --------------------------------------------------------------------------------
@@ -385,7 +397,7 @@
     # repository cleanup
     # param  integer  exitcode of command
     function t_rcCheckCleanup(){
-        echo -n "__PRUNE__ "
+        echo -n "__CLEANUP__ "
         case $1 in
             0) color ok;      echo "OK" ;;
             *) color error;   echo "FAILED ${2} - Cleanup error - returncode was $1" ;;

prune.sh

+#!/bin/bash
+$(dirname $0 )/transfer.sh prune
\ No newline at end of file

transfer.sh

@@ -30,6 +30,7 @@
 # 2021-05-19  ah,ds, v2.0  plugin driven to support multiple backup tools (duplicity + restic)
 # 2021-12-02  ah     v2.1  added parameter "prune" to cleanup only
 # 2022-02-10  ah     v2.2  update logging (removing tee)
+# 2022-10-01  ah     v2.3  customize prune and verify action
 # ================================================================================
 
 
@@ -44,7 +45,9 @@
 
   typeset -i rc=0
   typeset -i doBackup=1
-  typeset -i doPrune=1
+  typeset -i doPrune
+  typeset -i doVerify
+  typeset -i doValue
 
   if [ ! -r "${DIRFILE}" -o ! -r "${STORAGEFILE}" ]; then
     echo "SKIP backup of local files - one of the files is not readable (no error): ${DIRFILE} | ${STORAGEFILE}"
@@ -97,15 +100,58 @@
   # first param "full"
   METHOD=
 
+  ACTION=backup
 
   transferlog="${DIR_LOGS}/transfer-$(date +%Y%m%d-%H%M%S).log"
   lockfile="${DIR_LOGS}/transfer.running"
+  lastbackupfile="${DIR_LOGS}/last_backup"
+  lastprunefile="${DIR_LOGS}/last_prune"
+  lastverifyfile="${DIR_LOGS}/last_verify"
   rcfile=/tmp/transfer-rc.$$.tmp
 
+
 # --------------------------------------------------------------------------------
 # FUNCTIONS
 # --------------------------------------------------------------------------------
 
+# get age of a file in sec
+# https://www.axel-hahn.de/blog/2020/03/09/bash-snippet-alter-einer-datei/
+# param  string  filename to test
+# return integer
+function _getFileAge(){
+    echo $(($(date +%s) - $(date +%s -r "$1")))
+}
+
+# set default behaviur based on limit (from config) and its last
+# execution (read from touchfile)
+# Function is enabled if no touchfile was found or it is older than
+# given limit
+#
+# It is used to set flags for prune and verify
+# param  string  name of action; one of prune|verify
+# param  string  filename of touchfile of last execution
+function setAction(){
+  local action=$1
+  local myfile=$2
+
+  typeset -i local iLimit=$(_j_getvar ${STORAGEFILE} "$action-after")
+
+  if [ ! -f "${myfile}" ]; then
+    echo "Info: $action is ENABLED - no last $action detected"
+    doValue=1
+  else
+    typeset -i iLastDone=$( _getFileAge "${myfile}" )/60
+    typeset -i iLastDoneD=iLastDone/60/24
+    echo "Info: Last $action was $iLastDone min ago ($iLastDoneD days). Limit is $iLimit days."
+    if [ $iLastDoneD -ge $iLimit ]; then 
+      echo "Info: $action is ENABLED - last $action is outdated"
+      doValue=1
+    else
+      echo "Info: $action is not needed yet."
+      doValue=0
+    fi
+  fi
+}
 
 # --------------------------------------------------------------------------------
 # MAIN
@@ -116,31 +162,60 @@
 
   Transfer local files to a backup target.
 
-    target       ${STORAGE_BASEDIR}
+    target       $( echo ${STORAGE_BASEDIR} | sed 's#:[^:]*@#:**********@#' )
     backup tool  $STORAGE_BIN
 
   PARAMETERS:
     transfer.sh        - incremental backup
-    transfer.sh full  - full backup
+    transfer.sh full   - full backup (Duplicity only)
     transfer.sh dumps  - transfer local dumps only
     transfer.sh prune  - cleanup backup data only (no backup)
+    transfer.sh verify - verify backup data (no backup)
     transfer.sh help   - show this help (works with -h and -? too)
     "
     exit 0
   fi
+
+  # set defaults for prune and verify
+  echo ">>> Detect default behaviuor:"
+  setAction "prune"  $lastprunefile;  doPrune=$doValue
+  setAction "verify" $lastverifyfile; doVerify=$doValue
+  echo
+
+  echo ">>> Check parameters"
   if [ "$1" = "prune" ]; then
+    echo "Info: Forcing prune only by parameter."
+    ACTION=$1
+    doBackup=0
+    doPrune=1
+    doVerify=0
+    transferlog="${DIR_LOGS}/prune-$(date +%Y%m%d-%H%M%S).log"
+  fi
+  if [ "$1" = "verify" ]; then
+    echo "Info: Forcing verify only by parameter."
+    ACTION=$1
     doBackup=0
+    doPrune=0
+    doVerify=1
+    transferlog="${DIR_LOGS}/verify-$(date +%Y%m%d-%H%M%S).log"
   fi
   exec 1> >( tee -a "$transferlog" ) 2>&1
   echo "INFO: Start logging into $transferlog"  
 
   h1 `date` TRANSFER LOCAL DATA TO STORAGE
 
+  echo "TOOL     : $STORAGE_BIN"
+  echo "ACTION   : $ACTION"
+  echo "TARGET   : ${STORAGE_BASEDIR}" | sed 's#:[^:]*@#:**********@#'
+  echo
+
+  if [ "$ACTION" = "backup" ]; then
     echo "METHOD   : $METHOD"
-  echo "TARGET   : ${STORAGE_BASEDIR}"
     echo "REGISTER : ${STORAGE_REGISTER}"
-  echo "TOOL     : $STORAGE_BIN"
+    echo "PRUNE    : $doPrune"
+    echo "VERIFY   : $doVerify"
     echo
+  fi
 
   . `dirname $0`/plugins/transfer/$STORAGE_BIN.sh || exit 1
   test -z "$STORAGE_REGISTER" || . `dirname $0`/plugins/register/$STORAGE_REGISTER.sh || exit 1
@@ -264,6 +339,9 @@
       if [ -d "$mydir" ]; then
 
         BACKUP_DIR=$mydir
+
+        if [ "$ACTION" = "backup" ]; then
+          
           h2 "`date` STORE $BACKUP_DIR"
 
           # --- build parameters
@@ -320,10 +398,30 @@
 
           # --- post action
           h3 "`date` POST backup task for ${BACKUP_DIR}"
+          t_backupDirDoPostTasks
+          echo
+
+          touch ${lastbackupfile}
+        fi
+
+        # --- prune
         if [ $doPrune -eq 0 ]; then
           echo "SKIP prune"
         else
-          t_backupDirDoPostTasks
+          h3 "`date` PRUNE for ${BACKUP_DIR}"
+          t_backupDoPrune
+          touch ${lastprunefile}
+          echo
+        fi
+        echo
+
+        # --- verify
+        if [ $doVerify -eq 0 ]; then
+          echo "SKIP verify"
+        else
+          h3 "`date` VERIFY for ${BACKUP_DIR}"
+          t_backupDoVerify
+          touch ${lastverifyfile}
           echo
         fi
         echo
@@ -373,7 +471,7 @@
   color reset
   echo
   typeset -i TIMER_TRANSFER=`date +%s`-$TIMER_TRANSFER_START
-  echo `date` TRANSFER DONE in $TIMER_TRANSFER sec
+  echo `date` $ACTION DONE in $TIMER_TRANSFER sec
 
   ls -l $transferlog                                                                  
   exit $rc

verify.sh

+#!/bin/bash
+$(dirname $0 )/transfer.sh verify
\ No newline at end of file