Merge branch '5046-finetune-restic-params' into 'master'

5046 finetune restic params

See merge request !38

docs/30_Configuration/30_Filetransfer.md

@@ -169,17 +169,36 @@ s, m, h, D, W, M, or Y (indicating seconds, minutes, hours, days, weeks, months,
 Restic works like a repository and uses deduplication. The first run of a file backup
 is a full backup and all following runs make an incremental backup.
 
+There are 2 simple cleanup variants for restic forget:
+
+* The value for "keep-last" never deletes the last N most recent snapshots.
+* "restic_keep-within" keeps all created snapshots within a given period (eg. 1y5m7d2h) 
+
+If you need more control and want less snapshots as older they are:
+
 You have the possibility to keep all younger backups and have the possibility to delete
 older backups but to keep N weekly, monthly and yearly snapshots.
 
-The value for "keep-last" never deletes the last N most recent snapshots.
+Multiple values are allowed and work like an OR condition.
+See the docs for more details to the keep values of the forget command <https://restic.readthedocs.io/en/latest/060_forget.html>
 
 ```text
 # prune 
-restic_keep-last = 5
-restic_keep-hourly = 100
-restic_keep-daily = 90
-restic_keep-weekly = 12
-restic_keep-monthly = 12
-restic_keep-yearly = 10
+restic_keep-within = 6m
+# restic_keep-last = 5
+# restic_keep-hourly = 10
+# restic_keep-daily = 30
+# restic_keep-weekly = 12
+# restic_keep-monthly = 12
+# restic_keep-yearly = 10
 ```
+
+Supported values are:
+
+* restic_keep-last [N] - keep the last n snapshots
+* restic_keep-(hourly|daily|weekly|monthly|yearly) [N] - keep then last n snapshots of a period
+* restic_keep-within [duration] - keep snapshots that are newer than duration (eg. 1y5m7d2h) relative to the latest snapshot
+* restic_keep-within-(hourly|daily|weekly|monthly|yearly) [duration] - keep snapshots of given period that are newer than duration (eg. 1y5m7d2h) relative to the latest snapshot
+* restic_keep-tag [taglist] - keep snapshots with this taglist (can be specified only one in imlbackup) (default [])
+
+`restic_keep-within = 6m`
\ No newline at end of file

docs/30_Configuration/50_File_transfer.job.md

@@ -255,13 +255,26 @@ restic_tag = imlbackup
 restic_mountpoint = /mnt/restore
 
 # prune 
-restic_keep-hourly = 100
-restic_keep-daily = 90
-restic_keep-weekly = 12
-restic_keep-monthly = 12
-restic_keep-yearly = 10
+restic_keep-within = 6m
+# restic_keep-last = 5
+# restic_keep-hourly = 10
+# restic_keep-daily = 30
+# restic_keep-weekly = 12
+# restic_keep-monthly = 12
+# restic_keep-yearly = 10
 ```
 
+For pruning with restic forget the following values can be used. 
+Multiple values are allowed and work like an OR condition.
+
+See the docs for more details to the keep values of the forget command <https://restic.readthedocs.io/en/latest/060_forget.html>
+
+* restic_keep-last [N] - keep the last n snapshots
+* restic_keep-(hourly|daily|weekly|monthly|yearly) [N] - keep then last n snapshots of a period
+* restic_keep-within [duration] - keep snapshots that are newer than duration (eg. 1y5m7d2h) relative to the latest snapshot
+* restic_keep-within-(hourly|daily|weekly|monthly|yearly) [duration] - keep snapshots of given period that are newer than duration (eg. 1y5m7d2h) relative to the latest snapshot
+* restic_keep-tag [taglist] - keep snapshots with this taglist (can be specified only one in imlbackup) (default [])
+
 ## restore-path
 
 Default path where to put restored file when running *restore.sh*.

jobs/transfer.job.dist

@@ -147,11 +147,12 @@ restic_tag = imlbackup
 restic_mountpoint = /mnt/restore
 
 # prune 
-restic_keep-last = 5
-restic_keep-hourly = 100
-restic_keep-daily = 90
-restic_keep-weekly = 12
-restic_keep-monthly = 12
-restic_keep-yearly = 10
+restic_keep-within = 6m
+# restic_keep-last = 5
+# restic_keep-hourly = 10
+# restic_keep-daily = 30
+# restic_keep-weekly = 12
+# restic_keep-monthly = 12
+# restic_keep-yearly = 10
 
 # ----------------------------------------------------------------------

plugins/transfer/restic.sh

@@ -9,7 +9,8 @@
 # ah - Axel Hahn <axel.hahn@iml.unibe.ch>
 # 2021-05-19  ah    v0.1   INIT ... WIP
 # 2022-01-06  ah    v0.2   added support for Repository with REST and authentication 
-# 2022-02-09  ah    v0.3   show difff to last backup; update pruning
+# 2022-02-09  ah    v0.3   show diff to last backup; update pruning
+# 2022-02-09  ah    v0.3   update pruning; more keep-params
 # ================================================================================
 
 # --------------------------------------------------------------------------------
@@ -153,10 +154,11 @@
           ${ARGS_DEFAULT} \
           --tag $_tag \
           --group-by paths,tags \
+          --prune \
           --cleanup-cache"
 
         local _keep
-        for mykeep in last hourly daily weekly monthly yearly
+        for mykeep in last hourly daily weekly monthly yearly within within-hourly within-daily within-weekly within-monthly within-yearly tag
         do
             _keep=$( _j_getvar "${STORAGEFILE}" "${CFGPREFIX}keep-${mykeep}")
             if [ -n "$_keep" ]; then

transfer.sh

@@ -29,6 +29,7 @@
 # 2021-01-29  ah,ds, v1.12 abort on empty passphrase
 # 2021-05-19  ah,ds, v2.0  plugin driven to support multiple backup tools (duplicity + restic)
 # 2021-12-02  ah     v2.1  added parameter "prune" to cleanup only
+# 2022-02-10  ah     v2.2  update logging (removing tee)
 # ================================================================================
 
 
@@ -130,14 +131,16 @@
   if [ "$1" = "prune" ]; then
     doBackup=0
   fi
+  exec 1> >( tee -a "$transferlog" ) 2>&1
+  echo "INFO: Start logging into $transferlog"
 
-  h1 `date` TRANSFER LOCAL DATA TO STORAGE                      | tee -a $transferlog
+  h1 `date` TRANSFER LOCAL DATA TO STORAGE
 
-  echo "METHOD   : $METHOD"                                     | tee -a $transferlog
-  echo "TARGET   : ${STORAGE_BASEDIR}"                          | tee -a $transferlog
-  echo "REGISTER : ${STORAGE_REGISTER}"                         | tee -a $transferlog
-  echo "TOOL     : $STORAGE_BIN"                                | tee -a $transferlog
-  echo                                                          | tee -a $transferlog
+  echo "METHOD   : $METHOD"
+  echo "TARGET   : ${STORAGE_BASEDIR}"
+  echo "REGISTER : ${STORAGE_REGISTER}"
+  echo "TOOL     : $STORAGE_BIN"
+  echo
 
   . `dirname $0`/plugins/transfer/$STORAGE_BIN.sh || exit 1
   test -z "$STORAGE_REGISTER" || . `dirname $0`/plugins/register/$STORAGE_REGISTER.sh || exit 1
@@ -242,7 +245,7 @@
           sleep $iRnd
       fi
     done
-  fi | tee -a $transferlog
+  fi
 
 
   h2 "`date` PRE transfer tasks"
@@ -345,7 +348,7 @@
     echo $rc > ${rcfile}
     exit $rc
 
-  ) | tee -a $transferlog 
+  ) 
 
   # rc=${PIPESTATUS[0]}
   rc=`cat ${rcfile}`
@@ -364,22 +367,22 @@
     echo "SKIP"
   else
     unregisterBackupSlot `hostname -f` $rc
-  fi | tee -a $transferlog
+  fi
 
   h2 "`date` Backup finished"
-  echo STATUS $0 exit with final returncode rc=$rc                                    | tee -a $transferlog
-  echo                                                                                | tee -a $transferlog
+  echo STATUS $0 exit with final returncode rc=$rc
+  echo
   if [ $rc -eq 0 ]; then
     color ok
-    echo Backup OK                                                                    | tee -a $transferlog
+    echo Backup OK
   else
     color error
-    echo Backup FAILED :-/                                                            | tee -a $transferlog
+    echo Backup FAILED :-/
   fi
   color reset
-  echo                                                                                | tee -a $transferlog
+  echo
   typeset -i TIMER_TRANSFER=`date +%s`-$TIMER_TRANSFER_START
-  echo `date` TRANSFER DONE in $TIMER_TRANSFER sec                                    | tee -a $transferlog
+  echo `date` TRANSFER DONE in $TIMER_TRANSFER sec
 
   ls -l $transferlog                                                                  
   exit $rc