Merge branch '5046-finetune-restic-params' into 'master'

5046 finetune restic params

See merge request !32

backup.sh

@@ -179,7 +179,8 @@ EOFbackupinfo
       cat "$STORAGEFILE" >>"$JOB_LOGFILE"
       echo "INFO: $(date) - Sending data to storage... ${DIR_SELF}/transfer.sh $JOBDEF_TYPE" | tee -a "$JOB_LOGFILE"
       touch "${JOB_DONEFILE}.02.STARTTRANSFER"
-      "${DIR_SELF}"/transfer.sh $JOBDEF_TYPE "$JOBDEF_AUTO" 2>&1 | tee -a "$JOB_LOGFILE"
+      # "${DIR_SELF}"/transfer.sh $JOBDEF_TYPE "$JOBDEF_AUTO" 2>&1 | tee -a "$JOB_LOGFILE"
+      "${DIR_SELF}"/transfer.sh $JOBDEF_TYPE "$JOBDEF_AUTO" 2>&1
       rcTransfer=$?
 
     else
@@ -193,8 +194,9 @@ EOFbackupinfo
     touch "${JOB_DONEFILE}.03.DONE"
 
     echo
-    echo "log for this executed job is"
-    ls -l "$JOB_LOGFILE"
+    echo "INFO: $(date) - compress older logs"
+    echo find "${DIR_LOGS}" -name "*.log" -mtime +2 -print -exec gzip {} \;
+    find "${DIR_LOGS}" -name "*.log" -mtime +2 -print -exec gzip {} \;
     echo
 
     echo "INFO: $(date) - cleanup logs"
@@ -204,6 +206,9 @@ EOFbackupinfo
 
     echo STATUS $0 exit with final returncode rc=$rcBackup | tee -a "$JOB_LOGFILE"
     echo
+    echo "log for this executed job is"
+    ls -l "$JOB_LOGFILE"
+    echo
 
   else
     echo "Nothing to do."

docs/30_Configuration/30_Filetransfer.md

@@ -114,11 +114,16 @@ If your /root partition has a limited size and/ or you have a lot of files
 
 `restic_cachedir = [full path]`
 
-The default verbose of IML backup for restic is 2. During file transfer
-this shows unchanged/ new/ deleted files. Change it to a less value
-to generate less log data.
+Restic by default stores file indexes locally. To save space in `restic_cachedir`
+you can disable the file index but the backup will run much slower.
 
-`restic_verbose = 2`
+`restic_nochache = false`
+
+The default verbose of IML backup for restic is 0. 
+If you want to see unchanged/ new/ deleted files set it to "2". But This generates
+larger logs in [installdir]/logs/.
+
+`restic_verbose = 0`
 
 A file transfer can be tagged. Each restic backup gets a defined tag
 
@@ -167,8 +172,11 @@ is a full backup and all following runs make an incremental backup.
 You have the possibility to keep all younger backups and have the possibility to delete
 older backups but to keep N weekly, monthly and yearly snapshots.
 
+The value for "keep-last" never deletes the last N most recent snapshots.
+
 ```text
 # prune 
+restic_keep-last = 5
 restic_keep-hourly = 100
 restic_keep-daily = 90
 restic_keep-weekly = 12

docs/30_Configuration/50_File_transfer.job.md

@@ -247,8 +247,9 @@ bin = restic.
 ```text
 # use another cache directory than /root/.cache/restic
 restic_cachedir =  
+restic_nocache = false
 
-restic_verbose = 2
+restic_verbose = 0
 restic_tag = imlbackup
 
 restic_mountpoint = /mnt/restore

jobs/transfer.job.dist

@@ -139,13 +139,15 @@ duplicity_cachedir =
 
 # use another cache directory than /root/.cache/restic
 restic_cachedir =  
+restic_nocache = false
 
-restic_verbose = 2
+restic_verbose = 0
 restic_tag = imlbackup
 
 restic_mountpoint = /mnt/restore
 
 # prune 
+restic_keep-last = 5
 restic_keep-hourly = 100
 restic_keep-daily = 90
 restic_keep-weekly = 12

plugins/transfer/restic.sh

@@ -58,8 +58,20 @@
     }
     # return a string with backup parameters that will be added to defaults
     function t_getParamBackup(){
+        local _tag
+        local _nocacheFlag
+
         # tagging
-        echo -n --tag $( _j_getvar ${STORAGEFILE} "${CFGPREFIX}tag" )
+        _tag=$( _j_getvar ${STORAGEFILE} "${CFGPREFIX}tag" )
+        if [ "$_tag" != "" ]; then
+            echo -n "--tag $_tag "
+        fi
+
+        # no cache ... to create smaller local cache dirs, but backup 3 times slower 
+        _nocacheFlag=$( _j_getvar ${STORAGEFILE} "${CFGPREFIX}nocache" )
+        if [ "$_nocacheFlag" != "" ] && [ "$_nocacheFlag" != "0" ] && [ "$_nocacheFlag" != "false" ]; then
+            echo -n "--no-cache "
+        fi
     }
 
     # return a cli parameter for a single exlude directory
@@ -126,19 +138,6 @@
     # uses global vars from ../../transfer.sh
     function t_backupDoPostTasks(){
 
-        echo "--- CLEANUP local data:"
-        echo restic cache --cleanup
-        color cmd
-        restic cache --cleanup
-        local _myrc=$?
-        color reset
-        case $_myrc in
-            0) color ok;      echo "OK" ;;
-            *) color error;   echo "Cleanup error - returncode was $_myrc" ;;
-        esac
-        color reset
-        echo
-
         echo "--- UNLOCK ... just in case :-)" 
         echo restic unlock ${ARGS_DEFAULT}
         color cmd
@@ -148,23 +147,24 @@
 
         echo "--- PRUNE"
         local _tag=$( _j_getvar ${STORAGEFILE} "${CFGPREFIX}tag")
-
-        local _keep_h=$( _j_getvar ${STORAGEFILE} "${CFGPREFIX}keep-hourly")
-        local _keep_d=$( _j_getvar ${STORAGEFILE} "${CFGPREFIX}keep-daily")
-        local _keep_w=$( _j_getvar ${STORAGEFILE} "${CFGPREFIX}keep-weekly")
-        local _keep_m=$( _j_getvar ${STORAGEFILE} "${CFGPREFIX}keep-monthly")
-        local _keep_y=$( _j_getvar ${STORAGEFILE} "${CFGPREFIX}keep-yearly")
         
         local _mycmd="restic forget \
           ${ARGS_DEFAULT} \
           --tag $_tag \
-          --group-by "paths,tags" \
-          --keep-hourly $_keep_h \
-          --keep-daily $_keep_d \
-          --keep-weekly $_keep_w \
-          --keep-monthly $_keep_m \
-          --keep-yearly $_keep_y"
+          --group-by paths,tags \
+          --cleanup-cache"
+
+        local _keep
+        for mykeep in last hourly daily weekly monthly yearly
+        do
+            _keep=$( _j_getvar "${STORAGEFILE}" "${CFGPREFIX}keep-${mykeep}")
+            if [ -n "$_keep" ]; then
+                _mycmd="${_mycmd} --keep-${mykeep} ${_keep}"
+            fi
+        done
+
         echo $_mycmd
+        sleep 3
         color cmd 
         eval $_mycmd
         local _myrc=$?
@@ -201,7 +201,28 @@
     # post backup actions
     # uses global vars from ../../transfer.sh
     function t_backupDirDoPostTasks(){
-        echo "Nothing to do."
+        echo "--- SHOW CHANGES between last 2 snapshots" 
+        local _data
+        local _snapshotLast
+        local _snapshotNow
+
+        # get list of snapshots and filter the lines with a date YYYY-MM-DD
+        _data=$( t_restoreDoShowVolumes | grep "[12][0-9][0-9][0-1]-[0-2][0-9]-[0-3][0-9]" | tail -5 )
+        echo "..."
+        echo "$_data"
+
+        _snapshotLast=$( echo "$_data" | tail -2 | head -1 | cut -f 1 -d " ")
+        _snapshotNow=$(  echo "$_data" | tail -1           | cut -f 1 -d " ")
+
+        if [ "${_snapshotLast}" = "${_snapshotNow}" ]; then
+            echo "This was the initial (full) Backup"
+        else
+            color cmd
+            restic diff "${_snapshotLast}" "${_snapshotNow}"
+            color reset
+        fi
+        echo
+
     }
 
 # --------------------------------------------------------------------------------

transfer.sh

@@ -260,17 +260,18 @@
 
       if [ -d "$mydir" ]; then
 
-        h2 "`date` STORE $mydir" 
+        BACKUP_DIR=$mydir
+        h2 "`date` STORE $BACKUP_DIR"
 
         # --- build parameters
-        sSafeName=`j_getSafename "$mydir"`
-        sTarget="$( t_backupDirGetTarget $mydir )"
+        sSafeName=`j_getSafename "$BACKUP_DIR"`
+        sTarget="$( t_backupDirGetTarget $BACKUP_DIR )"
 
         ARGS_BACKUP="${sParamExclude} $( t_getParamBackup )"
 
 
         # detect custom backup sets and add its includes and excludes
-        backupid=`j_getSetnameOfPath "$mydir"`
+        backupid=`j_getSetnameOfPath "$BACKUP_DIR"`
         sSpaceReplace="___SPACE___"
 
 
@@ -287,18 +288,18 @@
 
 
         # --- pre task
-        h3 "`date` PRE backup task for ${mydir}"
+        h3 "`date` PRE backup task for ${BACKUP_DIR}"
         t_backupDirDoPreTasks
         # sCmdPre="$( t_backupDirDoPreTasks )"
 
 
         # --- backup
-        h3 "`date` Backup ${mydir}"
+        h3 "`date` Backup ${BACKUP_DIR}"
         if [ $doBackup -eq 0 ]; then
           echo "SKIP backup"
         else
           sCmd="$( t_backupDirGetCmdBackup )"
-          echo "what:    ${mydir}"
+          echo "what:    ${BACKUP_DIR}"
           echo "target:  ${sTarget}"
           echo "command: $sCmd"
           echo
@@ -312,11 +313,11 @@
 
           if [ $myrc -ne 0 ]; then
             color error
-            echo DIR ERROR ${mydir} rc=$myrc during file transfer
+            echo DIR ERROR ${BACKUP_DIR} rc=$myrc during file transfer
 
           else
             color ok
-            echo DIR OK ${mydir} was successful.
+            echo DIR OK ${BACKUP_DIR} was successful.
           fi
           color reset
         fi
@@ -324,7 +325,7 @@
 
 
         # --- post action
-        h3 "`date` POST backup task for ${mydir}"
+        h3 "`date` POST backup task for ${BACKUP_DIR}"
         if [ $doPrune -eq 0 ]; then
           echo "SKIP prune"
         else