diff --git a/docs/30_Configuration/50_File_transfer.job.md b/docs/30_Configuration/50_File_transfer.job.md index 42cd1d45ad67257b6504a035651a059aebcfa882..42368f6a709d343b857a88d9a1fb032ebea790d6 100644 --- a/docs/30_Configuration/50_File_transfer.job.md +++ b/docs/30_Configuration/50_File_transfer.job.md @@ -107,6 +107,14 @@ sambashares = 1 restore-path = /restore +# ---------------------------------------------------------------------- +# automatic tasks on each backup run +# ---------------------------------------------------------------------- + +# number of days when to prune and verify backup. +# set it to 0 to run prune and verify in each backup run. +prune-after = 3 +verify-after = 7 # ---------------------------------------------------------------------- # @@ -239,6 +247,15 @@ restore any data anymore. `passphrase = EnterYourSecretHere` +## Prune-after + +You can define how often to prune old data. +Set an integer for age in days here. If it is 0 then it will be executed on each backup run. + +`prune-after = 3` + +Remark: You can force the prune functionality by executing `transfer.sh prune` + ## restic_* Variables that are specific for restic. They will be used when you set @@ -334,3 +351,11 @@ Duplicity supports 2 modes. see also **bin** +## Verify-after + +You can define how often to verify integrity of the backup repoitory data. +Set an integer for age in days here. If it is 0 then it will be executed on each backup run. + +`verify-after = 3` + +Remark: You can force the verify functionality by executing `transfer.sh verify` \ No newline at end of file diff --git a/docs/40_Usage/10_Backup.md b/docs/40_Usage/10_Backup.md index 1ffc77487d329a22be36f816854f116c45916210..290950dbb39c2b5a4d3e88ef3ff0ceee23808e11 100644 --- a/docs/40_Usage/10_Backup.md +++ b/docs/40_Usage/10_Backup.md @@ -38,4 +38,3 @@ OPTIONS: -i, --inc force incrmenental backup ``` - diff --git a/docs/40_Usage/30_Filetransfer.md b/docs/40_Usage/30_Filetransfer.md index 80202c4ab6bde06aede89ea4c5f5ec1a2ec3a75c..aeac6cc4a32a8091e62ea983f9b069dbfa73752b 100644 --- a/docs/40_Usage/30_Filetransfer.md +++ b/docs/40_Usage/30_Filetransfer.md @@ -1,5 +1,5 @@ ``` -# ./transfer.sh -? +> ./transfer.sh -h ___ ___ ___ ___ _______ __ | | Y | | | _ .---.-.----| |--.--.--.-----. @@ -20,14 +20,14 @@ HELP: Transfer local files to a backup target. - target sftp://imlbackup@storage-connector.iml.unibe.ch//netshare/restic-backup + target rest:https://backup:**********@mysynology.dsmynas.net:8000/linux-pc backup tool restic PARAMETERS: - transfer.sh - incremental backup - transfer.sh full - full backup - transfer.sh dumps - transfer local dumps only - transfer.sh prune - cleanup backup data only (no backup) - transfer.sh help - show this help (works with -h and -? too) - + transfer.sh - incremental backup + transfer.sh full - full backup (Duplicity only) + transfer.sh dumps - transfer local dumps only + transfer.sh prune - cleanup backup data only (no backup) + transfer.sh verify - verify backup data (no backup) + transfer.sh help - show this help (works with -h and -? too) ``` diff --git a/docs/40_Usage/40_Cronjob.md b/docs/40_Usage/40_Cronjob.md index b17489c2de4fc62255802a1053b14ad9eb57fb98..3e71eb40a5fc770f1fc4142f9ed28a971e0929ef 100644 --- a/docs/40_Usage/40_Cronjob.md +++ b/docs/40_Usage/40_Cronjob.md @@ -1,3 +1,5 @@ +## Backup + To start a daily backup at 23:03: ```shell @@ -7,7 +9,8 @@ $ cat /etc/cron.d/client-backup ... or with using the cronwrapper -see <https://github.com/axelhahn/cronwrapper> +see ๐ <https://www.axel-hahn.de/docs/cronwrapper/> +๐งพ <https://github.com/axelhahn/cronwrapper> TTL is 1440 for 1440 min = 1d @@ -15,3 +18,29 @@ TTL is 1440 for 1440 min = 1d $ cat /etc/cron.d/client-backup 3 23 * * * root /usr/local/bin/cronwrapper.sh 1440 /opt/imlbackup/client/backup.sh 'iml-backup' ``` + +## Prune and verify + +By default prune and verify are executed on each backup run. This is a safe way. A very safe way. + +BUT: By doing this the backup with Restic is running for a few seconds and prune + verify take a much-much longer time. In many cases it is a good idea to split backup and prune + verify. + +**Example:** + +You want to start a daily (or hourly) backup. Pruning and verify shall run on Saturday and Sunday. + +In jobs/transfer.job set a limit greater 7 days: + +```txt +prune-after = 14 +verify-after = 14 +``` + +Then define your weekly yobs on Saturday and Sunday. + +```shell +$ cat /etc/cron.d/client-backup +3 23 * * * root /usr/local/bin/cronwrapper.sh 1440 /opt/imlbackup/client/backup.sh 'iml-backup' +7 18 * * 6 root /usr/local/bin/cronwrapper.sh 1440 /opt/imlbackup/client/prune.sh 'iml-backup-prune' +7 18 * * 0 root /usr/local/bin/cronwrapper.sh 1440 /opt/imlbackup/client/verify.sh 'iml-backup-verify' +``` diff --git a/docs/99_Glossary.md b/docs/99_Glossary.md new file mode 100644 index 0000000000000000000000000000000000000000..b95f767472eb69bf81ebef6e303b9ad48ccf1b43 --- /dev/null +++ b/docs/99_Glossary.md @@ -0,0 +1,19 @@ +## Explaination of some words + +* **backup** = save current data "somewhare".<br>Store your backup outside the local system to have a copy you can access if the system is damaged. + +* **cronjob** = start a command in a defined cycle. ie. eve3ry hour, every day, once a week and so on + +* **database dump** = backup a database / scheme into a single file + +* **full backup** = backup of all data of the local system. + +* **incremental backup** = backup of data that have changed since the last backup. + +* **prune** = delete old backup sets and thin data in the backup repository. <br>As an example: You can make an hourly backup and define to keep all of them for a week, daily backups for 90 days and monthly backups starting from 1 to 36 monthes (=3 years). The prune actions removes data in the backup storage to thin out your backup data. + +* **restore** = copy data back from a made backup to the local system. + +* **ttl** = time to live<br>Definition how long an information is valid before it needs to be updated again. + +* **verify** = check the integrity of backup data \ No newline at end of file diff --git a/jobs/transfer.job.dist b/jobs/transfer.job.dist index c4d6465ed84f5d04f1f15d3db43ef9fb3919225c..9c1fa30777e933f583e4f79ab17a8680eaa073f4 100644 --- a/jobs/transfer.job.dist +++ b/jobs/transfer.job.dist @@ -104,6 +104,14 @@ sambashares = 1 restore-path = /restore +# ---------------------------------------------------------------------- +# automatic tasks on each backup run +# ---------------------------------------------------------------------- + +# number of days when to prune and verify backup. +# set it to 0 to run prune and verify in each backup run. +prune-after = 3 +verify-after = 7 # ---------------------------------------------------------------------- # diff --git a/plugins/transfer/duplicity.sh b/plugins/transfer/duplicity.sh index 2840527e784796d967fbad6752f8b570cdabdcfc..2b1788c16734b024aa1182ddeeea06118b93d3c7 100644 --- a/plugins/transfer/duplicity.sh +++ b/plugins/transfer/duplicity.sh @@ -172,7 +172,12 @@ color reset exit 1 fi + } + # prune old data + # uses global vars from ../../transfer.sh + function t_backupDoPrune(){ + echo "--- FORGET some data" cmd="duplicity remove-older-than $STORAGE_KEEP --force ${ARGS_DEFAULT} ${STORAGE_TARGETPATH}" echo $cmd color cmd @@ -181,8 +186,14 @@ color reset t_rcCheckCleanup $myrc - } + # verify backup data + # uses global vars from ../../transfer.sh + function t_backupDoVerify(){ + echo "--- VERIFY is not implemented yet for duplicity." + } + + # -------------------------------------------------------------------------------- # RESTORE # -------------------------------------------------------------------------------- diff --git a/plugins/transfer/restic.sh b/plugins/transfer/restic.sh index cb6635c483f30434ab6faa16300f8a7515ddb291..663848b4f76611a05cdd2587e0ea4ee12d473a38 100644 --- a/plugins/transfer/restic.sh +++ b/plugins/transfer/restic.sh @@ -142,6 +142,11 @@ # detect return code ... and abort on any error. t_rcCheckInit $_myrc fi + echo restic unlock ${ARGS_DEFAULT} + color cmd + eval restic unlock ${ARGS_DEFAULT} + color reset + echo } @@ -157,6 +162,11 @@ color reset echo + } + + # prune old data + # uses global vars from ../../transfer.sh + function t_backupDoPrune(){ # -------------------- echo "--- FORGET some data" local _tag=$( _j_getvar ${STORAGEFILE} "${CFGPREFIX}tag") @@ -199,7 +209,10 @@ t_rcCheckPrune $_myrc echo - + } + # verify backup data + # uses global vars from ../../transfer.sh + function t_backupDoVerify(){ # -------------------- echo "--- VERIFY" # param --read-data takes a long time. Maybe use an extra job with it. @@ -214,7 +227,6 @@ t_rcCheckVerify $_myrc echo - } # -------------------------------------------------------------------------------- @@ -385,7 +397,7 @@ # repository cleanup # param integer exitcode of command function t_rcCheckCleanup(){ - echo -n "__PRUNE__ " + echo -n "__CLEANUP__ " case $1 in 0) color ok; echo "OK" ;; *) color error; echo "FAILED ${2} - Cleanup error - returncode was $1" ;; diff --git a/prune.sh b/prune.sh new file mode 100755 index 0000000000000000000000000000000000000000..8ea35cbb187a7e080b29312d5c222a1c03484010 --- /dev/null +++ b/prune.sh @@ -0,0 +1,2 @@ +#!/bin/bash +$(dirname $0 )/transfer.sh prune \ No newline at end of file diff --git a/transfer.sh b/transfer.sh index 216162ee2bc2768ab8e77ee323d850cd9e85889a..3cd45727788ec270d7227579cc0913dbe64c2f7b 100755 --- a/transfer.sh +++ b/transfer.sh @@ -30,6 +30,7 @@ # 2021-05-19 ah,ds, v2.0 plugin driven to support multiple backup tools (duplicity + restic) # 2021-12-02 ah v2.1 added parameter "prune" to cleanup only # 2022-02-10 ah v2.2 update logging (removing tee) +# 2022-10-01 ah v2.3 customize prune and verify action # ================================================================================ @@ -44,7 +45,9 @@ typeset -i rc=0 typeset -i doBackup=1 - typeset -i doPrune=1 + typeset -i doPrune + typeset -i doVerify + typeset -i doValue if [ ! -r "${DIRFILE}" -o ! -r "${STORAGEFILE}" ]; then echo "SKIP backup of local files - one of the files is not readable (no error): ${DIRFILE} | ${STORAGEFILE}" @@ -97,15 +100,58 @@ # first param "full" METHOD= + ACTION=backup transferlog="${DIR_LOGS}/transfer-$(date +%Y%m%d-%H%M%S).log" lockfile="${DIR_LOGS}/transfer.running" + lastbackupfile="${DIR_LOGS}/last_backup" + lastprunefile="${DIR_LOGS}/last_prune" + lastverifyfile="${DIR_LOGS}/last_verify" rcfile=/tmp/transfer-rc.$$.tmp + # -------------------------------------------------------------------------------- # FUNCTIONS # -------------------------------------------------------------------------------- +# get age of a file in sec +# https://www.axel-hahn.de/blog/2020/03/09/bash-snippet-alter-einer-datei/ +# param string filename to test +# return integer +function _getFileAge(){ + echo $(($(date +%s) - $(date +%s -r "$1"))) +} + +# set default behaviur based on limit (from config) and its last +# execution (read from touchfile) +# Function is enabled if no touchfile was found or it is older than +# given limit +# +# It is used to set flags for prune and verify +# param string name of action; one of prune|verify +# param string filename of touchfile of last execution +function setAction(){ + local action=$1 + local myfile=$2 + + typeset -i local iLimit=$(_j_getvar ${STORAGEFILE} "$action-after") + + if [ ! -f "${myfile}" ]; then + echo "Info: $action is ENABLED - no last $action detected" + doValue=1 + else + typeset -i iLastDone=$( _getFileAge "${myfile}" )/60 + typeset -i iLastDoneD=iLastDone/60/24 + echo "Info: Last $action was $iLastDone min ago ($iLastDoneD days). Limit is $iLimit days." + if [ $iLastDoneD -ge $iLimit ]; then + echo "Info: $action is ENABLED - last $action is outdated" + doValue=1 + else + echo "Info: $action is not needed yet." + doValue=0 + fi + fi +} # -------------------------------------------------------------------------------- # MAIN @@ -116,32 +162,61 @@ Transfer local files to a backup target. - target ${STORAGE_BASEDIR} + target $( echo ${STORAGE_BASEDIR} | sed 's#:[^:]*@#:**********@#' ) backup tool $STORAGE_BIN PARAMETERS: - transfer.sh - incremental backup - transfer.sh full - full backup - transfer.sh dumps - transfer local dumps only - transfer.sh prune - cleanup backup data only (no backup) - transfer.sh help - show this help (works with -h and -? too) + transfer.sh - incremental backup + transfer.sh full - full backup (Duplicity only) + transfer.sh dumps - transfer local dumps only + transfer.sh prune - cleanup backup data only (no backup) + transfer.sh verify - verify backup data (no backup) + transfer.sh help - show this help (works with -h and -? too) " exit 0 fi + + # set defaults for prune and verify + echo ">>> Detect default behaviuor:" + setAction "prune" $lastprunefile; doPrune=$doValue + setAction "verify" $lastverifyfile; doVerify=$doValue + echo + + echo ">>> Check parameters" if [ "$1" = "prune" ]; then + echo "Info: Forcing prune only by parameter." + ACTION=$1 doBackup=0 + doPrune=1 + doVerify=0 + transferlog="${DIR_LOGS}/prune-$(date +%Y%m%d-%H%M%S).log" + fi + if [ "$1" = "verify" ]; then + echo "Info: Forcing verify only by parameter." + ACTION=$1 + doBackup=0 + doPrune=0 + doVerify=1 + transferlog="${DIR_LOGS}/verify-$(date +%Y%m%d-%H%M%S).log" fi exec 1> >( tee -a "$transferlog" ) 2>&1 - echo "INFO: Start logging into $transferlog" + echo "INFO: Start logging into $transferlog" h1 `date` TRANSFER LOCAL DATA TO STORAGE - echo "METHOD : $METHOD" - echo "TARGET : ${STORAGE_BASEDIR}" - echo "REGISTER : ${STORAGE_REGISTER}" echo "TOOL : $STORAGE_BIN" + echo "ACTION : $ACTION" + echo "TARGET : ${STORAGE_BASEDIR}" | sed 's#:[^:]*@#:**********@#' echo + if [ "$ACTION" = "backup" ]; then + echo "METHOD : $METHOD" + echo "REGISTER : ${STORAGE_REGISTER}" + echo "PRUNE : $doPrune" + echo "VERIFY : $doVerify" + echo + fi + . `dirname $0`/plugins/transfer/$STORAGE_BIN.sh || exit 1 test -z "$STORAGE_REGISTER" || . `dirname $0`/plugins/register/$STORAGE_REGISTER.sh || exit 1 @@ -264,66 +339,89 @@ if [ -d "$mydir" ]; then BACKUP_DIR=$mydir - h2 "`date` STORE $BACKUP_DIR" - # --- build parameters - sSafeName=`j_getSafename "$BACKUP_DIR"` - sTarget="$( t_backupDirGetTarget $BACKUP_DIR )" + if [ "$ACTION" = "backup" ]; then + + h2 "`date` STORE $BACKUP_DIR" - ARGS_BACKUP="${sParamExclude} $( t_getParamBackup )" + # --- build parameters + sSafeName=`j_getSafename "$BACKUP_DIR"` + sTarget="$( t_backupDirGetTarget $BACKUP_DIR )" + ARGS_BACKUP="${sParamExclude} $( t_getParamBackup )" - # detect custom backup sets and add its includes and excludes - backupid=`j_getSetnameOfPath "$BACKUP_DIR"` - sSpaceReplace="___SPACE___" + # detect custom backup sets and add its includes and excludes + backupid=`j_getSetnameOfPath "$BACKUP_DIR"` + sSpaceReplace="___SPACE___" - if [ ! -z $backupid ]; then - for sItem in `_j_getvar ${DIRFILE} "${backupid}\-\-include" | sed "s#\ #${sSpaceReplace}#g"` - do - ARGS_BACKUP="${ARGS_BACKUP} $( t_getParamInlude $sItem)" - done - for sItem in `_j_getvar ${DIRFILE} "${backupid}\-\-exclude" | sed "s#\ #${sSpaceReplace}#g"` - do - ARGS_BACKUP="${ARGS_BACKUP} $( t_getParamExlude $sItem)" - done - fi + if [ ! -z $backupid ]; then + for sItem in `_j_getvar ${DIRFILE} "${backupid}\-\-include" | sed "s#\ #${sSpaceReplace}#g"` + do + ARGS_BACKUP="${ARGS_BACKUP} $( t_getParamInlude $sItem)" + done + for sItem in `_j_getvar ${DIRFILE} "${backupid}\-\-exclude" | sed "s#\ #${sSpaceReplace}#g"` + do + ARGS_BACKUP="${ARGS_BACKUP} $( t_getParamExlude $sItem)" + done + fi - # --- pre task - h3 "`date` PRE backup task for ${BACKUP_DIR}" - t_backupDirDoPreTasks - # sCmdPre="$( t_backupDirDoPreTasks )" + # --- pre task + h3 "`date` PRE backup task for ${BACKUP_DIR}" + t_backupDirDoPreTasks + # sCmdPre="$( t_backupDirDoPreTasks )" - # --- backup - h3 "`date` Backup ${BACKUP_DIR}" - if [ $doBackup -eq 0 ]; then - echo "SKIP backup" - else - sCmd="$( t_backupDirGetCmdBackup )" - echo "what: ${BACKUP_DIR}" - echo "target: ${sTarget}" - echo "command: $sCmd" - echo - color cmd - $sCmd - fetchrc - color reset + + # --- backup + h3 "`date` Backup ${BACKUP_DIR}" + if [ $doBackup -eq 0 ]; then + echo "SKIP backup" + else + sCmd="$( t_backupDirGetCmdBackup )" + echo "what: ${BACKUP_DIR}" + echo "target: ${sTarget}" + echo "command: $sCmd" + echo + color cmd + $sCmd + fetchrc + color reset + echo + + t_rcCheckBackup $myrc "${BACKUP_DIR}" + + fi echo - t_rcCheckBackup $myrc "${BACKUP_DIR}" - fi - echo + # --- post action + h3 "`date` POST backup task for ${BACKUP_DIR}" + t_backupDirDoPostTasks + echo + touch ${lastbackupfile} + fi - # --- post action - h3 "`date` POST backup task for ${BACKUP_DIR}" + # --- prune if [ $doPrune -eq 0 ]; then echo "SKIP prune" else - t_backupDirDoPostTasks + h3 "`date` PRUNE for ${BACKUP_DIR}" + t_backupDoPrune + touch ${lastprunefile} + echo + fi + echo + + # --- verify + if [ $doVerify -eq 0 ]; then + echo "SKIP verify" + else + h3 "`date` VERIFY for ${BACKUP_DIR}" + t_backupDoVerify + touch ${lastverifyfile} echo fi echo @@ -373,7 +471,7 @@ color reset echo typeset -i TIMER_TRANSFER=`date +%s`-$TIMER_TRANSFER_START - echo `date` TRANSFER DONE in $TIMER_TRANSFER sec + echo `date` $ACTION DONE in $TIMER_TRANSFER sec ls -l $transferlog exit $rc diff --git a/verify.sh b/verify.sh new file mode 100755 index 0000000000000000000000000000000000000000..954a5558fb6bd37ce7d2960460ddce5c6651a7af --- /dev/null +++ b/verify.sh @@ -0,0 +1,2 @@ +#!/bin/bash +$(dirname $0 )/transfer.sh verify \ No newline at end of file