diff --git a/README.md b/README.md index 702839e3ddc4c9bee0a6a69930ebf27247e61ef4..2c66b219a96bafe5a35c8d2f508c4107d82357ba 100644 --- a/README.md +++ b/README.md @@ -51,6 +51,11 @@ RESTIC * delete backups by rules to keep a count of hourly, daily, weekly, mothly, yearly backups * several backup targets (we currently use sftp:// http:// and file://) +### control simoultanous backups ### + +As an optional feature you can control the count simultanous written backups. +This requires additional effort next to the client installation. + ## Installation ## - Uncompress / clone the client to a local directory @@ -123,7 +128,7 @@ To make a database restore its dump must be located at this directory. To restor If you have local Mysql daemon or Pgsql you can test it by starting # dump all databases - ./localdump.sh + sudo ./localdump.sh # show written files find /var/iml-backup diff --git a/jobhelper.sh b/jobhelper.sh index 497b5bae81e8919bb55f2fe237b54e424c0de54a..6b895c138bf4a73cf0483fa62771ef2f87b2d113 100755 --- a/jobhelper.sh +++ b/jobhelper.sh @@ -416,60 +416,6 @@ function j_requireUser(){ fi } -# ------------------------------------------------------------ -# call storage helper; "private" function -# param string command, i.e. register|unregister [hostname] -# ------------------------------------------------------------ -function _j_storagehelper(){ - # get user and targethost - # rsync://[backupuser]@[storage]//[targetdir]/ - - sProtocol=`j_getFullTarget "" | cut -f 1 -d ":"` - if [ ${sProtocol} = "rsync" -o ${sProtocol} = "scp" -o ${sProtocol} = "sftp" ]; then - - sSshTarget=`j_getFullTarget "" | cut -f 3 -d "/"` - # echo target: $sSshTarget - if [ ! -z $sSshTarget ]; then - - sSshParams= - sFileSshPrivkey=`_j_getvar ${STORAGEFILE} "ssh-privatekey"` - if [ ! -z $sFileSshPrivkey ]; then - sSshParams="${sSshParams} -i ${sFileSshPrivkey}" - fi - - ssh ${sSshParams} $sSshTarget ./storage_helper.sh $* - - fi - else - echo INFO: storage protocol is $sProtocol - skipping ssh to storage_helper. - fi -} - -# ------------------------------------------------------------ -# transfer start: wait for a free slot -# ------------------------------------------------------------ -function j_transferStart(){ - iExit=1 - until [ $iExit -eq 0 ]; do - _j_storagehelper register `hostname -f` - iExit=$? - if [ $iExit -ne 0 ]; then - _j_storagehelper status - iRnd=$(($RANDOM%30+30)) - echo "I wait a bit ... random time ... $iRnd sec ..." - sleep $iRnd - fi - done -} - -# ------------------------------------------------------------ -# transfer end: free used slot on storage -# ------------------------------------------------------------ -function j_transferEnd(){ - _j_storagehelper unregister `hostname -f` -} - - # ---------------------------------------------------------------------- # INIT # ---------------------------------------------------------------------- diff --git a/jobs/transfer.job.dist b/jobs/transfer.job.dist index d3948e4c4c536792d445ef1bf0d0938922e1796b..e97d3f10ddebb2946808dff1198829667f038759 100644 --- a/jobs/transfer.job.dist +++ b/jobs/transfer.job.dist @@ -44,6 +44,14 @@ # HINT: if using scp:// on Debian 8 enable ssh-backend; default: no value # duplicity_ssh-backend = pexpect +# for local or locally mounted targets - you can add a test file that must be found +# to detect that a backup volume is mounted +# storage-file = /run/media/backup/I_am_mounted.txt + +# OPTIONAL: register for a backup slot +# storage-register = ssh +# storage-registercmd = ./storage_helper.sh + # ---------------------------------------------------------------------- # BASIC settings diff --git a/plugins/localdump/readme.md b/plugins/localdump/readme.md new file mode 100644 index 0000000000000000000000000000000000000000..45a651261cd2e1f30859f88263f690c6dd7f14b9 --- /dev/null +++ b/plugins/localdump/readme.md @@ -0,0 +1,26 @@ +# Help plugins/localdump/ + +Here are database plugins that can dump and restore types of databases. +They will be sourced by [APPDIR]/localdump.sh and cannot started directly. + +* couchdb2.sh - couchdb 2+3 - using cloudant +* couchdb.sh - couchdb1 - using a bash script +* ldap.sh (*) - openLdap - experimantal +* mysql.sh (*) - Mysql/ Mariadb +* pgsql.sh (*) - PostgreSql - using pg_dump +* sqlite.sh - Sqlite sqlite3 + +(*) If ever possible we use a zero config method. This plugin detects locally installed binaries +and running processes of a given name to run a database backup or not. + +See settings in `[APPDIR]/jobs/dirs.job`: + +```text +dir-localdumps = /var/iml-backup +keep-days = 7 +``` + +dir-localdumps configures the target base directory for dumps. Below are subdirectories for the database type. +In those are the dumps containing name of database scheme and a timestamp. All dumps are gzip compressed. + +keep-days contains an integer for the days to keep database dumps locally. Older dumps will be removed. diff --git a/plugins/register/readme.md b/plugins/register/readme.md new file mode 100644 index 0000000000000000000000000000000000000000..1885646b5c4dfc0ba0a0a015198cae92ddda65e4 --- /dev/null +++ b/plugins/register/readme.md @@ -0,0 +1,58 @@ +# Help plugins/register/ + +Here are plugins to control count of simultanous backups. + +For SSH based backup of more than - let's say - 10 servers you should +think about to enable that feature. + +A backup will register before starting a file transfer +and unregister when the backup is finished. + +Plugins: + +* ssh.sh - execute a remote command via ssh on backup target using backup user + +The configuration is in `[APPDIR]/jobs/transfer.job` + +```text +# register for a backup slot +storage-register = [plugin] +``` + +## Plugins + +### ssh + +Executes a ssh command. Its limitation is that it automatically connects +with the same user, its privatekey and the same host like ssh backend. + +You need just one additional config entry for the remote command to execute. + +Example config: + +```text +# register for a backup slot +storage-register = ssh +storage-registercmd = ./storage_helper.sh +``` + +See the project https://git-repo.iml.unibe.ch/iml-open-source/iml-backup-server +for a bash implementation of a counter with `storage_helper.sh`. + +## Developer notes + +You can create another registration process. You need to add your own +register plugin `[plugin].sh` that contains these functions: + +* registerBackupSlot \ + A command on a control instance can handles the counts. If the limit is reached it sends returncode <> 0. + The backup client will wait a random time before asking for a free slot again. + +* unregisterBackupSlot \ + A backup sends an unregister command to unlock a given slot. + The control instance needs to free the slot. + +* statusBackupSlot \ + Show currently used slots. + +The file needs no execute permissions - read permissions are enough i.e. `chmod 644 [plugin].sh`. diff --git a/plugins/register/ssh.sh b/plugins/register/ssh.sh new file mode 100755 index 0000000000000000000000000000000000000000..fcaaa1c4594bd814cafb5a9ac31b0b85a5939fc5 --- /dev/null +++ b/plugins/register/ssh.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +# ================================================================================ +# +# REGISTER :: SSH +# SSH command to register/ unregister +# +# -------------------------------------------------------------------------------- +# ah - Axel Hahn <axel.hahn@iml.unibe.ch> +# +# 2021-05-31 ah v1.0 first lines +# ================================================================================ + +# -------------------------------------------------------------------------------- +# FUNCTIONS +# -------------------------------------------------------------------------------- + +# @private function +# detect ssh params and storage-registercmd to run ssh remote command +# param string action; one of register|unregrister|status +# param string hostname; given for register and unregister +function _runssh(){ + + sProtocol=`j_getFullTarget "" | cut -f 1 -d ":"` + if [ ${sProtocol} = "rsync" -o ${sProtocol} = "scp" -o ${sProtocol} = "sftp" ]; then + + sSshTarget=`j_getFullTarget "" | cut -f 3 -d "/"` + if [ ! -z $sSshTarget ]; then + + sSshParams= + sFileSshPrivkey=`_j_getvar ${STORAGEFILE} "ssh-privatekey"` + if [ ! -z $sFileSshPrivkey ]; then + sSshParams="${sSshParams} -i ${sFileSshPrivkey}" + fi + + sSshCmd=`_j_getvar ${STORAGEFILE} "storage-registercmd"` + + if [ -z "$sSshCmd" ]; then + echo "WARNING: Missing storage-registercmd = ... in ${STORAGEFILE} for a command to execute via SSH." + else + echo ssh ${sSshParams} ${sSshTarget} ${sSshCmd} $* + color cmd + ssh ${sSshParams} ${sSshTarget} ${sSshCmd} $* + color reset + fi + fi + else + echo INFO: storage protocol is $sProtocol - skipping register via ssh. + fi +} + +# -------------------------------------------------------------------------------- +# functions called in [APPDIR]/transfer.sh + +function registerBackupSlot(){ + _runssh register $1 +} + +function unregisterBackupSlot(){ + _runssh unregister $1 +} +function statusBackupSlot(){ + _runssh status +} + +# -------------------------------------------------------------------------------- \ No newline at end of file diff --git a/plugins/transfer/readme.md b/plugins/transfer/readme.md new file mode 100644 index 0000000000000000000000000000000000000000..73e015c52865c5e8310076ace166cd52804ed4a0 --- /dev/null +++ b/plugins/transfer/readme.md @@ -0,0 +1,17 @@ +# Help plugins/transfer/ + +Here are plugins for backup tools that can backup and restore files and directories. + +* duplicity.sh - Duplicity +* restic.sh - Restic + +See `[APPDIR]/jobs/transfer.job` to configure them. + +## Duplicity + +Website: <http://duplicity.nongnu.org> + +## Restic + +Website: <https://restic.net/> +Docs: <https://restic.readthedocs.io/en/stable/> diff --git a/transfer.sh b/transfer.sh index 70a2ab6b5e8358e4953d0e813570ebe0967f54c1..35cf788821d9a826a576d1bc49615059e29e91eb 100755 --- a/transfer.sh +++ b/transfer.sh @@ -45,6 +45,8 @@ STORAGE_TESTFILE=`_j_getvar ${STORAGEFILE} "storage-file"` PASSPHRASE=`_j_getvar ${STORAGEFILE} "passphrase"` + STORAGE_REGISTER=`_j_getvar ${STORAGEFILE} "storage-register"` + # check if [ -z "$STORAGE_BIN" ]; then # STORAGE_BIN=restic @@ -99,15 +101,16 @@ - h1 `date` TRANSFER LOCAL DATA TO STORAGE | tee -a $transferlog + h1 `date` TRANSFER LOCAL DATA TO STORAGE | tee -a $transferlog - echo "METHOD: $METHOD" | tee -a $transferlog - echo "TARGET: ${STORAGE_BASEDIR}" | tee -a $transferlog - echo "TOOL : $STORAGE_BIN" | tee -a $transferlog - echo | tee -a $transferlog + echo "METHOD : $METHOD" | tee -a $transferlog + echo "TARGET : ${STORAGE_BASEDIR}" | tee -a $transferlog + echo "REGISTER : ${STORAGE_REGISTER}" | tee -a $transferlog + echo "TOOL : $STORAGE_BIN" | tee -a $transferlog + echo | tee -a $transferlog . `dirname $0`/plugins/transfer/$STORAGE_BIN.sh || exit 1 - + test -z "$STORAGE_REGISTER" || . `dirname $0`/plugins/register/$STORAGE_REGISTER.sh || exit 1 # -------------------------------------------------------------------------------- # ----- Check requirements @@ -197,7 +200,24 @@ # ----- PRE transfer h2 "`date` Wait for a free slot" - j_transferStart | tee -a $transferlog + if [ -z "$STORAGE_REGISTER" ]; then + echo "SKIP" + else + iExit=1 + until [ $iExit -eq 0 ]; do + registerBackupSlot `hostname -f` + iExit=$? + if [ $iExit -ne 0 ]; then + statusBackupSlot + iRnd=$(($RANDOM%30+30)) + echo "I wait a bit ... random time ... $iRnd sec ..." + sleep $iRnd + fi + done + fi | tee -a $transferlog + + # TODO: remove in jobhelper + # j_transferStart | tee -a $transferlog h2 "`date` PRE transfer tasks" t_backupDoPreTasks @@ -302,13 +322,17 @@ h2 "`date` POST transfer tasks" t_backupDoPostTasks - rm -f "${lockfile}" "${rcfile}" echo "Local lock file was removed." + h2 "`date` Unregister used slot" + if [ -z "$STORAGE_REGISTER" ]; then + echo "SKIP" + else + unregisterBackupSlot `hostname -f` + fi | tee -a $transferlog - j_transferEnd - + h2 "`date` Backup finished" echo STATUS $0 exit with final returncode rc=$rc | tee -a $transferlog echo | tee -a $transferlog if [ $rc -eq 0 ]; then