Merge branch 'version-2' into 'master'

move register process to plugin

See merge request !3

README.md

@@ -51,6 +51,11 @@ RESTIC
 * delete backups by rules to keep a count of hourly, daily, weekly, mothly, yearly backups
 * several backup targets (we currently use sftp:// http:// and file://)
 
+### control simoultanous backups ###
+
+As an optional feature you can control the count simultanous written backups.
+This requires additional effort next to the client installation.
+
 ## Installation ##
 
 - Uncompress / clone the client to a local directory
@@ -123,7 +128,7 @@ To make a database restore its dump must be located at this directory. To restor
 If you have local Mysql daemon or Pgsql you can test it by starting 
 
     # dump all databases
-    ./localdump.sh
+    sudo ./localdump.sh
 
     # show written files
     find /var/iml-backup

jobhelper.sh

@@ -416,60 +416,6 @@ function j_requireUser(){
   fi
 }
 
-# ------------------------------------------------------------
-# call storage helper; "private" function
-# param  string  command, i.e. register|unregister [hostname]
-# ------------------------------------------------------------
-function _j_storagehelper(){
-  # get user and targethost
-  # rsync://[backupuser]@[storage]//[targetdir]/
-
-  sProtocol=`j_getFullTarget "" | cut -f 1 -d ":"`
-  if [ ${sProtocol} = "rsync" -o ${sProtocol} = "scp" -o ${sProtocol} = "sftp" ]; then
-
-    sSshTarget=`j_getFullTarget "" | cut -f 3 -d "/"`
-    # echo target: $sSshTarget
-    if [ ! -z $sSshTarget ]; then
-
-      sSshParams=
-      sFileSshPrivkey=`_j_getvar ${STORAGEFILE} "ssh-privatekey"`
-      if [ ! -z $sFileSshPrivkey ]; then
-        sSshParams="${sSshParams} -i ${sFileSshPrivkey}"
-      fi
-
-      ssh ${sSshParams} $sSshTarget ./storage_helper.sh $*
-
-    fi
-  else
-    echo INFO: storage protocol is $sProtocol - skipping ssh to storage_helper.
-  fi
-}
-
-# ------------------------------------------------------------
-# transfer start: wait for a free slot
-# ------------------------------------------------------------
-function j_transferStart(){
-  iExit=1
-  until [  $iExit -eq 0 ]; do
-    _j_storagehelper register `hostname -f`
-    iExit=$?
-    if [ $iExit -ne 0 ]; then
-        _j_storagehelper status
-        iRnd=$(($RANDOM%30+30))
-        echo "I wait a bit ... random time ... $iRnd sec ..."
-        sleep $iRnd
-    fi
-  done
-}
-
-# ------------------------------------------------------------
-# transfer end: free used slot on storage
-# ------------------------------------------------------------
-function j_transferEnd(){
-    _j_storagehelper unregister `hostname -f`
-}
-
-
 # ----------------------------------------------------------------------
 # INIT
 # ----------------------------------------------------------------------

jobs/transfer.job.dist

@@ -44,6 +44,14 @@
 # HINT: if using scp:// on Debian 8 enable ssh-backend; default: no value
 # duplicity_ssh-backend = pexpect
 
+# for local or locally mounted targets - you can add a test file that must be found
+# to detect that a backup volume is mounted
+# storage-file = /run/media/backup/I_am_mounted.txt
+
+# OPTIONAL: register for a backup slot
+# storage-register = ssh
+# storage-registercmd = ./storage_helper.sh
+
 
 # ----------------------------------------------------------------------
 # BASIC settings

plugins/localdump/readme.md

+# Help plugins/localdump/
+
+Here are database plugins that can dump and restore types of databases.
+They will be sourced by [APPDIR]/localdump.sh and cannot started directly.
+
+* couchdb2.sh - couchdb 2+3 - using cloudant
+* couchdb.sh - couchdb1 - using a bash script
+* ldap.sh (*) - openLdap - experimantal
+* mysql.sh (*) - Mysql/ Mariadb
+* pgsql.sh (*) - PostgreSql - using pg_dump
+* sqlite.sh - Sqlite sqlite3
+
+(*) If ever possible we use a zero config method. This plugin detects locally installed binaries
+and running processes of a given name to run a database backup or not.
+
+See settings in `[APPDIR]/jobs/dirs.job`:
+
+```text
+dir-localdumps = /var/iml-backup
+keep-days = 7
+```
+
+dir-localdumps configures the target base directory for dumps. Below are subdirectories for the database type.
+In those are the dumps containing name of database scheme and a timestamp. All dumps are gzip compressed.
+
+keep-days contains an integer for the days to keep database dumps locally. Older dumps will be removed.

plugins/register/readme.md

+# Help plugins/register/
+
+Here are plugins to control count of simultanous backups.
+
+For SSH based backup of more than - let's say - 10 servers you should
+think about to enable that feature.
+
+A backup will register before starting a file transfer
+and unregister when the backup is finished.
+
+Plugins:
+
+* ssh.sh - execute a remote command via ssh on backup target using backup user
+
+The configuration is in `[APPDIR]/jobs/transfer.job`
+
+```text
+# register for a backup slot
+storage-register = [plugin]
+```
+
+## Plugins
+
+### ssh
+
+Executes a ssh command. Its limitation is that it automatically connects
+with the same user, its privatekey and the same host like ssh backend.
+
+You need just one additional config entry for the remote command to execute.
+
+Example config:
+
+```text
+# register for a backup slot
+storage-register = ssh
+storage-registercmd = ./storage_helper.sh
+```
+
+See the project https://git-repo.iml.unibe.ch/iml-open-source/iml-backup-server
+for a bash implementation of a counter with `storage_helper.sh`.
+
+## Developer notes
+
+You can create another registration process. You need to add your own
+register plugin `[plugin].sh` that contains these functions:
+
+* registerBackupSlot \
+  A command on a control instance can handles the counts. If the limit is reached it sends returncode <> 0.
+  The backup client will wait a random time before asking for a free slot again.
+
+* unregisterBackupSlot \
+  A backup sends an unregister command to unlock a given slot.
+  The control instance needs to free the slot.
+
+* statusBackupSlot \
+  Show currently used slots.
+
+The file needs no execute permissions - read permissions are enough i.e. `chmod 644 [plugin].sh`.

plugins/register/ssh.sh

+#!/usr/bin/env bash
+# ================================================================================
+#
+# REGISTER :: SSH
+# SSH command to register/ unregister
+#
+# --------------------------------------------------------------------------------
+# ah - Axel Hahn <axel.hahn@iml.unibe.ch>
+#
+# 2021-05-31  ah  v1.0  first lines
+# ================================================================================
+
+# --------------------------------------------------------------------------------
+# FUNCTIONS
+# --------------------------------------------------------------------------------
+
+# @private function
+# detect ssh params and storage-registercmd to run ssh remote command
+# param  string  action; one of register|unregrister|status
+# param  string  hostname; given for register and unregister
+function _runssh(){
+  
+  sProtocol=`j_getFullTarget "" | cut -f 1 -d ":"`
+  if [ ${sProtocol} = "rsync" -o ${sProtocol} = "scp" -o ${sProtocol} = "sftp" ]; then
+
+    sSshTarget=`j_getFullTarget "" | cut -f 3 -d "/"`
+    if [ ! -z $sSshTarget ]; then
+
+      sSshParams=
+      sFileSshPrivkey=`_j_getvar ${STORAGEFILE} "ssh-privatekey"`
+      if [ ! -z $sFileSshPrivkey ]; then
+        sSshParams="${sSshParams} -i ${sFileSshPrivkey}"
+      fi
+
+      sSshCmd=`_j_getvar ${STORAGEFILE} "storage-registercmd"`
+      
+      if [ -z "$sSshCmd" ]; then
+        echo "WARNING: Missing storage-registercmd = ... in ${STORAGEFILE} for a command to execute via SSH."
+      else
+        echo ssh ${sSshParams} ${sSshTarget} ${sSshCmd} $*
+        color cmd
+        ssh ${sSshParams} ${sSshTarget} ${sSshCmd} $*
+        color reset
+      fi
+    fi
+  else
+    echo INFO: storage protocol is $sProtocol - skipping register via ssh.
+  fi
+}
+
+# --------------------------------------------------------------------------------
+# functions called in [APPDIR]/transfer.sh
+
+function registerBackupSlot(){
+    _runssh register $1
+}
+
+function unregisterBackupSlot(){
+    _runssh unregister $1
+}
+function statusBackupSlot(){
+    _runssh status
+}
+
+# --------------------------------------------------------------------------------
\ No newline at end of file

plugins/transfer/readme.md

+# Help plugins/transfer/
+
+Here are plugins for backup tools that can backup and restore files and directories.
+
+* duplicity.sh - Duplicity
+* restic.sh - Restic
+
+See `[APPDIR]/jobs/transfer.job` to configure them.
+
+## Duplicity
+
+Website: <http://duplicity.nongnu.org>
+
+## Restic
+
+Website: <https://restic.net/>
+Docs: <https://restic.readthedocs.io/en/stable/>

transfer.sh

@@ -45,6 +45,8 @@
   STORAGE_TESTFILE=`_j_getvar ${STORAGEFILE} "storage-file"`
   PASSPHRASE=`_j_getvar ${STORAGEFILE} "passphrase"`
 
+  STORAGE_REGISTER=`_j_getvar ${STORAGEFILE} "storage-register"`
+
   # check
   if [ -z "$STORAGE_BIN" ]; then
     # STORAGE_BIN=restic
@@ -99,15 +101,16 @@
 
 
   
-  h1 `date` TRANSFER LOCAL DATA TO STORAGE                                             | tee -a $transferlog
+  h1 `date` TRANSFER LOCAL DATA TO STORAGE                      | tee -a $transferlog
   
-  echo "METHOD: $METHOD"                                                               | tee -a $transferlog
-  echo "TARGET: ${STORAGE_BASEDIR}"                                                    | tee -a $transferlog
-  echo "TOOL  : $STORAGE_BIN"                                                          | tee -a $transferlog
-  echo                                                                                 | tee -a $transferlog
+  echo "METHOD   : $METHOD"                                     | tee -a $transferlog
+  echo "TARGET   : ${STORAGE_BASEDIR}"                          | tee -a $transferlog
+  echo "REGISTER : ${STORAGE_REGISTER}"                         | tee -a $transferlog
+  echo "TOOL     : $STORAGE_BIN"                                | tee -a $transferlog
+  echo                                                          | tee -a $transferlog
 
   . `dirname $0`/plugins/transfer/$STORAGE_BIN.sh || exit 1
-
+  test -z "$STORAGE_REGISTER" || . `dirname $0`/plugins/register/$STORAGE_REGISTER.sh || exit 1
 
 # --------------------------------------------------------------------------------
 # ----- Check requirements
@@ -197,7 +200,24 @@
 # ----- PRE transfer
 
   h2 "`date` Wait for a free slot"
-  j_transferStart | tee -a $transferlog
+  if [ -z "$STORAGE_REGISTER" ]; then
+    echo "SKIP"
+  else
+    iExit=1
+    until [ $iExit -eq 0 ]; do
+      registerBackupSlot `hostname -f`
+      iExit=$?
+      if [ $iExit -ne 0 ]; then
+          statusBackupSlot
+          iRnd=$(($RANDOM%30+30))
+          echo "I wait a bit ... random time ... $iRnd sec ..."
+          sleep $iRnd
+      fi
+    done
+  fi | tee -a $transferlog
+
+  # TODO: remove in jobhelper
+  # j_transferStart | tee -a $transferlog
 
   h2 "`date` PRE transfer tasks"
   t_backupDoPreTasks
@@ -302,13 +322,17 @@
   h2 "`date` POST transfer tasks"
   t_backupDoPostTasks
 
-
   rm -f "${lockfile}" "${rcfile}"
   echo "Local lock file was removed."
 
+  h2 "`date` Unregister used slot"
+  if [ -z "$STORAGE_REGISTER" ]; then
+    echo "SKIP"
+  else
+    unregisterBackupSlot `hostname -f`
+  fi | tee -a $transferlog
 
-  j_transferEnd
-
+  h2 "`date` Backup finished"
   echo STATUS $0 exit with final returncode rc=$rc                                    | tee -a $transferlog
   echo                                                                                | tee -a $transferlog
   if [ $rc -eq 0 ]; then