From dc42ef1e38ba3dca7920c283d914f1b665dbb480 Mon Sep 17 00:00:00 2001
From: "Hahn Axel (hahn)" <axel.hahn@iml.unibe.ch>
Date: Mon, 31 May 2021 11:21:46 +0200
Subject: [PATCH] handle register a slot as plugin
---
jobhelper.sh | 54 -------------------------------
jobs/transfer.job.dist | 8 +++++
plugins/register/readme.md | 58 ++++++++++++++++++++++++++++++++++
plugins/register/ssh.sh | 65 ++++++++++++++++++++++++++++++++++++++
transfer.sh | 44 ++++++++++++++++++++------
5 files changed, 165 insertions(+), 64 deletions(-)
create mode 100644 plugins/register/readme.md
create mode 100755 plugins/register/ssh.sh
diff --git a/jobhelper.sh b/jobhelper.sh
index 497b5ba..6b895c1 100755
--- a/jobhelper.sh
+++ b/jobhelper.sh
@@ -416,60 +416,6 @@ function j_requireUser(){
fi
}
-# ------------------------------------------------------------
-# call storage helper; "private" function
-# param string command, i.e. register|unregister [hostname]
-# ------------------------------------------------------------
-function _j_storagehelper(){
- # get user and targethost
- # rsync://[backupuser]@[storage]//[targetdir]/
-
- sProtocol=`j_getFullTarget "" | cut -f 1 -d ":"`
- if [ ${sProtocol} = "rsync" -o ${sProtocol} = "scp" -o ${sProtocol} = "sftp" ]; then
-
- sSshTarget=`j_getFullTarget "" | cut -f 3 -d "/"`
- # echo target: $sSshTarget
- if [ ! -z $sSshTarget ]; then
-
- sSshParams=
- sFileSshPrivkey=`_j_getvar ${STORAGEFILE} "ssh-privatekey"`
- if [ ! -z $sFileSshPrivkey ]; then
- sSshParams="${sSshParams} -i ${sFileSshPrivkey}"
- fi
-
- ssh ${sSshParams} $sSshTarget ./storage_helper.sh $*
-
- fi
- else
- echo INFO: storage protocol is $sProtocol - skipping ssh to storage_helper.
- fi
-}
-
-# ------------------------------------------------------------
-# transfer start: wait for a free slot
-# ------------------------------------------------------------
-function j_transferStart(){
- iExit=1
- until [ $iExit -eq 0 ]; do
- _j_storagehelper register `hostname -f`
- iExit=$?
- if [ $iExit -ne 0 ]; then
- _j_storagehelper status
- iRnd=$(($RANDOM%30+30))
- echo "I wait a bit ... random time ... $iRnd sec ..."
- sleep $iRnd
- fi
- done
-}
-
-# ------------------------------------------------------------
-# transfer end: free used slot on storage
-# ------------------------------------------------------------
-function j_transferEnd(){
- _j_storagehelper unregister `hostname -f`
-}
-
-
# ----------------------------------------------------------------------
# INIT
# ----------------------------------------------------------------------
diff --git a/jobs/transfer.job.dist b/jobs/transfer.job.dist
index d3948e4..e97d3f1 100644
--- a/jobs/transfer.job.dist
+++ b/jobs/transfer.job.dist
@@ -44,6 +44,14 @@
# HINT: if using scp:// on Debian 8 enable ssh-backend; default: no value
# duplicity_ssh-backend = pexpect
+# for local or locally mounted targets - you can add a test file that must be found
+# to detect that a backup volume is mounted
+# storage-file = /run/media/backup/I_am_mounted.txt
+
+# OPTIONAL: register for a backup slot
+# storage-register = ssh
+# storage-registercmd = ./storage_helper.sh
+
# ----------------------------------------------------------------------
# BASIC settings
diff --git a/plugins/register/readme.md b/plugins/register/readme.md
new file mode 100644
index 0000000..1885646
--- /dev/null
+++ b/plugins/register/readme.md
@@ -0,0 +1,58 @@
+# Help plugins/register/
+
+Here are plugins to control count of simultanous backups.
+
+For SSH based backup of more than - let's say - 10 servers you should
+think about to enable that feature.
+
+A backup will register before starting a file transfer
+and unregister when the backup is finished.
+
+Plugins:
+
+* ssh.sh - execute a remote command via ssh on backup target using backup user
+
+The configuration is in `[APPDIR]/jobs/transfer.job`
+
+```text
+# register for a backup slot
+storage-register = [plugin]
+```
+
+## Plugins
+
+### ssh
+
+Executes a ssh command. Its limitation is that it automatically connects
+with the same user, its privatekey and the same host like ssh backend.
+
+You need just one additional config entry for the remote command to execute.
+
+Example config:
+
+```text
+# register for a backup slot
+storage-register = ssh
+storage-registercmd = ./storage_helper.sh
+```
+
+See the project https://git-repo.iml.unibe.ch/iml-open-source/iml-backup-server
+for a bash implementation of a counter with `storage_helper.sh`.
+
+## Developer notes
+
+You can create another registration process. You need to add your own
+register plugin `[plugin].sh` that contains these functions:
+
+* registerBackupSlot \
+ A command on a control instance can handles the counts. If the limit is reached it sends returncode <> 0.
+ The backup client will wait a random time before asking for a free slot again.
+
+* unregisterBackupSlot \
+ A backup sends an unregister command to unlock a given slot.
+ The control instance needs to free the slot.
+
+* statusBackupSlot \
+ Show currently used slots.
+
+The file needs no execute permissions - read permissions are enough i.e. `chmod 644 [plugin].sh`.
diff --git a/plugins/register/ssh.sh b/plugins/register/ssh.sh
new file mode 100755
index 0000000..fcaaa1c
--- /dev/null
+++ b/plugins/register/ssh.sh
@@ -0,0 +1,65 @@
+#!/usr/bin/env bash
+# ================================================================================
+#
+# REGISTER :: SSH
+# SSH command to register/ unregister
+#
+# --------------------------------------------------------------------------------
+# ah - Axel Hahn <axel.hahn@iml.unibe.ch>
+#
+# 2021-05-31 ah v1.0 first lines
+# ================================================================================
+
+# --------------------------------------------------------------------------------
+# FUNCTIONS
+# --------------------------------------------------------------------------------
+
+# @private function
+# detect ssh params and storage-registercmd to run ssh remote command
+# param string action; one of register|unregrister|status
+# param string hostname; given for register and unregister
+function _runssh(){
+
+ sProtocol=`j_getFullTarget "" | cut -f 1 -d ":"`
+ if [ ${sProtocol} = "rsync" -o ${sProtocol} = "scp" -o ${sProtocol} = "sftp" ]; then
+
+ sSshTarget=`j_getFullTarget "" | cut -f 3 -d "/"`
+ if [ ! -z $sSshTarget ]; then
+
+ sSshParams=
+ sFileSshPrivkey=`_j_getvar ${STORAGEFILE} "ssh-privatekey"`
+ if [ ! -z $sFileSshPrivkey ]; then
+ sSshParams="${sSshParams} -i ${sFileSshPrivkey}"
+ fi
+
+ sSshCmd=`_j_getvar ${STORAGEFILE} "storage-registercmd"`
+
+ if [ -z "$sSshCmd" ]; then
+ echo "WARNING: Missing storage-registercmd = ... in ${STORAGEFILE} for a command to execute via SSH."
+ else
+ echo ssh ${sSshParams} ${sSshTarget} ${sSshCmd} $*
+ color cmd
+ ssh ${sSshParams} ${sSshTarget} ${sSshCmd} $*
+ color reset
+ fi
+ fi
+ else
+ echo INFO: storage protocol is $sProtocol - skipping register via ssh.
+ fi
+}
+
+# --------------------------------------------------------------------------------
+# functions called in [APPDIR]/transfer.sh
+
+function registerBackupSlot(){
+ _runssh register $1
+}
+
+function unregisterBackupSlot(){
+ _runssh unregister $1
+}
+function statusBackupSlot(){
+ _runssh status
+}
+
+# --------------------------------------------------------------------------------
\ No newline at end of file
diff --git a/transfer.sh b/transfer.sh
index 70a2ab6..35cf788 100755
--- a/transfer.sh
+++ b/transfer.sh
@@ -45,6 +45,8 @@
STORAGE_TESTFILE=`_j_getvar ${STORAGEFILE} "storage-file"`
PASSPHRASE=`_j_getvar ${STORAGEFILE} "passphrase"`
+ STORAGE_REGISTER=`_j_getvar ${STORAGEFILE} "storage-register"`
+
# check
if [ -z "$STORAGE_BIN" ]; then
# STORAGE_BIN=restic
@@ -99,15 +101,16 @@
- h1 `date` TRANSFER LOCAL DATA TO STORAGE | tee -a $transferlog
+ h1 `date` TRANSFER LOCAL DATA TO STORAGE | tee -a $transferlog
- echo "METHOD: $METHOD" | tee -a $transferlog
- echo "TARGET: ${STORAGE_BASEDIR}" | tee -a $transferlog
- echo "TOOL : $STORAGE_BIN" | tee -a $transferlog
- echo | tee -a $transferlog
+ echo "METHOD : $METHOD" | tee -a $transferlog
+ echo "TARGET : ${STORAGE_BASEDIR}" | tee -a $transferlog
+ echo "REGISTER : ${STORAGE_REGISTER}" | tee -a $transferlog
+ echo "TOOL : $STORAGE_BIN" | tee -a $transferlog
+ echo | tee -a $transferlog
. `dirname $0`/plugins/transfer/$STORAGE_BIN.sh || exit 1
-
+ test -z "$STORAGE_REGISTER" || . `dirname $0`/plugins/register/$STORAGE_REGISTER.sh || exit 1
# --------------------------------------------------------------------------------
# ----- Check requirements
@@ -197,7 +200,24 @@
# ----- PRE transfer
h2 "`date` Wait for a free slot"
- j_transferStart | tee -a $transferlog
+ if [ -z "$STORAGE_REGISTER" ]; then
+ echo "SKIP"
+ else
+ iExit=1
+ until [ $iExit -eq 0 ]; do
+ registerBackupSlot `hostname -f`
+ iExit=$?
+ if [ $iExit -ne 0 ]; then
+ statusBackupSlot
+ iRnd=$(($RANDOM%30+30))
+ echo "I wait a bit ... random time ... $iRnd sec ..."
+ sleep $iRnd
+ fi
+ done
+ fi | tee -a $transferlog
+
+ # TODO: remove in jobhelper
+ # j_transferStart | tee -a $transferlog
h2 "`date` PRE transfer tasks"
t_backupDoPreTasks
@@ -302,13 +322,17 @@
h2 "`date` POST transfer tasks"
t_backupDoPostTasks
-
rm -f "${lockfile}" "${rcfile}"
echo "Local lock file was removed."
+ h2 "`date` Unregister used slot"
+ if [ -z "$STORAGE_REGISTER" ]; then
+ echo "SKIP"
+ else
+ unregisterBackupSlot `hostname -f`
+ fi | tee -a $transferlog
- j_transferEnd
-
+ h2 "`date` Backup finished"
echo STATUS $0 exit with final returncode rc=$rc | tee -a $transferlog
echo | tee -a $transferlog
if [ $rc -eq 0 ]; then
--
GitLab