From 541da5bf8abbc37d1ecaea5f65beac0532366841 Mon Sep 17 00:00:00 2001
From: "Hahn Axel (hahn)" <axel.hahn@unibe.ch>
Date: Thu, 1 Feb 2024 17:02:18 +0100
Subject: [PATCH] first lines of rest_pruner script
---
docs/30_Scripts.md | 94 +++++++++++++++++++++
inc_config.sh.dist | 3 +
rest_pruner.sh | 202 +++++++++++++++++++++++++++++++++++++++++++++
3 files changed, 299 insertions(+)
create mode 100755 rest_pruner.sh
diff --git a/docs/30_Scripts.md b/docs/30_Scripts.md
index da5497b..52ca90b 100644
--- a/docs/30_Scripts.md
+++ b/docs/30_Scripts.md
@@ -123,3 +123,97 @@ In the backup status the deactivated target is listed with D = Disabled and is g
BTW: the opposite way is possible to:
`./storage_helper.sh setactive`
+
+# rest_pruner.sh
+
+**!! This script is in version 0.1 - and work in progress !!**
+
+The pruner script is for restic rest server with append only option. It can prune all repositories on server side.
+
+## inc_config.sh
+
+The inc_config.sh mus contain 3 variables for pruning
+
+```bash
+ # for prune on restic rest server
+ prune_basedir=/netshare/restic-backup
+ prune_params="--group-by paths,tags --prune --keep-within 180d --max-unused unlimited --max-repack-size 100M --cleanup-cache"
+ prune_skipdays="7"
+```
+
+## rest_pruner.cfg
+
+To access the different local repositories we need the RESTIC_PASSWORD for each repository.
+The config file `rest_pruner.cfg` contains lines in the syntax
+
+`<USER>:<RESTIC_PASSWORD>`
+
+If a directory matches `${prune_basedir}/<USER>` then it will be pruned.
+
+You need a mechanism to create this file eg. by Ansible.
+
+For securiy reasons this file must be owned by root:root and must have the permissions 0400.
+
+```txt
+ls -l rest_pruner.cfg
+-r--------. 1 root root 159 Feb 1 13:35 rest_pruner.cfg
+```
+
+## Syntax
+
+```txt
+
+========== RESTIC REST PRUNER v0.1 ==========
+
+
+Pruner for restic rest server with append only option.
+This script prunes all repositories on server side.
+
+The config file [rest_pruner.cfg] contains <USER>:<RESTIC_PASSWORD>
+If a directory matches /netshare/restic-backup/<USER> then it will be pruned.
+
+SYNTAX:
+ rest_pruner.sh [OPTIONS] [FILTER]
+
+OPTIONS:
+ -h, --help show help and exit.
+
+PARAMETERS:
+ FILTER regex to filter directory list in
+ /netshare/restic-backup/*
+
+EXAMPLES:
+ rest_pruner.sh
+ Start pruning of all matching repositories
+ rest_pruner.sh mail
+ Prune servers that match "mail",
+ eg. my-mailhub.example.com
+
+```
+
+## How does it work
+
+It detects some requirements:
+
+* was it started by root? (The help is shown without being root)
+* permissions of rest_pruner.cfg
+* variables from inc_config.sh
+
+It loops over the starting dir `${prune_basedir}` and reads all its subdirectories.
+If a subdir matches a configuration entry (text before first ":" `<USER>:<RESTIC_PASSWORD>`) then the prune process will be started. It detects the username of the owner of the directory and executes
+
+`su - $_user - /bin/bash -c "<restic forget ...>"`
+
+Exitcodes:
+
+* 0 if the run was without errors.
+* >0 if a prune run of a reposaitory failed.
+* 1..3 if a requirement failed to start
+
+The output of the prune command is written into
+
+`_last_prune/<USER>.log`
+
+If a job failed it is renamed to
+
+`_last_prune/<USER>.log.error`
diff --git a/inc_config.sh.dist b/inc_config.sh.dist
index df10f4e..18621c0 100644
--- a/inc_config.sh.dist
+++ b/inc_config.sh.dist
@@ -27,5 +27,8 @@
sLogdir="${sSelfdir}/log"
sLogfile="$sLogdir/connections.log"
+ # for prune on restic rest server
+ prune_params="--group-by paths,tags --prune --keep-within 180d --max-unused unlimited --max-repack-size 100M --cleanup-cache"
+ prune_skipdays="7"
# ----------------------------------------------------------------------
diff --git a/rest_pruner.sh b/rest_pruner.sh
new file mode 100755
index 0000000..37cae6a
--- /dev/null
+++ b/rest_pruner.sh
@@ -0,0 +1,202 @@
+#!/bin/bash
+# ======================================================================
+#
+# RESTIC REST PRUNER
+#
+# ----------------------------------------------------------------------
+# 2024-02-01 v0.1 <axel.hahn@unibe.ch> first lines
+# ======================================================================
+
+
+cd "$( dirname $0 )" || exit
+_version=0.1
+logdir=_last_prune
+
+prune_basedir=
+prune_params=
+bOptDebug=0
+
+typeset -i iCountDirs=0
+typeset -i iCountMatch=0
+typeset -i iCountPrune=0
+
+
+typeset -i rcAll=0
+
+. "inc_config.sh" || exit 1
+cfgfile=rest_pruner.cfg || exit 1
+
+
+
+# ----------------------------------------------------------------------
+# FUNCTIONS
+# ----------------------------------------------------------------------
+
+# Show help text
+function _showHelp(){
+ local _self; _self=$( basename $0 )
+ cat <<EOH
+
+Pruner for restic rest server with append only option.
+This script prunes all repositories on server side.
+
+The config file [$cfgfile] contains <USER>:<RESTIC_PASSWORD>
+If a directory matches ${prune_basedir}/<USER> then it will be pruned.
+
+SYNTAX:
+ $_self [OPTIONS] [FILTER]
+
+OPTIONS:
+ -h, --help show help and exit.
+
+PARAMETERS:
+ FILTER regex to filter directory list in
+ ${prune_basedir}/*
+
+EXAMPLES:
+ $_self
+ Start pruning of all matching repositories
+ $_self mail
+ Prune servers that match "mail",
+ eg. my-mailhub.example.com
+
+EOH
+}
+
+# start prune of a given repository
+# global string cfgfile name of the config file
+# global integer rcAll sum of all prune exit status
+#
+# param string directory to prune
+function _prune(){
+ local _dir="$1"
+
+ local mybase; mybase=$( basename "${_dir}" )
+ local mypw; mypw=$( grep "^${mybase}:" "${cfgfile}" | cut -f2 -d ':')
+ local logfile="${logdir}/${mybase}.log"
+
+ iCountDirs+=1
+
+ if [ -n "$mypw" ]; then
+ local _user=$( stat -c "%U" "$_dir" )
+ echo "----- $_dir"
+
+ iCountMatch+=1
+ bDoRun=1
+
+ if ps -ef | grep "restic forget.*${_dir}" | grep -v "grep" | grep . ; then
+ echo "SKIP: a process is still running..."
+ bDoRun=0
+ else
+ rm -f "${logfile}.running"
+ echo "TODO: check age of last prune run."
+ fi
+
+ if [ "$bDoRun" -eq "1" ]; then
+ echo "Starting prune as user $_user ..."
+ su - $_user - /bin/bash -c "
+ echo START $( date ) $_dir
+ export RESTIC_PASSWORD=$mypw
+ restic forget -r $_dir $prune_params 2>&1
+ " | tee "${logfile}.running"
+ rc=${PIPESTATUS[0]}
+ rcAll+=$rc
+ echo END $( date ) exitcode $rc | tee -a "${logfile}.running"
+ if [ "$rc" -eq "0" ]; then
+ mv "${logfile}.running" "${logfile}"
+ else
+ mv "${logfile}.running" "${logfile}.error"
+ fi
+ fi
+ fi
+}
+
+# ----------------------------------------------------------------------
+# MAIN
+# ----------------------------------------------------------------------
+
+echo
+echo "========== RESTIC REST PRUNER v${_version} =========="
+echo
+
+# ----- check parameters
+
+while [[ "$#" -gt 0 ]]; do case $1 in
+ -h|--help) _showHelp; exit 0;;
+ -d|--debug) bOptDebug=1; shift;;
+ -p|--path) if ! grep ":{$2}:" <<< ":{$PATH}:" >/dev/null; then
+ PATH="$2:$PATH";
+ fi
+ shift; shift;;
+ -t|--target) export DOCKER_HOST="$2"; shift; shift;;
+ *) echo "ERROR: Unknown parameter: $1"; _showHelp; exit 2;
+esac; done
+
+# ----- verify needed settings
+
+if [ ! -f "$cfgfile" ]; then
+ echo "ERROR: The configuration [$cfgfile] does not exist (yet)."
+ exit 3
+fi
+
+if [ "$USER" != "root" ]; then
+ echo "ERROR: This script must be started as root."
+ exit 1
+fi
+
+cfgpermissions=$( stat -c '%A:%U:%G' "$cfgfile" )
+if [ "$cfgpermissions" != "-r--------:root:root" ]; then
+ echo "WARNING: The configuration file must be owned by root:root with permission 0400."
+ ls -l "$cfgfile"
+ echo "Try to fix it..."
+ chown root:root "$cfgfile"
+ chmod 0400 "$cfgfile"
+ ls -l "$cfgfile"
+ cfgpermissions=$( stat -c '%A:%U:%G' "$cfgfile" )
+ if [ "$cfgpermissions" != "-r--------:root:root" ]; then
+ echo "Fix failed. Aborting."
+ exit 3
+ fi
+ echo "OK"
+fi
+
+if [ -z "${prune_basedir}" ]; then
+ echo "ERROR: variable [prune_basedir] was not set in [inc_config.sh]"
+ exit 3
+fi
+
+if [ ! -d "${prune_basedir}" ]; then
+ echo "ERROR: variable [prune_basedir] in [inc_config.sh] points to a directory"
+ echo "that does not exist: ${prune_basedir}"
+ exit 3
+fi
+
+if [ -z "${prune_params}" ]; then
+ echo "ERROR: variable [prune_params] was not set in [inc_config.sh]"
+ exit 3
+fi
+
+if ! grep "\-\-keep\-" <<< "${prune_params}" >/dev/null; then
+ echo "ERROR: variable [prune_params] does not contain a --keep-* prameter."
+ exit 3
+fi
+
+if ! which restic >/dev/null; then
+ echo "ERROR: The restic client was not found."
+ exit 3
+fi
+
+# ----- Go
+
+test -d "${logdir}" || mkdir -p "${logdir}"
+filter=${1:-.}
+for mydir in $( find ${prune_basedir} -maxdepth 1 -type d | grep -E "$filter")
+do
+ _prune "${mydir}"
+done
+
+
+echo "done - exitcode $rcAll"
+exit $rcAll
+
+# ----------------------------------------------------------------------
--
GitLab