Merge branch 'add_prune_script' into 'master'

first lines of rest_pruner script

See merge request !2

docs/30_Scripts.md

@@ -123,3 +123,97 @@ In the backup status the deactivated target is listed with D = Disabled and is g
 BTW: the opposite way is possible to:
 
 `./storage_helper.sh setactive`
+
+# rest_pruner.sh
+
+**!! This script is in version 0.1 - and work in progress !!**
+
+The pruner script is for restic rest server with append only option. It can prune all repositories on server side.
+
+## inc_config.sh
+
+The inc_config.sh mus contain 3 variables for pruning
+
+```bash
+    # for prune on restic rest server
+    prune_basedir=/netshare/restic-backup
+    prune_params="--group-by paths,tags --prune --keep-within 180d --max-unused unlimited --max-repack-size 100M --cleanup-cache"
+    prune_skipdays="7"
+```
+
+## rest_pruner.cfg
+
+To access the different local repositories we need the RESTIC_PASSWORD for each repository.
+The config file `rest_pruner.cfg` contains lines in the syntax
+
+`<USER>:<RESTIC_PASSWORD>`
+
+If a directory matches `${prune_basedir}/<USER>` then it will be pruned.
+
+You need a mechanism to create this file eg. by Ansible.
+
+For securiy reasons this file must be owned by root:root and must have the permissions 0400.
+
+```txt
+ls -l rest_pruner.cfg
+-r--------. 1 root root 159 Feb  1 13:35 rest_pruner.cfg
+```
+
+## Syntax
+
+```txt
+
+========== RESTIC REST PRUNER v0.1 ==========
+
+
+Pruner for restic rest server with append only option.
+This script prunes all repositories on server side.
+
+The config file [rest_pruner.cfg] contains <USER>:<RESTIC_PASSWORD>
+If a directory matches /netshare/restic-backup/<USER> then it will be pruned.
+
+SYNTAX:
+  rest_pruner.sh [OPTIONS] [FILTER]
+
+OPTIONS:
+  -h, --help           show help and exit.
+
+PARAMETERS:
+  FILTER               regex to filter directory list in
+                       /netshare/restic-backup/*
+
+EXAMPLES:
+  rest_pruner.sh
+                       Start pruning of all matching repositories
+  rest_pruner.sh mail
+                       Prune servers that match "mail",
+                       eg. my-mailhub.example.com
+
+```
+
+## How does it work
+
+It detects some requirements:
+
+* was it started by root? (The help is shown without being root)
+* permissions of rest_pruner.cfg
+* variables from inc_config.sh
+
+It loops over the starting dir `${prune_basedir}` and reads all its subdirectories.
+If a subdir matches a configuration entry (text before first ":" `<USER>:<RESTIC_PASSWORD>`) then the prune process will be started. It detects the username of the owner of the directory and executes
+
+`su - $_user - /bin/bash -c "<restic forget ...>"`
+
+Exitcodes:
+
+* 0 if the run was without errors.
+* >0 if a prune run of a reposaitory failed.
+* 1..3 if a requirement failed to start
+
+The output of the prune command is written into
+
+`_last_prune/<USER>.log`
+
+If a job failed it is renamed to
+
+`_last_prune/<USER>.log.error`

inc_config.sh.dist

@@ -27,5 +27,8 @@
     sLogdir="${sSelfdir}/log"
     sLogfile="$sLogdir/connections.log"
 
+    # for prune on restic rest server
+    prune_params="--group-by paths,tags --prune --keep-within 180d --max-unused unlimited --max-repack-size 100M --cleanup-cache"
+    prune_skipdays="7"
 
 # ----------------------------------------------------------------------

rest_pruner.sh

+#!/bin/bash
+# ======================================================================
+#
+# RESTIC REST PRUNER
+#
+# ----------------------------------------------------------------------
+# 2024-02-01  v0.1  <axel.hahn@unibe.ch>  first lines
+# ======================================================================
+
+
+cd "$( dirname $0 )" || exit
+_version=0.1
+logdir=_last_prune
+
+prune_basedir=
+prune_params=
+bOptDebug=0
+
+typeset -i iCountDirs=0
+typeset -i iCountMatch=0
+typeset -i iCountPrune=0
+
+
+typeset -i rcAll=0
+
+. "inc_config.sh" || exit 1
+cfgfile=rest_pruner.cfg || exit 1
+
+
+
+# ----------------------------------------------------------------------
+# FUNCTIONS
+# ----------------------------------------------------------------------
+
+# Show help text
+function _showHelp(){
+    local _self; _self=$( basename $0 )
+    cat <<EOH
+
+Pruner for restic rest server with append only option.
+This script prunes all repositories on server side.
+
+The config file [$cfgfile] contains <USER>:<RESTIC_PASSWORD>
+If a directory matches ${prune_basedir}/<USER> then it will be pruned.
+
+SYNTAX:
+  $_self [OPTIONS] [FILTER]
+
+OPTIONS:
+  -h, --help           show help and exit.
+
+PARAMETERS:
+  FILTER               regex to filter directory list in
+                       ${prune_basedir}/*
+
+EXAMPLES:
+  $_self
+                       Start pruning of all matching repositories
+  $_self mail
+                       Prune servers that match "mail",
+                       eg. my-mailhub.example.com
+
+EOH
+}
+
+# start prune of a given repository
+# global  string   cfgfile  name of the config file
+# global  integer  rcAll    sum of all prune exit status
+#
+# param   string            directory to prune
+function _prune(){
+        local _dir="$1"
+
+        local mybase; mybase=$( basename "${_dir}" )
+        local mypw; mypw=$( grep "^${mybase}:" "${cfgfile}" | cut -f2 -d ':')
+        local logfile="${logdir}/${mybase}.log"
+
+        iCountDirs+=1
+
+        if [ -n "$mypw" ]; then
+                local _user=$( stat -c "%U" "$_dir" )
+                echo "----- $_dir"
+
+                iCountMatch+=1
+                bDoRun=1
+
+                if ps -ef | grep "restic forget.*${_dir}" | grep -v "grep" | grep . ; then
+                    echo "SKIP: a process is still running..."
+                    bDoRun=0
+                else
+                    rm -f "${logfile}.running"
+                    echo "TODO: check age of last prune run."
+                fi
+
+                if [ "$bDoRun" -eq "1" ]; then
+                    echo "Starting prune as user $_user ..."
+                    su - $_user - /bin/bash -c "
+                            echo START $( date ) $_dir
+                            export RESTIC_PASSWORD=$mypw
+                            restic forget -r $_dir $prune_params 2>&1
+                    " | tee "${logfile}.running"
+                    rc=${PIPESTATUS[0]}
+                    rcAll+=$rc
+                    echo END $( date ) exitcode $rc | tee -a "${logfile}.running"
+                    if [ "$rc" -eq "0" ]; then
+                            mv "${logfile}.running" "${logfile}"
+                    else
+                            mv "${logfile}.running" "${logfile}.error"
+                    fi
+                fi
+        fi
+}
+
+# ----------------------------------------------------------------------
+# MAIN
+# ----------------------------------------------------------------------
+
+echo
+echo "========== RESTIC REST PRUNER v${_version} =========="
+echo
+
+# ----- check parameters
+
+while [[ "$#" -gt 0 ]]; do case $1 in
+    -h|--help)        _showHelp; exit 0;;
+    -d|--debug)       bOptDebug=1; shift;;
+    -p|--path)        if ! grep ":{$2}:" <<< ":{$PATH}:" >/dev/null; then
+                        PATH="$2:$PATH"; 
+                      fi
+                      shift; shift;;
+    -t|--target)      export DOCKER_HOST="$2"; shift; shift;;
+    *) echo "ERROR: Unknown parameter: $1"; _showHelp; exit 2;
+esac; done
+
+# ----- verify needed settings
+
+if [ ! -f "$cfgfile" ]; then
+    echo "ERROR: The configuration [$cfgfile] does not exist (yet)."
+    exit 3
+fi
+
+if [ "$USER" != "root" ]; then
+        echo "ERROR: This script must be started as root."
+        exit 1
+fi
+
+cfgpermissions=$( stat -c '%A:%U:%G' "$cfgfile" )
+if [ "$cfgpermissions" != "-r--------:root:root" ]; then
+    echo "WARNING: The configuration file must be owned by root:root with permission 0400."
+    ls -l "$cfgfile"
+    echo "Try to fix it..."
+    chown root:root "$cfgfile"
+    chmod 0400 "$cfgfile"
+    ls -l "$cfgfile"
+    cfgpermissions=$( stat -c '%A:%U:%G' "$cfgfile" )
+    if [ "$cfgpermissions" != "-r--------:root:root" ]; then
+        echo "Fix failed. Aborting."
+        exit 3
+    fi
+    echo "OK"
+fi
+
+if [ -z "${prune_basedir}" ]; then
+    echo "ERROR: variable [prune_basedir] was not set in [inc_config.sh]"
+    exit 3
+fi
+
+if [ ! -d "${prune_basedir}" ]; then
+    echo "ERROR: variable [prune_basedir] in [inc_config.sh] points to a directory"
+    echo "that does not exist: ${prune_basedir}"
+    exit 3
+fi
+
+if [ -z "${prune_params}" ]; then
+    echo "ERROR: variable [prune_params] was not set in [inc_config.sh]"
+    exit 3
+fi
+
+if ! grep "\-\-keep\-" <<< "${prune_params}" >/dev/null;  then
+    echo "ERROR: variable [prune_params] does not contain a --keep-* prameter."
+    exit 3
+fi
+
+if ! which restic >/dev/null; then
+    echo "ERROR: The restic client was not found."
+    exit 3
+fi
+
+# ----- Go
+
+test -d "${logdir}" || mkdir -p "${logdir}"
+filter=${1:-.}
+for mydir in $( find ${prune_basedir} -maxdepth 1 -type d | grep -E "$filter")
+do
+        _prune "${mydir}"
+done
+
+
+echo "done - exitcode $rcAll"
+exit $rcAll
+
+# ----------------------------------------------------------------------