-
Hahn Axel (hahn) authoredHahn Axel (hahn) authored
storage_helper.sh 18.19 KiB
#!/bin/bash
# ======================================================================
#
# IML BACKUP STORAGE HELPER
#
# ----------------------------------------------------------------------
#
# SYNTAX:
# storage_helper.sh PARAMETERS
#
# PARAMETERS:
#
# register [hostname]
# init backup for hostname; backup client must interrupt/ repeat
# return code is <> 0
#
# unregister [hostname] [statuscode]
# send info that a backup was done to delete the connection flag
# for the given host.
#
# status
# show current connections
#
# usage
# show diskusage infos
#
# ----------------------------------------------------------------------
# 2019-11-01 hahn write more clear log messages
# 2021-06-xx hahn added setactive + setinactive
# 2021-06-16 hahn added backupstatus
# 2021-06-18 hahn store backup status in status files ... do not grep frpom logs
# 2021-06-21 hahn use single status file; added backup size
# 2021-01-22 hahn show inactive backups in status page; colored lines by status
# ======================================================================
# ----------------------------------------------------------------------
# CONFIG
# ----------------------------------------------------------------------
. `dirname $0`/inc_config.sh
# set in setBackupclient
sBackupClient=dummy
# value is set in _showConnectionCount
iConnections=0
# max age of last backup in h
iMaxbackupAge=36
# ----------------------------------------------------------------------
# FUNCTIONS
# ----------------------------------------------------------------------
# ------------------------------------------------------------
# color text
# ------------------------------------------------------------
function color(){
sColorcode=""
case $1 in
"reset") sColorcode="0"
;;
"head") sColorcode="33" # yellow
;;
"cmd") sColorcode="94" # light blue
;;
"input") sColorcode="92" # green
;;
"ok") sColorcode="92" # green
;;
"error") sColorcode="91" # red
;;
"warn") sColorcode="33" # yellow
;;
"active") sColorcode="94" # light blue
;;
"disabled") sColorcode="90" # gray
;;
esac
if [ ! -z ${sColorcode} ]; then
echo -ne "\e[${sColorcode}m"
fi
}
function _checkDir(){
typeset -i iCount
mydir=$1
if [ -d $mydir ]; then
iCount=`find $mydir -type f -cmin -720 | wc -l `
if [ $iCount -eq 0 ]; then
color error
fi
echo -n "$iCount "
echo -n "`du -hs $mydir #| awk ' { print $1 } '` "
if [ $iCount -eq 0 ]; then
echo -n "<<< WARNING: No changed files detected !!! rc=1"
color reset
iErrors=$iErrors+1
fi
echo
else
echo SKIP: $mydir is not a directory
fi
}
# ------------------------------------------------------------
# logging
# ------------------------------------------------------------
# add a line into logfile
# param string message (date will be added as prefix)
function _addLog(){
# echo `date` $* >>${sLogDir}${sLogfile}
echo `date "+%Y-%m-%d %H:%M:%S"` $* >>${sLogfile}
}
# get content of all logfiles
function _getAllLogs(){
zcat $sLogdir/*.gz ; cat $sLogfile
}
# get a list of all servers in the log
function _getLoggedServers(){
_getAllLogs | cut -f 2 -d '[' | cut -f 1 -d ']' | sort -u
}
# get timestamp of last backup start of a given server
# global string sBackupClient FQDN of current server
function _getLastBackupstart(){
#test -r "${sStatusDir}/${sBackupClient}_start" && stat -c %Y "${sStatusDir}/${sBackupClient}_start"
#echo $tsBackupStart
#local _srv=$1
#_getAllLogs | sort | grep "\[$_srv\]" | grep ACCEPTED | tail -1 | cut -f -2 -d " "
local tsBackupStart=
test -r "$sStatusDir/${sBackupClient}" && . "$sStatusDir/${sBackupClient}"
echo $tsBackupStart
}
# get timestamp of last backup end of a given server
# global string sBackupClient FQDN of current server
function _getLastBackupend(){
# test -r "${sStatusDir}/${sBackupClient}_end" && stat -c %Y "${sStatusDir}/${sBackupClient}_end"
# local _srv=$1
# _getAllLogs | sort | grep "\[$_srv\]" | grep REMOVED | tail -1 | cut -f -2 -d " "
local tsBackupEnd=
test -r "$sStatusDir/${sBackupClient}" && . "$sStatusDir/${sBackupClient}"
echo $tsBackupEnd
}
# get status code of last backup from status file
# global string sBackupClient FQDN of current server
function _getLastBackupstatus(){
if [ -r "${sStatusDir}/${sBackupClient}" ]; then
local backupRc=
. "${sStatusDir}/${sBackupClient}"
echo $backupRc
else
echo "-2"
fi
}
# get size of backup data from status file
# global string sBackupClient FQDN of current server
function _getBackupsize(){
local size=
test -r "$sStatusDir/${sBackupClient}" && . "$sStatusDir/${sBackupClient}"
echo $size
}
# check if a backup for current server is running
# it returns 0 for no and 1 for yes
# global string sBackupClient FQDN of current server
function _isRunning(){
sTouchfile=`getConnFilename $sBackupClient`
test -f "$sTouchfile" && echo 1
test -f "$sTouchfile" || echo 0
}
# check if a backup for current server is running
# it returns 0 for no and 1 for yes
# global string sBackupClient FQDN of current server
# global string sBackupBasedir backup directory
function _isInactive(){
ls ${sBackupBasedir}/*/$sBackupClient/inactive.txt >/dev/null 2>/dev/null
if [ $? -eq 0 ]; then
echo 1
else
echo 0
fi
}
# ------------------------------------------------------------
# slot handling
# ------------------------------------------------------------
# show current connction count
function _showConnectionCount(){
iConnections=`getConnectionCount`
sMax=$iMaxConnections
if [ $iMaxConnections -eq 0 ]; then
sMax="0 [unlimited]"
fi
echo STATUS: $iConnections existing connection\(s\). Allowed maximum is $sMax.
}
# helper: get filename for backup client connection
function getConnFilename(){
# sBackupClient=$1
echo $sConncectionDir/$sBackupClient
}
# allow a slot for a new connection
# see PUBLIC_register
function addConnection(){
sTouchfile=`getConnFilename $sBackupClient`
if [ -f "$sTouchfile" ]; then
echo INFO: server [$sBackupClient] has a slot already
_addLog "IGNORE request for a slot - [$sBackupClient] has a slot already"
else
touch "$sTouchfile"
echo "client='$sBackupClient'" > "$sTouchfile"
echo "# $( date """+%Y-%m-%d %H:%M:%S""" )" >> "$sTouchfile"
echo "tsBackupStart=$( date +%s )" >> "$sTouchfile"
_addLog "REQUEST a slot for server [$sBackupClient]"
fi
}
# remove slot of a connection
# see PUBLIC_unregister
# param int backup status
function freeConnection(){
local _status=$1
test -z "$1" && _status=-1
sTouchfile=`getConnFilename $sBackupClient`
if [ -f "$sTouchfile" ]; then
echo "# $( date """+%Y-%m-%d %H:%M:%S""" )" >> "$sTouchfile"
echo "tsBackupEnd=$( date +%s )" >> "$sTouchfile"
echo "backupRc=$_status" >> "$sTouchfile"
echo "size='$( du -hs ${sBackupBasedir}/*/$sBackupClient 2>/dev/null )'" >> "$sTouchfile"
mv "$sTouchfile" "$sStatusDir/${sBackupClient}"
else
echo INFO: server [$sBackupClient] had no reserved slot
_addLog "IGNORE freeing the connection - [$sBackupClient] had no reserved slot"
fi
}
# count of conection slots
function getConnectionCount(){
# who | grep "^${sBackupUser}" | wc -l
find $sConncectionDir -type f | wc -l
}
# helper function: set internal var for hostname of backup client
function setBackupclient(){
sBackupClient=`echo $1 | sed s"#[^a-zA-Z\.0-9\-]#_#g"`
if [ -z $sBackupClient ]; then
echo ERROR: no backup client was given
exit 1
fi
# read last backup status
typeset -i tsBackupStart=0
typeset -i tsBackupEnd=0
typeset -i backupRc=-2
# test -r "$sStatusDir/${sBackupClient}" && . "$sStatusDir/${sBackupClient}"
}
# list repositories ... used in setactive/ setinactive
function _listRepodirs(){
for mydir in $( ls -1d ${sBackupBasedir}/*/* | grep -vE '(_active|somethinglsetodisable)' )
do
echo -n "$mydir "
test -f "$mydir/inactive.txt" && ( color error ; echo -n "(inactive)" )
test -f "$mydir/inactive.txt" || ( color ok ; echo -n "(active)" )
color reset
echo
done
}
# ----------------------------------------------------------------------
# PUBLIC FUNCTIONS
# ----------------------------------------------------------------------
function PUBLIC_backupstatus(){
typeset -i local ierrors=0
local tbl="%s %-45s | %-20s | %-20s | %8s | %3s | %7s | %s \n"
_showConnectionCount
echo
echo "This table shows the time and duration [s] of the last backup for each server."
echo
printf "$tbl" " " "server" "start" "end" "duration" "rc" "age [h]" "size"
echo "-----------------------------------------------------------------------------------------------------------------------------"
for myserver in $( _getLoggedServers )
do
# get data
setBackupclient $myserver
typeset -i local istart="$( _getLastBackupstart )"
typeset -i local iend="$( _getLastBackupend )"
typeset -i local bIsRunning=$( _isRunning )
typeset -i local bIsInactive=$( _isInactive )
typeset -i local iLastStatus=$( _getLastBackupstatus )
local size=$( _getBackupsize )
local tstart=$( date +"%Y-%m-%d %H:%M:%S" -d @${istart} )
local tend=$( date +"%Y-%m-%d %H:%M:%S" -d @${iend} )
typeset -i local iduration=${iend}-${istart}
typeset -i local iage=$( date +%s )-${iend}
typeset -i local iagehours=$iage/60/60
local sduration=$iduration
test $iduration -lt 0 && sduration="RUNNING"
sStatusRun="$(color ok)."
test $iend -eq 0 && sStatusRun="$(color warn)?"
test $bIsRunning -eq 1 && sStatusRun="$(color active)R"
test $bIsInactive -eq 1 && sStatusRun="$(color disabled)D"
# check values
if [ $iend -gt 0 -a $iagehours -gt $iMaxbackupAge -a $bIsInactive -eq 0 ]; then
ierrors=$ierrors+1
sStatusRun="$(color error)E"
fi
if [ $iend -eq 0 ]; then
tstart="-"
tend="-"
sduration=""
iagehours=""
fi
# output
printf "$tbl" "$sStatusRun" "$myserver" "${tstart}" "${tend}" "${sduration}" "$iLastStatus" "${iagehours}" "$size"
done
color reset
echo
echo "Legend"
echo ". OK | ? not started | R running | D disabled | E error"
echo
echo "total : $( _getLoggedServers | wc -l ) servers"
echo "errors: $ierrors"
echo
echo "rc=$ierrors"
exit $ierrors
}
function PUBLIC_setactive(){
echo --- list of inactive backup repositories
_listRepodirs | grep '\(inactive\)'
echo
echo -n "Repo to activate again >"
read inputdir
if [ -f "$inputdir/inactive.txt" ]; then
cat "$inputdir/inactive.txt"
echo
echo Removing $inputdir/inactive.txt
rm -f "$inputdir/inactive.txt"
echo If it was a mistake run $( basename $0 ) setinactive
else
echo SKIP [$inputdir]
fi
echo
}
function PUBLIC_setinactive(){
echo --- list of backup repositories
_listRepodirs
echo
echo -n "Repo to deactivate >"
read inputdir
if [ -d "$inputdir" -a ! -f "$inputdir/inactive.txt" ]; then
echo -n "Optional short message >"
read mymessage
echo "$( date ) | $mymessage" > "$inputdir/inactive.txt"
ls -l "$inputdir/inactive.txt"
else
echo SKIP [$inputdir]
fi
echo
}
# return backup directory for a client
function PUBLIC_getBackupdir(){
setBackupclient $1
echo $sBackupBasedir/$sBackupClient
exit 0
}
# register a hostname to start backups
# param string FQDN
function PUBLIC_register(){
setBackupclient $1
echo "REGISTER $1"
echo
addConnection
# iConnections=`getConnectionCount`
_showConnectionCount >/dev/null
if [ $iConnections -le $iMaxConnections -o $iMaxConnections -eq 0 ]; then
sStatus="OK"
iExit=0
_addLog "ACCEPTED [$sBackupClient] got a slot."
else
sStatus="FAILED"
iExit=1
freeConnection
_addLog "REJECTED maximum of $iMaxConnections connections exceeded - request of server [$sBackupClient] failed."
fi
_showConnectionCount
echo $sStatus - rc=$iExit
exit $iExit
}
# unregister a backup slot
# param string FQDN
# param int optional: return code of the backup run
function PUBLIC_unregister(){
typeset -i local _status=$2
setBackupclient $1
echo "UNREGISTER $1"
echo
freeConnection $_status
_addLog "REMOVED slot for [$sBackupClient]"
_showConnectionCount
exit 0
}
# show used slots
function PUBLIC_status(){
_showConnectionCount
echo
if [ $iConnections -gt 0 ]; then
find $sConncectionDir -type f -exec ls -l {} \; | nl
echo
fi
exit 0
}
function PUBLIC_usage(){
typeset -i iErrors=0
# can be "full"
sDetail=$1
echo "DISK USAGE:"
echo
echo "time : `date`"
echo "Backups are located in : $sBackupBasedir"
echo "dirs : `find ${sBackupBasedir} -type d | wc -l`"
echo "files : `find ${sBackupBasedir} -type f | wc -l`"
echo "volume size : `df -h ${sBackupBasedir} | tail -1 | awk '{ print $2 }'`"
echo "backup size : `du -hs ${sBackupBasedir} | cut -f 1 `"
echo "free : `df -h ${sBackupBasedir} | tail -1 | awk '{ print $4 }'`"
echo
echo sizes:
# ls -l "${sBackupBasedir}" | grep "^d" | fgrep -v "`basename $sConncectionDir`"
du -hs ${sBackupBasedir}*
echo
echo "changed files last 12 hours and used diskspace:"
echo
_checkDir ${sBackupBasedir}
echo
if [ ! -z ${sDetail} ]; then
sFilter=${sDetail}
if [ "${sDetail}" = "full" ]; then
sFilter=
fi
echo "... and by server and backup sets:"
echo
for myserver in `ls -1 ${sBackupBasedir} | fgrep -v "_active" | grep "$sFilter"`
do
color head
echo ----- $myserver
color reset
echo
_checkDir ${sBackupBasedir}/$myserver
echo
for subdir in `ls -1 ${sBackupBasedir}/$myserver`
do
_checkDir ${sBackupBasedir}/$myserver/$subdir
done
echo
done
fi
# --- show result
echo -n "Status: "
if [ $iErrors -eq 0 ]; then
echo "OK"
else
color error
echo "$iErrors Errors detected."
fi
color reset
echo rc=$iErrors
exit $iErrors
}
# ----------------------------------------------------------------------
# MAIN
# ----------------------------------------------------------------------
if [ ! -d "$sConncectionDir" ]; then
mkdir -p "$sConncectionDir" || exit 1
fi
if [ ! -d "$sStatusDir" ]; then
mkdir -p "$sStatusDir" || exit 1
fi
color head
echo
echo "---------- :: STORAGE :: `hostname` :: ----------"
echo
color reset
sFunction="PUBLIC_$1"
fgrep "${sFunction}()" $0 >/dev/null
if [ $? -ne 0 ]; then
echo SYNTAX:
echo "`basename $0` [function]"
echo
echo " backupstatus"
echo " show all servers and their backup times"
echo
echo " register [hostname]"
echo " add a slot for a backup client"
echo
echo " unregister [hostname] [statuscode]"
echo " remove slot for a backup client."
echo " The statuscode is an integer of the return code."
echo
echo " setactive"
echo " Reactivate an inctave backup repo."
echo
echo " setinactive"
echo " Mark a backup repo as inactive."
echo " This prevents errors of missing backup data"
echo " if a host is obsolete and was shut down."
echo
echo " status"
echo " show current reserved slots"
echo
echo " usage [full|[filter to hostname]] - DEPRECATED"
echo " show used diskspace"
echo " you can set a string to show additionally details if these hosts"
echo " if you add \"full\" then details of all hosts will be shown"
echo
#echo SCAN... all functions are:
#grep "function\ PUBLIC_" $0 | cut -f 2 -d "_" | cut -f 1 -d "("
else
shift 1
$sFunction $*
fi
# ----------------------------------------------------------------------