diff --git a/icinga-cli.sh b/icinga-cli.sh index 670c262fae27691ff56fa0a84bcb47e41805a64e..973b9df8b0f278a50a71d2372fc779b1a0aba352 100755 --- a/icinga-cli.sh +++ b/icinga-cli.sh @@ -21,12 +21,12 @@ # 2023-02-17 v0.14 ah add hostnamme into CFGSTORAGE # 2023-02-17 v0.15 ah check inc_getconfig.sh exists and hostname -f has a value # 2023-11-01 v0.16 ah generate json with "jo" -# 2023-11-02 v0.17 ah generate json with "jo" +# 2023-11-02 v0.17 ah fix line breaks of sent output, shell check fixes # ====================================================================== _product="ICINGA PASSIVE CLIENT" -_version="0.16" +_version="0.17" _license="GNU GPL 3.0" _copyright='(c) Institute for Medical Education * University of Bern' @@ -103,14 +103,6 @@ function _getFileSnapshot(){ # function loopChecks(){ - # TODO-MEMORY-CHECK - # echo ${MY_NAME} | egrep "^(kvm4|icinga)" - # echo ${MY_NAME} | egrep "^(monitortest)" - # if [ $? -ne 0 ]; then - # echo "HARD EXIT - DO NOT EXECUTE ANY CHECK ON $MY_NAME" - # exit 1 - # fi - local lockfile lockfile="${dir_data}/loop.pid" local snapShotStart @@ -129,6 +121,8 @@ function loopChecks(){ fi _log "---------- starting in a permanent loop" + icingaHostMustExist + echo "Serviceloop started $(date) - process id $$" > "${lockfile}" if [ $? -ne 0 ]; then _elog "ABORT: Lock file is not writable ${lockfile}." @@ -145,23 +139,21 @@ function loopChecks(){ while true; do # typeset -i local iSleep=$(($RANDOM%$sleeptime)) # sleep minimum is half of $sleeptime - typeset -i local iSleep=$(($RANDOM%$sleeptime/2+$sleeptime/2)) + local iSleep + typeset -i iSleep=$(($RANDOM%$sleeptime/2+$sleeptime/2)) _log "sleeping $iSleep sec ..." sleep $iSleep _log "______________________________________________________________________" _log "" - _getFileSnapshot>$snapShotCurrent - if [ $? -ne 0 ]; then + if ! _getFileSnapshot > "$snapShotCurrent"; then _elog "ABORT: Snapshot file is not writable ${snapShotCurrent}." _elog $( ls -l "${snapShotCurrent}" ) exit 1 fi - diff $snapShotStart $snapShotCurrent >/dev/null - if [ $? -ne 0 ]; then - _elog "ABORT: Files were updated / overwritten. The loop must be restarted.\n`diff $snapShotStart $snapShotCurrent`" + if ! diff "$snapShotStart" "$snapShotCurrent" >/dev/null; then + _elog "ABORT: Files were updated / overwritten. The loop must be restarted.\n$(diff $snapShotStart $snapShotCurrent)" exit 1 fi - icingaHostMustExist processAllChecks done } @@ -172,13 +164,15 @@ function loopChecks(){ # function processAllChecks(){ # loop over all defined checks - typeset -i local iChecksTotal + local iChecksTotal; typeset -i iChecksTotal + local iCounter; typeset -i iCounter + local iLoopStart; typeset -i iLoopStart + local iLoopEnd; typeset -i iLoopEnd + local iLoopTime; typeset -i iLoopTime + iChecksTotal=$(getChecks | wc -l) - typeset -i local iCounter iCounter=0 - _rc_all=0 - typeset -i local iLoopStart iLoopStart=$(_getUnixTs) _log "" @@ -191,48 +185,19 @@ function processAllChecks(){ _log "" done - typeset -i local iLoopEnd iLoopEnd=$(_getUnixTs) - typeset -i local iLoopTime iLoopTime=$iLoopEnd-$iLoopStart _log "------ loop done - needed $iLoopTime sec - rc=$_rc_all" } -# ...................................................................... -# -# parse a config file and set global vars: -# checkName -# checkCommand -# checkInterval -# param string full path of a config file -# -function UNUSED_parseCheckConfig(){ - local _myconfig="$1" - - if [ ! -r "$_myconfig" ]; then - _elog "ERROR: config file is not readable [$_myconfig]" - exit 1 - fi - - # EXAMPLE a config contains ... - # checkname=check_cronstatus - # command=check_cronstatus -param1 -param2 - # interval=60 - - checkName=$(cat $_myconfig | grep ^checkname= | cut -f 2 -d "=") - checkCommand=$(cat $_myconfig | grep ^command= | cut -f 2 -d "=") - checkInterval=$(cat $_myconfig | grep ^interval= | cut -f 2 -d "=") - -} - # actions for icinga host # param string action; "get" only function icingaHost(){ local _logPrefix="${MY_NAME} :: API |" local _apiRequest=objects/hosts/${MY_NAME} local _localCache=${dir_data}/host_${MY_NAME}_deployed-at-icinga.txt - typeset -i local _iRefreshCache=120 + local _iRefreshCache; typeset -i _iRefreshCache=120 local sAction=$1 @@ -305,20 +270,23 @@ function processCheck(){ local _myconfig=$1 local _force=$2 - typeset -i local iPipes - typeset -i local iCheckStart - iCheckStart=$(_getUnixTs) - - _parseCheckConfig "${_myconfig}" - local _logPrefix="${checkName} |" - _log "${_logPrefix} INFO: every ${checkInterval} sec: ${checkCommand}" + local iPipes; typeset -i iPipes + local iCheckStart; typeset -i iCheckStart local _outfile=${dir_data}/service__check__${checkName}__output.txt local _output local _FoundErrors local _response=${dir_data}/service__check__${checkName}__icinga_response.txt - typeset -i local _rc=0 + local _rc; typeset -i _rc=0 + local iTsEnd; typeset -i iTsEnd + local iTsStart; typeset -i iTsStart + + iCheckStart=$(_getUnixTs) + + _parseCheckConfig "${_myconfig}" + local _logPrefix="${checkName} |" + _log "${_logPrefix} INFO: every ${checkInterval} sec: ${checkCommand}" _initHttp @@ -328,7 +296,6 @@ function processCheck(){ _log "${_logPrefix} INFO: Never executed before" doRun=1 else - # typeset -i iAgeLastRun=$(($(date +%s) - $(date +%s -r "$_outfile"))) typeset -i iAgeLastRun iAgeLastRun=$(_getFileAge "$_outfile") _log "${_logPrefix} INFO: last run was $iAgeLastRun sec ago ... vs Interval = $checkInterval ... sleeptime = $sleeptime" @@ -355,37 +322,35 @@ function processCheck(){ # --- this executes the check plugin ... # _log "${_logPrefix} starting $myFullscript $myparams" - typeset -i local iTsStart=`date +%s` - # $myFullscript $myparams | tee $_outfile - eval $myFullscript $myparams > $_outfile + iTsStart=$(date +%s) + eval $myFullscript $myparams > "$_outfile" rc=$? if [ ! -w $_outfile ]; then _elog "${_logPrefix} ERROR: output file $_outfile is not writable." _elog "${_logPrefix} $( ls -ld ${dir_data} $_outfile )" exit 1 fi - test $rc -gt 0 && _FoundErrors+="Exitcode of check is $rc (greater zero).\n" - typeset -i local iTsEnd=`date +%s` - # outPerfdata=`grep '|' $_outfile | cut -f 2 -d '|'` - outPerfdata=`grep '|' $_outfile | rev | cut -f 1 -d '|' | rev` + test $rc -gt 0 && _FoundErrors+="Exitcode of check is greater zero ... " + iTsEnd=$(date +%s) + outPerfdata=$(grep '|' $_outfile | rev | cut -f 1 -d '|' | rev | grep "=") _output="$( cat "$_outfile" )" _echo _echo -------- check output: _echo "$_output" iPipes=$( grep -o "|" < "$_outfile" | wc -l ) - _echo "Pipe chars: $iPipes" + # _echo "Pipe chars: $iPipes" if [[ $iPipes -gt 1 ]]; then _elog "WARNING - pipes were found in plugin output" _elog "command was: $myFullscript $myparams" _log "${_logPrefix} WARNING - pipes were found in plugin output" fi - # sed -i "s#|#:#g" "$_outfile" _echo - # echo -------- extracted performance data: - # echo $outPerfdata - # echo + + # + # --- extract performance data: + # if ! test -n "$outPerfdata" && echo "$outPerfdata" | grep "=[0-9\.]*;[0-9\.]*;[0-9\.]*;[0-9\.]*;" >/dev/null; then _elog "WARNING - this does not look like performance data: $outPerfdata" _elog "command was: $myFullscript $myparams" @@ -395,29 +360,24 @@ function processCheck(){ _log "${_logPrefix} check command finished with returncode $rc" _rc=$_rc+$rc + # # --- send check result to Icinga # fields of the object # https://icinga.com/docs/icinga2/latest/doc/12-icinga2-api/#process-check-result - - - commandAsJson="$(jq -nR --arg data """${myFullscript} $myparams""" '$data')" - outAsJson="$( jq -nR --arg data """${_output}""" '$data')" - - local JSONPARAMS=" - check_source=${MY_NAME} - check_command=${commandAsJson} - exit_status=$rc - ttl=$checkInterval - execution_start=$iTsStart - execution_end=$iTsEnd - performance_data="\"${outPerfdata}\"" - plugin_output=$outAsJson - " - - data=$( eval jo -p -d. $JSONPARAMS ) - - slot="`_getName4Svcathost ${checkName} | sed 's# #%20#g'`" + # + data="$( jo \ + check_source=${MY_NAME} \ + check_command="""${myFullscript} $myparams""" \ + exit_status=$rc \ + ttl=$checkInterval \ + execution_start=$iTsStart \ + execution_end=$iTsEnd \ + performance_data="\"${outPerfdata}\"" \ + plugin_output="""${_output}""" + )" + + slot="$(_getName4Svcathost ${checkName} | sed 's# #%20#g')" _log "${_logPrefix} starting POST of data to monitoring server" _echo POST actions/process-check-result?service=${MY_NAME}!${slot} "$data" @@ -430,18 +390,20 @@ function processCheck(){ fi # --- check if data were sent successfully - # fgrep "HTTP/1.1 200" ${_response} >/dev/null - # _testHttpOk ${_response} >/dev/null - http.isOk >/dev/null - if [ $? -eq 0 ]; then + if http.isOk >/dev/null; then _log "${_logPrefix} rc=$rc - OK, response was sent to Icinga" else - _elog "${_logPrefix} rc=$rc - WARNING: $(http.getStatuscode) the check response was NOT sent to Icinga" + local httpcode; httpcode=$(http.getStatuscode) _rc=$_rc+1 - _FoundErrors+="Response was not sent to Icinga." + case $httpcode in + 404) _FoundErrors+="Icinga response $httpcode: host or service check was not created in Icinga yet ... " ;; + *) _FoundErrors+="Icinga response was not sent to Icinga: $httpcode ... " ;; + esac _echo - _echo For Debugging: - _echo "$data" + _elog "${_logPrefix} rc=$rc - $_FoundErrors" + # _echo + # _echo For Debugging: + # _echo "$data" _log "$data" fi @@ -454,11 +416,13 @@ function processCheck(){ # add current result to global returncode _rc_all=$_rc_all+$_rc - typeset -i local iCheckEnd=`_getUnixTs` - typeset -i local iCheckTime=$iCheckEnd-$iCheckStart + local iCheckEnd; typeset -i iCheckEnd + local iCheckTime; typeset -i iCheckTime + + iCheckEnd=$(_getUnixTs) + iCheckTime=$iCheckEnd-$iCheckStart _log "${_logPrefix} finished after $iCheckTime sec with returncode $_rc" - test $_rc -eq 0 || (_echo; _echo "ERRORS: [${checkName}] $_FoundErrors"; _echo; _echo) } # ---------------------------------------------------------------------- # help @@ -466,7 +430,7 @@ function processCheck(){ # show help text function showHelp(){ -self=`basename $0` +self=$(basename $0) cat <<EOH @@ -489,7 +453,7 @@ GENERAL PARAMETERS show this help and abort. --version or -v - show the version abd abort + show the version abd abort. It must be the 1st parameter. SERVICE ACTIONS @@ -538,11 +502,7 @@ DEBUGGING EOH } -function showVersion(){ - echo "$_license" - echo "$_copyright" - echo -} + # ---------------------------------------------------------------------- # # MAIN @@ -551,6 +511,7 @@ function showVersion(){ . "$( dirname $0 )/inc_functions.sh" +# show version without header if [ "$1" = "--version" ] || [ "$1" = "-v" ]; then echo "$( basename $0) v$_version" echo "$_license .. $_copyright" @@ -601,9 +562,6 @@ if [ -z "${dir_cfg}" ]; then exit 1 fi -icingaHostMustExist - - touch ${logfile} @@ -615,11 +573,6 @@ do showHelp exit 0 ;; - '--version' | '-v') - showVersion - exit 0 - ;; - '--list') getChecks diff --git a/inc/confighandler.sh b/inc/confighandler.sh deleted file mode 100755 index 699ac70338dcb529b6ffcda87242720b7a53666b..0000000000000000000000000000000000000000 --- a/inc/confighandler.sh +++ /dev/null @@ -1,438 +0,0 @@ -#!/bin/bash -# ====================================================================== -# -# VARIABLE STORAGE written in Bash -# -# ---------------------------------------------------------------------- -# 2020-01-25 v0.1 <axel.hahn@iml.unibe.ch> -# 2020-01-26 v0.2 <axel.hahn@iml.unibe.ch> added --get, --add -# 2020-01-27 v0.3 <axel.hahn@iml.unibe.ch> added storages handling -# 2020-01-30 v0.5 <axel.hahn@iml.unibe.ch> added --setfile -# ====================================================================== - -_product="VARIABLE STORAGE" -_version=0.5 - -# _storagedir=/var/tmp/confighandler -_storagedir=/dev/shm/confighandler - -_storageprefix='flat-variable-store-' -_storagename='default' -tmpfile=/tmp/cfgmover.tmp - -_markFile='::FILE-CONTENT::' - -# ---------------------------------------------------------------------- -# FUNCTIONS -# ---------------------------------------------------------------------- - -# write a debug message to STDERR -function _wd(){ - echo -e "\e[33m# $*\e[0m" >&2 -} - -# write error message in red to STDERR -function _we(){ - echo -e "\e[31m# $*\e[0m" >&2 -} - -# ---------------------------------------------------------------------- -# storage functions -# ---------------------------------------------------------------------- - -# set a variable storage -# param string name of the storage; allowed chars are a-z and 0-9 -function storageset(){ - _storagename=`echo $1 | sed "s#[^a-z0-9]##g"` - - mkdir $_storagedir 2>/dev/null - chmod 777 $_storagedir - _wd "set storage [${_storagename}]" - # cfgfile=`dirname $0`/${_storageprefix}${_storagename}.cfg - cfgfile=$_storagedir/${_storageprefix}${_storagename}.cfg -} - -# list existing storages -function storagelist(){ - ls -1 ${_storagedir}/${_storageprefix}*.cfg | sed "s#${_storagedir}/${_storageprefix}##g" | sed "s#.cfg\$##g" -} - -# delete current storage data -function storagedelete(){ - rm -f "$cfgfile" -} - - -# ---------------------------------------------------------------------- -# variable functions -# ---------------------------------------------------------------------- - -function _removeprefix(){ - local _var=$1 - _wd "removing ${_var} ..." - cat $cfgfile 2>/dev/null | grep -v "^${_var}" > $tmpfile - mv $tmpfile $cfgfile -} - -function vargetvalue(){ - local _var=$1 - grep "^${_var}=" $cfgfile | cut -f 2 -d '=' -} - - -function varaddvalue(){ - local _var=$1 - shift 1 - local _val=$* - local _value=`vargetvalue $_var` - echo $_value | grep '^\[' >/dev/null - if [ $? -ne 0 ]; then - # echo ERROR: you can add a value to an array only - not on [$_value] - _we "ERROR: you can add a value to an array only - not on [$_value]" - else - _wd "OK, array detected: $_value" - _value=`echo $_value | sed "s#]##"`,$_val"]" - varset $_var "$_value" - fi -} - -function varremove(){ - local _var=$1 - # _removeprefix "${_var}=" - cat $cfgfile 2>/dev/null | grep -v "^${_var}=" > $tmpfile - mv $tmpfile $cfgfile -} - - -function varset(){ - local _var=$1 - shift 1 - local _val=$* - - varremove "${_var}" - _wd "setting ${_var} = ${_val} ..." - (echo ${_var}=${_val} ; cat $cfgfile 2>/dev/null ) >$tmpfile - sort $tmpfile > $cfgfile - rm -f $tmpfile -} -function varsetfile(){ - local _var=$1 - shift 1 - local _val=$* - varset $_var "${_markFile}${_val}" -} -function showstorage(){ - if [ -f $cfgfile ]; then - _wd "content of storage [${_storagename}]" - cat $cfgfile - else - _wd "storage [${_storagename}] is emtpy" - fi -} - -# ---------------------------------------------------------------------- -# export functions -# ---------------------------------------------------------------------- - -function _showFileAsJson(){ - local _file=$1 - which jq >/dev/null - if [ $? -eq 0 ]; then - jq -nR --arg data "`cat $_file`" '$data' - else - _we "ERROR: Json export of files ${_markFile} requires [jq]" - fi -} - -function _jsonloop(){ - local var=$1 - typeset -i iFollow=$2 - - # _wd "_jsonloop $1 $2" - - typeset -i level=`echo $var | grep -o '\.' | wc -l`+2 - - typeset -i local iChilds=0 - typeset -i local iCount=0 - typeset -i local iLeft=0 - - typeset -i lastlevel=$level-1 - local sLastvar=`echo ${var} | cut -f $lastlevel -d "."` - local space=`printf %${lastlevel}s |tr " " "\t"` - - # --- handle existing subkeys - grep "^${var}\." $cfgfile >/dev/null && ( - iCount=0 - iChilds=`grep "^${var}" $cfgfile | wc -l` - echo "${space}\"$sLastvar\": {" - for myhash in `grep "^${var}[\.=]" $cfgfile | cut -f 1 -d "=" | cut -f $level -d "." | sort -u` - do - iCount=$iCount+1 - iLeft=$iChilds-$iCount - _jsonloop "$var.$myhash" $iLeft - done - echo -n "${space}}" - ) - - # --- show values - - grep "^${var}=" $cfgfile >/dev/null && ( - value=`grep "^${var}=" $cfgfile | cut -f 2- -d "="` - echo -n "${space}\"$sLastvar\": " - echo ${value}| grep "${_markFile}" >/dev/null - if [ $? -eq 0 ]; then - _showFileAsJson `echo ${value} | sed "s#^${_markFile}##"` - else - echo -n "${value}" - fi - ) - if [ $iFollow -gt 0 ]; then - echo "," - else - echo - fi -} - -function exportJson(){ - # _wd "exportJson - init: find first level of vars" - typeset -i local iCount=0 - typeset -i local iLeft=0 - typeset -i local iChilds=`grep "^[a-zA-Z]" $cfgfile | cut -f 1 -d "=" | cut -f 1 -d '.' | sort -u | wc -l` - - echo "{" - - # first level names - for lev1var in `cat $cfgfile | grep "^.*\=" | cut -f 1 -d "=" | grep -v "\." | sort -u` - do - #_wd "exportJson - start run with var [$lev1var]" - iCount=$iCount+1 - iLeft=$iChilds-$iCount - _jsonloop "$lev1var" $iLeft - done - - # _wd "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" - # first level hashes - for lev1var in `cat $cfgfile | cut -f 1 -d "=" | grep "\." | cut -f 1 -d "." | sort -u` - do - #_wd "exportJson - start run with hash [$lev1var]" - iCount=$iCount+1 - iLeft=$iChilds-$iCount - _jsonloop "$lev1var" $iLeft - done - echo "}" - -} - - -# W.I.P -# JSON export with processing line by line -# -function exportJson2(){ - typeset -i local iSublevel=0 - typeset -i local iLast=0 - - typeset -i local lastlevel - local sLastvar - - grep '^[a-zA-Z]' $cfgfile | while read line - do - fullvar=`echo $line | cut -f 1 -d "="` - iSublevel=`echo $fullvar | grep -o '\.' | wc -l`+2 - lastlevel=$iSublevel-1 - sLastvar=`echo ${fullvar} | cut -f $lastlevel -d "."` - - local space=`printf %${lastlevel}s |tr " " "\t"` - - echo "${space} $line .. level $iSublevel .. $sLastvar" - done -} - -# ---------------------------------------------------------------------- -# help -# ---------------------------------------------------------------------- - -# show help text -function showHelp(){ -self=`basename $0` -cat <<EOH - -$_product v$_version - -This script handles variables and nested values (hashes and arrays). -You can create several configuration sets (storages) to isolate them. - -Help - - --help or -h or -? - show this help and abort. - - --version or -v - show the version. - -Storage parameters - - --storage NAME - Set a storage to use. Allowed chars are a-z and 0-9. - Default is a storage named "default". - You can set an evironment variable CFGSTORAGE to override - the default (= to skip --storage NAME). - - --list - List existing storages. - - --flush - delete current storage data - - --show - show the current config entries as plain text - - --json - show config entries as json. - Warning: it is implemented in Bash - can be slow on more data - -Create | read | update | delete variables - - --set VARNAME VALUE - Set a variable and its value. - If it does not exist then it will be created. - If it exist then it will be overwritten with the new value. - - For the values you can force the type: - * Use single and double quotes to set a string, i.e. - \$ $self --set my_string '"hello world"' - - * Quote the doublequotes to expand variables, i.e. - \$ $self --set my_string2 \"\${my_string2}\" - - * use no quoting vor integers and values - \$ $self --set number_cpu 8 - - * use barackets in quotes to set an array - \$ $self --set my_array '[8, 12, 16]' - - Use a "." (dot) in varname to create a hierarchy, i.e. - \$ $self --set host.os '"linux"' - - --setfile VARNAME FILENAME - Set an input file to show its content - - --add VARNAME VALUE - Add VALUE as array element to VARNAME. - It fails if VARNAME does not exist or isn't an array. - $ confighandler.sh --add my_array 20 - - --get VARNAME - Show value of the given variable. - - --delete VARNAME - delete a variable. - -You can concatenate all parameters. So you can use --show before -and after an --set action to see the differences. -\$ $self --show --set hello '"world"' --show - -You get additional infos on STDERR. To get plain data send STDERR -to /dev/null -\$ $self --show 2>/dev/null - -EOH -} - -# ---------------------------------------------------------------------- -# -# MAIN -# -# ---------------------------------------------------------------------- - - -if [ ! -z $CFGSTORAGE ]; then - storageset "$CFGSTORAGE" -else - storageset "default" -fi - -if [ $# -eq 0 ]; then - showHelp - exit 0 -fi - -while [ $# -gt 0 ]; -do - case "$1" in - '--help' | '-h' | '-?') - _wd ">>> $1" - showHelp - exit 0 - ;; - '--version' | '-v') - _wd ">>> $1" - echo $_product v$_version - ;; - - # ----- storage - - '--storage') - _wd ">>> $1 $2" - storageset "$2" - shift 1 - ;; - '--list') - _wd ">>> $1" - storagelist - ;; - '--flush') - _wd ">>> $1" - storagedelete - ;; - - # ----- variables - - '--set') - _wd ">>> $1 $2 $3" - varset $2 "$3" - shift 2 - ;; - '--setfile') - _wd ">>> $1 $2 $3" - varsetfile $2 "$3" - shift 2 - ;; - '--add') - _wd ">>> $1 $2 $3" - varaddvalue $2 "$3" - shift 2 - ;; - '--get') - _wd ">>> $1 $2" - vargetvalue $2 - shift 1 - ;; - '--delete') - _wd ">>> $1 $2" - varremove $2 - shift 1 - ;; - '--show') - _wd ">>> $1" - showstorage - ;; - - # ----- export - - '--json') - _wd ">>> $1" - exportJson - # time exportJson - # time exportJson2 - ;; - *) - _we "ERROR: unknown param detected: [$1]" - showHelp - echo "Aborting ..." - exit 1 - esac - _wd "" - shift 1 - -done -_wd "DONE"