Initial commit

_etc/api-director.cfg.dist

+#
+# API CONFIG to connect to REST API
+#
+# Description:
+# Connect to Icinga Director# Used to create/ read/ update/ delete my host
+# This config is included in /opt/imlmonitor/client/director-cli.sh
+#
+
+RestApiBaseUrl="https://icinga.example.com/icingaweb2/"
+RestApiUser="directorcli"
+RestApiPassword="password-of-director-api-user"
+RestApiDocs="https://icinga.com/docs/director/latest/doc/70-REST-API/"

_etc/api-icinga2.cfg.dist

+#
+# API CONFIG to connect to REST API
+#
+# Description:
+# Connect to Icinga API
+# Used to ...
+#   - create service of my host
+#   - to send monitor check data to icinga
+#
+# This config is included in /opt/imlmonitor/client/service-cli.sh 
+#
+
+RestApiBaseUrl="https://icinga-endpoint.example.com:5665/v1/"
+RestApiUser="clientchecks"
+RestApiPassword="password-of-Icinga-Api-User"
+RestApiDocs="https://icinga.com/docs/icinga2/latest/doc/12-icinga2-api/"

_etc/checks/CPU-usage-example.dist

+# ------------------------------------------------------------
+#
+# This is an example file for a client check.
+# Syntax is
+# [variable]=[Value]
+# - no spaces around "="
+# - no quoting of spaces required after "="
+#
+# You should generate this config file.
+#
+# ------------------------------------------------------------
+
+# vivible label in Icinga web frontend:
+checkname=CPU-usage
+
+# command to be executed. It will be searched in given pathes - see ../client.cfg
+# the parameters 
+command=check_cpu -w 90 -c 95 -i 50
+customparams=
+
+# interval of check execution; must be 60s and higher
+interval=60
+
+# icon to be used in icingaweb frontend
+icon=/images/nuvolax64/apps/kcmprocessor.png
+
+# number of failing checks before alert
+max_check_attempts=3
+

_etc/client.cfg.dist

+# ============================================================
+#
+# CONFIG file for execution of the client checks (aka plugins)
+#
+# Remark:
+# this is a bash file that it can be sourced
+#
+# ------------------------------------------------------------
+#
+# DO NOT CHANGE THIS FILE - IT IS GENERATED BY PUPPET
+#
+# instead change the template
+# modules/iml/templates/component/icinga/client/client.cfg.erb
+#
+# ------------------------------------------------------------
+# 2020-02-xx Axel Hahn
+# ============================================================
+
+
+
+# ------------------------------------------------------------
+#
+# selfname is set in puppet... define default dirs with it
+#
+selfname="icinga2-passive-client"
+
+  # directory of config files for api profiles and defined
+  # checks
+  dir_cfg="/etc/${selfname}"
+
+  # tmp data of checks, i.e. to keep last output
+  dir_data="/var/tmp/${selfname}"
+
+  # log dir ... *log will be rotated by logrotate config
+  dir_logs="/var/log/${selfname}"
+
+
+# ------------------------------------------------------------
+#
+# where to find check scripts ... first directory where a
+# check plugin was found wins
+#
+dir_plugins=" \
+  /opt/imlmonitor/client/plugins \
+  /usr/lib64/nagios/plugins \
+"
+
+# ------------------------------------------------------------
+# ports open to outside
+host_vars_tcpport="[22, 80, 443]"
+
+# ------------------------------------------------------------

cleanup-services.sh

+#/bin/bash
+# ======================================================================
+#
+# CLEANUP script to remove services on a host if a service was deleted
+# in LDAP
+#
+# IML specific: uses puppet to recreate all checks
+#
+# script is started as cron
+# ----------------------------------------------------------------------
+# 2020-03-06  v0.1  <axel.hahn@iml.unibe.ch>
+# 2020-03-09  v0.2  <axel.hahn@iml.unibe.ch>  waiting for pupet puppet
+# ======================================================================
+
+. `dirname $0`/inc_getconfig.sh || exit 1
+
+dirWithChecks=${dir_cfg}/checks/
+lockfile="${dir_data}/`basename $0`.pid"
+
+typeset -i iCounter=0
+typeset -i iMaxWait=120
+
+# ----------------------------------------------------------------------
+# MAIN
+# ----------------------------------------------------------------------
+
+if [ ! -d ${dirWithChecks} ]; then
+    echo ERROR: directory witch check configs does not exist [$dirWithChecks]
+    exit 1
+fi
+
+
+if [ -f "${lockfile}" ]; then
+    lockpid=`cat "${lockfile}" | cut -f 2 -d "-" | cut -f 4 -d " " | grep "[0-9]"`
+    ps -f --pid $lockpid >/dev/null
+    if [ $? -eq 0 ]; then
+        _log "ABORT: update seems to run already. See process with PID $lockpid"
+        echo
+        ps -f --pid $lockpid
+        echo
+        exit 0
+    fi
+fi
+echo "cleanup services started `date` - process id $$" > "${lockfile}"
+
+
+echo --- 1/3 :: DELETE all existing configs
+ls -l ${dir_cfg}/checks/*
+rm -f ${dir_cfg}/checks/*
+echo
+echo directory content after deletion:
+ls -l ${dir_cfg}/checks/*
+echo
+
+
+echo --- 2/3 :: WAITING max $iMaxWait min for a puppet run to recreate the checks
+rc=1
+while [ $rc -ne 0 ]; do
+    ls -l ${dir_cfg}/checks/* >/dev/null 2>&1
+    rc=$?
+    if [ $rc -ne 0 ]; then
+        iCounter=$iCounter+1
+        echo -n "$iCounter ... `date` - "
+        if [ $iCounter -gt $iMaxWait ]; then
+            echo "ERROR: puppet did not start within $iMaxWait min??? ABORTING here."
+            exit 1
+        fi
+        echo "waiting ..."
+        sleep 60
+    fi
+done
+echo "Found :-)"
+echo
+
+echo --- 3/3 :: CLEANUP links on icinga
+`dirname $0`/director-cli.sh --linkcleanup
+echo
+
+
+echo --- DONE
+rm -f $lockfile
+exit 0
+
+# ----------------------------------------------------------------------

confighandler2.sh

+#!/bin/bash
+# ======================================================================
+#
+# VARIABLE STORAGE written in Bash
+#
+# ----------------------------------------------------------------------
+# 2020-01-25  v0.1  <axel.hahn@iml.unibe.ch>
+# 2020-01-26  v0.2  <axel.hahn@iml.unibe.ch> added --get, --add
+# 2020-01-27  v0.3  <axel.hahn@iml.unibe.ch> added storages handling
+# 2020-01-30  v0.5  <axel.hahn@iml.unibe.ch> added --setfile
+# ======================================================================
+
+_product="VARIABLE STORAGE"
+_version=0.5
+
+_storagedir=/var/tmp/confighandler
+_storageprefix='flat-variable-store-'
+_storagename='default'
+tmpfile=/tmp/cfgmover.tmp
+
+_markFile='::FILE-CONTENT::'
+
+# ----------------------------------------------------------------------
+# FUNCTIONS
+# ----------------------------------------------------------------------
+
+# write a debug message to STDERR
+function _wd(){
+	echo -e "\e[33m# $*\e[0m" >&2
+}
+
+# write error message in red to STDERR
+function _we(){
+	echo -e "\e[31m# $*\e[0m" >&2
+}
+
+# ----------------------------------------------------------------------
+# storage functions
+# ----------------------------------------------------------------------
+
+# set a variable storage
+# param  string  name of the storage; allowed chars are a-z and 0-9
+function storageset(){
+	_storagename=`echo $1 | sed "s#[^a-z0-9]##g"`
+
+	mkdir $_storagedir 2>/dev/null
+        chmod 777 $_storagedir
+	_wd "set storage [${_storagename}]"
+	# cfgfile=`dirname $0`/${_storageprefix}${_storagename}.cfg
+	cfgfile=$_storagedir/${_storageprefix}${_storagename}.cfg
+}
+
+# list existing storages
+function storagelist(){
+	ls -1 ${_storagedir}/${_storageprefix}*.cfg | sed "s#${_storagedir}/${_storageprefix}##g" | sed "s#.cfg\$##g"
+}
+
+# delete current storage data
+function storagedelete(){
+	rm -f "$cfgfile"
+}
+
+
+# ----------------------------------------------------------------------
+# variable functions
+# ----------------------------------------------------------------------
+
+function _removeprefix(){
+	local _var=$1
+	_wd "removing ${_var} ..."
+	cat $cfgfile 2>/dev/null | grep -v "^${_var}" > $tmpfile
+	mv $tmpfile $cfgfile
+}
+
+function vargetvalue(){
+	local _var=$1
+	grep "^${_var}=" $cfgfile | cut -f 2 -d '='
+}
+
+
+function varaddvalue(){
+	local _var=$1
+        shift 1
+	local _val=$*
+	local _value=`vargetvalue $_var`
+	echo $_value | grep '^\[' >/dev/null
+	if [ $? -ne 0 ]; then
+		# echo ERROR: you can add a value to an array only - not on [$_value]
+		_we "ERROR: you can add a value to an array only - not on [$_value]"
+	else
+		_wd "OK, array detected: $_value"
+		_value=`echo $_value | sed "s#]##"`,$_val"]"
+		varset $_var "$_value"
+	fi
+}
+
+function varremove(){
+	local _var=$1
+	# _removeprefix "${_var}="
+	cat $cfgfile 2>/dev/null | grep -v "^${_var}=" > $tmpfile
+	mv $tmpfile $cfgfile
+}
+
+
+function varset(){
+	local _var=$1
+        shift 1
+	local _val=$*
+
+	varremove "${_var}"
+	_wd "setting ${_var} = ${_val} ..."
+	(echo ${_var}=${_val} ; cat $cfgfile 2>/dev/null ) >$tmpfile
+	sort $tmpfile > $cfgfile
+	rm -f $tmpfile
+}
+function varsetfile(){
+	local _var=$1
+        shift 1
+	local _val=$*
+	varset $_var "${_markFile}${_val}"
+}
+function showstorage(){
+	if [ -f $cfgfile ]; then
+		_wd "content of storage [${_storagename}]"
+		cat $cfgfile
+	else
+		_wd "storage [${_storagename}] is emtpy"
+	fi
+}
+
+# ----------------------------------------------------------------------
+# export functions
+# ----------------------------------------------------------------------
+
+function _showFileAsJson(){
+	local _file=$1
+	which jq >/dev/null
+	if [ $? -eq 0 ]; then
+		jq -nR --arg data "`cat $_file`" '$data'
+	else
+		_we "ERROR: Json export of files ${_markFile} requires [jq]"
+	fi
+}
+
+function _jsonloop(){
+	local var=$1
+	typeset -i iFollow=$2
+
+	# _wd "_jsonloop $1 $2"
+
+	typeset -i level=`echo $var | grep -o '\.' | wc -l`+2
+
+	typeset -i local iChilds=0
+	typeset -i local iCount=0
+	typeset -i local iLeft=0
+
+	typeset -i lastlevel=$level-1
+	local sLastvar=`echo ${var} | cut -f $lastlevel -d "."`
+	local space=`printf %${lastlevel}s |tr " " "\t"`
+
+	# --- handle existing subkeys
+	grep "^${var}\." $cfgfile >/dev/null && (
+		iCount=0
+		iChilds=`grep "^${var}" $cfgfile | wc -l`
+		echo "${space}\"$sLastvar\": {"
+		for myhash in `grep "^${var}[\.=]" $cfgfile | cut -f 1 -d "=" | cut -f $level -d "." | sort -u`
+		do
+			iCount=$iCount+1
+			iLeft=$iChilds-$iCount
+			_jsonloop "$var.$myhash" $iLeft
+		done
+		echo -n "${space}}"
+	)
+
+	# --- show values
+
+	grep "^${var}=" $cfgfile >/dev/null && (
+		value=`grep "^${var}=" $cfgfile | cut -f 2- -d "="`
+		echo -n "${space}\"$sLastvar\": "
+		echo ${value}| grep "${_markFile}" >/dev/null
+		if [ $? -eq 0 ]; then
+			_showFileAsJson `echo ${value} | sed "s#^${_markFile}##"`
+		else	
+			echo -n "${value}"
+		fi
+	)
+	if [ $iFollow -gt 0 ]; then
+		echo ","
+	else
+		echo
+	fi
+}
+
+function exportJson(){
+	# _wd "exportJson - init: find first level of vars"
+	typeset -i local iCount=0
+	typeset -i local iLeft=0
+	typeset -i local iChilds=`grep "^[a-zA-Z]" $cfgfile | cut -f 1 -d "=" | cut -f 1 -d '.' | sort -u | wc -l`
+
+	echo "{"
+
+	# first level names
+	for lev1var in `cat $cfgfile | grep "^.*\=" | cut -f 1 -d "=" | grep -v "\." | sort -u`
+	do
+		#_wd "exportJson - start run with var [$lev1var]"
+		iCount=$iCount+1
+		iLeft=$iChilds-$iCount
+		_jsonloop "$lev1var" $iLeft
+	done
+
+	# _wd "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+	# first level hashes
+	for lev1var in `cat $cfgfile | cut -f 1 -d "=" | grep  "\." | cut -f 1 -d "." | sort -u`
+	do
+		#_wd "exportJson - start run with hash [$lev1var]"
+		iCount=$iCount+1
+		iLeft=$iChilds-$iCount
+		_jsonloop "$lev1var" $iLeft
+	done
+	echo "}"
+
+}
+
+
+# W.I.P
+# JSON export with processing line by line
+#
+function exportJson2(){
+	typeset -i local iSublevel=0
+	typeset -i local iLast=0
+
+	typeset -i local lastlevel
+	local sLastvar
+
+	grep '^[a-zA-Z]' $cfgfile | while read line
+	do
+		fullvar=`echo $line | cut -f 1 -d "="`
+		iSublevel=`echo $fullvar | grep -o '\.' | wc -l`+2
+		lastlevel=$iSublevel-1
+		sLastvar=`echo ${fullvar} | cut -f $lastlevel -d "."`
+
+		local space=`printf %${lastlevel}s |tr " " "\t"`
+
+		echo "${space} $line .. level $iSublevel .. $sLastvar"
+	done
+}
+
+# ----------------------------------------------------------------------
+# help
+# ----------------------------------------------------------------------
+
+# show help text
+function showHelp(){
+self=`basename $0`
+cat <<EOH
+
+$_product v$_version
+
+This script handles variables and nested values (hashes and arrays).
+You can create several configuration sets (storages) to isolate them.
+
+Help
+
+	--help or -h or -?
+		show this help and abort.
+
+	--version or -v
+		show the version.
+
+Storage parameters
+
+	--storage NAME
+		Set a storage to use. Allowed chars are a-z and 0-9.
+		Default is a storage named "default".
+		You can set an evironment variable CFGSTORAGE to override
+		the default (= to skip --storage NAME).
+
+	--list
+		List existing storages.
+
+	--flush
+		delete current storage data
+
+	--show
+		show the current config entries as plain text
+
+	--json
+		show config entries as json.
+		Warning: it is implemented in Bash - can be slow on more data
+
+Create | read | update | delete variables
+
+	--set VARNAME VALUE
+		Set a variable and its value.
+		If it does not exist then it will be created.
+		If it exist then it will be overwritten with the new value.
+
+		For the values you can force the type:
+			* Use single and double quotes to set a string, i.e.
+				\$ $self --set my_string '"hello world"'
+
+			* Quote the doublequotes to expand variables, i.e.
+				\$ $self --set my_string2 \"\${my_string2}\"
+
+			* use no quoting vor integers and values
+				\$ $self --set number_cpu 8
+
+			* use barackets in quotes to set an array
+				\$ $self --set my_array '[8, 12, 16]'
+
+		Use a "." (dot) in varname to create a hierarchy, i.e.
+			\$ $self --set host.os '"linux"'
+
+	--setfile VARNAME FILENAME
+		Set an input file to show its content
+
+	--add VARNAME VALUE
+		Add VALUE as array element to VARNAME.
+		It fails if VARNAME does not exist or isn't an array.
+			 $ confighandler.sh --add my_array 20
+
+	--get VARNAME
+		Show value of the given variable.
+
+	--delete VARNAME
+		delete a variable.
+
+You can concatenate all parameters. So you can use --show before
+and after an --set action to see the differences.
+\$ $self --show --set hello '"world"' --show
+
+You get additional infos on STDERR. To get plain data send STDERR
+to /dev/null
+\$ $self --show 2>/dev/null
+
+EOH
+}
+
+# ----------------------------------------------------------------------
+#
+# MAIN
+#
+# ----------------------------------------------------------------------
+
+
+if [ ! -z $CFGSTORAGE ]; then
+	storageset "$CFGSTORAGE"
+else
+	storageset "default"
+fi
+
+if [ $# -eq 0 ]; then
+	showHelp
+	exit 0
+fi
+
+while [ $# -gt 0 ];
+do
+	case "$1" in
+		'--help' | '-h' | '-?')
+			_wd ">>> $1"
+			showHelp
+			exit 0
+			;;
+		'--version' | '-v')
+			_wd ">>> $1"
+			echo $_product v$_version
+			;;
+
+		# ----- storage
+
+		'--storage')
+			_wd ">>> $1 $2"
+			storageset "$2"
+			shift 1
+			;;
+		'--list')
+			_wd ">>> $1"
+			storagelist
+			;;
+		'--flush')
+			_wd ">>> $1"
+			storagedelete
+			;;
+
+		# ----- variables
+
+		'--set')
+			_wd ">>> $1 $2 $3"
+			varset $2 "$3"
+			shift 2
+			;;
+		'--setfile')
+			_wd ">>> $1 $2 $3"
+			varsetfile $2 "$3"
+			shift 2
+			;;
+		'--add')
+			_wd ">>> $1 $2 $3"
+			varaddvalue $2 "$3"
+			shift 2
+			;;
+		'--get')
+			_wd ">>> $1 $2"
+			vargetvalue $2
+			shift 1
+			;;
+		'--delete')
+			_wd ">>> $1 $2"
+			varremove $2
+			shift 1
+			;;
+		'--show')
+			_wd ">>> $1"
+			showstorage
+			;;
+
+		# ----- export
+
+		'--json')
+			_wd ">>> $1"
+			exportJson
+			# time exportJson
+			# time exportJson2
+			;;
+		*)
+			_we "ERROR: unknown param detected: [$1]"
+			showHelp
+			echo "Aborting ..."
+			exit 1
+	esac
+	_wd ""
+	shift 1
+
+done
+_wd "DONE"

icinga-cli.sh

+#!/bin/bash
+# ======================================================================
+#
+# ICINGA PASSIVE CLIENT
+#
+# Run all passive checks and send response to Icinga Endpoint
+#
+# Requirements
+# - curl
+# - jq
+# ----------------------------------------------------------------------
+#
+# ======================================================================
+
+
+_product="ICINGA PASSIVE CLIENT"
+_version="0.3"
+_license="GNU GPL 3.0"
+_copyright='(c) 2020 Institute for Medical Education * University of Bern'
+
+typeset -i debug=0
+
+# source config ...
+. `dirname $0`/inc_getconfig.sh
+
+. `dirname $0`/inc_functions.sh
+. `dirname $0`/inc/rest-api-client.sh
+
+# where to find check scripts ... first directory wins
+# dir_plugins="/opt/imlmonitor/client/plugins/ /usr/lib64/nagios/plugins"
+# dir_cfg="/etc/icinga2-passive-client"
+# dir_data="/var/tmp/icinga2-passive-client"
+# dir_logs="/var/log/icinga2-passive-client"
+logfile=${dir_logs}/execution.log
+
+ch=`dirname $0`/inc/confighandler.sh
+myHost=`hostname -f`
+
+# for loop mode only: max. random sleep time
+typeset -i sleeptime=30
+
+typeset -i _rc_all=0
+
+# ----------------------------------------------------------------------
+#
+# FUNCTIONS
+#
+# ----------------------------------------------------------------------
+# ..................................................................
+#
+# helper to make http base setup for host actions
+function _initHttp(){
+  # see inc_functions
+   _initHttpWithConfigfile "/etc/icinga2-passive-client/api-icinga2.cfg"
+  if [ $debug -ne 0 ]; then
+    http.setDebug 1
+  fi
+}
+
+
+# ......................................................................
+#
+# find first place of the check script in the known plugin dirs 
+# see ${dir_plugins} in checks.cfg
+# param  string  name of the check script without path
+#
+function findCheckScript(){
+	local _script=$1
+	for mydir in ${dir_plugins}
+	do
+		if [ -x ${mydir}/${_script} ]; then
+			echo ${mydir}/${_script}
+		fi
+	done | head -1
+}
+
+# helper used function in loopChecks
+# get a snapshot of a few files
+function _getFileSnapshot(){
+  ls -l `dirname $0`/* ${dir_cfg}/*
+}
+
+
+# ......................................................................
+#
+# Loop over executing all checks
+# no params
+#
+function loopChecks(){
+
+      # TODO-MEMORY-CHECK
+      # echo ${myHost} | egrep "^(kvm4|icinga)"
+      # echo ${myHost} | egrep "^(monitortest)"
+      # if [ $? -ne 0 ]; then
+      #   echo "HARD EXIT - DO NOT EXECUTE ANY CHECK ON $myHost"
+      #   exit 1
+      # fi
+
+      local lockfile="${dir_data}/loop.pid"
+      local snapShotStart=${dir_data}/`basename $0`-start.fingerprint
+      local snapShotCurrent=${dir_data}/`basename $0`-last.fingerprint
+      if [ -f "${lockfile}" ]; then
+        local lockpid=`cat "${lockfile}" | cut -f 2 -d "-" | cut -f 4 -d " " | grep "[0-9]"`
+        ps -f --pid $lockpid >/dev/null
+        if [ $? -eq 0 ]; then
+          _log "ABORT: Loop seems to run already. See process with PID $lockpid"
+          echo
+          ps -f --pid $lockpid
+          echo
+          exit 0
+        fi
+      fi
+
+      _log "---------- starting in a permanent loop"
+      echo "Serviceloop started `date` - process id $$" > "${lockfile}"
+      _getFileSnapshot>$snapShotStart
+      while true; do
+        # typeset -i local iSleep=$(($RANDOM%$sleeptime))
+        # sleep minimum is half of $sleeptime
+        typeset -i local iSleep=$(($RANDOM%$sleeptime/2+$sleeptime/2))
+        _log "sleeping $iSleep sec ..."
+        sleep $iSleep
+        _log "______________________________________________________________________"
+        _log ""
+        _getFileSnapshot>$snapShotCurrent
+        diff  $snapShotStart $snapShotCurrent >/dev/null
+        if [ $? -ne 0 ]; then
+          _log "ABORT: Files were updated / overwritten. The loop must be restarted.\n`diff  $snapShotStart $snapShotCurrent`"
+          exit 1
+        fi
+        icingaHostMustExist
+        processAllChecks
+        echo
+        echo
+      done
+}
+# ......................................................................
+#
+# execute all defined checks one by one
+# no params
+#
+function processAllChecks(){
+  # loop over all defined checks
+  typeset -i local iChecksTotal=`getChecks | wc -l`
+  typeset -i local iCounter=0
+
+  _rc_all=0
+  typeset -i local iLoopStart=`_getUnixTs`
+
+  _log ""
+  _log "------ looping over all checks"
+  getChecks
+  echo
+  for myconfig in `getChecks`
+  do
+    iCounter=$iCounter+1
+    _log "--- processing [$iCounter of $iChecksTotal] $myconfig"
+    processCheck $myconfig
+    _log ""
+    echo
+    echo ----------------------------------------------------------------------
+    echo
+  done
+  typeset -i local iLoopEnd=`_getUnixTs`
+  typeset -i local iLoopTime=$iLoopEnd-$iLoopStart
+
+  _log "------ loop done - needed $iLoopTime sec - rc=$_rc_all"
+}
+
+# ......................................................................
+#
+# parse a config file and set global vars:
+#   checkName
+#   checkCommand
+#   checkInterval
+# param  string  full path of a config file
+#
+function _parseCheckConfig(){
+  local _myconfig=$1
+
+  # EXAMPLE a config contains ...
+  # checkname=check_cronstatus
+  # command=check_cronstatus -param1 -param2
+  # interval=60
+
+  checkName=`cat $_myconfig | grep ^checkname= | cut -f 2 -d "="`
+  checkCommand=`cat $_myconfig | grep ^command= | cut -f 2 -d "="`
+  checkInterval=`cat $_myconfig | grep ^interval= | cut -f 2 -d "="`
+
+}
+
+
+function icingaHost(){
+  local _logPrefix="${myHost} :: API |"
+  local _apiRequest=objects/hosts/${myHost}
+  local _localCache=${dir_data}/host_${myHost}_deployed-at-icinga.txt
+  typeset -i local _iRefreshCache=120
+
+  local sAction=$1
+
+  _initHttp
+
+  case $sAction in
+    'get')
+      # update after caching was added in http-component
+      # http.responseImport $_localCache
+      # typeset -i local iAgeLastGet=`http.getRequestAge`
+      # _log "${_logPrefix} INFO: cache is $iAgeLastGet sec old ... TTL is _iRefreshCache=3600 sec"
+      # if [ $iAgeLastGet -eq 0 -o $iAgeLastGet -gt $_iRefreshCache ]; then
+      #   _log "${_logPrefix} INFO: request to Icinga GET $_apiRequest"
+      #   _getApiObject $_apiRequest $_localCache
+      # fi
+
+      # new:
+      http.setCacheTtl     $_iRefreshCache
+      http.setCacheFile    $_localCache
+      http.makeRequest GET $_apiRequest
+
+      # set return code of GET action
+      http.isOk >/dev/null
+      ;;
+    *)
+      _log "ERROR: unknown action parameter $sAction"
+  esac
+
+}
+
+function icingaHostMustExist(){
+  _log "check if the host [${myHost}] exists on Icinga ..."
+  icingaHost get
+  if [ $? -ne 0 ]; then
+    http.getResponse
+    if [ "`http.getStatuscode`" = "000" ]; then
+      _log "ERROR: Unable to reach the Icinga node. Stopping script current monitoring actions."
+      exit 1
+    fi
+    _log "ERROR: host object for ${myHost} is not available on Icinga service (yet) - Status: `http.getStatuscode`"
+    echo
+    echo "ABORTING"
+    echo
+    echo "To run checks ..."
+    echo "- you must create the host on director (check director-cli.sh --hr)"
+    echo "- the director must deploy the host to icinga daemon"
+    echo
+    rm -f ${dir_data}/service__check* 2>/dev/null
+    exit 1
+  fi
+  _log "OK, found."
+}
+
+
+# ......................................................................
+#
+# process a single check
+# param  string  name of config file
+# param  string  name of config file
+#
+function processCheck(){
+  local _myconfig=$1
+  local _force=$2
+
+  typeset -i local iCheckStart=`_getUnixTs`
+
+  _parseCheckConfig ${_myconfig}
+  local _logPrefix="${checkName} |"
+  _log "${_logPrefix} INFO: every ${checkInterval} sec: ${checkCommand}"
+
+  local _outfile=${dir_data}/service__check__${checkName}__output.txt
+  local _response=${dir_data}/service__check__${checkName}__icinga_response.txt
+  typeset -i local _rc=0
+
+
+  _initHttp
+
+  # --- check last run ... if never or > $interval then execute
+  doRun=0
+  if [ ! -f $_outfile ]; then
+    _log "${_logPrefix} INFO: Never executed before"
+    doRun=1
+  else
+    # typeset -i iAgeLastRun=$(($(date +%s) - $(date +%s -r "$_outfile")))
+    typeset -i iAgeLastRun=`_getFileAge "$_outfile"`
+    _log "${_logPrefix} INFO: last run was $iAgeLastRun sec ago ... vs Interval = $checkInterval ... sleeptime = $sleeptime"
+    iAgeLastRun=$iAgeLastRun+$sleeptime
+    if [ $iAgeLastRun -gt $checkInterval ]; then
+      doRun=1
+    fi
+
+    if [ ! -z "$_force" ]; then
+      doRun=1
+      _log "${_logPrefix} INFO: forced execution by given param "
+    fi
+  fi
+
+  if [ $doRun -ne 0 ]; then
+    myscript=`echo $checkCommand | cut -f 1 -d " "`
+    myFullscript=`findCheckScript $myscript`
+    if [ -z "$myFullscript" ]; then
+      _log "${_logPrefix} ERROR: $myscript was not found in any plugin dir"
+    else
+      myparams=`echo $checkCommand | grep " " | cut -f 2- -d " "`
+
+      #
+      # --- this executes the check plugin ...
+      #
+      _log "${_logPrefix} starting $myFullscript $myparams"
+      typeset -i local iTsStart=`date +%s`
+      # $myFullscript $myparams | tee $_outfile
+      eval $myFullscript $myparams > $_outfile
+      rc=$?
+      typeset -i local iTsEnd=`date +%s`
+      outPerfdata=`grep '|' $_outfile | cut -f 2 -d '|'`
+      echo
+      echo -------- check output:
+      cat $_outfile
+      echo
+      echo -------- extracted performance data:
+      echo $outPerfdata
+      echo
+      _log "${_logPrefix} check command finished with returncode $rc"
+
+      _rc=$_rc+$rc
+      #
+      # --- send check result to Icinga
+      #     fields of the object
+      #     https://icinga.com/docs/icinga2/latest/doc/12-icinga2-api/#process-check-result
+      export CFGSTORAGE="${checkName}output"
+
+      outputAsText="$(cat $_outfile)"
+      outputAsJson="$(jq -nR --arg data """${outputAsText}""" '$data')"
+      commandAsJson="$(jq -nR --arg data """${myFullscript} $myparams""" '$data')"
+      (
+        $ch --set      check_source      \"${myHost}\"
+        $ch --set      check_command     "${commandAsJson}"
+        $ch --set      exit_status       $rc
+        # $ch --set      plugin_output     "${outputAsJson}"
+        $ch --setfile  plugin_output     "${_outfile}"
+        $ch --set      performance_data  "\"${outPerfdata}\""
+        $ch --set      ttl               $checkInterval
+        $ch --set      execution_start   $iTsStart
+        $ch --set      execution_end     $iTsEnd
+      ) 2>/dev/null
+      # $ch --json
+      data=`$ch --json 2>/dev/null`
+
+      slot="`_getName4Svcathost ${checkName} | sed 's# #%20#g'`"
+
+      _log "${_logPrefix} starting POST of data to monitoring server"
+      echo POST actions/process-check-result?service=${myHost}!${slot} "$data"
+      _APIcall POST actions/process-check-result?service=${myHost}!${slot} "$data"
+      http.responseExport "$_response"
+
+      # --- check if data were sent successfully
+      # fgrep "HTTP/1.1 200" ${_response} >/dev/null
+      # _testHttpOk ${_response} >/dev/null
+      http.isOk 2>/dev/null
+      if [ $? -eq 0 ]; then
+        _log "${_logPrefix} OK, response was sent to Icinga"
+      else
+        _log "${_logPrefix} WARNING: the check response was NOT sent to Icinga"
+        _rc=$_rc+1
+        echo
+        echo For Debugging:
+        $ch --show --json
+      fi
+      $ch --flush 2>/dev/null
+
+    fi
+
+  else
+    _log "${_logPrefix} SKIP execution."
+  fi
+
+  # add current result to global returncode
+  _rc_all=$_rc_all+$_rc
+
+  typeset -i local iCheckEnd=`_getUnixTs`
+  typeset -i local iCheckTime=$iCheckEnd-$iCheckStart
+
+  _log "${_logPrefix} finished after $iCheckTime sec with returncode $_rc"
+  test $_rc -eq 0 || (echo; echo "    >>> Check ${checkName} was not OK. See Output block above!"; echo; echo)
+}
+# ----------------------------------------------------------------------
+# help
+# ----------------------------------------------------------------------
+
+# show help text
+function showHelp(){
+self=`basename $0`
+cat <<EOH
+
+
+INTRODUCTION
+
+$_product v$_version
+
+Handle and execute icinga passive checks.
+With this client can run a single check, all checks or make a permanent loop.
+A new local check will be added to Icinga while running it the first time.
+
+
+GENERAL PARAMETERS
+
+  --help or -h or -?
+    show this help and abort.
+
+  --version or -v
+    show the version abd abort
+
+
+SERVICE ACTIONS
+
+  --list
+    get a list of local config files
+
+  --loop
+    Start to check all passive checks in a permanent loop.
+    It makes a random sleep if $sleeptime sec between all loops. This shuffles
+    the access time of all clients making requests to the icinga server.
+
+    Multiple starts will be detected. This parameter is the optimal choice for a
+    cronjob.
+
+  --runonce
+    Start to check all passive checks once.
+    This method respects the interval per check. Only outdated checks will be
+    executet.
+    This is a second choice for a cronjob if the runtime of all checks
+    is much shorter than your cronjob interval to prevent multiple processes.
+    Multiple starts will NOT be detected.
+
+  --run CONFIGFILE
+    Run a check by pointing the config file (see --list)
+    This execution ignores the interval and forces the execution.
+
+
+CONFIG
+
+  Config file
+  /etc/icinga2-passive-client/client.cfg
+  It sets other used pathes.
+
+  Config files for checks are in ${dir_cfg}/checks/
+
+
+DEBUGGING
+
+  The Output of check and results from Icinga are in
+  ${dir_data}.
+
+  A log of all performed executed check runs by $self
+  are in $logfile.
+  BTW: do not forget to add a log rotation for it.
+
+EOH
+}
+
+function showVersion(){
+  echo "$_license"
+  echo "$_copyright"
+  echo
+}
+# ----------------------------------------------------------------------
+#
+# MAIN
+#
+# ----------------------------------------------------------------------
+
+
+cat <<EOBANNER
+______________________________________________________________________________________
+
+ _______        __
+|_     _|.----.|__|.-----.-----.---.-.
+ _|   |_ |  __||  ||     |  _  |  _  |
+|_______||____||__||__|__|___  |___._|
+                         |_____|
+ ______                     __                   ______ _____   __               __
+|   __ \.---.-.-----.-----.|__|.--.--.-----.    |      |     |_|__|.-----.-----.|  |_
+|    __/|  _  |__ --|__ --||  ||  |  |  -__|    |   ---|       |  ||  -__|     ||   _|
+|___|   |___._|_____|_____||__| \___/|_____|    |______|_______|__||_____|__|__||____|
+                                                                                 v${_version}
+
+  $_license .. $_copyright
+
+______________________________________________________________________________________
+
+
+EOBANNER
+
+
+if [ -z "${dir_cfg}" ]; then
+  echo ERROR: $_product is not installed/ configured yet on this machine.
+  exit 1
+fi
+
+icingaHostMustExist
+
+
+touch ${logfile}
+if [ $# -eq 0 ]; then
+  showHelp
+  exit 0
+fi
+
+while [ $# -gt 0 ];
+do
+  case "$1" in
+
+    '--help' | '-h' | '-?')
+      showHelp
+      exit 0
+      ;;
+    '--version' | '-v')
+      showVersion
+      exit 0
+      ;;
+
+
+    '--list')
+      getChecks
+      ;;
+
+    '--loop')
+      loopChecks
+      ;;
+
+    '--runonce')
+      processAllChecks
+      ;;
+
+    '--run')
+      processCheck "$2" "force"
+      shift 1
+      ;;
+
+    *)
+      echo "ERROR: unknown parameter detected."
+      exit 2
+  esac
+  shift 1
+done
+echo
+
+# remark:
+# $_rc_all is a collected status code in the loop of all actions
+# if it is 0 then all checks were OK and have been sent to Icinga
+# echo exit with status code $_rc_all
+# exit $_rc_all
+exit 0
+
+# ----------------------------------------------------------------------

inc/confighandler.sh

+#!/bin/bash
+# ======================================================================
+#
+# VARIABLE STORAGE written in Bash
+#
+# ----------------------------------------------------------------------
+# 2020-01-25  v0.1  <axel.hahn@iml.unibe.ch>
+# 2020-01-26  v0.2  <axel.hahn@iml.unibe.ch> added --get, --add
+# 2020-01-27  v0.3  <axel.hahn@iml.unibe.ch> added storages handling
+# 2020-01-30  v0.5  <axel.hahn@iml.unibe.ch> added --setfile
+# ======================================================================
+
+_product="VARIABLE STORAGE"
+_version=0.5
+
+# _storagedir=/var/tmp/confighandler
+_storagedir=/dev/shm/confighandler
+
+_storageprefix='flat-variable-store-'
+_storagename='default'
+tmpfile=/tmp/cfgmover.tmp
+
+_markFile='::FILE-CONTENT::'
+
+# ----------------------------------------------------------------------
+# FUNCTIONS
+# ----------------------------------------------------------------------
+
+# write a debug message to STDERR
+function _wd(){
+	echo -e "\e[33m# $*\e[0m" >&2
+}
+
+# write error message in red to STDERR
+function _we(){
+	echo -e "\e[31m# $*\e[0m" >&2
+}
+
+# ----------------------------------------------------------------------
+# storage functions
+# ----------------------------------------------------------------------
+
+# set a variable storage
+# param  string  name of the storage; allowed chars are a-z and 0-9
+function storageset(){
+	_storagename=`echo $1 | sed "s#[^a-z0-9]##g"`
+
+	mkdir $_storagedir 2>/dev/null
+        chmod 777 $_storagedir
+	_wd "set storage [${_storagename}]"
+	# cfgfile=`dirname $0`/${_storageprefix}${_storagename}.cfg
+	cfgfile=$_storagedir/${_storageprefix}${_storagename}.cfg
+}
+
+# list existing storages
+function storagelist(){
+	ls -1 ${_storagedir}/${_storageprefix}*.cfg | sed "s#${_storagedir}/${_storageprefix}##g" | sed "s#.cfg\$##g"
+}
+
+# delete current storage data
+function storagedelete(){
+	rm -f "$cfgfile"
+}
+
+
+# ----------------------------------------------------------------------
+# variable functions
+# ----------------------------------------------------------------------
+
+function _removeprefix(){
+	local _var=$1
+	_wd "removing ${_var} ..."
+	cat $cfgfile 2>/dev/null | grep -v "^${_var}" > $tmpfile
+	mv $tmpfile $cfgfile
+}
+
+function vargetvalue(){
+	local _var=$1
+	grep "^${_var}=" $cfgfile | cut -f 2 -d '='
+}
+
+
+function varaddvalue(){
+	local _var=$1
+        shift 1
+	local _val=$*
+	local _value=`vargetvalue $_var`
+	echo $_value | grep '^\[' >/dev/null
+	if [ $? -ne 0 ]; then
+		# echo ERROR: you can add a value to an array only - not on [$_value]
+		_we "ERROR: you can add a value to an array only - not on [$_value]"
+	else
+		_wd "OK, array detected: $_value"
+		_value=`echo $_value | sed "s#]##"`,$_val"]"
+		varset $_var "$_value"
+	fi
+}
+
+function varremove(){
+	local _var=$1
+	# _removeprefix "${_var}="
+	cat $cfgfile 2>/dev/null | grep -v "^${_var}=" > $tmpfile
+	mv $tmpfile $cfgfile
+}
+
+
+function varset(){
+	local _var=$1
+        shift 1
+	local _val=$*
+
+	varremove "${_var}"
+	_wd "setting ${_var} = ${_val} ..."
+	(echo ${_var}=${_val} ; cat $cfgfile 2>/dev/null ) >$tmpfile
+	sort $tmpfile > $cfgfile
+	rm -f $tmpfile
+}
+function varsetfile(){
+	local _var=$1
+        shift 1
+	local _val=$*
+	varset $_var "${_markFile}${_val}"
+}
+function showstorage(){
+	if [ -f $cfgfile ]; then
+		_wd "content of storage [${_storagename}]"
+		cat $cfgfile
+	else
+		_wd "storage [${_storagename}] is emtpy"
+	fi
+}
+
+# ----------------------------------------------------------------------
+# export functions
+# ----------------------------------------------------------------------
+
+function _showFileAsJson(){
+	local _file=$1
+	which jq >/dev/null
+	if [ $? -eq 0 ]; then
+		jq -nR --arg data "`cat $_file`" '$data'
+	else
+		_we "ERROR: Json export of files ${_markFile} requires [jq]"
+	fi
+}
+
+function _jsonloop(){
+	local var=$1
+	typeset -i iFollow=$2
+
+	# _wd "_jsonloop $1 $2"
+
+	typeset -i level=`echo $var | grep -o '\.' | wc -l`+2
+
+	typeset -i local iChilds=0
+	typeset -i local iCount=0
+	typeset -i local iLeft=0
+
+	typeset -i lastlevel=$level-1
+	local sLastvar=`echo ${var} | cut -f $lastlevel -d "."`
+	local space=`printf %${lastlevel}s |tr " " "\t"`
+
+	# --- handle existing subkeys
+	grep "^${var}\." $cfgfile >/dev/null && (
+		iCount=0
+		iChilds=`grep "^${var}" $cfgfile | wc -l`
+		echo "${space}\"$sLastvar\": {"
+		for myhash in `grep "^${var}[\.=]" $cfgfile | cut -f 1 -d "=" | cut -f $level -d "." | sort -u`
+		do
+			iCount=$iCount+1
+			iLeft=$iChilds-$iCount
+			_jsonloop "$var.$myhash" $iLeft
+		done
+		echo -n "${space}}"
+	)
+
+	# --- show values
+
+	grep "^${var}=" $cfgfile >/dev/null && (
+		value=`grep "^${var}=" $cfgfile | cut -f 2- -d "="`
+		echo -n "${space}\"$sLastvar\": "
+		echo ${value}| grep "${_markFile}" >/dev/null
+		if [ $? -eq 0 ]; then
+			_showFileAsJson `echo ${value} | sed "s#^${_markFile}##"`
+		else	
+			echo -n "${value}"
+		fi
+	)
+	if [ $iFollow -gt 0 ]; then
+		echo ","
+	else
+		echo
+	fi
+}
+
+function exportJson(){
+	# _wd "exportJson - init: find first level of vars"
+	typeset -i local iCount=0
+	typeset -i local iLeft=0
+	typeset -i local iChilds=`grep "^[a-zA-Z]" $cfgfile | cut -f 1 -d "=" | cut -f 1 -d '.' | sort -u | wc -l`
+
+	echo "{"
+
+	# first level names
+	for lev1var in `cat $cfgfile | grep "^.*\=" | cut -f 1 -d "=" | grep -v "\." | sort -u`
+	do
+		#_wd "exportJson - start run with var [$lev1var]"
+		iCount=$iCount+1
+		iLeft=$iChilds-$iCount
+		_jsonloop "$lev1var" $iLeft
+	done
+
+	# _wd "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+	# first level hashes
+	for lev1var in `cat $cfgfile | cut -f 1 -d "=" | grep  "\." | cut -f 1 -d "." | sort -u`
+	do
+		#_wd "exportJson - start run with hash [$lev1var]"
+		iCount=$iCount+1
+		iLeft=$iChilds-$iCount
+		_jsonloop "$lev1var" $iLeft
+	done
+	echo "}"
+
+}
+
+
+# W.I.P
+# JSON export with processing line by line
+#
+function exportJson2(){
+	typeset -i local iSublevel=0
+	typeset -i local iLast=0
+
+	typeset -i local lastlevel
+	local sLastvar
+
+	grep '^[a-zA-Z]' $cfgfile | while read line
+	do
+		fullvar=`echo $line | cut -f 1 -d "="`
+		iSublevel=`echo $fullvar | grep -o '\.' | wc -l`+2
+		lastlevel=$iSublevel-1
+		sLastvar=`echo ${fullvar} | cut -f $lastlevel -d "."`
+
+		local space=`printf %${lastlevel}s |tr " " "\t"`
+
+		echo "${space} $line .. level $iSublevel .. $sLastvar"
+	done
+}
+
+# ----------------------------------------------------------------------
+# help
+# ----------------------------------------------------------------------
+
+# show help text
+function showHelp(){
+self=`basename $0`
+cat <<EOH
+
+$_product v$_version
+
+This script handles variables and nested values (hashes and arrays).
+You can create several configuration sets (storages) to isolate them.
+
+Help
+
+	--help or -h or -?
+		show this help and abort.
+
+	--version or -v
+		show the version.
+
+Storage parameters
+
+	--storage NAME
+		Set a storage to use. Allowed chars are a-z and 0-9.
+		Default is a storage named "default".
+		You can set an evironment variable CFGSTORAGE to override
+		the default (= to skip --storage NAME).
+
+	--list
+		List existing storages.
+
+	--flush
+		delete current storage data
+
+	--show
+		show the current config entries as plain text
+
+	--json
+		show config entries as json.
+		Warning: it is implemented in Bash - can be slow on more data
+
+Create | read | update | delete variables
+
+	--set VARNAME VALUE
+		Set a variable and its value.
+		If it does not exist then it will be created.
+		If it exist then it will be overwritten with the new value.
+
+		For the values you can force the type:
+			* Use single and double quotes to set a string, i.e.
+				\$ $self --set my_string '"hello world"'
+
+			* Quote the doublequotes to expand variables, i.e.
+				\$ $self --set my_string2 \"\${my_string2}\"
+
+			* use no quoting vor integers and values
+				\$ $self --set number_cpu 8
+
+			* use barackets in quotes to set an array
+				\$ $self --set my_array '[8, 12, 16]'
+
+		Use a "." (dot) in varname to create a hierarchy, i.e.
+			\$ $self --set host.os '"linux"'
+
+	--setfile VARNAME FILENAME
+		Set an input file to show its content
+
+	--add VARNAME VALUE
+		Add VALUE as array element to VARNAME.
+		It fails if VARNAME does not exist or isn't an array.
+			 $ confighandler.sh --add my_array 20
+
+	--get VARNAME
+		Show value of the given variable.
+
+	--delete VARNAME
+		delete a variable.
+
+You can concatenate all parameters. So you can use --show before
+and after an --set action to see the differences.
+\$ $self --show --set hello '"world"' --show
+
+You get additional infos on STDERR. To get plain data send STDERR
+to /dev/null
+\$ $self --show 2>/dev/null
+
+EOH
+}
+
+# ----------------------------------------------------------------------
+#
+# MAIN
+#
+# ----------------------------------------------------------------------
+
+
+if [ ! -z $CFGSTORAGE ]; then
+	storageset "$CFGSTORAGE"
+else
+	storageset "default"
+fi
+
+if [ $# -eq 0 ]; then
+	showHelp
+	exit 0
+fi
+
+while [ $# -gt 0 ];
+do
+	case "$1" in
+		'--help' | '-h' | '-?')
+			_wd ">>> $1"
+			showHelp
+			exit 0
+			;;
+		'--version' | '-v')
+			_wd ">>> $1"
+			echo $_product v$_version
+			;;
+
+		# ----- storage
+
+		'--storage')
+			_wd ">>> $1 $2"
+			storageset "$2"
+			shift 1
+			;;
+		'--list')
+			_wd ">>> $1"
+			storagelist
+			;;
+		'--flush')
+			_wd ">>> $1"
+			storagedelete
+			;;
+
+		# ----- variables
+
+		'--set')
+			_wd ">>> $1 $2 $3"
+			varset $2 "$3"
+			shift 2
+			;;
+		'--setfile')
+			_wd ">>> $1 $2 $3"
+			varsetfile $2 "$3"
+			shift 2
+			;;
+		'--add')
+			_wd ">>> $1 $2 $3"
+			varaddvalue $2 "$3"
+			shift 2
+			;;
+		'--get')
+			_wd ">>> $1 $2"
+			vargetvalue $2
+			shift 1
+			;;
+		'--delete')
+			_wd ">>> $1 $2"
+			varremove $2
+			shift 1
+			;;
+		'--show')
+			_wd ">>> $1"
+			showstorage
+			;;
+
+		# ----- export
+
+		'--json')
+			_wd ">>> $1"
+			exportJson
+			# time exportJson
+			# time exportJson2
+			;;
+		*)
+			_we "ERROR: unknown param detected: [$1]"
+			showHelp
+			echo "Aborting ..."
+			exit 1
+	esac
+	_wd ""
+	shift 1
+
+done
+_wd "DONE"

inc/confighandler2.sh

+#!/bin/bash
+# ======================================================================
+#
+# VARIABLE STORAGE written in Bash
+#
+# ----------------------------------------------------------------------
+# 2020-01-25  v0.1  <axel.hahn@iml.unibe.ch>
+# 2020-01-26  v0.2  <axel.hahn@iml.unibe.ch> added --get, --add
+# 2020-01-27  v0.3  <axel.hahn@iml.unibe.ch> added storages handling
+# 2020-01-30  v0.5  <axel.hahn@iml.unibe.ch> added --setfile
+# ======================================================================
+
+_product="VARIABLE STORAGE"
+_version=0.5
+
+_storagedir=/var/tmp/confighandler
+_storagedir=/dev/shm/confighandler
+
+_storageprefix='flat-variable-store-'
+_storagename='default'
+tmpfile=/tmp/cfgmover.tmp
+
+_markFile='::FILE-CONTENT::'
+
+
+# ----------------------------------------------------------------------
+# FUNCTIONS
+# ----------------------------------------------------------------------
+
+# write a debug message to STDERR
+function _wd(){
+	echo -e "\e[33m# $*\e[0m" >&2
+}
+
+# write error message in red to STDERR
+function _we(){
+	echo -e "\e[31m# $*\e[0m" >&2
+}
+
+# ----------------------------------------------------------------------
+# storage functions
+# ----------------------------------------------------------------------
+
+# set a variable storage
+# param  string  name of the storage; allowed chars are a-z and 0-9
+function storageset(){
+	_storagename=`echo $1 | sed "s#[^a-z0-9]##g"`
+
+	mkdir $_storagedir 2>/dev/null
+    chmod 777 $_storagedir
+	_wd "set storage [${_storagename}]"
+	# cfgfile=`dirname $0`/${_storageprefix}${_storagename}.cfg
+	cfgfile=$_storagedir/${_storageprefix}${_storagename}.cfg
+}
+
+# list existing storages
+function storagelist(){
+	ls -1 ${_storagedir}/${_storageprefix}*.cfg | sed "s#${_storagedir}/${_storageprefix}##g" | sed "s#.cfg\$##g"
+}
+
+# delete current storage data
+function storagedelete(){
+	rm -f "$cfgfile"
+}
+
+
+# ----------------------------------------------------------------------
+# variable functions
+# ----------------------------------------------------------------------
+
+function _removeprefix(){
+	local _var=$1
+	_wd "removing ${_var} ..."
+	cat $cfgfile 2>/dev/null | grep -v "^${_var}" > $tmpfile
+	mv $tmpfile $cfgfile
+}
+
+function vargetvalue(){
+	local _var=$1
+	grep "^${_var}=" $cfgfile | cut -f 2 -d '='
+}
+
+
+function varaddvalue(){
+	local _var=$1
+        shift 1
+	local _val=$*
+	local _value=`vargetvalue $_var`
+	echo $_value | grep '^\[' >/dev/null
+	if [ $? -ne 0 ]; then
+		# echo ERROR: you can add a value to an array only - not on [$_value]
+		_we "ERROR: you can add a value to an array only - not on [$_value]"
+	else
+		_wd "OK, array detected: $_value"
+		_value=`echo $_value | sed "s#]##"`,$_val"]"
+		varset $_var "$_value"
+	fi
+}
+
+function varremove(){
+	local _var=$1
+	# _removeprefix "${_var}="
+	cat $cfgfile 2>/dev/null | grep -v "^${_var}=" > $tmpfile
+	mv $tmpfile $cfgfile
+}
+
+
+function varset(){
+	local _var=$1
+        shift 1
+	local _val=$*
+
+	varremove "${_var}"
+	_wd "setting ${_var} = ${_val} ..."
+	(echo ${_var}=${_val} ; cat $cfgfile 2>/dev/null ) >$tmpfile
+	sort $tmpfile > $cfgfile
+	rm -f $tmpfile
+}
+function varsetfile(){
+	local _var=$1
+        shift 1
+	local _val=$*
+	varset $_var "${_markFile}${_val}"
+}
+function showstorage(){
+	if [ -f $cfgfile ]; then
+		_wd "content of storage [${_storagename}]"
+		cat $cfgfile
+	else
+		_wd "storage [${_storagename}] is emtpy"
+	fi
+}
+
+# ----------------------------------------------------------------------
+# export functions
+# ----------------------------------------------------------------------
+
+function _showFileAsJson(){
+	local _file=$1
+	which jq >/dev/null
+	if [ $? -eq 0 ]; then
+		jq -nR --arg data "`cat $_file`" '$data'
+	else
+		_we "ERROR: Json export of files ${_markFile} requires [jq]"
+	fi
+}
+
+function _jsonloop(){
+	local var=$1
+	typeset -i iFollow=$2
+
+	# _wd "_jsonloop $1 $2"
+
+	typeset -i level=`echo $var | grep -o '\.' | wc -l`+2
+
+	typeset -i local iChilds=0
+	typeset -i local iCount=0
+	typeset -i local iLeft=0
+
+	typeset -i lastlevel=$level-1
+	local sLastvar=`echo ${var} | cut -f $lastlevel -d "."`
+	local space=`printf %${lastlevel}s |tr " " "\t"`
+
+	# --- handle existing subkeys
+	grep "^${var}\." $cfgfile >/dev/null && (
+		iCount=0
+		iChilds=`grep "^${var}" $cfgfile | wc -l`
+		echo "${space}\"$sLastvar\": {"
+		for myhash in `grep "^${var}[\.=]" $cfgfile | cut -f 1 -d "=" | cut -f $level -d "." | sort -u`
+		do
+			iCount=$iCount+1
+			iLeft=$iChilds-$iCount
+			_jsonloop "$var.$myhash" $iLeft
+		done
+		echo -n "${space}}"
+	)
+
+	# --- show values
+
+	grep "^${var}=" $cfgfile >/dev/null && (
+		value=`grep "^${var}=" $cfgfile | cut -f 2- -d "="`
+		echo -n "${space}\"$sLastvar\": "
+		echo ${value}| grep "${_markFile}" >/dev/null
+		if [ $? -eq 0 ]; then
+			_showFileAsJson `echo ${value} | sed "s#^${_markFile}##"`
+		else	
+			echo -n "${value}"
+		fi
+	)
+	if [ $iFollow -gt 0 ]; then
+		echo ","
+	else
+		echo
+	fi
+}
+
+function exportJson(){
+	# _wd "exportJson - init: find first level of vars"
+	typeset -i local iCount=0
+	typeset -i local iLeft=0
+	typeset -i local iChilds=`grep "^[a-zA-Z]" $cfgfile | cut -f 1 -d "=" | cut -f 1 -d '.' | sort -u | wc -l`
+
+	echo "{"
+
+	# first level names
+	for lev1var in `cat $cfgfile | grep "^.*\=" | cut -f 1 -d "=" | grep -v "\." | sort -u`
+	do
+		#_wd "exportJson - start run with var [$lev1var]"
+		iCount=$iCount+1
+		iLeft=$iChilds-$iCount
+		_jsonloop "$lev1var" $iLeft
+	done
+
+	# _wd "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+	# first level hashes
+	for lev1var in `cat $cfgfile | cut -f 1 -d "=" | grep  "\." | cut -f 1 -d "." | sort -u`
+	do
+		#_wd "exportJson - start run with hash [$lev1var]"
+		iCount=$iCount+1
+		iLeft=$iChilds-$iCount
+		_jsonloop "$lev1var" $iLeft
+	done
+	echo "}"
+
+}
+
+
+# W.I.P
+# JSON export with processing line by line
+#
+function exportJson2(){
+	typeset -i local iSublevel=0
+	typeset -i local iLast=0
+
+	typeset -i local lastlevel
+	local sLastvar
+
+	grep '^[a-zA-Z]' §fgfile | while read line
+	do
+		fullvar=`echo $line | cut -f 1 -d "="`
+		iSublevel=`echo $fullvar | grep -o '\.' | wc -l`+2
+		lastlevel=$iSublevel-1
+		sLastvar=`echo ${fullvar} | cut -f $lastlevel -d "."`
+
+		local space=`printf %${lastlevel}s |tr " " "\t"`
+
+		echo "${space} $line .. level $iSublevel .. $sLastvar"
+	done
+}
+
+# ----------------------------------------------------------------------
+# help
+# ----------------------------------------------------------------------
+
+# show help text
+function showHelp(){
+self=`basename $0`
+cat <<EOH
+
+$_product v$_version
+
+This script handles variables and nested values (hashes and arrays).
+You can create several configuration sets (storages) to isolate them.
+
+Help
+
+	--help or -h or -?
+		show this help and abort.
+
+	--version or -v
+		show the version.
+
+Storage parameters
+
+	--storage NAME
+		Set a storage to use. Allowed chars are a-z and 0-9.
+		Default is a storage named "default".
+		You can set an evironment variable CFGSTORAGE to override
+		the default (= to skip --storage NAME).
+
+	--list
+		List existing storages.
+
+	--flush
+		delete current storage data
+
+	--show
+		show the current config entries as plain text
+
+	--json
+		show config entries as json.
+		Warning: it is implemented in Bash - can be slow on more data
+
+Create | read | update | delete variables
+
+	--set VARNAME VALUE
+		Set a variable and its value.
+		If it does not exist then it will be created.
+		If it exist then it will be overwritten with the new value.
+
+		For the values you can force the type:
+			* Use single and double quotes to set a string, i.e.
+				\$ $self --set my_string '"hello world"'
+
+			* Quote the doublequotes to expand variables, i.e.
+				\$ $self --set my_string2 \"\${my_string2}\"
+
+			* use no quoting vor integers and values
+				\$ $self --set number_cpu 8
+
+			* use barackets in quotes to set an array
+				\$ $self --set my_array '[8, 12, 16]'
+
+		Use a "." (dot) in varname to create a hierarchy, i.e.
+			\$ $self --set host.os '"linux"'
+
+	--setfile VARNAME FILENAME
+		Set an input file to show its content
+
+	--add VARNAME VALUE
+		Add VALUE as array element to VARNAME.
+		It fails if VARNAME does not exist or isn't an array.
+			 $ confighandler.sh --add my_array 20
+
+	--get VARNAME
+		Show value of the given variable.
+
+	--delete VARNAME
+		delete a variable.
+
+You can concatenate all parameters. So you can use --show before
+and after an --set action to see the differences.
+\$ $self --show --set hello '"world"' --show
+
+You get additional infos on STDERR. To get plain data send STDERR
+to /dev/null
+\$ $self --show 2>/dev/null
+
+EOH
+}
+
+# ----------------------------------------------------------------------
+#
+# MAIN
+#
+# ----------------------------------------------------------------------
+
+
+if [ ! -z $CFGSTORAGE ]; then
+	storageset "$CFGSTORAGE"
+else
+	storageset "default"
+fi
+
+if [ $# -eq 0 ]; then
+	showHelp
+	exit 0
+fi
+
+while [ $# -gt 0 ];
+do
+	case "$1" in
+		'--help' | '-h' | '-?')
+			_wd ">>> $1"
+			showHelp
+			exit 0
+			;;
+		'--version' | '-v')
+			_wd ">>> $1"
+			echo $_product v$_version
+			;;
+
+		# ----- storage
+
+		'--storage')
+			_wd ">>> $1 $2"
+			storageset "$2"
+			shift 1
+			;;
+		'--list')
+			_wd ">>> $1"
+			storagelist
+			;;
+		'--flush')
+			_wd ">>> $1"
+			storagedelete
+			;;
+
+		# ----- variables
+
+		'--set')
+			_wd ">>> $1 $2 $3"
+			varset $2 "$3"
+			shift 2
+			;;
+		'--setfile')
+			_wd ">>> $1 $2 $3"
+			varsetfile $2 "$3"
+			shift 2
+			;;
+		'--add')
+			_wd ">>> $1 $2 $3"
+			varaddvalue $2 "$3"
+			shift 2
+			;;
+		'--get')
+			_wd ">>> $1 $2"
+			vargetvalue $2
+			shift 1
+			;;
+		'--delete')
+			_wd ">>> $1 $2"
+			varremove $2
+			shift 1
+			;;
+		'--show')
+			_wd ">>> $1"
+			showstorage
+			;;
+
+		# ----- export
+
+		'--json')
+			_wd ">>> $1"
+			exportJson
+			# time exportJson
+			# time exportJson2
+			;;
+		*)
+			_we "ERROR: unknown param detected: [$1]"
+			showHelp
+			echo "Aborting ..."
+			exit 1
+	esac
+	_wd ""
+	shift 1
+
+done
+_wd "DONE"

inc/test_cfghandler.sh

+#!/bin/bash
+
+
+
+export CFGSTORAGE="${checkName}output"
+
+myFullscript=check_whatever
+myparams="-w 75 -c 90"
+_outfile=$0
+rc=0
+
+
+# outputAsText="$(cat $_outfile)"
+# outputAsJson="$(jq -nR --arg data """${outputAsText}""" '$data')"
+commandAsJson="$(jq -nR --arg data """${myFullscript} $myparams""" '$data')"
+
+function setHandler(){
+    ch=$1
+}
+
+function makeTest(){
+    (
+        $ch --set      check_source      \"${myHost}\"
+        $ch --set      check_command     "${commandAsJson}"
+        $ch --setfile  plugin_output     "${_outfile}"
+        $ch --set      exit_status       $rc
+    ) 2>&1
+    $ch --json
+}
+
+setHandler `dirname $0`/confighandler.sh
+time makeTest
+
+setHandler `dirname $0`/confighandler2.sh
+time makeTest
\ No newline at end of file

inc_functions.sh

+
+# ======================================================================
+#
+# OUTPUT FUNCTIONS
+#
+# ======================================================================
+
+
+  # ..................................................................
+  # write debug output to STDERR
+  function _wd(){
+    if [ $cfg_debug = true ]; then
+          echo -e "\e[33m# DEBUG: $*\e[0m" >&2
+    fi
+  }
+
+  # ..................................................................
+  # write header string
+  function _h1(){ 
+    echo
+    echo
+    echo "###############| $* |###############" 
+    echo
+  }
+  function _h2(){
+    echo
+    echo "---------- $*"
+  }
+  function _h3(){
+    echo
+    echo ">>>>> $*"
+  }
+
+  # ......................................................................
+  # logging output. writes timestamp and the given message to STDOUT
+  # and ${logfile}
+  # params  string(s)  message to log
+  #
+  function _log(){
+    echo "`date` | $*" | tee -a ${logfile}
+  }
+
+# ======================================================================
+#
+# DATE AND TIME FUNCTIONS
+#
+# ======================================================================
+
+  # ......................................................................
+  # get age of a file in sec
+  # param  string  filename to test
+  function _getFileAge(){
+    echo $(($(date +%s) - $(date +%s -r "$1")))
+  }
+
+  # ......................................................................
+  # get current unix timestamp
+  # no param
+  function _getUnixTs(){
+    date +%s
+  }
+
+  # get a list of config file of all checks to execute
+  # no params
+  function getChecks(){
+    if [ -d ${dir_cfg}/checks/ ]; then
+      for myconfig in `ls -1 ${dir_cfg}/checks/*`
+      do
+        echo ${myconfig}
+      done
+    fi
+  }
+
+  # parse a config file and set global vars:
+  #   checkName
+  #   checkCommand
+  #   checkInterval
+  # param  string  full path of a config file
+  function _parseCheckConfig(){
+    local _myconfig=$1
+
+    # EXAMPLE a config contains ...
+    # checkname=check_cronstatus
+    # command=check_cronstatus -param1 -param2
+    # interval=60
+    # icon=/images/icon.png
+    # max_check_attempts=3
+
+    checkName=`cat $_myconfig | grep ^checkname= | cut -f 2 -d "=" | sed 's# #_#g'`
+    checkCommand=`cat $_myconfig | grep ^command= | cut -f 2 -d "="`
+    checkInterval=`cat $_myconfig | grep ^interval= | cut -f 2 -d "="`
+    checkIcon=`cat $_myconfig | grep ^icon= | cut -f 2 -d "="`
+    checkMaxAttempts=`cat $_myconfig | grep ^max_check_attempts= | cut -f 2 -d "="`
+  }
+
+# ======================================================================
+#
+# NETWORK FUNCTIONS
+#
+# ======================================================================
+
+function _getinet4(){
+    ip -4 addr | grep inet
+}
+
+# filter output of _getinet4
+# TODO: bonded devices
+# param  string  filter method
+#                by type
+#                - loopback  filter loopback
+#                - net       filter network addresses
+#                - peer      filter peer
+#                - physical  filter physical addresses
+#                - virtual   filter 
+#
+#                by address range:
+#                - private   filter ips from private network ranges
+#                - public    filter non private ips (opposite of "private")
+#                output
+#                - ip         must be the last: get ip address only
+# _getinet4 | _ipf 
+function _ipf(){
+    local _filter=$1
+
+    local _regexInclude='.*'
+    local _regexExclude='idonotmatchtoskip'
+
+    if [ -z $_filter ]; then
+        echo ERROR: filter _ipf requires a param
+    fi
+
+    # 10.0.0.0    – 10.255.255.255 	
+    # 172.16.0.0  – 172.31.255.255
+    # 192.168.0.0 – 192.168.255.255
+    local _privateranges='(10\.|172\.1[6-9]\.|172.2[0-9]\.|172\.3[01]\.|192.168\.)'
+
+    case $_filter in
+
+        # --- types
+        'loopback')
+            _regexInclude='\ scope\ host\ '
+            ;;
+        'physical')
+            _regexExclude="(\ scope\ host\ |\ peer\ |\:vip)"
+            ;;
+        'virtual')
+            _regexInclude="\:vip"
+            ;;
+        'net')
+            _regexInclude='\ scope\ global\ '
+            ;;
+        'peer')
+            _regexInclude='\ peer\ .*\ scope\ global\ '
+            ;;
+
+        # --- private/ public ips
+        'private')
+            _regexInclude="\ ${_privateranges}.*"
+            ;;
+        'public')
+            _regexInclude="\ scope\ global\ "
+            _regexExclude="\ ${_privateranges}\.*"
+            ;;
+
+        # --- output filter
+        'ip')
+            awk '{ print $2 }' | cut -f 1 -d '/'
+            return 0
+            ;;
+        *)
+            echo ERROR: filter [$_filter] is not supported ... showing all.
+    esac
+
+    grep -E  "$_regexInclude" | grep -Ev "$_regexExclude" | sed "s#^    ##g"
+}
+
+function _getIp(){
+  ( _getinet4 | _ipf public | _ipf physical | _ipf ip ; _getinet4 | _ipf private | _ipf physical | _ipf ip ) | head -1
+}
+
+function _getIpFrontend(){
+  _getinet4 | _ipf public | _ipf physical | _ipf ip
+}
+function _getIpVirtual(){
+  _getinet4 | _ipf virtual | _ipf ip
+}
+function _getIpPrivate(){
+  _getinet4 | _ipf private | _ipf ip
+}
+
+
+# ======================================================================
+#
+# REST API FUNCTIONS
+#
+# ======================================================================
+
+  # ..................................................................
+  # read config file and apply API base settings
+  # param  string  filename of config file
+  function _initHttpWithConfigfile(){
+
+    # read config like /etc/icinga2-passive-client/api-director.cfg
+    # ... it contains
+    # RestApiBaseUrl="https://HOST/path"
+    # RestApiUser="USER"
+    # RestApiPassword="PASSWORD"
+    # RestApiDocs="https://icinga.com/docs/something"
+    . "$1" || exit 1
+
+    http.init
+    http.setAuth     "${RestApiUser}:${RestApiPassword}"
+    http.setBaseUrl  "${RestApiBaseUrl}"
+    http.setDocs     "${RestApiDocs}"
+
+  }
+
+  # ..................................................................
+  # make a curl request and show the response header + body
+  # param  string  method - GET|POST|...see http.setMethod
+  # param  string  relative URL behind API base url
+  # param  string  optional: data for PUTs and POSTs
+  function _APIcall(){
+
+    http.makeRequest "$1" "$2" "$3"
+
+    http.getResponseHeader
+    http.getResponse
+
+  }
+
+  # ..................................................................
+  # make an rest API regeques with GET command to update local cache
+  # file of an object.
+  # The cache file will be deleted if the http response code is not OK.
+  #
+  # global  $APICLIENT  rest api wrapper
+  #
+  # param  string  url behind RestAPI base path
+  # param  string  filename of the local cache file
+  function _getApiObject(){
+    local _url=$1
+    local _outfile=$2
+
+    #http.setDebug   1
+    # http.setUrl     "${_url}"
+    # http.setMethod  "GET"
+    # http.setBody    ""
+    # http.makeRequest
+
+    http.makeRequest GET "${_url}" ""
+
+    if [ ! -z "`http.isok`" ]; then
+      _wd "OK, Object $_url exists - updating cache file $_outfile"
+      http.responseExport "${_outfile}"
+    else
+      _wd "FAILED :-/ GET $_url was NOT successful - removing $_outfile"
+      rm -f "${_outfile}"
+    fi
+
+  }
+
+
+  # show name of linked service on a host
+  # it is used to 
+  #   - link the service to a host on director
+  #   - send process data from client to icinga
+  #
+  # param  string  name of check (=checkName after _parseCheckConfig FILE)
+  # param  bool    flag: reverse renaming: linkname --> checkname
+  function _getName4Svcathost(){
+
+    echo $1
+    # prefix was removed ... reverse functionailty not needed (sed fails)
+    #
+    # local prefix="added_service_for_command_"
+    # local prefix=""
+    # if [ $# -eq 1 ]; then
+    #   echo "${prefix}${1}"
+    # else
+    #   echo $1 | sed "s#$prefix##g"
+    # fi
+  }

inc_getconfig.sh.dist

+. /etc/icinga2-passive-client/client.cfg

plugins @ 7591e745

+Subproject commit 7591e7456609b9313937662bfce4823669f60eda

rest-api-client

+#!/bin/bash
+# ======================================================================
+#
+# REST API CLIENT USING CURL
+#
+# requires
+# - curl
+# - config file with endpoint, api user, password
+# ----------------------------------------------------------------------
+# see
+# - https://icinga.com/docs/icinga2/latest/doc/12-icinga2-api/
+# - https://icinga.com/docs/director/latest/doc/70-REST-API/
+# ----------------------------------------------------------------------
+# 2020-01-24  v0.1  axel.hahn@iml.unibe.ch
+# 2020-02-05  v0.2  axel.hahn@iml.unibe.ch  get curl meta infos; split header
+# ======================================================================
+
+# --- fetch incoming params
+  RestApiCfg=$1
+  RestApiMethod=$2
+  ApiUrl=$3
+  Body="$4"
+
+# --- curl meta infos to collect 
+#     see variables in man curl --write-out param
+  curlMeta="\
+    http_code \
+    http_connect \
+    local_ip \
+    local_port \
+    num_connects \
+    num_redirects \
+    redirect_url \
+    remote_ip \
+    remote_port \
+    size_download \
+    size_header \
+    size_request \
+    size_upload \
+    speed_download \
+    speed_upload \
+    ssl_verify_result \
+    time_appconnect \
+    time_connect \
+    time_namelookup \
+    time_pretransfer \
+    time_redirect \
+    time_starttransfer \
+    time_total \
+    url_effective \
+"
+
+
+# ----------------------------------------------------------------------
+#
+# functions
+#
+# ----------------------------------------------------------------------
+
+  # ......................................................................
+  #
+  # write a debug message to STDERR
+  # Do no not change the prefix - is is read in inc_functions
+  #
+  # params  strings  output message
+  function _wd(){
+    echo -e "\e[33m# RESTAPI::DEBUG $*\e[0m" >&2
+  }
+
+  # ......................................................................
+  #
+  # write a http response header line
+  # Do no not change the prefix - is is read in inc_functions
+  #
+  # params  strings  output message
+  function _wHeader(){
+    echo -e "\e[34m# CURL::RESPONSE-HEADER | $*\e[0m" >&2
+  }
+
+  # ......................................................................
+  #
+  # write a http response header line
+  # params  strings  output message
+  function _wBody(){
+    # echo -e "\e[32m$*\e[0m"
+    echo $*
+  }
+
+  # ......................................................................
+  #
+  # write a http response header line
+  # Do no not change the prefix - is is read in inc_functions
+  #
+  # params  strings  output message
+  function _wData(){
+    echo -e "\e[36m# CURL::INFOS | $*\e[0m" >&2
+  }
+
+  # ......................................................................
+  #
+  # show error message with last return code and quit with this exitcode
+  # no params
+  function quit(){
+    rc=$?
+    echo >&2
+    echo -e "\e[31m# ERROR: command FAILED with rc $rc. \e[0m" >&2
+    if [ ! -z "${RestApiDocs}" ]; then
+      echo "HINT: see ${RestApiDocs}" >&2
+    fi
+    exit $rc
+  }
+
+  # ......................................................................
+  #
+  # show a help text
+  # no params
+  function showHelp(){
+    cat <<EOH
+
+SYNTAX:
+  - config file (added by a wrapper)
+  - method
+  - url
+  - body (JSON)
+The first 3 params required.
+
+HINT:
+To get rid of the commented lines in the output redirect STDERR.
+[command] 2>/dev/null
+
+EOH
+  }
+
+# ----------------------------------------------------------------------
+#
+# main
+#
+# ----------------------------------------------------------------------
+
+  _wd "REST API CLIENT"
+
+  which curl >/dev/null || quit
+
+  if [ $# -lt 3 ]; then
+    showHelp
+    exit 1
+  fi
+
+  # read config
+  . "${RestApiCfg}" || quit
+
+  test -z "$RestApiPassword" || RestApiUser="$RestApiUser:$RestApiPassword"
+
+
+  # ----------------------------------------------------------------------
+  # make request
+  # ----------------------------------------------------------------------
+
+  _wd "$RestApiMethod $RestApiBaseUrl$ApiUrl ..."
+
+  sCurlDataPrefix="RESTAPICLIENTMETADATA_`date +%s`_$$"
+  for myvar in $curlMeta
+  do
+    writevar="${writevar}|${myvar}:%{${myvar}}"
+  done
+  curlparams="-k -s --write-out \"\\n$sCurlDataPrefix${writevar}\\n\""
+
+  output=`
+    if [ -z "$Body" ]; then
+      curl ${curlparams} \
+        -u "$RestApiUser" \
+        -i $RestApiBaseUrl$ApiUrl \
+        -H 'Accept: application/json' \
+        -X $RestApiMethod
+    else
+      curl ${curlparams} \
+        -u "$RestApiUser" \
+        -i $RestApiBaseUrl$ApiUrl \
+        -H 'Accept: application/json' \
+        -X $RestApiMethod \
+        -d "$Body"
+    fi
+` || quit
+
+  _wd "OK - Curl finished the http request"
+
+  # ----------------------------------------------------------------------
+  # split response data
+  # ----------------------------------------------------------------------
+
+  isheader=true
+  body=
+
+  # keep leading spaces
+  IFS=''
+
+  _wd "curl meta infos"
+
+  # --- dump curl metadata
+  #   example line
+  #   RESTAPICLIENTMETADATA_1580905381_2336|http_code:200|time_total:0.391|speed_download:990.000|...
+  # echo "$output" | sed "s#$sCurlDataPrefix#\n$sCurlDataPrefix#" | grep  "$sCurlDataPrefix" | cut -f 2- -d "|" | sed "s#|#\n#g" | while read line; do
+  echo "$output" | grep  "$sCurlDataPrefix" | cut -f 2- -d "|" | sed "s#|#\n#g" | while read line; do
+    _wData $line
+  done
+
+  # --- loop over output and split header from body
+  while read line; do
+    if $isheader; then
+      if [[ $line = $'\r' ]]; then
+        isheader=false
+          _wd "http response body"
+      else
+        # header="$header"$'\n'"$line"
+        _wHeader $line
+      fi
+    else
+      # body="$body"$'\n'"$line"
+      _wBody $line
+    fi
+  # done < <(echo "$output" | sed "s#$sCurlDataPrefix#\n$sCurlDataPrefix#" | grep -v "$sCurlDataPrefix")
+  done < <(echo "$output" | grep -v "$sCurlDataPrefix")
+
+
+# ----------------------------------------------------------------------

test-netfkt.sh

+
+. `dirname $0`/inc_functions.sh
+
+function _exec(){
+    echo ... [$2] command: $1
+    eval $1
+    # echo rc=$?
+    echo
+}
+
+# _exec ls "testrun"
+
+
+_exec "_getinet4" "all ip v4 addresses"
+_exec "_getinet4 | _ipf private" "private range"
+
+
+_exec "_getinet4 | _ipf public" "public ips"
+
+_exec "_getinet4 | _ipf public | _ipf physical" "public physical addresses"
+_exec "_getinet4 | _ipf public | _ipf virtual"  "public virtual addresses"
+
+echo ----
+
+
+
+_exec "_getIpFrontend" "get frontend address"
+_exec "_getIpVirtual" "get virtual address"
+_exec "_getIpPrivate" "get private address"
+
+_exec "_getIp"  "get main IP"