Initial commit

_etc/api-director.cfg.dist

+#
+# API CONFIG to connect to REST API
+#
+# Description:
+# Connect to Icinga Director# Used to create/ read/ update/ delete my host
+# This config is included in /opt/imlmonitor/client/director-cli.sh
+#
+
+RestApiBaseUrl="https://icinga.example.com/icingaweb2/"
+RestApiUser="directorcli"
+RestApiPassword="password-of-director-api-user"
+RestApiDocs="https://icinga.com/docs/director/latest/doc/70-REST-API/"

_etc/api-icinga2.cfg.dist

+#
+# API CONFIG to connect to REST API
+#
+# Description:
+# Connect to Icinga API
+# Used to ...
+#   - create service of my host
+#   - to send monitor check data to icinga
+#
+# This config is included in /opt/imlmonitor/client/service-cli.sh 
+#
+
+RestApiBaseUrl="https://icinga-endpoint.example.com:5665/v1/"
+RestApiUser="clientchecks"
+RestApiPassword="password-of-Icinga-Api-User"
+RestApiDocs="https://icinga.com/docs/icinga2/latest/doc/12-icinga2-api/"

_etc/checks/CPU-usage-example.dist

+# ------------------------------------------------------------
+#
+# This is an example file for a client check.
+# Syntax is
+# [variable]=[Value]
+# - no spaces around "="
+# - no quoting of spaces required after "="
+#
+# You should generate this config file.
+#
+# ------------------------------------------------------------
+
+# vivible label in Icinga web frontend:
+checkname=CPU-usage
+
+# command to be executed. It will be searched in given pathes - see ../client.cfg
+# the parameters 
+command=check_cpu -w 90 -c 95 -i 50
+customparams=
+
+# interval of check execution; must be 60s and higher
+interval=60
+
+# icon to be used in icingaweb frontend
+icon=/images/nuvolax64/apps/kcmprocessor.png
+
+# number of failing checks before alert
+max_check_attempts=3
+

_etc/client.cfg.dist

+# ============================================================
+#
+# CONFIG file for execution of the client checks (aka plugins)
+#
+# Remark:
+# this is a bash file that it can be sourced
+#
+# ------------------------------------------------------------
+#
+# DO NOT CHANGE THIS FILE - IT IS GENERATED BY PUPPET
+#
+# instead change the template
+# modules/iml/templates/component/icinga/client/client.cfg.erb
+#
+# ------------------------------------------------------------
+# 2020-02-xx Axel Hahn
+# ============================================================
+
+
+
+# ------------------------------------------------------------
+#
+# selfname is set in puppet... define default dirs with it
+#
+selfname="icinga2-passive-client"
+
+  # directory of config files for api profiles and defined
+  # checks
+  dir_cfg="/etc/${selfname}"
+
+  # tmp data of checks, i.e. to keep last output
+  dir_data="/var/tmp/${selfname}"
+
+  # log dir ... *log will be rotated by logrotate config
+  dir_logs="/var/log/${selfname}"
+
+
+# ------------------------------------------------------------
+#
+# where to find check scripts ... first directory where a
+# check plugin was found wins
+#
+dir_plugins=" \
+  /opt/imlmonitor/client/plugins \
+  /usr/lib64/nagios/plugins \
+"
+
+# ------------------------------------------------------------
+# ports open to outside
+host_vars_tcpport="[22, 80, 443]"
+
+# ------------------------------------------------------------

cleanup-services.sh

+#/bin/bash
+# ======================================================================
+#
+# CLEANUP script to remove services on a host if a service was deleted
+# in LDAP
+#
+# IML specific: uses puppet to recreate all checks
+#
+# script is started as cron
+# ----------------------------------------------------------------------
+# 2020-03-06  v0.1  <axel.hahn@iml.unibe.ch>
+# 2020-03-09  v0.2  <axel.hahn@iml.unibe.ch>  waiting for pupet puppet
+# ======================================================================
+
+. `dirname $0`/inc_getconfig.sh || exit 1
+
+dirWithChecks=${dir_cfg}/checks/
+lockfile="${dir_data}/`basename $0`.pid"
+
+typeset -i iCounter=0
+typeset -i iMaxWait=120
+
+# ----------------------------------------------------------------------
+# MAIN
+# ----------------------------------------------------------------------
+
+if [ ! -d ${dirWithChecks} ]; then
+    echo ERROR: directory witch check configs does not exist [$dirWithChecks]
+    exit 1
+fi
+
+
+if [ -f "${lockfile}" ]; then
+    lockpid=`cat "${lockfile}" | cut -f 2 -d "-" | cut -f 4 -d " " | grep "[0-9]"`
+    ps -f --pid $lockpid >/dev/null
+    if [ $? -eq 0 ]; then
+        _log "ABORT: update seems to run already. See process with PID $lockpid"
+        echo
+        ps -f --pid $lockpid
+        echo
+        exit 0
+    fi
+fi
+echo "cleanup services started `date` - process id $$" > "${lockfile}"
+
+
+echo --- 1/3 :: DELETE all existing configs
+ls -l ${dir_cfg}/checks/*
+rm -f ${dir_cfg}/checks/*
+echo
+echo directory content after deletion:
+ls -l ${dir_cfg}/checks/*
+echo
+
+
+echo --- 2/3 :: WAITING max $iMaxWait min for a puppet run to recreate the checks
+rc=1
+while [ $rc -ne 0 ]; do
+    ls -l ${dir_cfg}/checks/* >/dev/null 2>&1
+    rc=$?
+    if [ $rc -ne 0 ]; then
+        iCounter=$iCounter+1
+        echo -n "$iCounter ... `date` - "
+        if [ $iCounter -gt $iMaxWait ]; then
+            echo "ERROR: puppet did not start within $iMaxWait min??? ABORTING here."
+            exit 1
+        fi
+        echo "waiting ..."
+        sleep 60
+    fi
+done
+echo "Found :-)"
+echo
+
+echo --- 3/3 :: CLEANUP links on icinga
+`dirname $0`/director-cli.sh --linkcleanup
+echo
+
+
+echo --- DONE
+rm -f $lockfile
+exit 0
+
+# ----------------------------------------------------------------------

confighandler2.sh

+#!/bin/bash
+# ======================================================================
+#
+# VARIABLE STORAGE written in Bash
+#
+# ----------------------------------------------------------------------
+# 2020-01-25  v0.1  <axel.hahn@iml.unibe.ch>
+# 2020-01-26  v0.2  <axel.hahn@iml.unibe.ch> added --get, --add
+# 2020-01-27  v0.3  <axel.hahn@iml.unibe.ch> added storages handling
+# 2020-01-30  v0.5  <axel.hahn@iml.unibe.ch> added --setfile
+# ======================================================================
+
+_product="VARIABLE STORAGE"
+_version=0.5
+
+_storagedir=/var/tmp/confighandler
+_storageprefix='flat-variable-store-'
+_storagename='default'
+tmpfile=/tmp/cfgmover.tmp
+
+_markFile='::FILE-CONTENT::'
+
+# ----------------------------------------------------------------------
+# FUNCTIONS
+# ----------------------------------------------------------------------
+
+# write a debug message to STDERR
+function _wd(){
+	echo -e "\e[33m# $*\e[0m" >&2
+}
+
+# write error message in red to STDERR
+function _we(){
+	echo -e "\e[31m# $*\e[0m" >&2
+}
+
+# ----------------------------------------------------------------------
+# storage functions
+# ----------------------------------------------------------------------
+
+# set a variable storage
+# param  string  name of the storage; allowed chars are a-z and 0-9
+function storageset(){
+	_storagename=`echo $1 | sed "s#[^a-z0-9]##g"`
+
+	mkdir $_storagedir 2>/dev/null
+        chmod 777 $_storagedir
+	_wd "set storage [${_storagename}]"
+	# cfgfile=`dirname $0`/${_storageprefix}${_storagename}.cfg
+	cfgfile=$_storagedir/${_storageprefix}${_storagename}.cfg
+}
+
+# list existing storages
+function storagelist(){
+	ls -1 ${_storagedir}/${_storageprefix}*.cfg | sed "s#${_storagedir}/${_storageprefix}##g" | sed "s#.cfg\$##g"
+}
+
+# delete current storage data
+function storagedelete(){
+	rm -f "$cfgfile"
+}
+
+
+# ----------------------------------------------------------------------
+# variable functions
+# ----------------------------------------------------------------------
+
+function _removeprefix(){
+	local _var=$1
+	_wd "removing ${_var} ..."
+	cat $cfgfile 2>/dev/null | grep -v "^${_var}" > $tmpfile
+	mv $tmpfile $cfgfile
+}
+
+function vargetvalue(){
+	local _var=$1
+	grep "^${_var}=" $cfgfile | cut -f 2 -d '='
+}
+
+
+function varaddvalue(){
+	local _var=$1
+        shift 1
+	local _val=$*
+	local _value=`vargetvalue $_var`
+	echo $_value | grep '^\[' >/dev/null
+	if [ $? -ne 0 ]; then
+		# echo ERROR: you can add a value to an array only - not on [$_value]
+		_we "ERROR: you can add a value to an array only - not on [$_value]"
+	else
+		_wd "OK, array detected: $_value"
+		_value=`echo $_value | sed "s#]##"`,$_val"]"
+		varset $_var "$_value"
+	fi
+}
+
+function varremove(){
+	local _var=$1
+	# _removeprefix "${_var}="
+	cat $cfgfile 2>/dev/null | grep -v "^${_var}=" > $tmpfile
+	mv $tmpfile $cfgfile
+}
+
+
+function varset(){
+	local _var=$1
+        shift 1
+	local _val=$*
+
+	varremove "${_var}"
+	_wd "setting ${_var} = ${_val} ..."
+	(echo ${_var}=${_val} ; cat $cfgfile 2>/dev/null ) >$tmpfile
+	sort $tmpfile > $cfgfile
+	rm -f $tmpfile
+}
+function varsetfile(){
+	local _var=$1
+        shift 1
+	local _val=$*
+	varset $_var "${_markFile}${_val}"
+}
+function showstorage(){
+	if [ -f $cfgfile ]; then
+		_wd "content of storage [${_storagename}]"
+		cat $cfgfile
+	else
+		_wd "storage [${_storagename}] is emtpy"
+	fi
+}
+
+# ----------------------------------------------------------------------
+# export functions
+# ----------------------------------------------------------------------
+
+function _showFileAsJson(){
+	local _file=$1
+	which jq >/dev/null
+	if [ $? -eq 0 ]; then
+		jq -nR --arg data "`cat $_file`" '$data'
+	else
+		_we "ERROR: Json export of files ${_markFile} requires [jq]"
+	fi
+}
+
+function _jsonloop(){
+	local var=$1
+	typeset -i iFollow=$2
+
+	# _wd "_jsonloop $1 $2"
+
+	typeset -i level=`echo $var | grep -o '\.' | wc -l`+2
+
+	typeset -i local iChilds=0
+	typeset -i local iCount=0
+	typeset -i local iLeft=0
+
+	typeset -i lastlevel=$level-1
+	local sLastvar=`echo ${var} | cut -f $lastlevel -d "."`
+	local space=`printf %${lastlevel}s |tr " " "\t"`
+
+	# --- handle existing subkeys
+	grep "^${var}\." $cfgfile >/dev/null && (
+		iCount=0
+		iChilds=`grep "^${var}" $cfgfile | wc -l`
+		echo "${space}\"$sLastvar\": {"
+		for myhash in `grep "^${var}[\.=]" $cfgfile | cut -f 1 -d "=" | cut -f $level -d "." | sort -u`
+		do
+			iCount=$iCount+1
+			iLeft=$iChilds-$iCount
+			_jsonloop "$var.$myhash" $iLeft
+		done
+		echo -n "${space}}"
+	)
+
+	# --- show values
+
+	grep "^${var}=" $cfgfile >/dev/null && (
+		value=`grep "^${var}=" $cfgfile | cut -f 2- -d "="`
+		echo -n "${space}\"$sLastvar\": "
+		echo ${value}| grep "${_markFile}" >/dev/null
+		if [ $? -eq 0 ]; then
+			_showFileAsJson `echo ${value} | sed "s#^${_markFile}##"`
+		else	
+			echo -n "${value}"
+		fi
+	)
+	if [ $iFollow -gt 0 ]; then
+		echo ","
+	else
+		echo
+	fi
+}
+
+function exportJson(){
+	# _wd "exportJson - init: find first level of vars"
+	typeset -i local iCount=0
+	typeset -i local iLeft=0
+	typeset -i local iChilds=`grep "^[a-zA-Z]" $cfgfile | cut -f 1 -d "=" | cut -f 1 -d '.' | sort -u | wc -l`
+
+	echo "{"
+
+	# first level names
+	for lev1var in `cat $cfgfile | grep "^.*\=" | cut -f 1 -d "=" | grep -v "\." | sort -u`
+	do
+		#_wd "exportJson - start run with var [$lev1var]"
+		iCount=$iCount+1
+		iLeft=$iChilds-$iCount
+		_jsonloop "$lev1var" $iLeft
+	done
+
+	# _wd "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+	# first level hashes
+	for lev1var in `cat $cfgfile | cut -f 1 -d "=" | grep  "\." | cut -f 1 -d "." | sort -u`
+	do
+		#_wd "exportJson - start run with hash [$lev1var]"
+		iCount=$iCount+1
+		iLeft=$iChilds-$iCount
+		_jsonloop "$lev1var" $iLeft
+	done
+	echo "}"
+
+}
+
+
+# W.I.P
+# JSON export with processing line by line
+#
+function exportJson2(){
+	typeset -i local iSublevel=0
+	typeset -i local iLast=0
+
+	typeset -i local lastlevel
+	local sLastvar
+
+	grep '^[a-zA-Z]' $cfgfile | while read line
+	do
+		fullvar=`echo $line | cut -f 1 -d "="`
+		iSublevel=`echo $fullvar | grep -o '\.' | wc -l`+2
+		lastlevel=$iSublevel-1
+		sLastvar=`echo ${fullvar} | cut -f $lastlevel -d "."`
+
+		local space=`printf %${lastlevel}s |tr " " "\t"`
+
+		echo "${space} $line .. level $iSublevel .. $sLastvar"
+	done
+}
+
+# ----------------------------------------------------------------------
+# help
+# ----------------------------------------------------------------------
+
+# show help text
+function showHelp(){
+self=`basename $0`
+cat <<EOH
+
+$_product v$_version
+
+This script handles variables and nested values (hashes and arrays).
+You can create several configuration sets (storages) to isolate them.
+
+Help
+
+	--help or -h or -?
+		show this help and abort.
+
+	--version or -v
+		show the version.
+
+Storage parameters
+
+	--storage NAME
+		Set a storage to use. Allowed chars are a-z and 0-9.
+		Default is a storage named "default".
+		You can set an evironment variable CFGSTORAGE to override
+		the default (= to skip --storage NAME).
+
+	--list
+		List existing storages.
+
+	--flush
+		delete current storage data
+
+	--show
+		show the current config entries as plain text
+
+	--json
+		show config entries as json.
+		Warning: it is implemented in Bash - can be slow on more data
+
+Create | read | update | delete variables
+
+	--set VARNAME VALUE
+		Set a variable and its value.
+		If it does not exist then it will be created.
+		If it exist then it will be overwritten with the new value.
+
+		For the values you can force the type:
+			* Use single and double quotes to set a string, i.e.
+				\$ $self --set my_string '"hello world"'
+
+			* Quote the doublequotes to expand variables, i.e.
+				\$ $self --set my_string2 \"\${my_string2}\"
+
+			* use no quoting vor integers and values
+				\$ $self --set number_cpu 8
+
+			* use barackets in quotes to set an array
+				\$ $self --set my_array '[8, 12, 16]'
+
+		Use a "." (dot) in varname to create a hierarchy, i.e.
+			\$ $self --set host.os '"linux"'
+
+	--setfile VARNAME FILENAME
+		Set an input file to show its content
+
+	--add VARNAME VALUE
+		Add VALUE as array element to VARNAME.
+		It fails if VARNAME does not exist or isn't an array.
+			 $ confighandler.sh --add my_array 20
+
+	--get VARNAME
+		Show value of the given variable.
+
+	--delete VARNAME
+		delete a variable.
+
+You can concatenate all parameters. So you can use --show before
+and after an --set action to see the differences.
+\$ $self --show --set hello '"world"' --show
+
+You get additional infos on STDERR. To get plain data send STDERR
+to /dev/null
+\$ $self --show 2>/dev/null
+
+EOH
+}
+
+# ----------------------------------------------------------------------
+#
+# MAIN
+#
+# ----------------------------------------------------------------------
+
+
+if [ ! -z $CFGSTORAGE ]; then
+	storageset "$CFGSTORAGE"
+else
+	storageset "default"
+fi
+
+if [ $# -eq 0 ]; then
+	showHelp
+	exit 0
+fi
+
+while [ $# -gt 0 ];
+do
+	case "$1" in
+		'--help' | '-h' | '-?')
+			_wd ">>> $1"
+			showHelp
+			exit 0
+			;;
+		'--version' | '-v')
+			_wd ">>> $1"
+			echo $_product v$_version
+			;;
+
+		# ----- storage
+
+		'--storage')
+			_wd ">>> $1 $2"
+			storageset "$2"
+			shift 1
+			;;
+		'--list')
+			_wd ">>> $1"
+			storagelist
+			;;
+		'--flush')
+			_wd ">>> $1"
+			storagedelete
+			;;
+
+		# ----- variables
+
+		'--set')
+			_wd ">>> $1 $2 $3"
+			varset $2 "$3"
+			shift 2
+			;;
+		'--setfile')
+			_wd ">>> $1 $2 $3"
+			varsetfile $2 "$3"
+			shift 2
+			;;
+		'--add')
+			_wd ">>> $1 $2 $3"
+			varaddvalue $2 "$3"
+			shift 2
+			;;
+		'--get')
+			_wd ">>> $1 $2"
+			vargetvalue $2
+			shift 1
+			;;
+		'--delete')
+			_wd ">>> $1 $2"
+			varremove $2
+			shift 1
+			;;
+		'--show')
+			_wd ">>> $1"
+			showstorage
+			;;
+
+		# ----- export
+
+		'--json')
+			_wd ">>> $1"
+			exportJson
+			# time exportJson
+			# time exportJson2
+			;;
+		*)
+			_we "ERROR: unknown param detected: [$1]"
+			showHelp
+			echo "Aborting ..."
+			exit 1
+	esac
+	_wd ""
+	shift 1
+
+done
+_wd "DONE"

director-cli.sh

+#!/bin/bash
+# ======================================================================
+#
+# director helper
+#
+#   C
+#   R
+#   U
+#   D
+#     actions for a host, commands, services
+#
+# This script contains specific logic for IML
+#   - use puppet facts
+#   - UniBe network and group names - see hostCreate()
+#
+# ======================================================================
+
+. `dirname $0`/inc_getconfig.sh
+. `dirname $0`/inc_functions.sh
+. `dirname $0`/inc/rest-api-client.sh
+
+tmpfile=/tmp/outcurl.tmp
+tmpfile2=/tmp/outcurl2.tmp
+
+APICLIENT=`dirname $0`/api2director
+
+MY_NAME=`facter fqdn | cut -f -2 -d ">"`
+MY_IP=`facter ipaddress | cut -f -2 -d ">"`
+# MY_IP=`_getIpFrontend`
+
+
+typeset cfg_debug=false
+typeset cfg_dryrun=false
+
+ch="`dirname $0`/inc/confighandler.sh"
+
+
+IDC_host__cachefile="${dir_data}/host_${MY_NAME}__at-director.txt"
+IDC_service__cachefile="${dir_data}/all_defined_services__at-director.txt"
+IDC_svcathost__cachefile="${dir_data}/services_on_host__at-director.txt"
+
+
+
+# ======================================================================
+#
+# FUNCTIONS
+#
+# ======================================================================
+
+
+  # helper to make http base setup for host actions
+  function _initHttp(){
+    # see inc_functions
+    _initHttpWithConfigfile "/etc/icinga2-passive-client/api-director.cfg"
+    if [ $cfg_debug = true ]; then
+      http.setDebug 1
+    fi
+  }
+
+
+  function flushDatadir(){
+    if [ ! -z "${dir_data}" -a -d "${dir_data}" ]; then
+      _wd "deleting ${dir_data} ..."
+      rm -f ${dir_data}/*.txt
+    fi
+  }
+
+# ----------------------------------------------------------------------
+# functions for objects
+# ----------------------------------------------------------------------
+
+  # ............................................................
+  # set $ch to store all object vars
+  function _generateJsonForHost(){
+
+
+    # --- host infos
+    # local MY_OSName=`facter os -y | grep " distid:" | cut -f 2 -d ":" | cut -f 2 -d " " | tr '[:upper:]' '[:lower:]'`
+    # local MY_OSMajorVersion=`facter os -y | grep " majdistrelease:" | cut -f 2 -d ":" | cut -f 2 -d "'"`
+
+    local MY_Platform=`facter kernel | tr '[:upper:]' '[:lower:]'`
+    . `dirname $0`/plugins/inc_pluginfunctions
+    local MY_OSName=`ph.getOS`
+    local MY_OSMajorVersion=`ph.getOSMajor`
+
+    export CFGSTORAGE="directorhost"
+    (
+      $ch --flush
+      $ch --set  object_name     \"$MY_NAME\"
+      $ch --set  object_type     '"object"'
+      $ch --set  address         \"$MY_IP\"
+      $ch --set  display_name    \"$MY_NAME\"
+
+      $ch --set  icon_image      \"/images/os/${MY_OSName}.png\"
+      $ch --set  icon_image_alt  \"${MY_Platform}\:\ ${MY_OSName}\ ${MY_OSMajorVersion}\"
+    ) 2>/dev/null
+
+    # --- detect phase
+    # local phase="live"
+    # for myphase in preview stage demo
+    # do
+    #   echo $MY_NAME | grep "\.$myphase\." >/dev/null && phase=$myphase
+    # done
+
+    (
+      # ----- facter data to host vars
+      $ch --set  vars.platform  \"${MY_Platform}\"
+      $ch --set  vars.os        \"${MY_OSName}${MY_OSMajorVersion}\"
+
+      # ----- set host type
+
+        # host in the UNIBE network:
+        echo $MY_IP | grep "^130.92" >/dev/null && (
+          $ch --set  imports        '["host in network"]'
+
+          # port checks initiated by icinga server to monitor client
+          if [ ! -z "${host_vars_tcpport}" -a "${host_vars_tcpport}" != "[]" ]; then
+            host_vars_tcpport=`echo ${host_vars_tcpport} | sed "s# ##g"`
+            $ch --set  vars.tcp_port  ${host_vars_tcpport}
+          fi
+        )
+
+        # ONE private network
+        echo $MY_IP | grep "^10\." >/dev/null && $ch --set  imports   '["host passive only"]'
+
+
+      # ----- /host type
+
+
+      # ----- host groups
+      #       all host groups must exist in director - otherwise the creation
+      #       of a host will fail
+      #       see https://icinga.one.iml.unibe.ch/icingaweb2/director/dashboard?name=hosts#!/icingaweb2/director/hostgroups
+
+      $ch --set  groups        '["iml", "iml-server"]'
+
+
+      # ... and add some others
+      # $ch --add  groups        \"iml-phase-$phase\"
+
+    ) 2>/dev/null
+
+    # ----- generate data and send to DIRECTOR
+    # DATA=`$ch --json 2>/dev/null`
+    # $ch --flush 2>/dev/null
+  }
+
+  # ............................................................
+  # set $ch to store all object vars
+  # UNUSED see response of GET director/commands/templates
+  # function _generateJsonForCommand(){
+  #   export CFGSTORAGE="command-${IDC_command__obj_name}"
+  #   (
+  #     $ch --flush
+  #     $ch --set  object_name     "\"${IDC_command__obj_name}\""
+  #     $ch --set  object_type     \"template\"
+  # 
+  #   ) 2>/dev/null
+  # }
+
+  # ............................................................
+  # set $ch to store all object vars
+  function _generateJsonForServicetemplate(){
+    export CFGSTORAGE="service-${IDC_service__obj_name}"
+    (
+      $ch --flush
+      $ch --set  object_name     "\"${IDC_service__obj_name}\""
+      $ch --set  check_command   \"${IDC_command__obj_name}\"
+      $ch --set  object_type     \"template\"
+
+      $ch --set  enable_active_checks  false
+      $ch --set  enable_passive_checks true
+
+      $ch --set  check_interval        "\"${checkInterval}s\""  # see _parseCheckConfig FILE
+
+      # graphite
+      # $ch --set  enable_perfdata       true 
+      # $ch --set  vars.check_command    \"${checkName}\"         # for graphite plugin
+
+      if [ ! -z "${checkIcon}" ]; then
+        $ch --set  icon_image      \"${checkIcon}\"
+        $ch --set  icon_image_alt  \"${checkName}\"
+      fi
+
+      if [ ! -z "${checkMaxAttempts}" ]; then
+        $ch --set  max_check_attempts  \"${checkMaxAttempts}\"
+      fi
+      
+
+    ) 2>/dev/null
+  }
+
+  # ............................................................
+  # set $ch to store all object vars
+  function _generateJsonForSvclink(){
+    export CFGSTORAGE="servicelink-${IDC_svcathost__obj_name}"
+    (
+      $ch --flush
+      $ch --set  object_name     "\"${IDC_svcathost__obj_name}\""
+      $ch --set  object_type     \"object\"
+      $ch --set  host            "\"${MY_NAME}\""
+      $ch --set  imports         "[ \"${IDC_service__obj_name}\" ]"
+    ) 2>/dev/null
+  }
+
+  # ............................................................
+  # CRUD actions for an director object: host, service template, linked service
+  # examples:
+  #   ObjAction create host
+  #   ObjAction list svclink
+  #
+  # param  string  name of action; one of create|read|update|delete|exists|list
+  # param  string  name of object; one of host|service|svclink
+  # param  string  Dryrun (set any not empty value to show infos without execution)
+  function ObjAction(){
+
+    local _paramAction=$1
+    local _paramObj=$2
+    local _paramDryrun=$cfg_dryrun
+    test -z "$3" || _paramDryrun=true
+
+    local _object_name=         # name of the current object (for ouput only)
+
+    local _sMethod=             # http method; GET|POST|PUT|DELETE
+    local _sUrl=                # relative url (part behind REST API base url)
+    local _jsondata=
+
+    local _bNeedsCheck=false    # requires $IDC* vars (a read check config first)
+    local _bSendData=false      # true for PUT and POST
+
+    local _sUrlList=
+    local _sUrlCreate=
+    local _sUrlExists=
+    local _sUrlRead=
+    local _sUrlUpdate=
+    local _sUrlDelete=
+
+    local _existFilter=
+
+    local _bShowResponse=true
+    local _bShowFilter=false
+    # local _sCachefile=
+
+    _initHttp
+
+    # --- init object based vars
+    case "${_paramObj}" in
+
+      'host')
+        _object_name=$MY_NAME
+        _sUrlCreate=director/host
+        _sUrlExists=director/host?name=${_object_name}
+        _sUrlRead=director/host?name=${_object_name}
+        _sUrlUpdate=director/host?name=${_object_name}
+        _sUrlDelete=director/host?name=${_object_name}
+
+        _existFilter="object_name"
+        _generateJsonForHost
+        ;;
+
+      'service')
+        _object_name=${IDC_service__obj_name}
+        _bNeedsCheck=true
+
+        _sUrlList=director/services/templates
+        _sUrlCreate=director/service
+        _sUrlExists=director/service?name=${_object_name}
+        _sUrlRead=director/service?name=${_object_name}
+        _sUrlUpdate=director/service?name=${_object_name}
+        _sUrlDelete=director/service?name=${_object_name}
+
+        _existFilter='"object_name": '
+        _generateJsonForServicetemplate
+        ;;
+
+      'svclink')
+        _object_name=${IDC_svcathost__obj_name}
+        _bNeedsCheck=true
+        _sUrlList=director/services?host=$MY_NAME
+        _sUrlCreate=director/service
+        _sUrlExists="director/service?name=${IDC_svcathost__obj_name}&host=$MY_NAME"
+        _sUrlRead="director/service?name=${IDC_svcathost__obj_name}&host=$MY_NAME"
+        _sUrlUpdate="director/service?name=${IDC_svcathost__obj_name}&host=$MY_NAME"
+        _sUrlDelete="director/service?name=${IDC_svcathost__obj_name}&host=$MY_NAME"
+
+        # exists:
+        # _existFilter="object_name.*${IDC_svcathost__obj_name}"
+        _existFilter='"object_name": '
+
+        _generateJsonForSvclink
+        ;;
+
+      *)
+        echo "ERROR: object [${_paramObj}] cannot be handled in ${FUNCNAME[0]} (yet?)."
+        exit 1
+    esac
+
+
+    # --- init method based vars
+    case "${_paramAction}" in
+      'list')
+        _sMethod=GET
+        _sUrl=$_sUrlList
+        _bShowResponse=false
+        _bShowFilter=true
+
+        _bNeedsCheck=false # 
+        ;;
+      'exists')
+        _sMethod=GET
+        _sUrl=$_sUrlExists
+        _bShowResponse=false
+        ;;
+      'create')
+        _sMethod=PUT
+        _sUrl=$_sUrlCreate
+        _bSendData=true
+        ;;
+      'read')
+        _sMethod=GET
+        _sUrl=$_sUrlRead
+        ;;
+      'update')
+        _sMethod=POST
+        _sUrl=$_sUrlUpdate
+        _bSendData=true
+        ;;
+      'delete')
+        _sMethod=DELETE
+        _sUrl=$_sUrlDelete
+        ;;
+      *)
+        echo "ERROR: method [${_paramAction}] does not exist."
+        exit 1
+    esac
+
+    if [ -z "$_sUrl" ]; then
+      echo "SKIP: Action [${_paramAction}] is not supported (yet?) for object type [${_paramObj}]"
+      return
+    fi
+
+    if [ $_bNeedsCheck = true -a -z "${IDC_svcathost__obj_name}" ]; then
+      echo "SKIP: you need to load a check before accessing [${_paramObj}]"
+      return
+    fi
+
+
+    # --- get json data of object
+    if [ $_bSendData = true ]; then
+      _jsondata=`$ch --json 2>/dev/null`
+    fi
+    $ch --flush 2>/dev/null
+
+    # --- http request
+    if [ ${_paramDryrun} = false ]; then
+      _wd ">>>>> $_paramAction $_paramObj [${_object_name}] >> $_sMethod $_sUrl $_jsondata"
+      http.makeRequest "$_sMethod" "$_sUrl" "$_jsondata"
+      if [ $_bShowResponse = true ]; then
+        http.getResponseHeader
+        http.getResponse
+      fi
+
+      http.isOk >/dev/null
+    else
+      echo "DRYRUN: >>>>> $_paramAction $_paramObj [${_object_name}] >> $_sMethod $_sUrl $_jsondata"
+      echo "... _bShowResponse: $_bShowResponse"
+      echo "... _bSendData    : $_bSendData"
+    fi
+
+    # --- on list action: filter response
+    if [ "${_paramAction}" = "list" ]; then
+      if [ ${_paramDryrun} = false ]; then
+        _wd ">>>>> filter response by [object_name]"
+        if [ $_bShowFilter = true ]; then
+          http.getResponse | grep object_name | cut -f 2- -d ":" | sed 's#^ "##g' | sed 's#",$##'
+        else
+          http.getResponse | grep object_name | cut -f 2- -d ":" | sed 's#^ "##g' | sed 's#",$##' >/dev/null
+        fi
+
+      else
+
+        echo "DRYRUN: >>>>> filter response by [object_name]"
+        echo
+      fi
+    fi
+
+    # --- on exist action: filter response
+    if [ "${_paramAction}" = "exists" -a ! -z "${_existFilter}" ]; then
+      if [ ${_paramDryrun} = false ]; then
+        _wd ">>>>> filter response by [$_existFilter]"
+        if [ $_bShowFilter = true ]; then
+          http.getResponse | grep $_existFilter
+        else
+          http.getResponse | grep $_existFilter >/dev/null
+        fi
+      else
+
+        echo "DRYRUN: >>>>> filter response by [$_existFilter]"
+        echo
+      fi
+    fi
+  }
+
+# ----------------------------------------------------------------------
+# functions for Host
+# ----------------------------------------------------------------------
+
+
+
+  # ............................................................
+  # helper to create a base config for the current host
+  function _initHostdata(){
+    export CFGSTORAGE="directorhost"
+    (
+      $ch --flush
+      $ch --set  object_name   \"$MY_NAME\"
+      $ch --set  object_type   '"object"'
+      $ch --set  address       \"$MY_IP\"
+      $ch --set  display_name  \"$MY_NAME\"
+    ) 2>/dev/null
+  }
+
+
+  # ............................................................
+  # create a host with PUT on director API
+  function hostCreate(){
+    _h2 "create host"
+
+    ObjAction create host
+    if [ -z "`http.isOk`" ]; then
+      echo "ERROR, host was NOT created."
+    else
+      echo "OK, host was created successfully."
+    fi
+  }
+
+
+  # ............................................................
+  # get data of current host from director API
+  # and by the way it updates the local host infos too.
+  function hostRead(){
+    _h2 "read host"
+    ObjAction read host
+    if [ -z "`http.isOk`" ]; then
+      echo "ERROR, host was NOT found."
+    fi
+  }
+
+
+  # ............................................................
+  # update current host
+  # param  JSON part starting with ", " and some json data
+  function hostUpdate(){
+    _h2 "update host - set $1 $2"
+
+    ObjAction update host
+    if [ -z "`http.isOk`" ]; then
+      case `http.getStatuscode` in
+        "304")
+          echo "OK, no update"
+          ;;
+        *)
+          echo "ERROR during update of the host data"
+      esac
+    else
+      echo "OK, host was updated"
+    fi
+  }
+  # ............................................................
+  # ensure that a host exists
+  function hostCreateOrUpdate(){
+    ObjAction exists host
+    if [ $? -ne 0 ]; then
+      hostCreate
+    else
+      hostUpdate
+    fi
+  }
+
+
+
+  # ............................................................
+  # delete the current host in the director
+  function hostDelete(){
+    _h2 "delete host"
+    ObjAction delete host
+    case `http.getStatuscode` in
+      "200")
+        echo "OK, host was deleted"
+        ;;
+      "404")
+        echo "ERROR, host does not exist"
+        ;;
+      *)
+        echo "ERROR, unable to delete host"
+    esac
+    flushDatadir
+  }
+
+
+
+# ----------------------------------------------------------------------
+# functions for services
+# director/service
+# ----------------------------------------------------------------------
+
+  # ............................................................
+  # helper for services: generate variable names for a check
+  # vars have prefix IDC for "Icinga Director"
+  # uses global var checkName
+  function _generateVarsByCheckname(){
+    if [ -z "${checkName}" ]; then
+      echo ERROR: checkName is empty - I guess _parseCheckConfig was not executed.
+      exit 1
+    fi
+
+    IDC_command__obj_name="${checkName}"
+    IDC_service__obj_name="service-template_for_command_${checkName}"
+    IDC_svcathost__obj_name="`_getName4Svcathost ${checkName}`"
+    # IDC_service__obj_name="${checkName}"
+    # IDC_svcathost__obj_name="${checkName}"
+
+    IDC_command__cachefile="${dir_data}/command_${checkName}__at-director.txt"
+    # IDC_service__cachefile="${dir_data}/all_defined_services__at-director.txt"
+    # IDC_svcathost__cachefile="${dir_data}/all_services_on_host__at-director.txt"
+  }
+
+
+
+# ............................................................
+  # create a service in Icinga director
+  # uses global variables only
+  function serviceCreateOrUpdate(){
+    _h2 "${FUNCNAME[0]}()"
+
+    local sMode=update
+    local sMethod=POST
+    local _sUrl="director/service?name=${IDC_service__obj_name}"
+
+    ObjAction exists service
+    if [ $? -ne 0 ]; then
+      ObjAction create service
+    else
+      ObjAction update service
+    fi
+    if [ $? -ne 0 ]; then
+      echo "ERROR :/"
+    else
+      echo "OK"
+    fi
+  }
+
+
+  # ............................................................
+  # create a service in Icinga director
+  # param   string  filename of check config
+  function serviceCreateByCfgFile(){
+      _h2 "${FUNCNAME[0]}($1) - create single service of given file"
+      _parseCheckConfig "${1}"
+      _generateVarsByCheckname "${checkName}"
+
+      # create command if it does not exist
+      serviceCreateOrUpdate
+
+      ObjAction exists service
+      if [ $? -eq 0 ]; then
+        ObjAction exists svclink
+        if [ $? -ne 0 ]; then
+          ObjAction create svclink
+        else
+          echo "SKIP: linked service on host [${IDC_svcathost__obj_name}] exists"
+          # TODO, uncomment -- wenn es sinnvolle Features gibt
+          # ObjAction update svclink
+        fi
+      else
+        echo "SKIP link ... service template for ${checkName} not ready."
+      fi
+  }
+
+  # ............................................................
+  # create all functions; this functon is called with cli parameter
+  # no params
+  function servicesCreateAll(){
+    _h2 "${FUNCNAME[0]}() - create all services"
+
+    # loop over all services and create
+    for mycheckfile in `getChecks`
+    do
+      serviceCreateByCfgFile "${mycheckfile}"
+      echo
+    done
+  }
+
+
+  # ............................................................
+  # cleanup services - delete unneded links
+  function svclinkCleanup(){
+    _h2 "Cleanup linked service templates"
+    tmpRemote=/tmp/remoteLinks
+    tmpLocal=/tmp/localChecks
+
+
+    # --- perpare I: create file with remote service template links
+    ObjAction list svclink >$tmpRemote
+
+    # --- perpare II: create file with local configs and object names for its link
+    rm -f $tmpLocal 2>/dev/null
+    for mycheckfile in `getChecks`
+    do
+      _parseCheckConfig "${mycheckfile}"
+      _generateVarsByCheckname "${checkName}"
+      echo "$mycheckfile:${IDC_svcathost__obj_name}" >>$tmpLocal
+    done
+
+    # _h3 "local checks"
+    # cat $tmpLocal
+    # _h3 "remote linked service templates"
+    # ObjAction list svclink
+
+    # --- Compare ...
+    # _h3 "Compare"
+    cat $tmpRemote | while read remoteLink
+    do
+      grep $remoteLink $tmpLocal >/dev/null
+      if [ $? -eq 0 ]; then
+        echo "OK: $remoteLink"
+      else
+        echo "DELETE: link [$remoteLink] is not used by any local check anymore."
+        checkName=`_getName4Svcathost $remoteLink reverse`
+        _generateVarsByCheckname "${checkName}"
+        ObjAction delete svclink
+      fi
+    done
+    rm -f $tmpLocal $tmpRemote
+    echo --- done.
+  }
+
+
+# ----------------------------------------------------------------------
+# functions for Director
+# ----------------------------------------------------------------------
+
+  # ............................................................
+  # kick the director to deploy config changes now
+  function directorDeploy(){
+    _h2 deploy
+    _initHttp
+
+    _wd POST director/config/deploy
+    _APIcall POST director/config/deploy
+    if [ -z "`http.isOk`" ]; then
+      echo "ERROR deploy config was not queued."
+    else
+      echo "OK, deploy was triggered"
+    fi
+  }
+
+# ..................................................................
+#
+# show a help
+function showHelp(){
+script=`basename $0`
+cat <<ENDOFHELP
+
+HELP:
+
+Host actions
+
+  --hc
+  --hostcreate
+    Create the host [$MY_NAME] in the icinga director
+
+  --hr
+  --hostread
+    Read the host information of [$MY_NAME] in the icinga director
+
+  --hu
+  --hostupdate
+    Update host in the Icinga director
+
+  --he
+  --hostensure
+    Create host if it does not exist otherwise update it in the
+    Icinga director
+
+  --hd
+  --hostdelete
+    Delete [$MY_NAME] in the icinga director
+
+
+Check actions
+
+  --listchecks
+    list all local config files of known checks for [$MY_NAME]
+
+  --linkcleanup
+    verify added service templates on [$MY_NAME] with locally
+    defined checks and remove unneeded items.
+
+  --sca
+  --servicescreateall
+    Loop over defined checks and add all checks to the host.
+    See also:
+      --listchecks to get a list of config files.
+
+  --sc CFGFILE
+  --servicecreate CFGFILE
+    create service by naming a config file
+    See also:
+      --listchecks to get a list of config files.
+
+Cache
+
+    --flushcache
+      remove files in [$dir_data]
+
+Director actions
+
+  --deploy
+    icinga update (deploy director data)
+
+Other parameters
+
+  --debug
+    enable debug output.
+
+  --nodebug
+    disable debug output.
+
+  --dryrun
+    enable dryrun - it shows actions without execution.
+
+
+EXAMPLES
+
+  # $script --dryrun --he
+  Show actions and api calls for "--he" parameter (create or update host)
+
+ENDOFHELP
+}
+
+# ------------------------------------------------------------
+#
+# MAIN
+#
+# ------------------------------------------------------------
+
+echo
+echo "##### DIRECTOR HELPER $MY_NAME - $MY_IP"
+echo
+
+cd `dirname $0`
+ls ./`basename $0` >/dev/null || exit 1
+
+# ensure that ./inc_getconfig.sh was loaded
+if [ -z "${dir_cfg}" ]; then
+  echo ERROR: Client is not installed/ configured yet on this machine.
+  exit 1
+fi
+
+if [ $# -eq 0 ]; then
+  showHelp
+  exit 0
+fi
+
+
+while [ $# -gt 0 ];
+do
+  case "$1" in
+
+    '--help' | '-h' | '-?')
+      showHelp
+      exit 0
+      ;;
+
+    '--debug')
+      cfg_debug=true
+      ;;
+
+    '--nodebug')
+      cfg_debug=false
+      ;;
+
+    '--dryrun')
+      cfg_dryrun=true
+      ;;
+
+    # ----- host actions
+
+    '--he' | '--hostensure')
+      hostCreateOrUpdate
+      ;;
+    '--hc' | '--hostcreate')
+      hostCreate
+      ;;
+    '--hr' | '--hostread')
+      hostRead
+      ;;
+    '--hu' | '--hostupdate')
+      hostUpdate
+      ;;
+    '--hd' | '--hostdelete')
+      hostDelete
+      ;;
+
+    # ----- check actions
+
+    '--listchecks')
+      getChecks
+      ;;
+
+    '--sca' | '--servicescreateall')
+      servicesCreateAll
+      ;;
+
+    '--sc' | '--servicecreate')
+      serviceCreateByCfgFile "${2}"
+      shift
+      ;;
+
+    '--linkcleanup')
+      svclinkCleanup
+      shift
+      ;;
+
+    '--flushcache')
+      flushDatadir
+      ;;
+
+    # ----- director
+
+    '--deploy')
+      directorDeploy
+      exit 0
+      ;;
+
+    '--testrun')
+      _h1 "svclink - without a config"
+      ObjAction "create" "svclink" dry
+      ObjAction "exists" "svclink" dry
+      ObjAction "read"   "svclink" dry
+      ObjAction "update" "svclink" dry
+      ObjAction "delete" "svclink" dry
+
+      _h1 "svclink - with reading a config"
+      _parseCheckConfig "/etc/icinga2-passive-client/checks/CPU-usage"
+      _generateVarsByCheckname "${checkName}"
+
+      ObjAction "create" "svclink" dry
+      ObjAction "exists" "svclink" dry
+      ObjAction "read"   "svclink" dry
+      ObjAction "update" "svclink" dry
+      ObjAction "delete" "svclink" dry
+
+      _h1 "host"
+      ObjAction "create" "host" dry
+      ObjAction "exists" "host" dry
+      ObjAction "read"   "host" dry
+      ObjAction "update" "host" dry
+      ObjAction "delete" "host" dry
+
+      ;;
+
+    # ---- ab hier TODO
+    u)
+      shift 1
+      hostUpdate "$*"
+      exit 0
+      ;;
+    *)
+      echo "ERROR: unknown parameter detected. No idea what to do with [$1]."
+      echo "Exiting..."
+      exit 2
+
+  esac
+  shift 1
+done
+
+exit 0
+# ======================================================================

icinga-cli.sh

+#!/bin/bash
+# ======================================================================
+#
+# ICINGA PASSIVE CLIENT
+#
+# Run all passive checks and send response to Icinga Endpoint
+#
+# Requirements
+# - curl
+# - jq
+# ----------------------------------------------------------------------
+#
+# ======================================================================
+
+
+_product="ICINGA PASSIVE CLIENT"
+_version="0.3"
+_license="GNU GPL 3.0"
+_copyright='(c) 2020 Institute for Medical Education * University of Bern'
+
+typeset -i debug=0
+
+# source config ...
+. `dirname $0`/inc_getconfig.sh
+
+. `dirname $0`/inc_functions.sh
+. `dirname $0`/inc/rest-api-client.sh
+
+# where to find check scripts ... first directory wins
+# dir_plugins="/opt/imlmonitor/client/plugins/ /usr/lib64/nagios/plugins"
+# dir_cfg="/etc/icinga2-passive-client"
+# dir_data="/var/tmp/icinga2-passive-client"
+# dir_logs="/var/log/icinga2-passive-client"
+logfile=${dir_logs}/execution.log
+
+ch=`dirname $0`/inc/confighandler.sh
+myHost=`hostname -f`
+
+# for loop mode only: max. random sleep time
+typeset -i sleeptime=30
+
+typeset -i _rc_all=0
+
+# ----------------------------------------------------------------------
+#
+# FUNCTIONS
+#
+# ----------------------------------------------------------------------
+# ..................................................................
+#
+# helper to make http base setup for host actions
+function _initHttp(){
+  # see inc_functions
+   _initHttpWithConfigfile "/etc/icinga2-passive-client/api-icinga2.cfg"
+  if [ $debug -ne 0 ]; then
+    http.setDebug 1
+  fi
+}
+
+
+# ......................................................................
+#
+# find first place of the check script in the known plugin dirs 
+# see ${dir_plugins} in checks.cfg
+# param  string  name of the check script without path
+#
+function findCheckScript(){
+	local _script=$1
+	for mydir in ${dir_plugins}
+	do
+		if [ -x ${mydir}/${_script} ]; then
+			echo ${mydir}/${_script}
+		fi
+	done | head -1
+}
+
+# helper used function in loopChecks
+# get a snapshot of a few files
+function _getFileSnapshot(){
+  ls -l `dirname $0`/* ${dir_cfg}/*
+}
+
+
+# ......................................................................
+#
+# Loop over executing all checks
+# no params
+#
+function loopChecks(){
+
+      # TODO-MEMORY-CHECK
+      # echo ${myHost} | egrep "^(kvm4|icinga)"
+      # echo ${myHost} | egrep "^(monitortest)"
+      # if [ $? -ne 0 ]; then
+      #   echo "HARD EXIT - DO NOT EXECUTE ANY CHECK ON $myHost"
+      #   exit 1
+      # fi
+
+      local lockfile="${dir_data}/loop.pid"
+      local snapShotStart=${dir_data}/`basename $0`-start.fingerprint
+      local snapShotCurrent=${dir_data}/`basename $0`-last.fingerprint
+      if [ -f "${lockfile}" ]; then
+        local lockpid=`cat "${lockfile}" | cut -f 2 -d "-" | cut -f 4 -d " " | grep "[0-9]"`
+        ps -f --pid $lockpid >/dev/null
+        if [ $? -eq 0 ]; then
+          _log "ABORT: Loop seems to run already. See process with PID $lockpid"
+          echo
+          ps -f --pid $lockpid
+          echo
+          exit 0
+        fi
+      fi
+
+      _log "---------- starting in a permanent loop"
+      echo "Serviceloop started `date` - process id $$" > "${lockfile}"
+      _getFileSnapshot>$snapShotStart
+      while true; do
+        # typeset -i local iSleep=$(($RANDOM%$sleeptime))
+        # sleep minimum is half of $sleeptime
+        typeset -i local iSleep=$(($RANDOM%$sleeptime/2+$sleeptime/2))
+        _log "sleeping $iSleep sec ..."
+        sleep $iSleep
+        _log "______________________________________________________________________"
+        _log ""
+        _getFileSnapshot>$snapShotCurrent
+        diff  $snapShotStart $snapShotCurrent >/dev/null
+        if [ $? -ne 0 ]; then
+          _log "ABORT: Files were updated / overwritten. The loop must be restarted.\n`diff  $snapShotStart $snapShotCurrent`"
+          exit 1
+        fi
+        icingaHostMustExist
+        processAllChecks
+        echo
+        echo
+      done
+}
+# ......................................................................
+#
+# execute all defined checks one by one
+# no params
+#
+function processAllChecks(){
+  # loop over all defined checks
+  typeset -i local iChecksTotal=`getChecks | wc -l`
+  typeset -i local iCounter=0
+
+  _rc_all=0
+  typeset -i local iLoopStart=`_getUnixTs`
+
+  _log ""
+  _log "------ looping over all checks"
+  getChecks
+  echo
+  for myconfig in `getChecks`
+  do
+    iCounter=$iCounter+1
+    _log "--- processing [$iCounter of $iChecksTotal] $myconfig"
+    processCheck $myconfig
+    _log ""
+    echo
+    echo ----------------------------------------------------------------------
+    echo
+  done
+  typeset -i local iLoopEnd=`_getUnixTs`
+  typeset -i local iLoopTime=$iLoopEnd-$iLoopStart
+
+  _log "------ loop done - needed $iLoopTime sec - rc=$_rc_all"
+}
+
+# ......................................................................
+#
+# parse a config file and set global vars:
+#   checkName
+#   checkCommand
+#   checkInterval
+# param  string  full path of a config file
+#
+function _parseCheckConfig(){
+  local _myconfig=$1
+
+  # EXAMPLE a config contains ...
+  # checkname=check_cronstatus
+  # command=check_cronstatus -param1 -param2
+  # interval=60
+
+  checkName=`cat $_myconfig | grep ^checkname= | cut -f 2 -d "="`
+  checkCommand=`cat $_myconfig | grep ^command= | cut -f 2 -d "="`
+  checkInterval=`cat $_myconfig | grep ^interval= | cut -f 2 -d "="`
+
+}
+
+
+function icingaHost(){
+  local _logPrefix="${myHost} :: API |"
+  local _apiRequest=objects/hosts/${myHost}
+  local _localCache=${dir_data}/host_${myHost}_deployed-at-icinga.txt
+  typeset -i local _iRefreshCache=120
+
+  local sAction=$1
+
+  _initHttp
+
+  case $sAction in
+    'get')
+      # update after caching was added in http-component
+      # http.responseImport $_localCache
+      # typeset -i local iAgeLastGet=`http.getRequestAge`
+      # _log "${_logPrefix} INFO: cache is $iAgeLastGet sec old ... TTL is _iRefreshCache=3600 sec"
+      # if [ $iAgeLastGet -eq 0 -o $iAgeLastGet -gt $_iRefreshCache ]; then
+      #   _log "${_logPrefix} INFO: request to Icinga GET $_apiRequest"
+      #   _getApiObject $_apiRequest $_localCache
+      # fi
+
+      # new:
+      http.setCacheTtl     $_iRefreshCache
+      http.setCacheFile    $_localCache
+      http.makeRequest GET $_apiRequest
+
+      # set return code of GET action
+      http.isOk >/dev/null
+      ;;
+    *)
+      _log "ERROR: unknown action parameter $sAction"
+  esac
+
+}
+
+function icingaHostMustExist(){
+  _log "check if the host [${myHost}] exists on Icinga ..."
+  icingaHost get
+  if [ $? -ne 0 ]; then
+    http.getResponse
+    if [ "`http.getStatuscode`" = "000" ]; then
+      _log "ERROR: Unable to reach the Icinga node. Stopping script current monitoring actions."
+      exit 1
+    fi
+    _log "ERROR: host object for ${myHost} is not available on Icinga service (yet) - Status: `http.getStatuscode`"
+    echo
+    echo "ABORTING"
+    echo
+    echo "To run checks ..."
+    echo "- you must create the host on director (check director-cli.sh --hr)"
+    echo "- the director must deploy the host to icinga daemon"
+    echo
+    rm -f ${dir_data}/service__check* 2>/dev/null
+    exit 1
+  fi
+  _log "OK, found."
+}
+
+
+# ......................................................................
+#
+# process a single check
+# param  string  name of config file
+# param  string  name of config file
+#
+function processCheck(){
+  local _myconfig=$1
+  local _force=$2
+
+  typeset -i local iCheckStart=`_getUnixTs`
+
+  _parseCheckConfig ${_myconfig}
+  local _logPrefix="${checkName} |"
+  _log "${_logPrefix} INFO: every ${checkInterval} sec: ${checkCommand}"
+
+  local _outfile=${dir_data}/service__check__${checkName}__output.txt
+  local _response=${dir_data}/service__check__${checkName}__icinga_response.txt
+  typeset -i local _rc=0
+
+
+  _initHttp
+
+  # --- check last run ... if never or > $interval then execute
+  doRun=0
+  if [ ! -f $_outfile ]; then
+    _log "${_logPrefix} INFO: Never executed before"
+    doRun=1
+  else
+    # typeset -i iAgeLastRun=$(($(date +%s) - $(date +%s -r "$_outfile")))
+    typeset -i iAgeLastRun=`_getFileAge "$_outfile"`
+    _log "${_logPrefix} INFO: last run was $iAgeLastRun sec ago ... vs Interval = $checkInterval ... sleeptime = $sleeptime"
+    iAgeLastRun=$iAgeLastRun+$sleeptime
+    if [ $iAgeLastRun -gt $checkInterval ]; then
+      doRun=1
+    fi
+
+    if [ ! -z "$_force" ]; then
+      doRun=1
+      _log "${_logPrefix} INFO: forced execution by given param "
+    fi
+  fi
+
+  if [ $doRun -ne 0 ]; then
+    myscript=`echo $checkCommand | cut -f 1 -d " "`
+    myFullscript=`findCheckScript $myscript`
+    if [ -z "$myFullscript" ]; then
+      _log "${_logPrefix} ERROR: $myscript was not found in any plugin dir"
+    else
+      myparams=`echo $checkCommand | grep " " | cut -f 2- -d " "`
+
+      #
+      # --- this executes the check plugin ...
+      #
+      _log "${_logPrefix} starting $myFullscript $myparams"
+      typeset -i local iTsStart=`date +%s`
+      # $myFullscript $myparams | tee $_outfile
+      eval $myFullscript $myparams > $_outfile
+      rc=$?
+      typeset -i local iTsEnd=`date +%s`
+      outPerfdata=`grep '|' $_outfile | cut -f 2 -d '|'`
+      echo
+      echo -------- check output:
+      cat $_outfile
+      echo
+      echo -------- extracted performance data:
+      echo $outPerfdata
+      echo
+      _log "${_logPrefix} check command finished with returncode $rc"
+
+      _rc=$_rc+$rc
+      #
+      # --- send check result to Icinga
+      #     fields of the object
+      #     https://icinga.com/docs/icinga2/latest/doc/12-icinga2-api/#process-check-result
+      export CFGSTORAGE="${checkName}output"
+
+      outputAsText="$(cat $_outfile)"
+      outputAsJson="$(jq -nR --arg data """${outputAsText}""" '$data')"
+      commandAsJson="$(jq -nR --arg data """${myFullscript} $myparams""" '$data')"
+      (
+        $ch --set      check_source      \"${myHost}\"
+        $ch --set      check_command     "${commandAsJson}"
+        $ch --set      exit_status       $rc
+        # $ch --set      plugin_output     "${outputAsJson}"
+        $ch --setfile  plugin_output     "${_outfile}"
+        $ch --set      performance_data  "\"${outPerfdata}\""
+        $ch --set      ttl               $checkInterval
+        $ch --set      execution_start   $iTsStart
+        $ch --set      execution_end     $iTsEnd
+      ) 2>/dev/null
+      # $ch --json
+      data=`$ch --json 2>/dev/null`
+
+      slot="`_getName4Svcathost ${checkName} | sed 's# #%20#g'`"
+
+      _log "${_logPrefix} starting POST of data to monitoring server"
+      echo POST actions/process-check-result?service=${myHost}!${slot} "$data"
+      _APIcall POST actions/process-check-result?service=${myHost}!${slot} "$data"
+      http.responseExport "$_response"
+
+      # --- check if data were sent successfully
+      # fgrep "HTTP/1.1 200" ${_response} >/dev/null
+      # _testHttpOk ${_response} >/dev/null
+      http.isOk 2>/dev/null
+      if [ $? -eq 0 ]; then
+        _log "${_logPrefix} OK, response was sent to Icinga"
+      else
+        _log "${_logPrefix} WARNING: the check response was NOT sent to Icinga"
+        _rc=$_rc+1
+        echo
+        echo For Debugging:
+        $ch --show --json
+      fi
+      $ch --flush 2>/dev/null
+
+    fi
+
+  else
+    _log "${_logPrefix} SKIP execution."
+  fi
+
+  # add current result to global returncode
+  _rc_all=$_rc_all+$_rc
+
+  typeset -i local iCheckEnd=`_getUnixTs`
+  typeset -i local iCheckTime=$iCheckEnd-$iCheckStart
+
+  _log "${_logPrefix} finished after $iCheckTime sec with returncode $_rc"
+  test $_rc -eq 0 || (echo; echo "    >>> Check ${checkName} was not OK. See Output block above!"; echo; echo)
+}
+# ----------------------------------------------------------------------
+# help
+# ----------------------------------------------------------------------
+
+# show help text
+function showHelp(){
+self=`basename $0`
+cat <<EOH
+
+
+INTRODUCTION
+
+$_product v$_version
+
+Handle and execute icinga passive checks.
+With this client can run a single check, all checks or make a permanent loop.
+A new local check will be added to Icinga while running it the first time.
+
+
+GENERAL PARAMETERS
+
+  --help or -h or -?
+    show this help and abort.
+
+  --version or -v
+    show the version abd abort
+
+
+SERVICE ACTIONS
+
+  --list
+    get a list of local config files
+
+  --loop
+    Start to check all passive checks in a permanent loop.
+    It makes a random sleep if $sleeptime sec between all loops. This shuffles
+    the access time of all clients making requests to the icinga server.
+
+    Multiple starts will be detected. This parameter is the optimal choice for a
+    cronjob.
+
+  --runonce
+    Start to check all passive checks once.
+    This method respects the interval per check. Only outdated checks will be
+    executet.
+    This is a second choice for a cronjob if the runtime of all checks
+    is much shorter than your cronjob interval to prevent multiple processes.
+    Multiple starts will NOT be detected.
+
+  --run CONFIGFILE
+    Run a check by pointing the config file (see --list)
+    This execution ignores the interval and forces the execution.
+
+
+CONFIG
+
+  Config file
+  /etc/icinga2-passive-client/client.cfg
+  It sets other used pathes.
+
+  Config files for checks are in ${dir_cfg}/checks/
+
+
+DEBUGGING
+
+  The Output of check and results from Icinga are in
+  ${dir_data}.
+
+  A log of all performed executed check runs by $self
+  are in $logfile.
+  BTW: do not forget to add a log rotation for it.
+
+EOH
+}
+
+function showVersion(){
+  echo "$_license"
+  echo "$_copyright"
+  echo
+}
+# ----------------------------------------------------------------------
+#
+# MAIN
+#
+# ----------------------------------------------------------------------
+
+
+cat <<EOBANNER
+______________________________________________________________________________________
+
+ _______        __
+|_     _|.----.|__|.-----.-----.---.-.
+ _|   |_ |  __||  ||     |  _  |  _  |
+|_______||____||__||__|__|___  |___._|
+                         |_____|
+ ______                     __                   ______ _____   __               __
+|   __ \.---.-.-----.-----.|__|.--.--.-----.    |      |     |_|__|.-----.-----.|  |_
+|    __/|  _  |__ --|__ --||  ||  |  |  -__|    |   ---|       |  ||  -__|     ||   _|
+|___|   |___._|_____|_____||__| \___/|_____|    |______|_______|__||_____|__|__||____|
+                                                                                 v${_version}
+
+  $_license .. $_copyright
+
+______________________________________________________________________________________
+
+
+EOBANNER
+
+
+if [ -z "${dir_cfg}" ]; then
+  echo ERROR: $_product is not installed/ configured yet on this machine.
+  exit 1
+fi
+
+icingaHostMustExist
+
+
+touch ${logfile}
+if [ $# -eq 0 ]; then
+  showHelp
+  exit 0
+fi
+
+while [ $# -gt 0 ];
+do
+  case "$1" in
+
+    '--help' | '-h' | '-?')
+      showHelp
+      exit 0
+      ;;
+    '--version' | '-v')
+      showVersion
+      exit 0
+      ;;
+
+
+    '--list')
+      getChecks
+      ;;
+
+    '--loop')
+      loopChecks
+      ;;
+
+    '--runonce')
+      processAllChecks
+      ;;
+
+    '--run')
+      processCheck "$2" "force"
+      shift 1
+      ;;
+
+    *)
+      echo "ERROR: unknown parameter detected."
+      exit 2
+  esac
+  shift 1
+done
+echo
+
+# remark:
+# $_rc_all is a collected status code in the loop of all actions
+# if it is 0 then all checks were OK and have been sent to Icinga
+# echo exit with status code $_rc_all
+# exit $_rc_all
+exit 0
+
+# ----------------------------------------------------------------------

inc/confighandler.sh

+#!/bin/bash
+# ======================================================================
+#
+# VARIABLE STORAGE written in Bash
+#
+# ----------------------------------------------------------------------
+# 2020-01-25  v0.1  <axel.hahn@iml.unibe.ch>
+# 2020-01-26  v0.2  <axel.hahn@iml.unibe.ch> added --get, --add
+# 2020-01-27  v0.3  <axel.hahn@iml.unibe.ch> added storages handling
+# 2020-01-30  v0.5  <axel.hahn@iml.unibe.ch> added --setfile
+# ======================================================================
+
+_product="VARIABLE STORAGE"
+_version=0.5
+
+# _storagedir=/var/tmp/confighandler
+_storagedir=/dev/shm/confighandler
+
+_storageprefix='flat-variable-store-'
+_storagename='default'
+tmpfile=/tmp/cfgmover.tmp
+
+_markFile='::FILE-CONTENT::'
+
+# ----------------------------------------------------------------------
+# FUNCTIONS
+# ----------------------------------------------------------------------
+
+# write a debug message to STDERR
+function _wd(){
+	echo -e "\e[33m# $*\e[0m" >&2
+}
+
+# write error message in red to STDERR
+function _we(){
+	echo -e "\e[31m# $*\e[0m" >&2
+}
+
+# ----------------------------------------------------------------------
+# storage functions
+# ----------------------------------------------------------------------
+
+# set a variable storage
+# param  string  name of the storage; allowed chars are a-z and 0-9
+function storageset(){
+	_storagename=`echo $1 | sed "s#[^a-z0-9]##g"`
+
+	mkdir $_storagedir 2>/dev/null
+        chmod 777 $_storagedir
+	_wd "set storage [${_storagename}]"
+	# cfgfile=`dirname $0`/${_storageprefix}${_storagename}.cfg
+	cfgfile=$_storagedir/${_storageprefix}${_storagename}.cfg
+}
+
+# list existing storages
+function storagelist(){
+	ls -1 ${_storagedir}/${_storageprefix}*.cfg | sed "s#${_storagedir}/${_storageprefix}##g" | sed "s#.cfg\$##g"
+}
+
+# delete current storage data
+function storagedelete(){
+	rm -f "$cfgfile"
+}
+
+
+# ----------------------------------------------------------------------
+# variable functions
+# ----------------------------------------------------------------------
+
+function _removeprefix(){
+	local _var=$1
+	_wd "removing ${_var} ..."
+	cat $cfgfile 2>/dev/null | grep -v "^${_var}" > $tmpfile
+	mv $tmpfile $cfgfile
+}
+
+function vargetvalue(){
+	local _var=$1
+	grep "^${_var}=" $cfgfile | cut -f 2 -d '='
+}
+
+
+function varaddvalue(){
+	local _var=$1
+        shift 1
+	local _val=$*
+	local _value=`vargetvalue $_var`
+	echo $_value | grep '^\[' >/dev/null
+	if [ $? -ne 0 ]; then
+		# echo ERROR: you can add a value to an array only - not on [$_value]
+		_we "ERROR: you can add a value to an array only - not on [$_value]"
+	else
+		_wd "OK, array detected: $_value"
+		_value=`echo $_value | sed "s#]##"`,$_val"]"
+		varset $_var "$_value"
+	fi
+}
+
+function varremove(){
+	local _var=$1
+	# _removeprefix "${_var}="
+	cat $cfgfile 2>/dev/null | grep -v "^${_var}=" > $tmpfile
+	mv $tmpfile $cfgfile
+}
+
+
+function varset(){
+	local _var=$1
+        shift 1
+	local _val=$*
+
+	varremove "${_var}"
+	_wd "setting ${_var} = ${_val} ..."
+	(echo ${_var}=${_val} ; cat $cfgfile 2>/dev/null ) >$tmpfile
+	sort $tmpfile > $cfgfile
+	rm -f $tmpfile
+}
+function varsetfile(){
+	local _var=$1
+        shift 1
+	local _val=$*
+	varset $_var "${_markFile}${_val}"
+}
+function showstorage(){
+	if [ -f $cfgfile ]; then
+		_wd "content of storage [${_storagename}]"
+		cat $cfgfile
+	else
+		_wd "storage [${_storagename}] is emtpy"
+	fi
+}
+
+# ----------------------------------------------------------------------
+# export functions
+# ----------------------------------------------------------------------
+
+function _showFileAsJson(){
+	local _file=$1
+	which jq >/dev/null
+	if [ $? -eq 0 ]; then
+		jq -nR --arg data "`cat $_file`" '$data'
+	else
+		_we "ERROR: Json export of files ${_markFile} requires [jq]"
+	fi
+}
+
+function _jsonloop(){
+	local var=$1
+	typeset -i iFollow=$2
+
+	# _wd "_jsonloop $1 $2"
+
+	typeset -i level=`echo $var | grep -o '\.' | wc -l`+2
+
+	typeset -i local iChilds=0
+	typeset -i local iCount=0
+	typeset -i local iLeft=0
+
+	typeset -i lastlevel=$level-1
+	local sLastvar=`echo ${var} | cut -f $lastlevel -d "."`
+	local space=`printf %${lastlevel}s |tr " " "\t"`
+
+	# --- handle existing subkeys
+	grep "^${var}\." $cfgfile >/dev/null && (
+		iCount=0
+		iChilds=`grep "^${var}" $cfgfile | wc -l`
+		echo "${space}\"$sLastvar\": {"
+		for myhash in `grep "^${var}[\.=]" $cfgfile | cut -f 1 -d "=" | cut -f $level -d "." | sort -u`
+		do
+			iCount=$iCount+1
+			iLeft=$iChilds-$iCount
+			_jsonloop "$var.$myhash" $iLeft
+		done
+		echo -n "${space}}"
+	)
+
+	# --- show values
+
+	grep "^${var}=" $cfgfile >/dev/null && (
+		value=`grep "^${var}=" $cfgfile | cut -f 2- -d "="`
+		echo -n "${space}\"$sLastvar\": "
+		echo ${value}| grep "${_markFile}" >/dev/null
+		if [ $? -eq 0 ]; then
+			_showFileAsJson `echo ${value} | sed "s#^${_markFile}##"`
+		else	
+			echo -n "${value}"
+		fi
+	)
+	if [ $iFollow -gt 0 ]; then
+		echo ","
+	else
+		echo
+	fi
+}
+
+function exportJson(){
+	# _wd "exportJson - init: find first level of vars"
+	typeset -i local iCount=0
+	typeset -i local iLeft=0
+	typeset -i local iChilds=`grep "^[a-zA-Z]" $cfgfile | cut -f 1 -d "=" | cut -f 1 -d '.' | sort -u | wc -l`
+
+	echo "{"
+
+	# first level names
+	for lev1var in `cat $cfgfile | grep "^.*\=" | cut -f 1 -d "=" | grep -v "\." | sort -u`
+	do
+		#_wd "exportJson - start run with var [$lev1var]"
+		iCount=$iCount+1
+		iLeft=$iChilds-$iCount
+		_jsonloop "$lev1var" $iLeft
+	done
+
+	# _wd "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+	# first level hashes
+	for lev1var in `cat $cfgfile | cut -f 1 -d "=" | grep  "\." | cut -f 1 -d "." | sort -u`
+	do
+		#_wd "exportJson - start run with hash [$lev1var]"
+		iCount=$iCount+1
+		iLeft=$iChilds-$iCount
+		_jsonloop "$lev1var" $iLeft
+	done
+	echo "}"
+
+}
+
+
+# W.I.P
+# JSON export with processing line by line
+#
+function exportJson2(){
+	typeset -i local iSublevel=0
+	typeset -i local iLast=0
+
+	typeset -i local lastlevel
+	local sLastvar
+
+	grep '^[a-zA-Z]' $cfgfile | while read line
+	do
+		fullvar=`echo $line | cut -f 1 -d "="`
+		iSublevel=`echo $fullvar | grep -o '\.' | wc -l`+2
+		lastlevel=$iSublevel-1
+		sLastvar=`echo ${fullvar} | cut -f $lastlevel -d "."`
+
+		local space=`printf %${lastlevel}s |tr " " "\t"`
+
+		echo "${space} $line .. level $iSublevel .. $sLastvar"
+	done
+}
+
+# ----------------------------------------------------------------------
+# help
+# ----------------------------------------------------------------------
+
+# show help text
+function showHelp(){
+self=`basename $0`
+cat <<EOH
+
+$_product v$_version
+
+This script handles variables and nested values (hashes and arrays).
+You can create several configuration sets (storages) to isolate them.
+
+Help
+
+	--help or -h or -?
+		show this help and abort.
+
+	--version or -v
+		show the version.
+
+Storage parameters
+
+	--storage NAME
+		Set a storage to use. Allowed chars are a-z and 0-9.
+		Default is a storage named "default".
+		You can set an evironment variable CFGSTORAGE to override
+		the default (= to skip --storage NAME).
+
+	--list
+		List existing storages.
+
+	--flush
+		delete current storage data
+
+	--show
+		show the current config entries as plain text
+
+	--json
+		show config entries as json.
+		Warning: it is implemented in Bash - can be slow on more data
+
+Create | read | update | delete variables
+
+	--set VARNAME VALUE
+		Set a variable and its value.
+		If it does not exist then it will be created.
+		If it exist then it will be overwritten with the new value.
+
+		For the values you can force the type:
+			* Use single and double quotes to set a string, i.e.
+				\$ $self --set my_string '"hello world"'
+
+			* Quote the doublequotes to expand variables, i.e.
+				\$ $self --set my_string2 \"\${my_string2}\"
+
+			* use no quoting vor integers and values
+				\$ $self --set number_cpu 8
+
+			* use barackets in quotes to set an array
+				\$ $self --set my_array '[8, 12, 16]'
+
+		Use a "." (dot) in varname to create a hierarchy, i.e.
+			\$ $self --set host.os '"linux"'
+
+	--setfile VARNAME FILENAME
+		Set an input file to show its content
+
+	--add VARNAME VALUE
+		Add VALUE as array element to VARNAME.
+		It fails if VARNAME does not exist or isn't an array.
+			 $ confighandler.sh --add my_array 20
+
+	--get VARNAME
+		Show value of the given variable.
+
+	--delete VARNAME
+		delete a variable.
+
+You can concatenate all parameters. So you can use --show before
+and after an --set action to see the differences.
+\$ $self --show --set hello '"world"' --show
+
+You get additional infos on STDERR. To get plain data send STDERR
+to /dev/null
+\$ $self --show 2>/dev/null
+
+EOH
+}
+
+# ----------------------------------------------------------------------
+#
+# MAIN
+#
+# ----------------------------------------------------------------------
+
+
+if [ ! -z $CFGSTORAGE ]; then
+	storageset "$CFGSTORAGE"
+else
+	storageset "default"
+fi
+
+if [ $# -eq 0 ]; then
+	showHelp
+	exit 0
+fi
+
+while [ $# -gt 0 ];
+do
+	case "$1" in
+		'--help' | '-h' | '-?')
+			_wd ">>> $1"
+			showHelp
+			exit 0
+			;;
+		'--version' | '-v')
+			_wd ">>> $1"
+			echo $_product v$_version
+			;;
+
+		# ----- storage
+
+		'--storage')
+			_wd ">>> $1 $2"
+			storageset "$2"
+			shift 1
+			;;
+		'--list')
+			_wd ">>> $1"
+			storagelist
+			;;
+		'--flush')
+			_wd ">>> $1"
+			storagedelete
+			;;
+
+		# ----- variables
+
+		'--set')
+			_wd ">>> $1 $2 $3"
+			varset $2 "$3"
+			shift 2
+			;;
+		'--setfile')
+			_wd ">>> $1 $2 $3"
+			varsetfile $2 "$3"
+			shift 2
+			;;
+		'--add')
+			_wd ">>> $1 $2 $3"
+			varaddvalue $2 "$3"
+			shift 2
+			;;
+		'--get')
+			_wd ">>> $1 $2"
+			vargetvalue $2
+			shift 1
+			;;
+		'--delete')
+			_wd ">>> $1 $2"
+			varremove $2
+			shift 1
+			;;
+		'--show')
+			_wd ">>> $1"
+			showstorage
+			;;
+
+		# ----- export
+
+		'--json')
+			_wd ">>> $1"
+			exportJson
+			# time exportJson
+			# time exportJson2
+			;;
+		*)
+			_we "ERROR: unknown param detected: [$1]"
+			showHelp
+			echo "Aborting ..."
+			exit 1
+	esac
+	_wd ""
+	shift 1
+
+done
+_wd "DONE"

inc/confighandler2.sh

+#!/bin/bash
+# ======================================================================
+#
+# VARIABLE STORAGE written in Bash
+#
+# ----------------------------------------------------------------------
+# 2020-01-25  v0.1  <axel.hahn@iml.unibe.ch>
+# 2020-01-26  v0.2  <axel.hahn@iml.unibe.ch> added --get, --add
+# 2020-01-27  v0.3  <axel.hahn@iml.unibe.ch> added storages handling
+# 2020-01-30  v0.5  <axel.hahn@iml.unibe.ch> added --setfile
+# ======================================================================
+
+_product="VARIABLE STORAGE"
+_version=0.5
+
+_storagedir=/var/tmp/confighandler
+_storagedir=/dev/shm/confighandler
+
+_storageprefix='flat-variable-store-'
+_storagename='default'
+tmpfile=/tmp/cfgmover.tmp
+
+_markFile='::FILE-CONTENT::'
+
+
+# ----------------------------------------------------------------------
+# FUNCTIONS
+# ----------------------------------------------------------------------
+
+# write a debug message to STDERR
+function _wd(){
+	echo -e "\e[33m# $*\e[0m" >&2
+}
+
+# write error message in red to STDERR
+function _we(){
+	echo -e "\e[31m# $*\e[0m" >&2
+}
+
+# ----------------------------------------------------------------------
+# storage functions
+# ----------------------------------------------------------------------
+
+# set a variable storage
+# param  string  name of the storage; allowed chars are a-z and 0-9
+function storageset(){
+	_storagename=`echo $1 | sed "s#[^a-z0-9]##g"`
+
+	mkdir $_storagedir 2>/dev/null
+    chmod 777 $_storagedir
+	_wd "set storage [${_storagename}]"
+	# cfgfile=`dirname $0`/${_storageprefix}${_storagename}.cfg
+	cfgfile=$_storagedir/${_storageprefix}${_storagename}.cfg
+}
+
+# list existing storages
+function storagelist(){
+	ls -1 ${_storagedir}/${_storageprefix}*.cfg | sed "s#${_storagedir}/${_storageprefix}##g" | sed "s#.cfg\$##g"
+}
+
+# delete current storage data
+function storagedelete(){
+	rm -f "$cfgfile"
+}
+
+
+# ----------------------------------------------------------------------
+# variable functions
+# ----------------------------------------------------------------------
+
+function _removeprefix(){
+	local _var=$1
+	_wd "removing ${_var} ..."
+	cat $cfgfile 2>/dev/null | grep -v "^${_var}" > $tmpfile
+	mv $tmpfile $cfgfile
+}
+
+function vargetvalue(){
+	local _var=$1
+	grep "^${_var}=" $cfgfile | cut -f 2 -d '='
+}
+
+
+function varaddvalue(){
+	local _var=$1
+        shift 1
+	local _val=$*
+	local _value=`vargetvalue $_var`
+	echo $_value | grep '^\[' >/dev/null
+	if [ $? -ne 0 ]; then
+		# echo ERROR: you can add a value to an array only - not on [$_value]
+		_we "ERROR: you can add a value to an array only - not on [$_value]"
+	else
+		_wd "OK, array detected: $_value"
+		_value=`echo $_value | sed "s#]##"`,$_val"]"
+		varset $_var "$_value"
+	fi
+}
+
+function varremove(){
+	local _var=$1
+	# _removeprefix "${_var}="
+	cat $cfgfile 2>/dev/null | grep -v "^${_var}=" > $tmpfile
+	mv $tmpfile $cfgfile
+}
+
+
+function varset(){
+	local _var=$1
+        shift 1
+	local _val=$*
+
+	varremove "${_var}"
+	_wd "setting ${_var} = ${_val} ..."
+	(echo ${_var}=${_val} ; cat $cfgfile 2>/dev/null ) >$tmpfile
+	sort $tmpfile > $cfgfile
+	rm -f $tmpfile
+}
+function varsetfile(){
+	local _var=$1
+        shift 1
+	local _val=$*
+	varset $_var "${_markFile}${_val}"
+}
+function showstorage(){
+	if [ -f $cfgfile ]; then
+		_wd "content of storage [${_storagename}]"
+		cat $cfgfile
+	else
+		_wd "storage [${_storagename}] is emtpy"
+	fi
+}
+
+# ----------------------------------------------------------------------
+# export functions
+# ----------------------------------------------------------------------
+
+function _showFileAsJson(){
+	local _file=$1
+	which jq >/dev/null
+	if [ $? -eq 0 ]; then
+		jq -nR --arg data "`cat $_file`" '$data'
+	else
+		_we "ERROR: Json export of files ${_markFile} requires [jq]"
+	fi
+}
+
+function _jsonloop(){
+	local var=$1
+	typeset -i iFollow=$2
+
+	# _wd "_jsonloop $1 $2"
+
+	typeset -i level=`echo $var | grep -o '\.' | wc -l`+2
+
+	typeset -i local iChilds=0
+	typeset -i local iCount=0
+	typeset -i local iLeft=0
+
+	typeset -i lastlevel=$level-1
+	local sLastvar=`echo ${var} | cut -f $lastlevel -d "."`
+	local space=`printf %${lastlevel}s |tr " " "\t"`
+
+	# --- handle existing subkeys
+	grep "^${var}\." $cfgfile >/dev/null && (
+		iCount=0
+		iChilds=`grep "^${var}" $cfgfile | wc -l`
+		echo "${space}\"$sLastvar\": {"
+		for myhash in `grep "^${var}[\.=]" $cfgfile | cut -f 1 -d "=" | cut -f $level -d "." | sort -u`
+		do
+			iCount=$iCount+1
+			iLeft=$iChilds-$iCount
+			_jsonloop "$var.$myhash" $iLeft
+		done
+		echo -n "${space}}"
+	)
+
+	# --- show values
+
+	grep "^${var}=" $cfgfile >/dev/null && (
+		value=`grep "^${var}=" $cfgfile | cut -f 2- -d "="`
+		echo -n "${space}\"$sLastvar\": "
+		echo ${value}| grep "${_markFile}" >/dev/null
+		if [ $? -eq 0 ]; then
+			_showFileAsJson `echo ${value} | sed "s#^${_markFile}##"`
+		else	
+			echo -n "${value}"
+		fi
+	)
+	if [ $iFollow -gt 0 ]; then
+		echo ","
+	else
+		echo
+	fi
+}
+
+function exportJson(){
+	# _wd "exportJson - init: find first level of vars"
+	typeset -i local iCount=0
+	typeset -i local iLeft=0
+	typeset -i local iChilds=`grep "^[a-zA-Z]" $cfgfile | cut -f 1 -d "=" | cut -f 1 -d '.' | sort -u | wc -l`
+
+	echo "{"
+
+	# first level names
+	for lev1var in `cat $cfgfile | grep "^.*\=" | cut -f 1 -d "=" | grep -v "\." | sort -u`
+	do
+		#_wd "exportJson - start run with var [$lev1var]"
+		iCount=$iCount+1
+		iLeft=$iChilds-$iCount
+		_jsonloop "$lev1var" $iLeft
+	done
+
+	# _wd "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+	# first level hashes
+	for lev1var in `cat $cfgfile | cut -f 1 -d "=" | grep  "\." | cut -f 1 -d "." | sort -u`
+	do
+		#_wd "exportJson - start run with hash [$lev1var]"
+		iCount=$iCount+1
+		iLeft=$iChilds-$iCount
+		_jsonloop "$lev1var" $iLeft
+	done
+	echo "}"
+
+}
+
+
+# W.I.P
+# JSON export with processing line by line
+#
+function exportJson2(){
+	typeset -i local iSublevel=0
+	typeset -i local iLast=0
+
+	typeset -i local lastlevel
+	local sLastvar
+
+	grep '^[a-zA-Z]' §fgfile | while read line
+	do
+		fullvar=`echo $line | cut -f 1 -d "="`
+		iSublevel=`echo $fullvar | grep -o '\.' | wc -l`+2
+		lastlevel=$iSublevel-1
+		sLastvar=`echo ${fullvar} | cut -f $lastlevel -d "."`
+
+		local space=`printf %${lastlevel}s |tr " " "\t"`
+
+		echo "${space} $line .. level $iSublevel .. $sLastvar"
+	done
+}
+
+# ----------------------------------------------------------------------
+# help
+# ----------------------------------------------------------------------
+
+# show help text
+function showHelp(){
+self=`basename $0`
+cat <<EOH
+
+$_product v$_version
+
+This script handles variables and nested values (hashes and arrays).
+You can create several configuration sets (storages) to isolate them.
+
+Help
+
+	--help or -h or -?
+		show this help and abort.
+
+	--version or -v
+		show the version.
+
+Storage parameters
+
+	--storage NAME
+		Set a storage to use. Allowed chars are a-z and 0-9.
+		Default is a storage named "default".
+		You can set an evironment variable CFGSTORAGE to override
+		the default (= to skip --storage NAME).
+
+	--list
+		List existing storages.
+
+	--flush
+		delete current storage data
+
+	--show
+		show the current config entries as plain text
+
+	--json
+		show config entries as json.
+		Warning: it is implemented in Bash - can be slow on more data
+
+Create | read | update | delete variables
+
+	--set VARNAME VALUE
+		Set a variable and its value.
+		If it does not exist then it will be created.
+		If it exist then it will be overwritten with the new value.
+
+		For the values you can force the type:
+			* Use single and double quotes to set a string, i.e.
+				\$ $self --set my_string '"hello world"'
+
+			* Quote the doublequotes to expand variables, i.e.
+				\$ $self --set my_string2 \"\${my_string2}\"
+
+			* use no quoting vor integers and values
+				\$ $self --set number_cpu 8
+
+			* use barackets in quotes to set an array
+				\$ $self --set my_array '[8, 12, 16]'
+
+		Use a "." (dot) in varname to create a hierarchy, i.e.
+			\$ $self --set host.os '"linux"'
+
+	--setfile VARNAME FILENAME
+		Set an input file to show its content
+
+	--add VARNAME VALUE
+		Add VALUE as array element to VARNAME.
+		It fails if VARNAME does not exist or isn't an array.
+			 $ confighandler.sh --add my_array 20
+
+	--get VARNAME
+		Show value of the given variable.
+
+	--delete VARNAME
+		delete a variable.
+
+You can concatenate all parameters. So you can use --show before
+and after an --set action to see the differences.
+\$ $self --show --set hello '"world"' --show
+
+You get additional infos on STDERR. To get plain data send STDERR
+to /dev/null
+\$ $self --show 2>/dev/null
+
+EOH
+}
+
+# ----------------------------------------------------------------------
+#
+# MAIN
+#
+# ----------------------------------------------------------------------
+
+
+if [ ! -z $CFGSTORAGE ]; then
+	storageset "$CFGSTORAGE"
+else
+	storageset "default"
+fi
+
+if [ $# -eq 0 ]; then
+	showHelp
+	exit 0
+fi
+
+while [ $# -gt 0 ];
+do
+	case "$1" in
+		'--help' | '-h' | '-?')
+			_wd ">>> $1"
+			showHelp
+			exit 0
+			;;
+		'--version' | '-v')
+			_wd ">>> $1"
+			echo $_product v$_version
+			;;
+
+		# ----- storage
+
+		'--storage')
+			_wd ">>> $1 $2"
+			storageset "$2"
+			shift 1
+			;;
+		'--list')
+			_wd ">>> $1"
+			storagelist
+			;;
+		'--flush')
+			_wd ">>> $1"
+			storagedelete
+			;;
+
+		# ----- variables
+
+		'--set')
+			_wd ">>> $1 $2 $3"
+			varset $2 "$3"
+			shift 2
+			;;
+		'--setfile')
+			_wd ">>> $1 $2 $3"
+			varsetfile $2 "$3"
+			shift 2
+			;;
+		'--add')
+			_wd ">>> $1 $2 $3"
+			varaddvalue $2 "$3"
+			shift 2
+			;;
+		'--get')
+			_wd ">>> $1 $2"
+			vargetvalue $2
+			shift 1
+			;;
+		'--delete')
+			_wd ">>> $1 $2"
+			varremove $2
+			shift 1
+			;;
+		'--show')
+			_wd ">>> $1"
+			showstorage
+			;;
+
+		# ----- export
+
+		'--json')
+			_wd ">>> $1"
+			exportJson
+			# time exportJson
+			# time exportJson2
+			;;
+		*)
+			_we "ERROR: unknown param detected: [$1]"
+			showHelp
+			echo "Aborting ..."
+			exit 1
+	esac
+	_wd ""
+	shift 1
+
+done
+_wd "DONE"

inc/rest-api-client.sh

+#!/bin/bash
+# ======================================================================
+#
+# REST API CLIENT USING CURL
+#
+# REQUIREMENTS
+# - Bash (Linux or MS Windows i.e with Cygwin)
+# - curl
+# - sha1sum (optional; for export functionality with AUTOFILE only)
+# ----------------------------------------------------------------------
+# (1) source this script
+# (2) enter "http.help" to get a list of available commands
+# ----------------------------------------------------------------------
+# 2020-02-07  v0.2  axel.hahn@iml.unibe.ch  BETABETA
+# 2020-02-12  v0.4  axel.hahn@iml.unibe.ch  Caching
+# 2020-03-02  v0.5  axel.hahn@iml.unibe.ch  a few more response check functions
+# ======================================================================
+
+# --- fetch incoming params
+  RestApiCfg=$1
+  RestApiMethod=$2
+  ApiUrl=$3
+  Body="$4"
+
+  http_cfg__about="Bash REST API client v0.5"
+  typeset -i http_cfg__debug=0
+  typeset -i http_cfg__cacheTtl=0
+  http_cfg__cacheDir=/var/tmp/http-cache
+  http_cfg__UA="${http_cfg__about}"
+  http_cfg__prjurl="https://git-repo.iml.unibe.ch/iml-open-source/bash-rest-api-client"
+
+# --- curl meta infos to collect
+#     see variables in man curl --write-out param
+  curlMeta="\
+    http_code \
+    http_connect \
+    local_ip \
+    local_port \
+    num_connects \
+    num_redirects \
+    redirect_url \
+    remote_ip \
+    remote_port \
+    size_download \
+    size_header \
+    size_request \
+    size_upload \
+    speed_download \
+    speed_upload \
+    ssl_verify_result \
+    time_appconnect \
+    time_connect \
+    time_namelookup \
+    time_pretransfer \
+    time_redirect \
+    time_starttransfer \
+    time_total \
+    url_effective \
+"
+
+
+# ----------------------------------------------------------------------
+#
+# functions
+#
+# ----------------------------------------------------------------------
+
+  # ......................................................................
+  #
+  # write a debug message to STDERR
+  # Do no not change the prefix - is is read in inc_functions
+  #
+  # params  strings  output message
+  function http._wd(){
+    if [ $http_cfg__debug -gt 0 ]; then
+      echo -e "\e[33m# RESTAPI::DEBUG $*\e[0m" >&2
+    fi
+  }
+
+  # ......................................................................
+  #
+  # write an error message to STDERR
+  # Do no not change the prefix - is is read in inc_functions
+  #
+  # params  strings  output message
+  function http._we(){
+    echo -e "\e[31m# RESTAPI::ERROR $*\e[0m" >&2
+  }
+
+  function http(){
+  cat <<EOH
+
+$http_cfg__about
+
+A REST API Client with curl
+
+Enter http.help to show all commands.
+
+EOH
+    # $0 is not the current file if we source a script
+    # grep "function http.[a-z]" $0 | sort
+  }
+
+  function http.init(){
+
+    which curl >/dev/null || http.quit
+
+    # request vars
+
+    http_req__auth=
+    http_req__auth=
+    http_req__body=
+    http_req__method=GET
+    http_req__url=
+    http_req__fullurl=
+    http_req__docs=
+
+    http_req__dataprefix="RESTAPICLIENTMETADATA_`date +%s`_$$"
+    local writevar=
+    for myvar in $curlMeta
+    do
+      writevar="${writevar}|${myvar}:%{${myvar}}"
+    done
+    http_curl__writeout="\\n${http_req__dataprefix}${writevar}\\n"
+
+    # cache
+    http_req__mode=undefined
+    http_cfg__cacheTtl=0
+    http_cfg__cacheFile=
+
+    # response
+    http_resp__all=
+    http_resp__neutral=
+    mkdir ${http_cfg__cacheDir} 2>/dev/null
+    chmod 777 ${http_cfg__cacheDir} 2>/dev/null
+  }
+
+  # execute the request
+  # param  string  optional: full url
+  function http.makeRequest(){
+    http._wd "${FUNCNAME[0]}($1)"
+
+    # --- handle optional prams
+    if [ $# -ne 0 ]; then
+      echo $1 | grep "^[A-Z]*$" >/dev/null
+      if [ $? -eq 0 ]; then
+        http.setMethod "$1"
+        shift 1
+      fi
+      http.setUrl "$1"
+      http.setBody "$2"
+    fi
+    # test -z "$1" || http.setFullUrl "$1"
+
+    # --- detect caching
+    http_req__mode=REQUEST
+    useCache=0
+    makeRequest=1
+    if [ $http_cfg__cacheTtl -gt 0 -a "${http_req__method}" = "GET" ]; then
+      useCache=1
+      test -z "${http_cfg__cacheFile}" && http_cfg__cacheFile=`http._genOutfilename "${http_cfg__cacheDir}/AUTOFILE"`
+      if [ -f "${http_cfg__cacheFile}" ]; then
+        http.responseImport "${http_cfg__cacheFile}"
+        typeset -i local iAge=`http.getRequestAge`
+        http._wd "INFO: Age of cache is $iAge sec  - vs TTL $http_cfg__cacheTtl sec - file $http_cfg__cacheFile"
+        if [ $iAge -gt 0 -a $iAge -lt $http_cfg__cacheTtl ]; then
+          http._wd "INFO: Using cache"
+          makeRequest=0
+          http_req__mode=CACHE
+        else
+          http._wd "INFO: Cache file will be updated after making the request"
+          rm -f "${http_cfg__cacheFile}" 2>/dev/null
+        fi
+      fi
+    fi
+
+
+    # --- make the request
+    if [ $makeRequest -eq 1 ]; then
+      http_req__start=`date +%s`
+      http._wd "${FUNCNAME[0]}($1) ${http_req__method} ${http_req__fullurl}"
+      http_resp__all=$(
+        if [ -z "${http_req__body}" ]; then
+          curl -k -s \
+            -A "${http_cfg__UA}" \
+            -w "${http_curl__writeout}" \
+            -H 'Accept: application/json' \
+            ${http_req__auth} \
+            -i "${http_req__fullurl}" \
+            -X "${http_req__method}"
+        else
+          curl -k -s \
+            -A "${http_cfg__UA}" \
+            -w "${http_curl__writeout}" \
+            -H 'Accept: application/json' \
+            ${http_req__auth} \
+            -i "${http_req__fullurl}" \
+            -X "${http_req__method}" \
+            -d "${http_req__body}"
+        fi
+        ) || http.quit
+      http._wd "OK - Curl finished the http request ... processing data"
+      http_resp__neutral=`http._fetchAllAndReformat`
+      if [ $useCache -eq 1 ]; then
+        http._wd "INFO: writing cache ..."
+        http.responseExport "${http_cfg__cacheFile}"
+      fi
+    fi
+    http._wd "Request function finished; Code `http.getStatuscode`"
+  }
+
+  # ......................................................................
+  #
+  # show error message with last return code and quit with this exitcode
+  # no params
+  function http.quit(){
+    http._wd "${FUNCNAME[0]}($1)"
+    rc=$?
+    echo >&2
+    echo -e "\e[31m# ERROR: command FAILED with rc $rc. \e[0m" >&2
+    if [ ! -z "${RestApiDocs}" ]; then
+      echo "HINT: see ${RestApiDocs}" >&2
+    fi
+    # dont make exit in a sourced file
+    # exit $rc
+  }
+
+
+  # load a config file
+  function http.loadcfg(){
+    http._wd "${FUNCNAME[0]}($1) !!! DEPRECATED !!!"
+    # reset expected vars from config
+    RestApiUser=
+    RestApiPassword=
+    RestApiBaseUrl=
+    RestApiDocs=
+
+    # source config file
+    . "${1}" || http.quit
+
+    # set "internal" vars
+    if [-z "$RestApiPassword" ]; then
+      http.setAuth "$RestApiUser:$RestApiPassword"
+    else
+      http.setAuth
+    fi
+    http.setBaseUrl "${RestApiBaseUrl}"
+    http.setDocs "${RestApiDocs}"
+  }
+
+  # ======================================================================
+  # GETTER
+  # ======================================================================
+  function http._fetchResponseHeaderOrBody(){
+    http._wd "${FUNCNAME[0]}($1)"
+    local isheader=true
+
+    # keep leading spaces
+    IFS=''
+
+    echo "${http_resp__all}" | grep -v "${http_req__dataprefix}" | while read -r line; do
+      if $isheader; then
+        if [[ $line = $'\r' ]]; then
+            isheader=false
+        else
+          test "$1" = "header" && echo $line
+        fi
+      else
+        # body="$body"$'\n'"$line"
+        test "$1" = "body" && echo $line
+      fi
+    done
+  }
+  function http._fetchResponseData(){
+    http._wd "${FUNCNAME[0]}($1)"
+    echo "${http_resp__all}" | sed "s#${http_req__dataprefix}#\n${http_req__dataprefix}#" | grep "${http_req__dataprefix}" | tail -1 | cut -f 2- -d "|" | sed "s#|#\n#g" | grep -v "${http_req__dataprefix}" | while read -r line; do
+      echo $line
+    done
+  }
+  function http._fetchAllAndReformat(){
+    http._wd "${FUNCNAME[0]}($1)"
+    IFS=''
+    line="#------------------------------------------------------------"
+
+    echo "#_META_|about:$http_cfg__about"
+    echo "#_META_|host:`hostname -f`"
+    echo $line
+    echo "#_REQUEST_|fullurl:$http_req__fullurl"
+    echo "#_REQUEST_|method:$http_req__method"
+    echo "#_REQUEST_|time:`date`"
+    echo "#_REQUEST_|timestamp:`date +%s`"
+    echo "#_REQUEST_|auth:`echo $http_req__auth | sed 's#:.*#:xxxxxxxx#'`"
+    echo "#_REQUEST_|body:$http_req__body"
+    echo "#_REQUEST_|baseurl:$http_req__baseurl"
+    echo "#_REQUEST_|url:$http_req__url"
+    echo "#_REQUEST_|docs:$http_req__docs"
+    echo $line
+    http._fetchResponseHeaderOrBody header  | sed "s,^,#_HEADER_|,g"
+    echo $line
+    http._fetchResponseData                 | sed "s,^,#_DATA_|,g"
+    echo $line
+    http._fetchResponseHeaderOrBody body    | sed "s,^,#_BODY_|,g"
+    echo $line END
+  }
+
+  function http._getFilteredResponse(){
+    http._wd "${FUNCNAME[0]}($1)"
+    echo "${http_resp__neutral}" | grep "^#_${1}_|"  | cut -f 2- -d "|"
+  }
+
+  # ---------- PUBLIC REQUEST GETTER
+
+  function http.getRequestTs(){
+    http._wd "${FUNCNAME[0]}($1)"
+    http._getFilteredResponse REQUEST | grep "^timestamp" | cut -f 2 -d ":"
+  }
+
+  # get age of the response in sec.
+  # It is especially useful after responseImport
+  function http.getRequestAge(){
+    http._wd "${FUNCNAME[0]}($1)"
+    typeset -i local iAge=`date +%s`-`http.getRequestTs`
+    echo $iAge
+  }
+
+  # ---------- PUBLIC RESPONSE GETTER
+
+  # get response body
+  function http.getResponse(){
+    http._wd "${FUNCNAME[0]}($1)"
+    http._getFilteredResponse BODY
+  }
+  # get curl data of this request with status, transferred bytes, speed, ...
+  function http.getResponseData(){
+    http._wd "${FUNCNAME[0]}($1)"
+    http._getFilteredResponse DATA
+  }
+  # get response header
+  function http.getResponseHeader(){
+    http._wd "${FUNCNAME[0]}($1)"
+    http._getFilteredResponse HEADER
+  }
+
+  # get raw response (not available after import)
+  function http.getResponseRaw(){
+    http._wd "${FUNCNAME[0]}($1)"
+    echo "${http_resp__all}"
+  }
+
+  # get Http status as string OK|Redirect|Error
+  function http.getStatus(){
+    http._wd "${FUNCNAME[0]}($1)"
+    http.isOk       >/dev/null && echo OK
+    http.isRedirect >/dev/null && echo Redirect
+    http.isError    >/dev/null && echo Error
+  }
+
+  # get Http status code of the request as 3 digit number
+  function http.getStatuscode(){
+    http._wd "${FUNCNAME[0]}($1)"
+    local _filter=$1
+    http.getResponseData | grep "^http_code:" | cut -f 2 -d ":"
+  }
+
+  # was response a 2xx status code?
+  # output is a statuscode if it matches ... or empty
+  # Additionally you can verify the return code
+  # $? -eq 0 means YES
+  # $? -ne 0 means NO
+  function http.isOk(){
+    http._wd "${FUNCNAME[0]}($1)"
+    http.getStatuscode | grep '2[0-9][0-9]'
+  }
+  # was the repsonse a redirect?
+  function http.isRedirect(){
+    http._wd "${FUNCNAME[0]}($1)"
+    http.getStatuscode | grep '3[0-9][0-9]'
+  }
+
+  # was the repsonse a client error (4xx or 5xx)
+  function http.isError(){
+    http._wd "${FUNCNAME[0]}($1)"
+    http.getStatuscode | grep '[45][0-9][0-9]'
+  }
+  # was the repsonse a client error (4xx)
+  function http.isClientError(){
+    http._wd "${FUNCNAME[0]}($1)"
+    http.getStatuscode | grep '4[0-9][0-9]'
+  }
+  # was the repsonse a client error (5xx)
+  function http.isServerError(){
+    http._wd "${FUNCNAME[0]}($1)"
+    http.getStatuscode | grep '5[0-9][0-9]'
+  }
+
+
+  # dump information about request and response
+  function http.dump(){
+    http._wd "${FUNCNAME[0]}($1)"
+    http.responseExport
+  }
+
+  # ======================================================================
+  # Import/ Export
+  # ======================================================================
+
+  # helper to replace "AUTOFILE" with something uniq using full url
+  # param  string  import or export filename
+  function http._genOutfilename(){
+    http._wd "${FUNCNAME[0]}($1)"
+    echo $1 | grep "AUTOFILE" >/dev/null
+    if [ $? -ne 0 ]; then
+      echo $1
+    else
+      local sum=`echo ${http_req__fullurl} | sha1sum `
+      local autofile=`echo "${sum}__${http_req__fullurl}" | sed "s#[^a-z0-9]#_#g"`
+      echo $1 | sed "s#AUTOFILE#${autofile}#"
+    fi
+  }
+
+
+  # export to a file
+  function http.responseExport(){
+    http._wd "${FUNCNAME[0]}($1)"
+    if [ -z $1 ]; then
+      echo "${http_resp__neutral}"
+    else
+      local outfile=`http._genOutfilename "$1"`
+      http._wd "${FUNCNAME[0]}($1) writing to outfile $outfile"
+      echo "${http_resp__neutral}" >$outfile
+    fi
+  }
+
+  # import a former response from a file
+  function http.responseImport(){
+    http._wd "${FUNCNAME[0]}($1)"
+    local infile=`http._genOutfilename "$1"`
+    if [ -r "${infile}" ]; then
+      grep "^#_META_|about:$http_cfg__about" "${infile}" >/dev/null
+      if [ $? -eq 0 ]; then
+         http_resp__neutral=`cat "${infile}"`
+      else
+         echo "ERROR: Ooops [${infile}] does not seem to be an export dump."
+         http.quit
+      fi
+    else
+      echo "ERROR: Ooops the file [${infile}] is not readable."
+      http.quit
+    fi
+  }
+  # delete an exported file; this is especially useful if you use
+  # AUTOFILE functionality
+  function http.responseDelete(){
+    http._wd "${FUNCNAME[0]}($1)"
+    local infile=`http._genOutfilename "$1"`
+    if [ -r "${infile}" ]; then
+      grep "^#_META_|about:$http_cfg__about" "${infile}" >/dev/null
+      if [ $? -eq 0 ]; then
+        rm -f "${infile}"
+        if [ $? -eq 0 ]; then
+          http._wd "OK, ${infile} was deleted."
+        else
+          http._wd "ERROR: unable to delete existing ${infile}. Check permissions."
+        fi
+       else
+        http._wd "SKIP: ${infile} is not an export file."
+      fi
+    else
+      http._wd "SKIP: ${infile} is not readable."
+    fi
+  }
+
+  # ======================================================================
+  # SETTER
+  # ======================================================================
+
+  # set authentication
+  # param  string  USER:PASSWORD
+  function http.setAuth(){
+    http._wd "${FUNCNAME[0]}($1)"
+    if [ -z "$1" ]; then
+      http_req__auth=
+    else
+      http_req__auth="-u $1"
+    fi
+  }
+  # set body to send for PUTs and POSTs
+  # param  string  body
+  function http.setBody(){
+    http._wd "${FUNCNAME[0]}($1)"
+    http_req__body=$1
+  }
+  # set a base url of an API
+  # Remark: Then use http.setUrl to complet the url to request
+  # param  string  url
+  function http.setBaseUrl(){
+    http._wd "${FUNCNAME[0]}($1)"
+    http_req__baseurl=$1
+    http.setFullUrl
+  }
+  # Enable or disable debug mode
+  # param  integer  0|1
+  function http.setDebug(){
+    http._wd "${FUNCNAME[0]}($1)"
+    http_cfg__debug=$1
+  }
+  function http.setDocs(){
+    http._wd "${FUNCNAME[0]}($1)"
+    http_req__docs=$1
+  }
+
+  # set the method to use; GET|POST|PUT|DELETE
+  # param  string  name of method
+  function http.setMethod(){
+    http._wd "${FUNCNAME[0]}($1)"
+    http_req__method=$1
+  }
+
+  # set a full url to request
+  # param  string  url
+  function http.setFullUrl(){
+    http._wd "${FUNCNAME[0]}($1)"
+    if [ -z "$1" ]; then
+      http_req__fullurl=${http_req__baseurl}${http_req__url}
+    else
+      http_req__fullurl=$1
+    fi
+  }
+  # complete the base url
+  # param  string  url part behind base url
+  function http.setUrl(){
+    http._wd "${FUNCNAME[0]}($1)"
+    http_req__url=$1
+    http.setFullUrl
+  }
+
+  # ----- caching
+
+  function http.setCacheTtl(){
+    http._wd "${FUNCNAME[0]}($1)"
+    http_cfg__cacheTtl=$1
+  }
+
+  function http.setCacheFile(){
+    http._wd "${FUNCNAME[0]}($1)"
+    http_cfg__cacheFile="$1"
+  }
+
+  function http.flushCache(){
+    http._wd "${FUNCNAME[0]}($1)"
+    rm -f ${http_cfg__cacheDir}/*
+  }
+
+  # ......................................................................
+  #
+  # show a help text
+  # no params
+  function http.help(){
+    cat <<EOH
+
+$http_cfg__about
+
+This is a bash solution to script REST API calls.
+
+Source:$http_cfg__prjurl
+License: GNU GPL 3
+
+
+INSTRUCTION:
+
+- Source the file once
+- Then you can run functions starting with "http."
+
+    http.init
+      Start a new request. It resets internal vars of the last response
+      (if there was one).
+
+    http.setDebug 0|1
+      Enable or disable debugging infos during processing. It is written
+      to STDERR.
+
+- initialize a request
+
+    setAuth AUTH:PASSWORD
+      set authentication
+
+    http.setBody DATA
+      set a body for POST/ PUT requests.
+
+    http.setBaseUrl URL
+      Set a base url to an api.
+      renmark:
+      Use http.setUrl to built a complete url.
+
+    http.setDocs URL
+
+    http.setMethod METHOD
+      Set a http method. Use an uppercase string for GET|POST|PUT|DELETE|...
+
+    http.setFullUrl URL
+      Set a complete url for a request.
+
+    http.setUrl REQUEST?QUERY
+      Set a relative url for a request.
+      This requires to use http.setBaseUrl before.
+
+- caching functions
+
+    http.setCacheTtl SECONDS
+      Enable caching with values > 0
+      Remark: only GET requests will be cached.
+      Default: 0 (no caching)
+
+    http.setCacheFile FILENAME
+      Set a file where to read/ store a request
+      Default: empty; autogenerated file below $http_cfg__cacheDir
+
+    http.flushCache
+      Delete all files in $http_cfg__cacheDir
+
+- make the request
+
+    http.makeRequest [[METHOD] [URL] [BODY]]
+      The parameters are optional. Without parameter the rquest will be
+      started with given data in http.set* functions described above.
+      If minimum one pram is given then they are handled:
+        METHOD  optional: set a method (must be uppercase) - see http.setMethod
+        URL     set a relative url - see http.setUrl
+        BODY    optional: set a body - see http.setBody
+
+      The request will be skipped and uses a cached content if ...
+        - METHOD is GET
+        - http.setCacheTtl set a value > 0
+        - the cache file exists and is younger than the given TTL
+
+- handle response
+
+      http.getResponse
+        Get the Response Body
+
+      http.getResponseData
+        Get Meta infos from curl
+
+      http.getResponseHeader
+        Get The http reponse header
+
+- check http status code
+
+      http.getStatuscode
+        Get the http status code of a request
+
+      http.isOk
+        Check if the http response code is a 2xx
+
+      http.isRedirect
+        Check if the http response code is a 3xx
+
+      http.isError
+        Check if the http response code is a 4xx or 5xx
+
+      http.isClientError
+        Check if the http response code is a 4xx
+
+      http.isServerError
+        Check if the http response code is a 5xx
+
+      http.getRequestAge
+        Get the age of the request in seconds.
+        Remark: This function is useful after an import
+        see http.responseImport.
+
+      http.getRequestTs
+        Get the Unix timestamp of the request
+
+- import/ export
+
+      http.responseExport [FILE]
+        dump the response data
+        Without parameter it is written on STDOUT.
+        You can set a filename to write it to a file.
+        The filename can contain "AUTOFILE" this string
+        will be replaced with a uniq string.
+        (requires sha1sum and a set url)
+        Example:
+        http.makeRequest "https://example.com/api/"
+        http.responseExport /tmp/something_AUTOFILE_.txt
+
+      http.responseImport FILE
+        Import an export file.
+        To use the AUTOFILE mechanism from export set
+        the url first.
+        Example:
+        http.setFullUrl "https://example.com/api/"
+        http.responseImport /tmp/something_AUTOFILE_.txt
+
+      http.responseDelete FILE
+        Delete a file after http.responseExport.
+        It is useful if you use the AUTOFILE mechanism.
+
+EOH
+  }
+
+# ----------------------------------------------------------------------
+#
+# main
+#
+# ----------------------------------------------------------------------
+
+  http.init
+
+# ----------------------------------------------------------------------

inc/test_cfghandler.sh

+#!/bin/bash
+
+
+
+export CFGSTORAGE="${checkName}output"
+
+myFullscript=check_whatever
+myparams="-w 75 -c 90"
+_outfile=$0
+rc=0
+
+
+# outputAsText="$(cat $_outfile)"
+# outputAsJson="$(jq -nR --arg data """${outputAsText}""" '$data')"
+commandAsJson="$(jq -nR --arg data """${myFullscript} $myparams""" '$data')"
+
+function setHandler(){
+    ch=$1
+}
+
+function makeTest(){
+    (
+        $ch --set      check_source      \"${myHost}\"
+        $ch --set      check_command     "${commandAsJson}"
+        $ch --setfile  plugin_output     "${_outfile}"
+        $ch --set      exit_status       $rc
+    ) 2>&1
+    $ch --json
+}
+
+setHandler `dirname $0`/confighandler.sh
+time makeTest
+
+setHandler `dirname $0`/confighandler2.sh
+time makeTest
\ No newline at end of file

inc_functions.sh

+
+# ======================================================================
+#
+# OUTPUT FUNCTIONS
+#
+# ======================================================================
+
+
+  # ..................................................................
+  # write debug output to STDERR
+  function _wd(){
+    if [ $cfg_debug = true ]; then
+          echo -e "\e[33m# DEBUG: $*\e[0m" >&2
+    fi
+  }
+
+  # ..................................................................
+  # write header string
+  function _h1(){ 
+    echo
+    echo
+    echo "###############| $* |###############" 
+    echo
+  }
+  function _h2(){
+    echo
+    echo "---------- $*"
+  }
+  function _h3(){
+    echo
+    echo ">>>>> $*"
+  }
+
+  # ......................................................................
+  # logging output. writes timestamp and the given message to STDOUT
+  # and ${logfile}
+  # params  string(s)  message to log
+  #
+  function _log(){
+    echo "`date` | $*" | tee -a ${logfile}
+  }
+
+# ======================================================================
+#
+# DATE AND TIME FUNCTIONS
+#
+# ======================================================================
+
+  # ......................................................................
+  # get age of a file in sec
+  # param  string  filename to test
+  function _getFileAge(){
+    echo $(($(date +%s) - $(date +%s -r "$1")))
+  }
+
+  # ......................................................................
+  # get current unix timestamp
+  # no param
+  function _getUnixTs(){
+    date +%s
+  }
+
+  # get a list of config file of all checks to execute
+  # no params
+  function getChecks(){
+    if [ -d ${dir_cfg}/checks/ ]; then
+      for myconfig in `ls -1 ${dir_cfg}/checks/*`
+      do
+        echo ${myconfig}
+      done
+    fi
+  }
+
+  # parse a config file and set global vars:
+  #   checkName
+  #   checkCommand
+  #   checkInterval
+  # param  string  full path of a config file
+  function _parseCheckConfig(){
+    local _myconfig=$1
+
+    # EXAMPLE a config contains ...
+    # checkname=check_cronstatus
+    # command=check_cronstatus -param1 -param2
+    # interval=60
+    # icon=/images/icon.png
+    # max_check_attempts=3
+
+    checkName=`cat $_myconfig | grep ^checkname= | cut -f 2 -d "=" | sed 's# #_#g'`
+    checkCommand=`cat $_myconfig | grep ^command= | cut -f 2 -d "="`
+    checkInterval=`cat $_myconfig | grep ^interval= | cut -f 2 -d "="`
+    checkIcon=`cat $_myconfig | grep ^icon= | cut -f 2 -d "="`
+    checkMaxAttempts=`cat $_myconfig | grep ^max_check_attempts= | cut -f 2 -d "="`
+  }
+
+# ======================================================================
+#
+# NETWORK FUNCTIONS
+#
+# ======================================================================
+
+function _getinet4(){
+    ip -4 addr | grep inet
+}
+
+# filter output of _getinet4
+# TODO: bonded devices
+# param  string  filter method
+#                by type
+#                - loopback  filter loopback
+#                - net       filter network addresses
+#                - peer      filter peer
+#                - physical  filter physical addresses
+#                - virtual   filter 
+#
+#                by address range:
+#                - private   filter ips from private network ranges
+#                - public    filter non private ips (opposite of "private")
+#                output
+#                - ip         must be the last: get ip address only
+# _getinet4 | _ipf 
+function _ipf(){
+    local _filter=$1
+
+    local _regexInclude='.*'
+    local _regexExclude='idonotmatchtoskip'
+
+    if [ -z $_filter ]; then
+        echo ERROR: filter _ipf requires a param
+    fi
+
+    # 10.0.0.0    – 10.255.255.255 	
+    # 172.16.0.0  – 172.31.255.255
+    # 192.168.0.0 – 192.168.255.255
+    local _privateranges='(10\.|172\.1[6-9]\.|172.2[0-9]\.|172\.3[01]\.|192.168\.)'
+
+    case $_filter in
+
+        # --- types
+        'loopback')
+            _regexInclude='\ scope\ host\ '
+            ;;
+        'physical')
+            _regexExclude="(\ scope\ host\ |\ peer\ |\:vip)"
+            ;;
+        'virtual')
+            _regexInclude="\:vip"
+            ;;
+        'net')
+            _regexInclude='\ scope\ global\ '
+            ;;
+        'peer')
+            _regexInclude='\ peer\ .*\ scope\ global\ '
+            ;;
+
+        # --- private/ public ips
+        'private')
+            _regexInclude="\ ${_privateranges}.*"
+            ;;
+        'public')
+            _regexInclude="\ scope\ global\ "
+            _regexExclude="\ ${_privateranges}\.*"
+            ;;
+
+        # --- output filter
+        'ip')
+            awk '{ print $2 }' | cut -f 1 -d '/'
+            return 0
+            ;;
+        *)
+            echo ERROR: filter [$_filter] is not supported ... showing all.
+    esac
+
+    grep -E  "$_regexInclude" | grep -Ev "$_regexExclude" | sed "s#^    ##g"
+}
+
+function _getIp(){
+  ( _getinet4 | _ipf public | _ipf physical | _ipf ip ; _getinet4 | _ipf private | _ipf physical | _ipf ip ) | head -1
+}
+
+function _getIpFrontend(){
+  _getinet4 | _ipf public | _ipf physical | _ipf ip
+}
+function _getIpVirtual(){
+  _getinet4 | _ipf virtual | _ipf ip
+}
+function _getIpPrivate(){
+  _getinet4 | _ipf private | _ipf ip
+}
+
+
+# ======================================================================
+#
+# REST API FUNCTIONS
+#
+# ======================================================================
+
+  # ..................................................................
+  # read config file and apply API base settings
+  # param  string  filename of config file
+  function _initHttpWithConfigfile(){
+
+    # read config like /etc/icinga2-passive-client/api-director.cfg
+    # ... it contains
+    # RestApiBaseUrl="https://HOST/path"
+    # RestApiUser="USER"
+    # RestApiPassword="PASSWORD"
+    # RestApiDocs="https://icinga.com/docs/something"
+    . "$1" || exit 1
+
+    http.init
+    http.setAuth     "${RestApiUser}:${RestApiPassword}"
+    http.setBaseUrl  "${RestApiBaseUrl}"
+    http.setDocs     "${RestApiDocs}"
+
+  }
+
+  # ..................................................................
+  # make a curl request and show the response header + body
+  # param  string  method - GET|POST|...see http.setMethod
+  # param  string  relative URL behind API base url
+  # param  string  optional: data for PUTs and POSTs
+  function _APIcall(){
+
+    http.makeRequest "$1" "$2" "$3"
+
+    http.getResponseHeader
+    http.getResponse
+
+  }
+
+  # ..................................................................
+  # make an rest API regeques with GET command to update local cache
+  # file of an object.
+  # The cache file will be deleted if the http response code is not OK.
+  #
+  # global  $APICLIENT  rest api wrapper
+  #
+  # param  string  url behind RestAPI base path
+  # param  string  filename of the local cache file
+  function _getApiObject(){
+    local _url=$1
+    local _outfile=$2
+
+    #http.setDebug   1
+    # http.setUrl     "${_url}"
+    # http.setMethod  "GET"
+    # http.setBody    ""
+    # http.makeRequest
+
+    http.makeRequest GET "${_url}" ""
+
+    if [ ! -z "`http.isok`" ]; then
+      _wd "OK, Object $_url exists - updating cache file $_outfile"
+      http.responseExport "${_outfile}"
+    else
+      _wd "FAILED :-/ GET $_url was NOT successful - removing $_outfile"
+      rm -f "${_outfile}"
+    fi
+
+  }
+
+
+  # show name of linked service on a host
+  # it is used to 
+  #   - link the service to a host on director
+  #   - send process data from client to icinga
+  #
+  # param  string  name of check (=checkName after _parseCheckConfig FILE)
+  # param  bool    flag: reverse renaming: linkname --> checkname
+  function _getName4Svcathost(){
+
+    echo $1
+    # prefix was removed ... reverse functionailty not needed (sed fails)
+    #
+    # local prefix="added_service_for_command_"
+    # local prefix=""
+    # if [ $# -eq 1 ]; then
+    #   echo "${prefix}${1}"
+    # else
+    #   echo $1 | sed "s#$prefix##g"
+    # fi
+  }

inc_getconfig.sh.dist

+. /etc/icinga2-passive-client/client.cfg

plugins @ 7591e745

+Subproject commit 7591e7456609b9313937662bfce4823669f60eda

rest-api-client

+#!/bin/bash
+# ======================================================================
+#
+# REST API CLIENT USING CURL
+#
+# requires
+# - curl
+# - config file with endpoint, api user, password
+# ----------------------------------------------------------------------
+# see
+# - https://icinga.com/docs/icinga2/latest/doc/12-icinga2-api/
+# - https://icinga.com/docs/director/latest/doc/70-REST-API/
+# ----------------------------------------------------------------------
+# 2020-01-24  v0.1  axel.hahn@iml.unibe.ch
+# 2020-02-05  v0.2  axel.hahn@iml.unibe.ch  get curl meta infos; split header
+# ======================================================================
+
+# --- fetch incoming params
+  RestApiCfg=$1
+  RestApiMethod=$2
+  ApiUrl=$3
+  Body="$4"
+
+# --- curl meta infos to collect 
+#     see variables in man curl --write-out param
+  curlMeta="\
+    http_code \
+    http_connect \
+    local_ip \
+    local_port \
+    num_connects \
+    num_redirects \
+    redirect_url \
+    remote_ip \
+    remote_port \
+    size_download \
+    size_header \
+    size_request \
+    size_upload \
+    speed_download \
+    speed_upload \
+    ssl_verify_result \
+    time_appconnect \
+    time_connect \
+    time_namelookup \
+    time_pretransfer \
+    time_redirect \
+    time_starttransfer \
+    time_total \
+    url_effective \
+"
+
+
+# ----------------------------------------------------------------------
+#
+# functions
+#
+# ----------------------------------------------------------------------
+
+  # ......................................................................
+  #
+  # write a debug message to STDERR
+  # Do no not change the prefix - is is read in inc_functions
+  #
+  # params  strings  output message
+  function _wd(){
+    echo -e "\e[33m# RESTAPI::DEBUG $*\e[0m" >&2
+  }
+
+  # ......................................................................
+  #
+  # write a http response header line
+  # Do no not change the prefix - is is read in inc_functions
+  #
+  # params  strings  output message
+  function _wHeader(){
+    echo -e "\e[34m# CURL::RESPONSE-HEADER | $*\e[0m" >&2
+  }
+
+  # ......................................................................
+  #
+  # write a http response header line
+  # params  strings  output message
+  function _wBody(){
+    # echo -e "\e[32m$*\e[0m"
+    echo $*
+  }
+
+  # ......................................................................
+  #
+  # write a http response header line
+  # Do no not change the prefix - is is read in inc_functions
+  #
+  # params  strings  output message
+  function _wData(){
+    echo -e "\e[36m# CURL::INFOS | $*\e[0m" >&2
+  }
+
+  # ......................................................................
+  #
+  # show error message with last return code and quit with this exitcode
+  # no params
+  function quit(){
+    rc=$?
+    echo >&2
+    echo -e "\e[31m# ERROR: command FAILED with rc $rc. \e[0m" >&2
+    if [ ! -z "${RestApiDocs}" ]; then
+      echo "HINT: see ${RestApiDocs}" >&2
+    fi
+    exit $rc
+  }
+
+  # ......................................................................
+  #
+  # show a help text
+  # no params
+  function showHelp(){
+    cat <<EOH
+
+SYNTAX:
+  - config file (added by a wrapper)
+  - method
+  - url
+  - body (JSON)
+The first 3 params required.
+
+HINT:
+To get rid of the commented lines in the output redirect STDERR.
+[command] 2>/dev/null
+
+EOH
+  }
+
+# ----------------------------------------------------------------------
+#
+# main
+#
+# ----------------------------------------------------------------------
+
+  _wd "REST API CLIENT"
+
+  which curl >/dev/null || quit
+
+  if [ $# -lt 3 ]; then
+    showHelp
+    exit 1
+  fi
+
+  # read config
+  . "${RestApiCfg}" || quit
+
+  test -z "$RestApiPassword" || RestApiUser="$RestApiUser:$RestApiPassword"
+
+
+  # ----------------------------------------------------------------------
+  # make request
+  # ----------------------------------------------------------------------
+
+  _wd "$RestApiMethod $RestApiBaseUrl$ApiUrl ..."
+
+  sCurlDataPrefix="RESTAPICLIENTMETADATA_`date +%s`_$$"
+  for myvar in $curlMeta
+  do
+    writevar="${writevar}|${myvar}:%{${myvar}}"
+  done
+  curlparams="-k -s --write-out \"\\n$sCurlDataPrefix${writevar}\\n\""
+
+  output=`
+    if [ -z "$Body" ]; then
+      curl ${curlparams} \
+        -u "$RestApiUser" \
+        -i $RestApiBaseUrl$ApiUrl \
+        -H 'Accept: application/json' \
+        -X $RestApiMethod
+    else
+      curl ${curlparams} \
+        -u "$RestApiUser" \
+        -i $RestApiBaseUrl$ApiUrl \
+        -H 'Accept: application/json' \
+        -X $RestApiMethod \
+        -d "$Body"
+    fi
+` || quit
+
+  _wd "OK - Curl finished the http request"
+
+  # ----------------------------------------------------------------------
+  # split response data
+  # ----------------------------------------------------------------------
+
+  isheader=true
+  body=
+
+  # keep leading spaces
+  IFS=''
+
+  _wd "curl meta infos"
+
+  # --- dump curl metadata
+  #   example line
+  #   RESTAPICLIENTMETADATA_1580905381_2336|http_code:200|time_total:0.391|speed_download:990.000|...
+  # echo "$output" | sed "s#$sCurlDataPrefix#\n$sCurlDataPrefix#" | grep  "$sCurlDataPrefix" | cut -f 2- -d "|" | sed "s#|#\n#g" | while read line; do
+  echo "$output" | grep  "$sCurlDataPrefix" | cut -f 2- -d "|" | sed "s#|#\n#g" | while read line; do
+    _wData $line
+  done
+
+  # --- loop over output and split header from body
+  while read line; do
+    if $isheader; then
+      if [[ $line = $'\r' ]]; then
+        isheader=false
+          _wd "http response body"
+      else
+        # header="$header"$'\n'"$line"
+        _wHeader $line
+      fi
+    else
+      # body="$body"$'\n'"$line"
+      _wBody $line
+    fi
+  # done < <(echo "$output" | sed "s#$sCurlDataPrefix#\n$sCurlDataPrefix#" | grep -v "$sCurlDataPrefix")
+  done < <(echo "$output" | grep -v "$sCurlDataPrefix")
+
+
+# ----------------------------------------------------------------------

test-netfkt.sh

+
+. `dirname $0`/inc_functions.sh
+
+function _exec(){
+    echo ... [$2] command: $1
+    eval $1
+    # echo rc=$?
+    echo
+}
+
+# _exec ls "testrun"
+
+
+_exec "_getinet4" "all ip v4 addresses"
+_exec "_getinet4 | _ipf private" "private range"
+
+
+_exec "_getinet4 | _ipf public" "public ips"
+
+_exec "_getinet4 | _ipf public | _ipf physical" "public physical addresses"
+_exec "_getinet4 | _ipf public | _ipf virtual"  "public virtual addresses"
+
+echo ----
+
+
+
+_exec "_getIpFrontend" "get frontend address"
+_exec "_getIpVirtual" "get virtual address"
+_exec "_getIpPrivate" "get private address"
+
+_exec "_getIp"  "get main IP"