#!/bin/bash
# ======================================================================
#
# ICINGA PASSIVE CLIENT
#
# Run all passive checks and send response to Icinga Endpoint
#
# Requirements
# - curl
# - jq
# ----------------------------------------------------------------------
# ah = axel.hahn@iml.unibe.ch
# 2021-03-..  init
# 2022-01-11  v0.7   ah  shellcheck
# 2022-02-16  v0.8   ah  add --cfg param
# 2022-03-04  v0.9   ah  abort on http 5xx error
# 2022-03-14  v0.10  ah  less output and add _elog to run as a service
# 2022-07-08  v0.11  ah  check pipes in output and performance data
# ======================================================================


_product="ICINGA PASSIVE CLIENT"
_version="0.11"
_license="GNU GPL 3.0"
_copyright='(c) 2020 Institute for Medical Education * University of Bern'

typeset -i debug=0

# source config ...
. "$( dirname $0 )/inc_getconfig.sh"



# where to find check scripts ... first directory wins
# dir_plugins="/opt/imlmonitor/client/plugins/ /usr/lib64/nagios/plugins"
# dir_cfg="/etc/icinga2-passive-client"
# dir_data="/var/tmp/icinga2-passive-client"
# dir_logs="/var/log/icinga2-passive-client"
logfile="${dir_logs}/execution.log"

ch="$( dirname $0 )/inc/confighandler.sh"
myHost=$(hostname -f)

# for loop mode only: max. random sleep time
typeset -i sleeptime=30

typeset -i _rc_all=0

# ----------------------------------------------------------------------
#
# FUNCTIONS
#
# ----------------------------------------------------------------------
# ..................................................................
#
# helper to make http base setup for host actions
function _initHttp(){
  # see inc_functions
   _initHttpWithConfigfile "${dir_cfg}/api-icinga2.cfg"
  if [ $debug -ne 0 ]; then
    http.setDebug 1
  fi
}


# ......................................................................
#
# find first place of the check script in the known plugin dirs 
# see ${dir_plugins} in checks.cfg
# param  string  name of the check script without path
#
function findCheckScript(){
	local _script=$1
	for mydir in ${dir_plugins}
	do
		if [ -x "${mydir}/${_script}" ]; then
			echo "${mydir}/${_script}"
		fi
	done | head -1
}

# helper used function in loopChecks
# get a snapshot of a few files
function _getFileSnapshot(){
  ls -l $(dirname $0)/* ${dir_cfg}/*
}


# ......................................................................
#
# Loop over executing all checks
# no params
#
function loopChecks(){

      # TODO-MEMORY-CHECK
      # echo ${myHost} | egrep "^(kvm4|icinga)"
      # echo ${myHost} | egrep "^(monitortest)"
      # if [ $? -ne 0 ]; then
      #   echo "HARD EXIT - DO NOT EXECUTE ANY CHECK ON $myHost"
      #   exit 1
      # fi

      local lockfile
      lockfile="${dir_data}/loop.pid"
      local snapShotStart
      snapShotStart=${dir_data}/$(basename $0)-start.fingerprint
      local snapShotCurrent
      snapShotCurrent=${dir_data}/$(basename $0)-last.fingerprint
      if [ -f "${lockfile}" ]; then
        local lockpid
        lockpid=$(cat "${lockfile}" | cut -f 2 -d "-" | cut -f 4 -d " " | grep "[0-9]")
        ps -f --pid "$lockpid" | grep "$(basename $0)" | grep loop >/dev/null
        if [ $? -eq 0 ]; then
          _elog "ABORT: Loop seems to run already. See process with PID $lockpid"
          _elog $( ps -f --pid "$lockpid" )
          exit 0
        fi
      fi

      _log "---------- starting in a permanent loop"
      echo "Serviceloop started $(date) - process id $$" > "${lockfile}"
      if [ $? -ne 0 ]; then
          _elog "ABORT: Lock file is not writable ${lockfile}."
          _elog $( ls -l "${lockfile}" );
          exit 1
      fi

      _getFileSnapshot>"${snapShotStart}"
      if [ $? -ne 0 ]; then
          _elog "ABORT: Snapshot file is not writable ${snapShotStart}."
          _elog $( ls -l "${snapShotStart}" )
          exit 1
      fi
      while true; do
        # typeset -i local iSleep=$(($RANDOM%$sleeptime))
        # sleep minimum is half of $sleeptime
        typeset -i local iSleep=$(($RANDOM%$sleeptime/2+$sleeptime/2))
        _log "sleeping $iSleep sec ..."
        sleep $iSleep
        _log "______________________________________________________________________"
        _log ""
        _getFileSnapshot>$snapShotCurrent
        if [ $? -ne 0 ]; then
            _elog "ABORT: Snapshot file is not writable ${snapShotCurrent}."
            _elog $( ls -l "${snapShotCurrent}" )
            exit 1
        fi
        diff  $snapShotStart $snapShotCurrent >/dev/null
        if [ $? -ne 0 ]; then
          _elog "ABORT: Files were updated / overwritten. The loop must be restarted.\n`diff  $snapShotStart $snapShotCurrent`"
          exit 1
        fi
        icingaHostMustExist
        processAllChecks
      done
}
# ......................................................................
#
# execute all defined checks one by one
# no params
#
function processAllChecks(){
  # loop over all defined checks
  typeset -i local iChecksTotal
  iChecksTotal=$(getChecks | wc -l)
  typeset -i local iCounter
  iCounter=0

  _rc_all=0
  typeset -i local iLoopStart
  iLoopStart=$(_getUnixTs)

  _log ""
  _log "------ looping over all checks"
  for myconfig in $(getChecks)
  do
    iCounter=$iCounter+1
    _log "--- processing [$iCounter of $iChecksTotal] $myconfig"
    processCheck "$myconfig"
    _log ""

  done
  typeset -i local iLoopEnd
  iLoopEnd=$(_getUnixTs)
  typeset -i local iLoopTime
  iLoopTime=$iLoopEnd-$iLoopStart

  _log "------ loop done - needed $iLoopTime sec - rc=$_rc_all"
}

# ......................................................................
#
# parse a config file and set global vars:
#   checkName
#   checkCommand
#   checkInterval
# param  string  full path of a config file
#
function _parseCheckConfig(){
  local _myconfig="$1"

  if [ ! -r "$_myconfig" ]; then
    _elog "ERROR: config file is not readable [$_myconfig]"
    exit 1
  fi

  # EXAMPLE a config contains ...
  # checkname=check_cronstatus
  # command=check_cronstatus -param1 -param2
  # interval=60

  checkName=$(cat $_myconfig | grep ^checkname= | cut -f 2 -d "=")
  checkCommand=$(cat $_myconfig | grep ^command= | cut -f 2 -d "=")
  checkInterval=$(cat $_myconfig | grep ^interval= | cut -f 2 -d "=")

}

# actions for icinga host
# param  string  action; "get" only
function icingaHost(){
  local _logPrefix="${myHost} :: API |"
  local _apiRequest=objects/hosts/${myHost}
  local _localCache=${dir_data}/host_${myHost}_deployed-at-icinga.txt
  typeset -i local _iRefreshCache=120

  local sAction=$1

  _initHttp

  case $sAction in
    'get')
      # update after caching was added in http-component
      # http.responseImport $_localCache
      # typeset -i local iAgeLastGet=`http.getRequestAge`
      # _log "${_logPrefix} INFO: cache is $iAgeLastGet sec old ... TTL is _iRefreshCache=3600 sec"
      # if [ $iAgeLastGet -eq 0 -o $iAgeLastGet -gt $_iRefreshCache ]; then
      #   _log "${_logPrefix} INFO: request to Icinga GET $_apiRequest"
      #   _getApiObject $_apiRequest $_localCache
      # fi

      # new:
      http.setCacheTtl     $_iRefreshCache
      http.setCacheFile    $_localCache
      http.makeRequest GET $_apiRequest
      if http.isServerError >/dev/null; then
        _elog "CRITICAL ERROR: Icinga2 API request failed with a server error GET $_apiRequest"
        exit 1
      fi

      # set return code of GET action
      http.isOk >/dev/null
      ;;
    *)
      _log "ERROR: unknown action parameter $sAction"
  esac

}

# for check on the beginning of the script:
# execute a check only if the host exists on icinga2
# global  string  myHost 
# global  string  dir_data
function icingaHostMustExist(){
  _log "check if the host [${myHost}] exists on Icinga ..."
  icingaHost get
  if [ $? -ne 0 ]; then
    _echo $( http.getResponse )
    if [ "$(http.getStatuscode)" = "000" ]; then
      _elog "ERROR: Unable to reach the Icinga node. Stopping script current monitoring actions."
      exit 1
    fi
    _elog "ERROR: host object for ${myHost} is not available on Icinga service (yet) - Status: $(http.getStatuscode)"
    _echo
    _echo "ABORTING"
    _echo
    _echo "To run checks ..."
    _echo "- you must create the host on director (check director-cli.sh --hr)"
    _echo "- the director must deploy the host to icinga daemon"
    _echo
    rm -f "${dir_data}"/service__check* 2>/dev/null
    exit 1
  fi
  _log "OK, found."
}


# ......................................................................
#
# process a single check
# param  string  name of config file
# param  string  name of config file
#
function processCheck(){
  local _myconfig=$1
  local _force=$2

  typeset -i local iPipes
  typeset -i local iCheckStart
  iCheckStart=$(_getUnixTs)

  _parseCheckConfig "${_myconfig}"
  local _logPrefix="${checkName} |"
  _log "${_logPrefix} INFO: every ${checkInterval} sec: ${checkCommand}"

  local _outfile=${dir_data}/service__check__${checkName}__output.txt
  local _response=${dir_data}/service__check__${checkName}__icinga_response.txt
  typeset -i local _rc=0


  _initHttp

  # --- check last run ... if never or > $interval then execute
  doRun=0
  if [ ! -f "$_outfile" ]; then
    _log "${_logPrefix} INFO: Never executed before"
    doRun=1
  else
    # typeset -i iAgeLastRun=$(($(date +%s) - $(date +%s -r "$_outfile")))
    typeset -i iAgeLastRun
    iAgeLastRun=$(_getFileAge "$_outfile")
    _log "${_logPrefix} INFO: last run was $iAgeLastRun sec ago ... vs Interval = $checkInterval ... sleeptime = $sleeptime"
    iAgeLastRun=$iAgeLastRun+$sleeptime
    if [ $iAgeLastRun -gt $checkInterval ]; then
      doRun=1
    fi

    if [ ! -z "$_force" ]; then
      doRun=1
      _log "${_logPrefix} INFO: forced execution by given param "
    fi
  fi

  if [ $doRun -ne 0 ]; then
    myscript=$(echo "$checkCommand" | cut -f 1 -d " ")
    myFullscript=$(findCheckScript "$myscript")
    if [ -z "$myFullscript" ]; then
      _log "${_logPrefix} ERROR: $myscript was not found in any plugin dir"
    else
      myparams=$( echo $checkCommand | grep " " | cut -f 2- -d " " )

      #
      # --- this executes the check plugin ...
      #
      _log "${_logPrefix} starting $myFullscript $myparams"
      typeset -i local iTsStart=`date +%s`
      # $myFullscript $myparams | tee $_outfile
      eval $myFullscript $myparams > $_outfile
      rc=$?
      if [ ! -w $_outfile ]; then
            _elog "${_logPrefix} ERROR: output file $_outfile is not writable."
            _elog "${_logPrefix} $( ls -ld ${dir_data} $_outfile )"
            exit 1
      fi
      typeset -i local iTsEnd=`date +%s`
      # outPerfdata=`grep '|' $_outfile | cut -f 2 -d '|'`
      outPerfdata=`grep '|' $_outfile | rev | cut -f 1 -d '|' | rev`
      _echo
      _echo -------- check output:
      _echo $( cat "$_outfile" )

      iPipes=$( sed 's#[^|]##g' "$_outfile" | grep "." | wc -L )
      _echo "Pipe chars: $iPipes"
      if [[ $iPipes -gt 1 ]]; then
        _elog "ABORT - pipes were found in plugin output"
        exit 1
      fi

      _echo
      # echo -------- extracted performance data:
      # echo $outPerfdata
      # echo
      if ! echo "$outPerfdata" | grep "=[0-9\.]*;[0-9\.]*;[0-9\.]*;[0-9\.]*;" >/dev/null; then
        _elog "ABORT - this does not look like performance data: $outPerfdata"
        exit 1
      fi
      _log "${_logPrefix} check command finished with returncode $rc"

      _rc=$_rc+$rc
      #
      # --- send check result to Icinga
      #     fields of the object
      #     https://icinga.com/docs/icinga2/latest/doc/12-icinga2-api/#process-check-result
      export CFGSTORAGE="${checkName}output"

      outputAsText="$(cat $_outfile)"
      # outputAsJson="$(jq -nR --arg data """${outputAsText}""" '$data')"
      commandAsJson="$(jq -nR --arg data """${myFullscript} $myparams""" '$data')"
      (
        $ch --set      check_source      \"${myHost}\"
        $ch --set      check_command     "${commandAsJson}"
        $ch --set      exit_status       $rc
        # $ch --set      plugin_output     "${outputAsJson}"
        $ch --setfile  plugin_output     "${_outfile}"
        $ch --set      performance_data  "\"${outPerfdata}\""
        $ch --set      ttl               $checkInterval
        $ch --set      execution_start   $iTsStart
        $ch --set      execution_end     $iTsEnd
      ) 2>/dev/null
      # $ch --json
      data=`$ch --json 2>/dev/null`

      slot="`_getName4Svcathost ${checkName} | sed 's# #%20#g'`"

      _log "${_logPrefix} starting POST of data to monitoring server"
      _echo POST actions/process-check-result?service=${myHost}!${slot} "$data"
      _APIcall POST actions/process-check-result?service=${myHost}!${slot} "$data"
      http.responseExport "$_response"
      if [ ! -w "$_response" ]; then
            _elog "${_logPrefix} ERROR: responsefile $_response is not writable."
            _elog "${_logPrefix} $( ls -ld ${dir_data} $_response )"
            exit 1
      fi

      # --- check if data were sent successfully
      # fgrep "HTTP/1.1 200" ${_response} >/dev/null
      # _testHttpOk ${_response} >/dev/null
      http.isOk >/dev/null
      if [ $? -eq 0 ]; then
        _log "${_logPrefix} rc=$rc - OK, response was sent to Icinga"
      else
        _elog "${_logPrefix} rc=$rc - WARNING: the check response was NOT sent to Icinga"
        _rc=$_rc+1
        _echo
        _echo For Debugging:
        _echo "$( $ch --show --json )"
        _log "$( $ch --show --json )"
      fi
      $ch --flush 2>/dev/null

    fi

  else
    _log "${_logPrefix} SKIP execution."
  fi

  # add current result to global returncode
  _rc_all=$_rc_all+$_rc

  typeset -i local iCheckEnd=`_getUnixTs`
  typeset -i local iCheckTime=$iCheckEnd-$iCheckStart

  _log "${_logPrefix} finished after $iCheckTime sec with returncode $_rc"
  test $_rc -eq 0 || (_echo; _echo "    >>> Check ${checkName} was not OK. See Output block above!"; _echo; _echo)
}
# ----------------------------------------------------------------------
# help
# ----------------------------------------------------------------------

# show help text
function showHelp(){
self=`basename $0`
cat <<EOH


INTRODUCTION

$_product v$_version

Handle and execute icinga passive checks.
With this client can run a single check, all checks or make a permanent loop.
A new local check will be added to Icinga while running it the first time.


GENERAL PARAMETERS

  --cfg CONFIGFILE
    load a costom config file; default: ./inc_getconfig.sh
    This must be the 1st parameter to be processed.

  --help or -h or -?
    show this help and abort.

  --version or -v
    show the version abd abort

SERVICE ACTIONS

  --list
    get a list of local config files

  --loop
    Start to check all passive checks in a permanent loop.
    It makes a random sleep if $sleeptime sec between all loops. This shuffles
    the access time of all clients making requests to the icinga server.

    Multiple starts will be detected. This parameter is the optimal choice for a
    cronjob.

  --runonce
    Start to check all passive checks once.
    This method respects the interval per check. Only outdated checks will be
    executet.
    This is a second choice for a cronjob if the runtime of all checks
    is much shorter than your cronjob interval to prevent multiple processes.
    Multiple starts will NOT be detected.

  --run CONFIGFILE
    Run a check by pointing the config file (see --list)
    This execution ignores the interval and forces the execution.


CONFIG

  Config file
  /etc/icinga2-passive-client/client.cfg
  It sets other used pathes.

  Config files for checks are in ${dir_cfg}/checks/


DEBUGGING

  The Output of check and results from Icinga are in
  ${dir_data}.

  A log of all performed executed check runs by $self
  are in $logfile.
  BTW: do not forget to add a log rotation for it.

EOH
}

function showVersion(){
  echo "$_license"
  echo "$_copyright"
  echo
}
# ----------------------------------------------------------------------
#
# MAIN
#
# ----------------------------------------------------------------------

. "$( dirname $0 )/inc_functions.sh"

_echo "
______________________________________________________________________________________

 _______        __
|_     _|.----.|__|.-----.-----.---.-.
 _|   |_ |  __||  ||     |  _  |  _  |
|_______||____||__||__|__|___  |___._|
                         |_____|
 ______                     __                   ______ _____   __               __
|   __ \.---.-.-----.-----.|__|.--.--.-----.    |      |     |_|__|.-----.-----.|  |_
|    __/|  _  |__ --|__ --||  ||  |  |  -__|    |   ---|       |  ||  -__|     ||   _|
|___|   |___._|_____|_____||__| \___/|_____|    |______|_______|__||_____|__|__||____|
                                                                                 v${_version}

  $_license .. $_copyright

______________________________________________________________________________________

"

_elog "Starting $_product $_version"


if [ "$1" = "--cfg" ] && [ -n "$2" ]; then
  echo "INFO: loading custom config [$2]..."
  . "${2}"
  shift 2
else
  . "$( dirname $0 )/inc_getconfig.sh"
fi

. "$( dirname $0 )/inc/rest-api-client.sh"

if [ $# -eq 0 ]; then
  showHelp
  exit 0
fi

if [ -z "${dir_cfg}" ]; then
  echo ERROR: $_product is not installed/ configured yet on this machine.
  exit 1
fi

icingaHostMustExist


touch ${logfile}


while [ $# -gt 0 ];
do
  case "$1" in

    '--help' | '-h' | '-?')
      showHelp
      exit 0
      ;;
    '--version' | '-v')
      showVersion
      exit 0
      ;;


    '--list')
      getChecks
      ;;

    '--loop')
      loopChecks
      ;;

    '--runonce')
      processAllChecks
      ;;

    '--run')
      processCheck "$2" "force"
      shift 1
      ;;

    *)
      echo "ERROR: unknown parameter detected."
      exit 2
  esac
  shift 1
done
echo

# remark:
# $_rc_all is a collected status code in the loop of all actions
# if it is 0 then all checks were OK and have been sent to Icinga
# echo exit with status code $_rc_all
# exit $_rc_all
exit 0

# ----------------------------------------------------------------------
