Merge branch '5127-detect-5xx-server-error' into 'master'

abort on http 5xx error

See merge request !4

director-cli.sh

@@ -16,6 +16,7 @@
 # ----------------------------------------------------------------------
 # ah = axel.hahn@iml.unibe.ch
 # 2022-02-16  v0.2  ah  add --cfg param
+# 2022-03-04  v0.3  ah  abort on http 5xx error
 # ======================================================================
 
 tmpfile=/tmp/outcurl.tmp
@@ -365,6 +366,10 @@ typeset cfg_dryrun=false
     if [ ${_paramDryrun} = false ]; then
       _wd ">>>>> $_paramAction $_paramObj [${_object_name}] >> $_sMethod $_sUrl $_jsondata"
       http.makeRequest "$_sMethod" "$_sUrl" "$_jsondata"
+      if http.isServerError >/dev/null; then
+        echo "CRITICAL ERROR: Director API request failed with a server error $_sMethod $_sUrl"
+        exit 1
+      fi
       if [ $_bShowResponse = true ]; then
         http.getResponseHeader
         http.getResponse

icinga-cli.sh

@@ -11,13 +11,15 @@
 # ----------------------------------------------------------------------
 # ah = axel.hahn@iml.unibe.ch
 # 2021-03-..  init
-# 2022-01-11  v0.7  ah  shellcheck
-# 2022-02-16  v0.8  ah  add --cfg param
+# 2022-01-11  v0.7   ah  shellcheck
+# 2022-02-16  v0.8   ah  add --cfg param
+# 2022-03-04  v0.9   ah  abort on http 5xx error
+# 2022-03-14  v0.10  ah  less output and add _elog to run as a service
 # ======================================================================
 
 
 _product="ICINGA PASSIVE CLIENT"
-_version="0.8"
+_version="0.10"
 _license="GNU GPL 3.0"
 _copyright='(c) 2020 Institute for Medical Education * University of Bern'
 
@@ -109,10 +111,8 @@ function loopChecks(){
         lockpid=$(cat "${lockfile}" | cut -f 2 -d "-" | cut -f 4 -d " " | grep "[0-9]")
         ps -f --pid "$lockpid" | grep "$(basename $0)" | grep loop >/dev/null
         if [ $? -eq 0 ]; then
-          _log "ABORT: Loop seems to run already. See process with PID $lockpid"
-          echo
-          ps -f --pid "$lockpid"
-          echo
+          _elog "ABORT: Loop seems to run already. See process with PID $lockpid"
+          _elog $( ps -f --pid "$lockpid" )
           exit 0
         fi
       fi
@@ -120,15 +120,15 @@ function loopChecks(){
       _log "---------- starting in a permanent loop"
       echo "Serviceloop started $(date) - process id $$" > "${lockfile}"
       if [ $? -ne 0 ]; then
-          _log "ABORT: Lock file is not writable ${lockfile}."
-          ls -l "${lockfile}"
+          _elog "ABORT: Lock file is not writable ${lockfile}."
+          _elog $( ls -l "${lockfile}" );
           exit 1
       fi
 
       _getFileSnapshot>"${snapShotStart}"
       if [ $? -ne 0 ]; then
-          _log "ABORT: Snapshot file is not writable ${snapShotStart}."
-          ls -l "${snapShotStart}"
+          _elog "ABORT: Snapshot file is not writable ${snapShotStart}."
+          _elog $( ls -l "${snapShotStart}" )
           exit 1
       fi
       while true; do
@@ -141,19 +141,17 @@ function loopChecks(){
         _log ""
         _getFileSnapshot>$snapShotCurrent
         if [ $? -ne 0 ]; then
-            _log "ABORT: Snapshot file is not writable ${snapShotCurrent}."
-            ls -l "${snapShotCurrent}"
+            _elog "ABORT: Snapshot file is not writable ${snapShotCurrent}."
+            _elog $( ls -l "${snapShotCurrent}" )
             exit 1
         fi
         diff  $snapShotStart $snapShotCurrent >/dev/null
         if [ $? -ne 0 ]; then
-          _log "ABORT: Files were updated / overwritten. The loop must be restarted.\n`diff  $snapShotStart $snapShotCurrent`"
+          _elog "ABORT: Files were updated / overwritten. The loop must be restarted.\n`diff  $snapShotStart $snapShotCurrent`"
           exit 1
         fi
         icingaHostMustExist
         processAllChecks
-        echo
-        echo
       done
 }
 # ......................................................................
@@ -175,16 +173,13 @@ function processAllChecks(){
   _log ""
   _log "------ looping over all checks"
   getChecks
-  echo
   for myconfig in $(getChecks)
   do
     iCounter=$iCounter+1
     _log "--- processing [$iCounter of $iChecksTotal] $myconfig"
     processCheck "$myconfig"
     _log ""
-    echo
-    echo ----------------------------------------------------------------------
-    echo
+
   done
   typeset -i local iLoopEnd
   iLoopEnd=$(_getUnixTs)
@@ -206,7 +201,7 @@ function _parseCheckConfig(){
   local _myconfig="$1"
 
   if [ ! -r "$_myconfig" ]; then
-    echo "ERROR: config file is not readable [$_myconfig]"
+    _elog "ERROR: config file is not readable [$_myconfig]"
     exit 1
   fi
 
@@ -248,6 +243,10 @@ function icingaHost(){
       http.setCacheTtl     $_iRefreshCache
       http.setCacheFile    $_localCache
       http.makeRequest GET $_apiRequest
+      if http.isServerError >/dev/null; then
+        _elog "CRITICAL ERROR: Icinga2 API request failed with a server error GET $_apiRequest"
+        exit 1
+      fi
 
       # set return code of GET action
       http.isOk >/dev/null
@@ -268,10 +267,10 @@ function icingaHostMustExist(){
   if [ $? -ne 0 ]; then
     http.getResponse
     if [ "$(http.getStatuscode)" = "000" ]; then
-      _log "ERROR: Unable to reach the Icinga node. Stopping script current monitoring actions."
+      _elog "ERROR: Unable to reach the Icinga node. Stopping script current monitoring actions."
       exit 1
     fi
-    _log "ERROR: host object for ${myHost} is not available on Icinga service (yet) - Status: $(http.getStatuscode)"
+    _elog "ERROR: host object for ${myHost} is not available on Icinga service (yet) - Status: $(http.getStatuscode)"
     echo
     echo "ABORTING"
     echo
@@ -348,8 +347,8 @@ function processCheck(){
       eval $myFullscript $myparams > $_outfile
       rc=$?
       if [ ! -w $_outfile ]; then
-            _log "${_logPrefix} ERROR: output file $_outfile is not writable."
-            _log "${_logPrefix} $( ls -ld ${dir_data} $_outfile )"
+            _elog "${_logPrefix} ERROR: output file $_outfile is not writable."
+            _elog "${_logPrefix} $( ls -ld ${dir_data} $_outfile )"
             exit 1
       fi
       typeset -i local iTsEnd=`date +%s`
@@ -395,8 +394,8 @@ function processCheck(){
       _APIcall POST actions/process-check-result?service=${myHost}!${slot} "$data"
       http.responseExport "$_response"
       if [ ! -w "$_response" ]; then
-            _log "${_logPrefix} ERROR: responsefile $_response is not writable."
-            _log "${_logPrefix} $( ls -ld ${dir_data} $_response )"
+            _elog "${_logPrefix} ERROR: responsefile $_response is not writable."
+            _elog "${_logPrefix} $( ls -ld ${dir_data} $_response )"
             exit 1
       fi
 

inc_functions.sh

@@ -32,12 +32,19 @@
   }
 
   # ......................................................................
-  # logging output. writes timestamp and the given message to STDOUT
-  # and ${logfile}
+  # logging output. writes timestamp and the given message to ${logfile}
   # params  string(s)  message to log
   #
   function _log(){
-    echo "`date` | $*" | tee -a ${logfile}
+    echo "`date` | $*" >> ${logfile}
+  }
+
+  # echo and logging output
+  # params  string(s)  message to log
+  #
+  function _elog(){
+    echo "$*"
+    _log "$*"
   }
 
 # ======================================================================
@@ -232,6 +239,10 @@ function _getIpPrivate(){
   function _APIcall(){
 
     http.makeRequest "$1" "$2" "$3"
+    if http.isServerError >/dev/null; then
+      echo "CRITICAL ERROR: API request failed with a server error $1 $2"
+      exit 1
+    fi
 
     http.getResponseHeader
     http.getResponse