Merge branch '6468-docs-and-harmonize' into 'master'

Update icinga checks and docs

See merge request !127

check_ceph_osd

@@ -26,12 +26,13 @@
 # 2022-10-21  v1.3  <axel.hahn@unibe.ch>     remove grep: warning: stray \ before white space
 # 2023-04-24  v1.4  <axel.hahn@unibe.ch>     update for newer ceph versions
 # 2023-06-19  v1.5  <axel.hahn@unibe.ch>     add help and param support; no more tmpfile
+# 2023-07-27  v1.6  <axel.hahn@unibe.ch>     shorten ceph exec; show output on error; shell fixes
 # ======================================================================
 
 . $(dirname $0)/inc_pluginfunctions
 
 self_APPNAME=$( basename $0 | tr [:lower:] [:upper:] )
-self_APPVERSION=1.5
+self_APPVERSION=1.6
 
 # column number in output where to find the up/ down info
 iColUpDown=5
@@ -94,19 +95,18 @@ case "$1" in
     *)
 esac
 
-data=$( sudo /bin/ceph osd tree 2>&1 )
-if [ $? -ne 0 ]; then
+if ! data=$( sudo /bin/ceph osd tree 2>&1 ); then
+    echo "$data"
     ph.abort "UNKNOWN: ceph is not available or no sudo permissions to execute ceph commands."
 fi
 
 # set default / override from command line params
-typeset -i iWarnLimit=$(     ph.getValueWithParam 1 w "$@")
-typeset -i iCriticalLimit=$( ph.getValueWithParam 2 c "$@")
-
-typeset -i iOsdTotal=$( getOsd | wc -l)
-typeset -i iOsdDown=$(  getOsd | awk '{ print $iColUpDown }' | grep "down" | wc -l)
-typeset -i iOsdUp=$(    getOsd | awk '{ print $iColUpDown }' | grep "up"   | wc -l)
+typeset -i iWarnLimit;     iWarnLimit=$(     ph.getValueWithParam 1 w "$@")
+typeset -i iCriticalLimit; iCriticalLimit=$( ph.getValueWithParam 2 c "$@")
 
+typeset -i iOsdTotal;      iOsdTotal=$( getOsd | wc -l)
+typeset -i iOsdDown;       iOsdDown=$(  getOsd | awk '{ print $iColUpDown }' | grep "down" | wc -l)
+typeset -i iOsdUp;         iOsdUp=$(    getOsd | awk '{ print $iColUpDown }' | grep "up"   | wc -l)
 
 if [ $iOsdDown -lt $iWarnLimit ]; then
     ph.setStatus "ok"

check_clientbackup

@@ -17,6 +17,7 @@
 # 2022-10-21  v1.10 <axel.hahn@unibe.ch>     remove grep: warning: stray \ before white space
 #                                            show last backup, prune, verify
 # 2023-06-22  v1.11  <axel.hahn@unibe.ch>    add help and param support; add parameter for install directory
+# 2023-06-22  v1.12  <axel.hahn@unibe.ch>    add docs url of IML backup
 # ======================================================================
 
 . $( dirname $0 )/inc_pluginfunctions
@@ -43,6 +44,10 @@ Licence: GNU GPL 3
 ______________________________________________________________________
 
 Show status of IML backup.
+See https://os-docs.iml.unibe.ch/iml-backup/
+
+It returns with state UNKNOWN if no backup has been performed yet.
+It switches to ERROR an error occured during backup process.
 
 SYNTAX:
 $_self [-d DIRECTORY]

check_couchdb-lb

@@ -3,7 +3,6 @@
 #
 # NAGIOS CLIENT CHECK :: see https://projects.iml.unibe.ch/projects/sysadmin/work_packages/3839/activity
 #
-
 # ======================================================================
 
 
@@ -19,8 +18,8 @@ for myHost in $lbList
 do
     ping -c 1 $myHost >$tmpfile || ph.setStatus "critical"
     grep "Redirect Host" $tmpfile >/dev/null && ph.setStatus "critical"
-    ip=`head -1 $tmpfile | cut -f 2 -d '(' | cut -f 1 -d ')'`
-    loss=`grep "packet loss" $tmpfile | cut -f 6 -d " "`
+    ip=$(head -1 $tmpfile | cut -f 2 -d '(' | cut -f 1 -d ')')
+    loss=$(grep "packet loss" $tmpfile | cut -f 6 -d " ")
 
     ph.status "Check LB $myHost $ip - $loss loss"
     test "$loss" = "0%" || cat $tmpfile 

check_disk-io

@@ -2,44 +2,58 @@
 # ======================================================================
 #
 # Check DISK IO over all disks
+#
 # data besed on /proc/diskstats
 # https://www.kernel.org/doc/Documentation/iostats.txt
 #
 # based on /sys/block/*/stat
 # https://www.kernel.org/doc/Documentation/block/stat.txt
 #
+# Requires: bc, lsblk
+#
 # ----------------------------------------------------------------------
 # 2020-07-17  v1.0  <axel.hahn@iml.unibe.ch>
+# 2023-07-27  v1.1  <axel.hahn@iml.unibe.ch>  shell fixes; remove unsupported warn and critical
 # ======================================================================
 
 
-. `dirname $0`/inc_pluginfunctions
+. $( dirname $0 )/inc_pluginfunctions
 
+self_APPNAME=$( basename $0 | tr [:lower:] [:upper:] )
+self_APPVERSION=1.1
 
 # ----------------------------------------------------------------------
 # FUNCTIONS
 # ----------------------------------------------------------------------
 
 # diskinfo based on lsblk
-# 
+# param  string  comma separated list of names (no spaces)
 function _diskInfo(){
     local _fields=$1
-    test -z $_fields && _fields='NAME,MAJ:MIN,TYPE,SIZE,FSTYPE,MOUNTPOINT,STATE,ROTA,VENDOR,MODEL,SERIAL,HCTL'
-    lsblk -ai --output $_fields
+    test -z "$_fields" && _fields='NAME,MAJ:MIN,TYPE,SIZE,FSTYPE,MOUNTPOINT,STATE,ROTA,VENDOR,MODEL,SERIAL,HCTL'
+    lsblk -ai --output "$_fields"
 }
 
+# get a list of local disks
 function getDisks(){
     _diskInfo "NAME,TYPE" | grep "disk" | awk '{ print $1 }' | sed "s#[^a-z0-9]##g"
 }
-function getPartitions(){
-    _diskInfo "NAME,TYPE" | grep "part" | awk '{ print $1 }' | sed "s#[^a-z0-9]##g"
-}
 
+# UNUSED get a list of local partitions
+# function getPartitions(){
+#     _diskInfo "NAME,TYPE" | grep "part" | awk '{ print $1 }' | sed "s#[^a-z0-9]##g"
+# }
+
+# show help
 function showHelp(){
+    _self=$(basename $0)
 cat <<EOF
 ______________________________________________________________________
 
-CHECK_DISK IO AND LATENCY
+$self_APPNAME 
+v$self_APPVERSION
+
+Check dis io and latency
 
 (c) Institute for Medical Education - University of Bern
 Licence: GNU GPL 3
@@ -47,17 +61,18 @@ ______________________________________________________________________
 
 Disk infos based on /sys/block/[NAME]/stat
 See https://www.kernel.org/doc/Documentation/block/stat.txt
+and https://www.kernel.org/doc/Documentation/iostats.txt
+
+The system data are counters that are difficult to read.
+The output of this check for each value a delta value per second since 
+last check.
 
 SYNTAX:
-`basename $0` -m MODE [-w WARN_LIMIT] [-c CRITICAL_LIMIT]
+$_self -m MODE
 
 OPTIONS:
 
     -m MODE        set mode for type of output (required)
-
-    -w VALUE       warning level  (default: 0 for none)
-    -c VALUE       critical level (default: 0 for none)
-
     -h or --help   show this help.
 
 PARAMETERS:
@@ -68,7 +83,7 @@ PARAMETERS:
         wait       total wait time for all requests
 
 EXAMPLE:
-`basename $0` -m io
+$_self -m io
 
 EOF
 }
@@ -77,18 +92,8 @@ EOF
 # MAIN
 # ----------------------------------------------------------------------
 
-# TESTAREA 51
-# _diskInfo
-# echo "--- disks: "
-# getDisks
-# echo "--- partitions: "
-# getPartitions
-
-ph.require bc lsblk
-
 typeset -i iDelta=0
 
-
 case "$1" in
     "--help"|"-h")
         showHelp
@@ -97,12 +102,10 @@ case "$1" in
     *)
 esac
 
-# set default / override from command line params
-typeset -i iWarnLimit=`     ph.getValueWithParam 0 w "$@"`
-typeset -i iCriticalLimit=` ph.getValueWithParam 0 c "$@"`
+ph.require bc lsblk
 
 # --- set mode
-sMode=`                     ph.getValueWithParam '' m "$@"`
+sMode=$(                     ph.getValueWithParam '' m "$@")
 
 # --- labels and its columns in /sys/block/$myDisk/stat
 
@@ -143,20 +146,20 @@ case "$sMode" in
     *)
         echo "ERROR: missing or wrong MODE parameter -m"
         showHelp
-        exit 0
+        exit 1
 esac
 
 
 
 
 
-tmpfile1=`mktemp`
+tmpfile1=$(mktemp)
 
 # --- add data for each disk
-for myDisk in `getDisks`
+for myDisk in $(getDisks)
 do
-    echo >>$tmpfile1
-    echo "--- $myDisk" >> $tmpfile1
+    echo >>"$tmpfile1"
+    echo "--- $myDisk" >> "$tmpfile1"
 
     diskdata=$( cat /sys/block/$myDisk/stat )
     # echo $diskdata >> $tmpfile1
@@ -166,35 +169,35 @@ do
         column=${aColums[$index]}
         value=$( echo $diskdata | cut -f $column -d " " )
 
-        iDelta=`ph.perfdeltaspeed "$label" $value`
+        iDelta=$(ph.perfdeltaspeed "$label" $value)
         
         typeset -i aTotals[$index]=${aTotals[$index]}+$iDelta
-        # echo "    $label $iDelta per sec ... total: $value" >> $tmpfile1
-        printf "%30s %10d \n" "$label:" "$iDelta" >> $tmpfile1
+        # echo "    $label $iDelta per sec ... total: $value" >> "$tmpfile1"
+        printf "%30s %10d \n" "$label:" "$iDelta" >> "$tmpfile1"
 
     done
 done
 
 # --- add total
-echo >>$tmpfile1
-echo "--- TOTAL" >> $tmpfile1
+echo >>"$tmpfile1"
+echo "--- TOTAL" >> "$tmpfile1"
 for index in ${!aNames[*]}
 do
 
     label="${aNames[$index]}"
     value=${aTotals[$index]}
     # echo "    $label: $value" >> $tmpfile1
-    printf "%30s %10d \n" "$label:" "$value" >> $tmpfile1
+    printf "%30s %10d \n" "$label:" "$value" >> "$tmpfile1"
     ph.perfadd "$label"   "$value"
 
 done
-echo >>$tmpfile1
+echo >>"$tmpfile1"
 
 # --- output
 ph.status "Disk data ... $info " # READ `toUnit $iTotalRead M` MB/s << [DISC] << `toUnit $iTotalWrite M` MB/s WRITE"
 
-cat $tmpfile1
+cat "$tmpfile1"
 
 # --- cleanup and bye
-rm -f $tmpfile1
+rm -f "$tmpfile1"
 ph.exit

check_dns_responsetime

@@ -11,24 +11,22 @@
 #                                             test tcp 53 first
 #                                             check result depends on 1st nameserver only
 # 2022-10-24  v1.2  <axel.hahn@unibe.ch>      shell fixes; remove pipe char in output
+# 2023-07-27  v1.3  <axel.hahn@iml.unibe.ch>  shell fixes; add help page
 # ======================================================================
 
 
-. $(dirname $0)/inc_pluginfunctions
+. $( dirname $0 )/inc_pluginfunctions
+
+self_APPNAME=$( basename $0 | tr [:lower:] [:upper:] )
+self_APPVERSION=1.3
+
 tmpfile=/tmp/check_netstat_out_$$
 infofile=/tmp/check_netstat_out_2_$$
 myHost=$( hostname -f )
 
-echo $myHost | cut -f 3- -d "." | grep "\." >/dev/null
-if [ $? -ne 0 ]; then
-    ph.setStatus unknown
-    ph.status "DNS check for [$myHost] - SKIP: hostname -f returned a FQDN with less than 2 dots"
-    ph.exit
-fi
-
 # set default / override from command line params
-typeset -i iWarnLimit=$(     ph.getValueWithParam  300 w "$@")
-typeset -i iCriticalLimit=$( ph.getValueWithParam 1000 c "$@")
+typeset -i iWarnLimit;     iWarnLimit=$(     ph.getValueWithParam  300 w "$@")
+typeset -i iCriticalLimit; iCriticalLimit=$( ph.getValueWithParam 1000 c "$@")
 
 rm -f $tmpfile $infofile 2>/dev/null
 
@@ -38,23 +36,67 @@ typeset -i iTime=0
 typeset -i iCounter=0
 typeset -i iNotReachable=0
 
+# show help
+function showHelp(){
+    _self=$(basename $0)
+cat <<EOF
+______________________________________________________________________
+
+$self_APPNAME 
+v$self_APPVERSION
+
+(c) Institute for Medical Education - University of Bern
+Licence: GNU GPL 3
+______________________________________________________________________
+
+Loop over all defined dns servers (in /etc/resolv.conf) and check
+each:
+  - detect of port 53 is available (DNS service)
+  - 5x check time of a nslookup for current host (${myHost})
+
+The a warning / critical response will be reached if the maximum time
+of nslookup on the first dns server reaches a limit.
+
+The critical response will be returned if the first of the 
+nameservers is not available.
+
+An unknown response will be returned if the current hostname 
+(hostname -f) is no FQDN.
+
+SYNTAX:
+$_self [ -w VALUE -c VALUE -h ]
+
+OPTIONS:
+
+    -w VALUE       warning level  (default: 300)
+    -c VALUE       critical level (default: 1000)
+    -h or --help   Show this help.
+
+EOF
+}
+
 # ----------------------------------------------------------------------
 # MAIN
 # ----------------------------------------------------------------------
 
 
 # --- check param -h
-if [ "$1" = "-h" ]; then
-    echo "
-    usage: $0 [ -w value -c value -h ]
-
-        -w  Warning level
-        -c  Critical level
-        -h  this help
-    "
-    exit 0
+case "$1" in
+    "--help"|"-h")
+        showHelp
+        exit 0
+        ;;
+    *)
+esac
+
+if ! echo $myHost | cut -f 3- -d "." | grep "\." >/dev/null; then
+    ph.setStatus unknown
+    ph.status "DNS check for [$myHost] - SKIP: hostname -f returned a FQDN with less than 2 dots"
+    ph.exit
 fi
 
+ph.require nslookup
+
 for mydns in $(grep ^nameserver /etc/resolv.conf | awk '{ print $2 } ' )
 do
     iCounter=$iCounter+1
@@ -63,13 +105,12 @@ do
 
     echo "---------- $iCounter - $mydns " >>$infofile
     # todo loop
-    >/dev/tcp/${mydns}/53 >>$infofile 2>&1 
-    if [ $? -ne 0 ]; then
+    if ! >/dev/tcp/${mydns}/53 >>$infofile 2>&1; then
       iNotReachable=$iNotReachable+1
       test $iCounter -eq 1 && ph.setStatus critical
       echo "ERROR: ${mydns} is not reachable on tcp 53" >>$infofile
     else
-        for i in `seq 5`
+        for i in $(seq 5)
         do
             (time nslookup ${myHost} $mydns) >$tmpfile 2>&1
 
@@ -84,7 +125,7 @@ do
         # --- set status
         test $iCounter -eq 1 && ph.setStatusByLimit $iSrvMax $iWarnLimit $iCriticalLimit
 
-        label=$(echo $mydns | sed "s#\.#-#g" )
+        label=${mydns//\./-}
         ph.perfadd "response-$label" "${iSrvMax}"
         test $iSrvMax -gt $iMax && iMax=$iSrvMax
     fi
@@ -93,7 +134,8 @@ do
 done
 
 
-ph.status "DNS check for $myHost - found maximum was $iMax ms - $iNotReachable of $iCounter nameservers not reachable"
+test $iNotReachable -eq 0 && ph.status "DNS check for $myHost - found maximum was $iMax ms - OK: all nameservers are reachable"
+test $iNotReachable -ne 0 && ph.status "DNS check for $myHost - found maximum was $iMax ms - INFO: $iNotReachable of $iCounter nameservers not reachable"
 cat $infofile
 
 rm -f $tmpfile $infofile

check_fs_errors

@@ -10,10 +10,14 @@
 # ----------------------------------------------------------------------
 # 2021-03-23  v1.0  <axel.hahn@iml.unibe.ch>
 # 2021-03-30  v1.1  <axel.hahn@iml.unibe.ch>  max age of detected errors: since yesterday (commented)
+# 2023-07-27  v1.2  <axel.hahn@iml.unibe.ch>  shell fixes; update help page
 # ======================================================================
 
 
-. `dirname $0`/inc_pluginfunctions
+. $( dirname $0 )/inc_pluginfunctions
+
+self_APPNAME=$( basename $0 | tr [:lower:] [:upper:] )
+self_APPVERSION=1.2
 
 
 # ----------------------------------------------------------------------
@@ -21,18 +25,22 @@
 # ----------------------------------------------------------------------
 
 function showHelp(){
+    _self=$(basename $0)
 cat <<EOF
 ______________________________________________________________________
 
-CHECK_FS_ERRORS check if kernel logs inconsistency messages
+$self_APPNAME 
+v$self_APPVERSION
 
 (c) Institute for Medical Education - University of Bern
 Licence: GNU GPL 3
 ______________________________________________________________________
 
+Check if kernel logs inconsistency messages in the journallog.
+It requires sudo permission on /bin/journalctl
 
 SYNTAX:
-`basename $0` [-h]
+$_self [-h]
 
 OPTIONS:
 
@@ -43,7 +51,7 @@ PARAMETERS:
     none
 
 EXAMPLE:
-`basename $0`
+$_self
 
 EOF
 }
@@ -76,7 +84,7 @@ test ! -z "$out" && ph.setStatus "critical"
 # ----- OUTPUT
 
 ph.status "check if kernel logs inconsistency messages"
-echo $out
+echo "$out"
 
 
 # ----- CLEANUP AND BYE!

check_fs_writable

@@ -8,10 +8,14 @@
 #
 # ----------------------------------------------------------------------
 # 2021-03-23  v0.0  <axel.hahn@iml.unibe.ch>
+# 2023-07-27  v1.2  <axel.hahn@iml.unibe.ch>  shell fixes; update help page
 # ======================================================================
 
 
-. `dirname $0`/inc_pluginfunctions
+. $( dirname $0 )/inc_pluginfunctions
+
+self_APPNAME=$( basename $0 | tr [:lower:] [:upper:] )
+self_APPVERSION=1.2
 
 dirs2test="/tmp /var/tmp"
 out=""
@@ -21,18 +25,25 @@ out=""
 # ----------------------------------------------------------------------
 
 function showHelp(){
+    _self=$(basename $0)
 cat <<EOF
 ______________________________________________________________________
 
-CHECK_FS_READONLY check if filesystem is readonly - v0.0
+$self_APPNAME 
+v$self_APPVERSION
 
 (c) Institute for Medical Education - University of Bern
 Licence: GNU GPL 3
 ______________________________________________________________________
 
+Check if a filesystem is readonly in given directories.
+In each of the given directories a random file will be touched.
+
+The response is critical of one of the directory is not writable
+for the icinga client user.
 
 SYNTAX:
-`basename $0` [directory [more directories]]
+$_self [directory [more directories]]
 
 OPTIONS:
 
@@ -41,9 +52,12 @@ OPTIONS:
 PARAMETERS:
 
     DIRECTORY where to touch a temporary file
+              default directories:
+              $dirs2test
 
 EXAMPLE:
-`basename $0` /tmp /root /var/www
+
+    $_self /tmp /root /var/www
 
 EOF
 }
@@ -73,11 +87,11 @@ test $# -gt 0 && dirs2test="$*"
 for mydir in $dirs2test
 do
   touchfile=$mydir/icinga_touch_testfile__${RANDOM}_${RANDOM}
-  out="$out
+  if ! out="$out
 
 --- touching something into $mydir
-  $( touch $touchfile && ls -l $touchfile && rm -f $touchfile 2>&1 )"
-  if [ $? -ne 0 ]; then
+    $( touch $touchfile 2>&1 && ls -l $touchfile && rm -f $touchfile )"
+  then
     ph.setStatus "critical"
   fi
 done
@@ -86,7 +100,6 @@ done
 # ----- OUTPUT
 
 ph.status "check if filesystem is writable in $dirs2test $out"
-#echo "$out"
 
 
 # ----- CLEANUP AND BYE!

docs/20_Checks/_index.md

@@ -17,9 +17,11 @@ There is one include script used by all checks:
 * check_couchdb-lb
 * [check_cpu](check_cpu.md)
 * [check_cronstatus](check_cronstatus.md)
-* check_disk-io
-* check_dns_responsetime
+* [check_disk-io](check_disk-io.md)
+* [check_dns_responsetime](check_dns_responsetime.md)
 * [check_eol](check_eol.md)
+* [check_fs_errors](check_fs_errors.md)
+* [check_fs_writable](check_fs_writable.md)
 * check_haproxy_health
 * check_haproxy_status
 * check_memory

docs/20_Checks/check_ceph_diskfree.md

@@ -15,7 +15,7 @@ icingaclient ALL=(ALL) NOPASSWD: /bin/ceph
 
 ## Syntax
 
-```
+```txt
 $ check_ceph_diskfree -h
 ______________________________________________________________________
 

docs/20_Checks/check_ceph_osd.md

@@ -18,7 +18,7 @@ critical level.
 ______________________________________________________________________
 
 CHECK_CEPH_OSD 
-v1.5
+v1.6
 
 (c) Institute for Medical Education - University of Bern
 Licence: GNU GPL 3

docs/20_Checks/check_clientbackup.md

@@ -4,9 +4,9 @@
 
 **check_client_backup** is a plugin to show the backup status of IML Backup.
 
-📄 Source: https://git-repo.iml.unibe.ch/iml-open-source/iml-backup/
+📄 Source: <https://git-repo.iml.unibe.ch/iml-open-source/iml-backup/>
 📜 License: GNU GPL 3.0
-📖 Docs: see docs folder or online https://os-docs.iml.unibe.ch/iml-backup/
+📖 Docs: see docs folder or online <https://os-docs.iml.unibe.ch/iml-backup/>
 
 ## Requirements
 
@@ -19,7 +19,7 @@
 ______________________________________________________________________
 
 CHECK_CLIENTBACKUP 
-v1.11
+v1.12
 
 (c) Institute for Medical Education - University of Bern
 Licence: GNU GPL 3

docs/20_Checks/check_disk_io.md

+# CHECK_DISK-IO
+
+## Introduction
+
+**check_disk-io** is a plugin to show io, ticks and wait time.
+It shows a multiline output and sends performance data.
+
+## Requirements
+
+* `lsblk` List information about block devices
+* `bc` calculator
+
+## Syntax
+
+```txt
+______________________________________________________________________
+
+CHECK_DISK-IO 
+v1.1
+
+Check dis io and latency
+
+(c) Institute for Medical Education - University of Bern
+Licence: GNU GPL 3
+______________________________________________________________________
+
+Disk infos based on /sys/block/[NAME]/stat
+See https://www.kernel.org/doc/Documentation/block/stat.txt
+and https://www.kernel.org/doc/Documentation/iostats.txt
+
+The system data are counters that are difficult to read.
+The output of this check for each value a delta value per second since 
+last check.
+
+SYNTAX:
+check_disk-io -m MODE
+
+OPTIONS:
+
+    -m MODE        set mode for type of output (required)
+    -h or --help   show this help.
+
+PARAMETERS:
+
+    MODE
+        io         read I/Os, write I/Os, discard I/0s
+        ticks      read ticks, write ticks, discard ticks
+        wait       total wait time for all requests
+
+EXAMPLE:
+check_disk-io -m io
+
+```
+
+### Parameters
+
+`-m <MODE>` where MODE is a string to define kind of output.
+
+## Examples
+
+`$ check_disk-io -m io` returns
+
+```txt
+OK: Disk data ... read I/Os, write I/Os, discard I/0s, number of I/Os currently in flight 
+
+--- sda
+              disk-sda-ReadIO:          0 
+             disk-sda-WriteIO:          0 
+           disk-sda-DiscardIO:          0 
+            disk-sda-FlightIO:          0 
+
+--- nvme0n1
+          disk-nvme0n1-ReadIO:          3 
+         disk-nvme0n1-WriteIO:         16 
+       disk-nvme0n1-DiscardIO:          4 
+        disk-nvme0n1-FlightIO:          0 
+
+--- TOTAL
+                       ReadIO:          3 
+                      WriteIO:         16 
+                    DiscardIO:          4 
+                     FlightIO:          0 
+
+ |readio=3;; writeio=16;; discardio=4;; flightio=0;;
+```

docs/20_Checks/check_dns_responsetime.md

+# CHECK_DNS_RESPONSETIME
+
+## Introduction
+
+**check_dns_responsetime** check dns response time of all dns servers found in local /etc/resolv.conf
+
+## Requirements
+
+* `nslookup` query Internet name servers
+
+## Syntax
+
+```txt
+______________________________________________________________________
+
+CHECK_DNS_RESPONSETIME 
+v1.3
+
+(c) Institute for Medical Education - University of Bern
+Licence: GNU GPL 3
+______________________________________________________________________
+
+Loop over all defined dns servers (in /etc/resolv.conf) and check
+each:
+  - detect of port 53 is available (DNS service)
+  - 5x check time of a nslookup for current host (www.example.com)
+
+The a warning / critical response will be reached if the maximum time
+of nslookup on the first dns server reaches a limit.
+
+The critical response will be returned if the first of the 
+nameservers is not available.
+
+An unknown response will be returned if the current hostname 
+(hostname -f) is no FQDN.
+
+SYNTAX:
+check_dns_responsetime [ -w VALUE -c VALUE -h ]
+
+OPTIONS:
+
+    -w VALUE       warning level  (default: 300)
+    -c VALUE       critical level (default: 1000)
+    -h or --help   Show this help.
+
+```
+
+### Parameters
+
+(none)
+
+## Examples
+
+`$ check_disk-io -m io` returns
+
+```txt
+./check_dns_responsetime
+OK: DNS check for www.example.com - found maximum was 46 ms - OK: all nameservers are reachable
+
+---------- 1 - 8.8.4.4 
+8.8.4.4 #1 >>> 36 ms
+8.8.4.4 #2 >>> 28 ms
+8.8.4.4 #3 >>> 24 ms
+8.8.4.4 #4 >>> 30 ms
+8.8.4.4 #5 >>> 31 ms
+max: 36 ms
+ ^
+ :
+ +--- 1st nameserver is relevant for total status of the check. Limits are warning=300 and critical=1000
+
+
+---------- 2 - 8.8.8.8 
+8.8.8.8 #1 >>> 39 ms
+8.8.8.8 #2 >>> 35 ms
+8.8.8.8 #3 >>> 33 ms
+8.8.8.8 #4 >>> 46 ms
+8.8.8.8 #5 >>> 43 ms
+max: 46 ms
+
+ |response-8-8-4-4=36;; response-8-8-8-8=46;;```

docs/20_Checks/check_fs_errors.md

+# Check filesystem errors
+
+## Introduction
+
+**check_fs_errors** checks if kernel logs inconsistency messages in the journallog.
+It requires sudo permission on /bin/journalctl
+
+If it would find error messages it switches to critical and shows the found journallog entries.
+
+## Requirements
+
+* sudo permission on /bin/journalctl to scan for the information
+
+```txt
+icingaclient ALL=(ALL) NOPASSWD: /bin/journalctl
+```
+
+## Syntax
+
+```txt
+______________________________________________________________________
+
+CHECK_FS_ERRORS 
+v1.2
+
+(c) Institute for Medical Education - University of Bern
+Licence: GNU GPL 3
+______________________________________________________________________
+
+Check if kernel logs inconsistency messages in the journallog.
+It requires sudo permission on /bin/journalctl
+
+SYNTAX:
+check_fs_errors [-h]
+
+OPTIONS:
+
+    -h or --help   show this help.
+
+PARAMETERS:
+
+    none
+
+EXAMPLE:
+check_fs_errors
+```
+
+### Parameters
+
+(none)
+
+## Examples
+
+`$ check_fs_errors` returns
+
+```txt
+OK: check if kernel logs inconsistency messages
+```

docs/20_Checks/check_fs_writable.md

+# Check writable filesystem
+
+## Introduction
+
+**check_fs_writebale** checks in the given directories a file can be touched.
+The response is critical of one of the directory is not writable for the icinga client user. 
+
+## Requirements
+
+(none)
+
+## Syntax
+
+```txt
+./check_fs_writable -h
+______________________________________________________________________
+
+CHECK_FS_WRITABLE 
+v1.2
+
+(c) Institute for Medical Education - University of Bern
+Licence: GNU GPL 3
+______________________________________________________________________
+
+Check if a filesystem is readonly in given directories.
+In each of the given directories a random file will be touched.
+
+The response is critical of one of the directory is not writable
+for the icinga client user.
+
+SYNTAX:
+check_fs_writable [directory [more directories]]
+
+OPTIONS:
+
+    -h or --help   show this help.
+
+PARAMETERS:
+
+    DIRECTORY where to touch a temporary file
+              default directories:
+              /tmp /var/tmp
+
+EXAMPLE:
+
+    check_fs_writable /tmp /root /var/www
+
+```
+
+### Parameters
+
+Add directories to check.
+Set a directory that is writable for world or prepared to be accessible for the icinga user.
+
+## Examples
+
+`$ check_fs_writable` returns
+
+```txt
+OK: check if filesystem is writable in /tmp /var/tmp 
+
+--- touching something into /tmp
+    -rw-r--r-- 1 axel axel 0 27. Jul 16:00 /tmp/icinga_touch_testfile__7749_15502
+
+--- touching something into /var/tmp
+    -rw-r--r-- 1 axel axel 0 27. Jul 16:00 /var/tmp/icinga_touch_testfile__30551_11189
+
+```