diff --git a/check_docker_stats b/check_docker_stats new file mode 100755 index 0000000000000000000000000000000000000000..34bc33d04596a6c78d177a61fef247c99ebe2af3 --- /dev/null +++ b/check_docker_stats @@ -0,0 +1,210 @@ +#!/bin/bash +# ====================================================================== +# +# Check DOCKER INFOS +# +# requirements: +# - docker +# - sudo permissions on docker command +# - jq +# - bc +# +# ---------------------------------------------------------------------- +# Cli docs: +# https://docs.docker.com/engine/reference/commandline/docker/ +# ---------------------------------------------------------------------- +# 2024-01-26 v0.1 <axel.hahn@unibe.ch> init +# ====================================================================== + + +. $(dirname $0)/inc_pluginfunctions +. $(dirname $0)/inc_dockerfunctions.sh + +self_APPVERSION=1.0 + +# ---------------------------------------------------------------------- +# FUNCTIONS +# ---------------------------------------------------------------------- + + +# show help +function _showHelp(){ + local _self=$( basename $0 ) + cat <<EOH +$( ph.showImlHelpHeader ) + +Show resources of running docker instances. + +This check provides performance data except on mode "all". + +USAGE: + $_self [OPTIONS] + +OPTIONS: + + General: + -h, --help this help + -d, --debug Debug; Flag: show all docker system infos + + Connect to docker: + -p, --path Custom directory for docker binary + -t, --target Custom docker target; value for DOCKER_HOST + Needed only if Docker does not run on a unix socket or + multiple users run a rootless docker daemon. + + mode: + -m, --mode MODE Mode what kind of information to show. + The output of container list is sorted by the maximum + value first. + + all default; This view is unsorted. + This check has no performance data. + + cpu CPU usage total of all containers + mem Memory max usage of most consuming container + + memtotal sum of memory usage + netio sum of netio of all containers IN and OUT + blockio sum of blockio of all containers IN and OUT + +EXAMPLES: + $_self + Show stats of all containers + + $_self -p /usr/bin + Show stats of all containers. The docker binary will be searched in + given path first - then in all other dirs of \$PATH + + $_self -d + Show stats of all containers and json with all docker system infos. + + $_self -m cpu + Show cpu usage total of all containers. The container list is sorted + by cpu usage. The most consuming container is on top. + +EOH +} + +# ---------------------------------------------------------------------- +# MAIN +# ---------------------------------------------------------------------- + +bOptDebug=0 +sMode="all" +while [[ "$#" -gt 0 ]]; do case $1 in + -h|--help) _showHelp; exit 0;; + -d|--debug) bOptDebug=1; shift;; + -p|--path) PATH="$2:$PATH"; shift; shift;; + -m|--mode) sMode="$2"; shift; shift;; + -t|--target) export DOCKER_HOST="$2"; shift; shift;; + *) echo "ERROR: Unknown parameter: $1"; showHelp; exit 1; +esac; done + +sLabel="Docker stats" + +case "$sMode" in + "cpu") sLabel="Docker CPU usage total of all containers: %s percent"; sUnit="1"; sumkey="CPUPerc"; sColumns=".Container, .${sumkey}, .Name" ;; + "mem") sLabel="Docker Memory max usage in most consuming container: %s percent"; sUnit="1"; maxkey="MemPerc"; sColumns=".Container, .${maxkey}, .MemUsage, .Name" ;; + "netio") sLabel="Docker Network I/O of all containers: %s MB IN .. %s MB OUT"; sUnit="M"; sumkey="NetIO"; f2=1; sColumns=".Container, .${sumkey}, .Name" ;; + "memtotal") sLabel="Docker Memory usage of all containers: %s MB"; sUnit="M"; sumkey="MemUsage";f2=0; sColumns=".Container, .${sumkey}, .Name" ;; + "blockio") sLabel="Docker Network I/O of all containers: %s MB IN .. %s MB OUT"; sUnit="M"; sumkey="BlockIO"; f2=1; sColumns=".Container, .${sumkey}, .Name" ;; + "all") ;; + *) ph.setStatus critical + echo "ERROR: mode [$sMode] is unknown. Use -h to get help." + ph.exit +esac + +test -z "$sColumns" && sColumns=".Container, .CPUPerc, .MemUsage, .MemPerc, .NetIO, .PIDs, .Name" + + +ph.require "docker" +ph.require "jq" +ph.require "bc" + +_detectDockerenv + +# --- get stats data + +data=$( sudo -n --preserve-env docker stats --no-stream --format '{{ json . }}' ) +_detectDockererror "$data" + + +# --- checks to detect maximum + +if [ -n "${maxkey}" ]; then + iMax=0 + sMax=$( echo "$data" | tr -d '%' | jq -r ".${maxkey}" | sort -hr | head -1 | sed "s#^\.#0#" ) + iMax=$( ph.toUnit "$sMax" "1" 0 ) + ph.perfadd "${sMode}-max" "$iMax" "" "" 0 "100" + sLabel=$( printf "${sLabel}" ${sMax} ) +fi + + +# --- checks with sum of a value + +if [ -n "${sumkey}" ]; then + sum=0 + # split 2 value field - get the 1st one. example: "81.1kB / 1.66MB" --> "81.1k" + for myval in $( echo "$data" | tr -d '%' | jq -r ".${sumkey}" | cut -f 1 -d " " | tr -d "B" ) + do + valuniq=$( ph.toUnit "$myval" "" ) + sum=$( bc <<< "$valuniq + ${sum}" ) + done + test "$sUnit" = "1" && sumhr=$( ph.toUnit "$sum" "1" 0 ) + test "$sUnit" = "M" && sumhr=$( ph.toUnit "$sum" "M" 3 ) + + # --- checks with IN and OUT + + if [ "$f2" = "1" ]; then + sum2=0 + # split 2 value field - get the 2nd one. example: "81.1kB / 1.66MB" --> "1.66M" + for myval in $( echo "$data" | tr -d '%' | jq -r ".${sumkey}" | cut -f 3 -d " " | tr -d "B" ) + do + valuniq=$( ph.toUnit "$myval" "" ) + sum2=$( bc <<< "$valuniq + ${sum2}" ) + done + ph.perfadd "${sMode}-1" "$sum" "" "" + ph.perfadd "${sMode}-2" "$sum2" "" "" + sum2hr=$( ph.toUnit "$sum2" "M" 3 ) + + sLabel=$( printf "${sLabel}" ${sumhr} ${sum2hr} ) + + else + ph.perfadd "${sMode}-sum" "$sum" "" "" + sLabel=$( printf "${sLabel}" ${sumhr} ) + fi +fi + + +# --- output +line="------------------------------------------------------------------------------------------" + +ph.status "$sLabel" + +if [ "$sMode" = "all" ]; then + echo "ID CPU % MEM USAGE / LIMIT MEM % NET I/O PIDs Name" + echo "$line" + jq -r "[$sColumns] | @tsv" <<< "$data" +fi + +if [ -n "${maxkey}" ]; then + echo "ID ${maxkey}" + echo "$line" + jq -r "[$sColumns] | @tsv" <<< "$data" | sort -k +2 -h -r +fi + +if [ -n "${sumkey}" ]; then + echo "ID ${sumkey}" + echo "$line" + jq -r "[$sColumns] | @tsv" <<< "$data" | sort -k +2 -h -r +fi + +echo + +# if -d was given then show debug infos too +test $bOptDebug -eq 1 && ( echo; echo "DEBUG: full docker system infos as json"; echo "$data" | jq ; _debugInfos) + +ph.exit + + +# ---------------------------------------------------------------------- diff --git a/docs/20_Checks/check_docker_info.md b/docs/20_Checks/check_docker_info.md index 9a9748a1cf93969aa844c83581d67d5c2c08d7af..38fed82ee5f706112f3fb65fe70aadbd2956594d 100644 --- a/docs/20_Checks/check_docker_info.md +++ b/docs/20_Checks/check_docker_info.md @@ -9,6 +9,7 @@ This check sends performance data. ## Requirements * `docker` Docker must be installed +* `jq` Docker must be installed * sudo permissions on docker command ```txt diff --git a/docs/20_Checks/check_docker_stats.md b/docs/20_Checks/check_docker_stats.md new file mode 100644 index 0000000000000000000000000000000000000000..058eae8cb373ca8a23e351f98199ddad98562993 --- /dev/null +++ b/docs/20_Checks/check_docker_stats.md @@ -0,0 +1,243 @@ +# CHECK_DOCKER_INFO + +## Introduction + +**check_docker_stats** shows the docker status of containers. +This check sends performance data. + +## Requirements + +* `docker` Docker must be installed +* `jq` must be installed - commandline JSON processor +* `bc` must be installed - an arbitrary precision calculator language +* sudo permissions on docker command + +```txt +icingaclient ALL=(ALL) NOPASSWD:SETENV: /usr/bin/docker +``` + +## Includes + +Additional needed files in the current folder that this check can run: + +* inc_dockerfunctions.sh +* inc_pluginfunctions + +## Rootless docker + +It works with docker setups as root and can handle rootless docker instances. + +To bring it up and running with an unpriviledged icinga user it must be able to access docker. Copy the docker binary eg. from /home/dockeruser/bin/docker to /usr/bin/. Then use the parameter `-p /usr/bin`. + +If DOCKER_HOST is not set then the docker socket is detected from a running docker instance and will be fetched from the process list. + +## Syntax + +```txt +______________________________________________________________________ + +CHECK_DOCKER_STATS +v1.0 + +(c) Institute for Medical Education - University of Bern +Licence: GNU GPL 3 + +https://os-docs.iml.unibe.ch/icinga-checks/Checks/check_docker_stats.html +______________________________________________________________________ + +Show resources of running docker instances. + +This check provides performance data except on mode "all". + +USAGE: + check_docker_stats [OPTIONS] + +OPTIONS: + + General: + -h, --help this help + -d, --debug Debug; Flag: show all docker system infos + + Connect to docker: + -p, --path Custom directory for docker binary + -t, --target Custom docker target; value for DOCKER_HOST + Needed only if Docker does not run on a unix socket or + multiple users run a rootless docker daemon. + + mode: + -m, --mode MODE Mode what kind of information to show. + The output of container list is sorted by the maximum + value first. + + all default; This view is unsorted. + This check has no performance data. + + cpu CPU usage total of all containers + mem Memory max usage of most consuming container + + memtotal sum of memory usage + netio sum of netio of all containers IN and OUT + blockio sum of blockio of all containers IN and OUT + +EXAMPLES: + check_docker_stats + Show stats of all containers + + check_docker_stats -p /usr/bin + Show stats of all containers. The docker binary will be searched in + given path first - then in all other dirs of $PATH + + check_docker_stats -d + Show stats of all containers and json with all docker system infos. + + check_docker_stats -m cpu + Show cpu usage total of all containers. The container list is sorted + by cpu usage. The most consuming container is on top. + +``` + +### Parameters + +None. + +## Examples + +### Stats + +`$ ./check_docker_info` returns + +```txt +OK: Docker stats + +ID CPU % MEM USAGE / LIMIT MEM % NET I/O PIDs Name +------------------------------------------------------------------------------------------ +7cd2e62febe5 0.00% 15.79MiB / 128MiB 12.34% 673kB / 13.9MB 4 portainer_monitoring_node-exporter.qm9ur2pzl7ngw2o6wb2s7fx4i.d6txef4ivx5wizt7ixg7z6rsn +b1733b4d1008 1.07% 40.52MiB / 128MiB 31.66% 915kB / 25.9MB 11 portainer_monitoring_cadvisor.qm9ur2pzl7ngw2o6wb2s7fx4i.vel7o1qcd2ezit82ep7r5qqlt +ad4d32e29aef 0.01% 98.01MiB / 3.828GiB 2.50% 104MB / 4.71MB 8 portainer_monitoring_prometheus.1.rlwvmh5hbo13sujt1rd5szqr6 +99a82bf29c82 0.02% 61.07MiB / 3.828GiB 1.56% 4.82MB / 13.1MB 11 portainer_monitoring_grafana.1.1fux1t4q7jy7ja5szun3hbq3p +72023160d7b1 0.00% 2.945MiB / 3.828GiB 0.08% 4.27MB / 7.39MB 3 myapp_viewer.1.76pvzu0srf13nx6vcd99kqjhf +b87843ce6220 0.00% 2.871MiB / 3.828GiB 0.07% 4.08MB / 2.2MB 3 myapp_author.1.0rsdcjt8tjokcvobf9jgqk0el +77e9f55b1113 0.19% 27.79MiB / 3.828GiB 0.71% 88.2MB / 122MB 6 portainer_agent.qm9ur2pzl7ngw2o6wb2s7fx4i.i7cohx1xim0ljmlpaeamd2k9s +``` + +### CPU + +`/check_docker_stats -m cpu` + +```txt +OK: Docker CPU usage total of all containers: 1.50 percent + +ID CPUPerc +------------------------------------------------------------------------------------------ +b1733b4d1008 1.24% portainer_monitoring_cadvisor.qm9ur2pzl7ngw2o6wb2s7fx4i.vel7o1qcd2ezit82ep7r5qqlt +77e9f55b1113 0.25% portainer_agent.qm9ur2pzl7ngw2o6wb2s7fx4i.i7cohx1xim0ljmlpaeamd2k9s +99a82bf29c82 0.01% portainer_monitoring_grafana.1.1fux1t4q7jy7ja5szun3hbq3p +b87843ce6220 0.00% myapp_author.1.0rsdcjt8tjokcvobf9jgqk0el +ad4d32e29aef 0.00% portainer_monitoring_prometheus.1.rlwvmh5hbo13sujt1rd5szqr6 +7cd2e62febe5 0.00% portainer_monitoring_node-exporter.qm9ur2pzl7ngw2o6wb2s7fx4i.d6txef4ivx5wizt7ixg7z6rsn +72023160d7b1 0.00% myapp_viewer.1.76pvzu0srf13nx6vcd99kqjhf + + |cpu-sum=1.50;; +``` + +### Memory percent + +Show most memory consuming in percent container on top. It checks the memory of the container. + +`/check_docker_stats -m mem` + +```txt +OK: Docker Memory max usage in most consuming container: 30.38 percent + +ID MemPerc +------------------------------------------------------------------------------------------ +b1733b4d1008 30.38% 38.89MiB / 128MiB portainer_monitoring_cadvisor.qm9ur2pzl7ngw2o6wb2s7fx4i.vel7o1qcd2ezit82ep7r5qqlt +7cd2e62febe5 12.21% 15.63MiB / 128MiB portainer_monitoring_node-exporter.qm9ur2pzl7ngw2o6wb2s7fx4i.d6txef4ivx5wizt7ixg7z6rsn +ad4d32e29aef 2.21% 86.67MiB / 3.828GiB portainer_monitoring_prometheus.1.rlwvmh5hbo13sujt1rd5szqr6 +99a82bf29c82 1.56% 61.07MiB / 3.828GiB portainer_monitoring_grafana.1.1fux1t4q7jy7ja5szun3hbq3p +77e9f55b1113 0.71% 27.8MiB / 3.828GiB portainer_agent.qm9ur2pzl7ngw2o6wb2s7fx4i.i7cohx1xim0ljmlpaeamd2k9s +72023160d7b1 0.08% 2.945MiB / 3.828GiB myapp_viewer.1.76pvzu0srf13nx6vcd99kqjhf +b87843ce6220 0.07% 2.871MiB / 3.828GiB myapp_author.1.0rsdcjt8tjokcvobf9jgqk0el + + |mem-max=30.38;;;0;100 +``` + +### Memory total + +`/check_docker_stats -m memtotal` + +```txt +OK: Docker Memory usage of all containers: 224.977 MB + +ID MemUsage +------------------------------------------------------------------------------------------ +ad4d32e29aef 86.43MiB / 3.828GiB portainer_monitoring_prometheus.1.rlwvmh5hbo13sujt1rd5szqr6 +99a82bf29c82 61.08MiB / 3.828GiB portainer_monitoring_grafana.1.1fux1t4q7jy7ja5szun3hbq3p +b1733b4d1008 39.26MiB / 128MiB portainer_monitoring_cadvisor.qm9ur2pzl7ngw2o6wb2s7fx4i.vel7o1qcd2ezit82ep7r5qqlt +77e9f55b1113 27.67MiB / 3.828GiB portainer_agent.qm9ur2pzl7ngw2o6wb2s7fx4i.i7cohx1xim0ljmlpaeamd2k9s +7cd2e62febe5 15.65MiB / 128MiB portainer_monitoring_node-exporter.qm9ur2pzl7ngw2o6wb2s7fx4i.d6txef4ivx5wizt7ixg7z6rsn +72023160d7b1 2.945MiB / 3.828GiB myapp_viewer.1.76pvzu0srf13nx6vcd99kqjhf +b87843ce6220 2.871MiB / 3.828GiB myapp_author.1.0rsdcjt8tjokcvobf9jgqk0el + + |memtotal-sum=235906000.000;; +``` + +### Netio + +`/check_docker_stats -m netio` + +```txt +OK: Docker Network I/O of all containers: 209.363 MB IN .. 191.180 MB OUT + +ID NetIO +------------------------------------------------------------------------------------------ +ad4d32e29aef 106MB / 4.8MB portainer_monitoring_prometheus.1.rlwvmh5hbo13sujt1rd5szqr6 +77e9f55b1113 88.4MB / 123MB portainer_agent.qm9ur2pzl7ngw2o6wb2s7fx4i.i7cohx1xim0ljmlpaeamd2k9s +99a82bf29c82 4.91MB / 13.1MB portainer_monitoring_grafana.1.1fux1t4q7jy7ja5szun3hbq3p +72023160d7b1 4.35MB / 7.44MB myapp_viewer.1.76pvzu0srf13nx6vcd99kqjhf +b87843ce6220 4.16MB / 2.24MB myapp_author.1.0rsdcjt8tjokcvobf9jgqk0el +b1733b4d1008 932kB / 26.4MB portainer_monitoring_cadvisor.qm9ur2pzl7ngw2o6wb2s7fx4i.vel7o1qcd2ezit82ep7r5qqlt +7cd2e62febe5 686kB / 14.2MB portainer_monitoring_node-exporter.qm9ur2pzl7ngw2o6wb2s7fx4i.d6txef4ivx5wizt7ixg7z6rsn + + |netio-1=219533064.32;; netio-2=200466759.68;; + ``` + +### Blockio + +`/check_docker_stats -m bliockio` + +```txt +OK: Docker Network I/O of all containers: 5.922 MB IN .. 95.773 MB OUT + +ID BlockIO +------------------------------------------------------------------------------------------ +b1733b4d1008 2.06MB / 0B portainer_monitoring_cadvisor.qm9ur2pzl7ngw2o6wb2s7fx4i.vel7o1qcd2ezit82ep7r5qqlt +99a82bf29c82 1.99MB / 786kB portainer_monitoring_grafana.1.1fux1t4q7jy7ja5szun3hbq3p +ad4d32e29aef 1.17MB / 95MB portainer_monitoring_prometheus.1.rlwvmh5hbo13sujt1rd5szqr6 +77e9f55b1113 479kB / 8.19kB portainer_agent.qm9ur2pzl7ngw2o6wb2s7fx4i.i7cohx1xim0ljmlpaeamd2k9s +7cd2e62febe5 258kB / 0B portainer_monitoring_node-exporter.qm9ur2pzl7ngw2o6wb2s7fx4i.d6txef4ivx5wizt7ixg7z6rsn +b87843ce6220 0B / 8.19kB myapp_author.1.0rsdcjt8tjokcvobf9jgqk0el +72023160d7b1 0B / 8.19kB myapp_viewer.1.76pvzu0srf13nx6vcd99kqjhf + + |blockio-1=6210566.72;; blockio-2=100425290.00;; + ``` + +## Troubleshooting + +### Missing environment + +```txt +sudo: sorry, you are not allowed to preserve the environment +UNKNOWN: No data. Unable to fetch Docker information. +``` + +Solution: you did not set SETENV in the sudoers config file + +icingaclient ALL=(ALL) NOPASSWD:**SETENV:** /usr/bin/docker + +### Docker not found + +If you get an error message that docker was not found then it is not reachable in ofe of the directories of $PATH. +This can happen especialy in rootless docker instances. + +Solution: Use the parameter `-p /usr/bin` to define a target. Copy the docker binary there. diff --git a/inc_dockerfunctions.sh b/inc_dockerfunctions.sh index 86383e80cbb080b3bf44d45320a0b93d9c9f8fb9..3b94193e059c199ea158624d20abaa3aad22362c 100644 --- a/inc_dockerfunctions.sh +++ b/inc_dockerfunctions.sh @@ -28,14 +28,14 @@ _is_docker_detected=0 # param string json data # param string jq filter function _filterJson(){ - echo "$1" | jq "$2" + echo "$1" | jq -r "$2" } # filter json data with jq (by expecting a single result) and remove quotes # param string json data # param string jq filter function _getString(){ - _filterJson "$1" "$2" | tr -d '"' | sed "s#^null\$##" + _filterJson "$1" "$2" | sed "s#^null\$##g" } # if DOCKER_HOST is not set we try to detect the user running "containerd" @@ -74,12 +74,10 @@ function _detectDockererror(){ ph.exit fi - local sSrvErrror; - sSrvErrror=$( _filterJson "$data" ".ServerErrors" ) - if [ "$sSrvErrror" != "null" ] ; then + if grep '"ServerErrors"' <<<"$data" >/dev/null; then ph.setStatus unknown ph.status "Unable to connect to Docker:" - echo "$sSrvErrror" | grep "^ " + _filterJson "$data" ".ServerErrors" | grep "^ " ph.exit fi diff --git a/inc_pluginfunctions b/inc_pluginfunctions index 3f64983a8793b37c0a1f954a0b70b99706323170..55b1b19dafb370ce2931f14c2f491e0c614cf3da 100644 --- a/inc_pluginfunctions +++ b/inc_pluginfunctions @@ -299,9 +299,11 @@ function ph._getExp(){ # test "$_unit" = "m" && echo 10^-3 # test "$_unit" = "c" && echo 10^-2 # test "$_unit" = "d" && echo 10^-1 - # test "$_unit" = "k" && echo 10^3 - # test "$_unit" = "M" && echo 10^6 - # test "$_unit" = "G" && echo 10^9 + test "$_unit" = "k" && echo 10^3 + test "$_unit" = "Mi" && echo 10^6 + test "$_unit" = "Gi" && echo 10^9 + test "$_unit" = "Ti" && echo 10^9 + test "$_unit" = "Pi" && echo 10^9 # echo "ERROR: unsupported value: $_value" # exit 1