Skip to content
Snippets Groups Projects
Commit 6627bf5a authored by Hahn Axel (hahn)'s avatar Hahn Axel (hahn)
Browse files

Merge branch 'docker-checks' into 'master' 

Docker checks: add check for docker stats

See merge request !235
parents 871e6e77 3bfdd72d
No related branches found
No related tags found
1 merge request!235Docker checks: add check for docker stats
Hide whitespace changes
Inline Side-by-side
check_docker_stats 0 → 100755
+ 210
− 0
View file @ 6627bf5a
#!/bin/bash
# ======================================================================
#
# Check DOCKER INFOS
#
# requirements:
# - docker
# - sudo permissions on docker command
# - jq
# - bc
#
# ----------------------------------------------------------------------
# Cli docs:
# https://docs.docker.com/engine/reference/commandline/docker/
# ----------------------------------------------------------------------
# 2024-01-26 v0.1 <axel.hahn@unibe.ch> init
# ======================================================================
. $(dirname $0)/inc_pluginfunctions
. $(dirname $0)/inc_dockerfunctions.sh
self_APPVERSION=1.0
# ----------------------------------------------------------------------
# FUNCTIONS
# ----------------------------------------------------------------------
# show help
function _showHelp(){
local _self=$( basename $0 )
cat <<EOH
$( ph.showImlHelpHeader )
Show resources of running docker instances.
This check provides performance data except on mode "all".
USAGE:
$_self [OPTIONS]
OPTIONS:
General:
-h, --help this help
-d, --debug Debug; Flag: show all docker system infos
Connect to docker:
-p, --path Custom directory for docker binary
-t, --target Custom docker target; value for DOCKER_HOST
Needed only if Docker does not run on a unix socket or
multiple users run a rootless docker daemon.
mode:
-m, --mode MODE Mode what kind of information to show.
The output of container list is sorted by the maximum
value first.
all default; This view is unsorted.
This check has no performance data.
cpu CPU usage total of all containers
mem Memory max usage of most consuming container
memtotal sum of memory usage
netio sum of netio of all containers IN and OUT
blockio sum of blockio of all containers IN and OUT
EXAMPLES:
$_self
Show stats of all containers
$_self -p /usr/bin
Show stats of all containers. The docker binary will be searched in
given path first - then in all other dirs of \$PATH
$_self -d
Show stats of all containers and json with all docker system infos.
$_self -m cpu
Show cpu usage total of all containers. The container list is sorted
by cpu usage. The most consuming container is on top.
EOH
}
# ----------------------------------------------------------------------
# MAIN
# ----------------------------------------------------------------------
bOptDebug=0
sMode="all"
while [[ "$#" -gt 0 ]]; do case $1 in
-h|--help) _showHelp; exit 0;;
-d|--debug) bOptDebug=1; shift;;
-p|--path) PATH="$2:$PATH"; shift; shift;;
-m|--mode) sMode="$2"; shift; shift;;
-t|--target) export DOCKER_HOST="$2"; shift; shift;;
*) echo "ERROR: Unknown parameter: $1"; showHelp; exit 1;
esac; done
sLabel="Docker stats"
case "$sMode" in
"cpu") sLabel="Docker CPU usage total of all containers: %s percent"; sUnit="1"; sumkey="CPUPerc"; sColumns=".Container, .${sumkey}, .Name" ;;
"mem") sLabel="Docker Memory max usage in most consuming container: %s percent"; sUnit="1"; maxkey="MemPerc"; sColumns=".Container, .${maxkey}, .MemUsage, .Name" ;;
"netio") sLabel="Docker Network I/O of all containers: %s MB IN .. %s MB OUT"; sUnit="M"; sumkey="NetIO"; f2=1; sColumns=".Container, .${sumkey}, .Name" ;;
"memtotal") sLabel="Docker Memory usage of all containers: %s MB"; sUnit="M"; sumkey="MemUsage";f2=0; sColumns=".Container, .${sumkey}, .Name" ;;
"blockio") sLabel="Docker Network I/O of all containers: %s MB IN .. %s MB OUT"; sUnit="M"; sumkey="BlockIO"; f2=1; sColumns=".Container, .${sumkey}, .Name" ;;
"all") ;;
*) ph.setStatus critical
echo "ERROR: mode [$sMode] is unknown. Use -h to get help."
ph.exit
esac
test -z "$sColumns" && sColumns=".Container, .CPUPerc, .MemUsage, .MemPerc, .NetIO, .PIDs, .Name"
ph.require "docker"
ph.require "jq"
ph.require "bc"
_detectDockerenv
# --- get stats data
data=$( sudo -n --preserve-env docker stats --no-stream --format '{{ json . }}' )
_detectDockererror "$data"
# --- checks to detect maximum
if [ -n "${maxkey}" ]; then
iMax=0
sMax=$( echo "$data" | tr -d '%' | jq -r ".${maxkey}" | sort -hr | head -1 | sed "s#^\.#0#" )
iMax=$( ph.toUnit "$sMax" "1" 0 )
ph.perfadd "${sMode}-max" "$iMax" "" "" 0 "100"
sLabel=$( printf "${sLabel}" ${sMax} )
fi
# --- checks with sum of a value
if [ -n "${sumkey}" ]; then
sum=0
# split 2 value field - get the 1st one. example: "81.1kB / 1.66MB" --> "81.1k"
for myval in $( echo "$data" | tr -d '%' | jq -r ".${sumkey}" | cut -f 1 -d " " | tr -d "B" )
do
valuniq=$( ph.toUnit "$myval" "" )
sum=$( bc <<< "$valuniq + ${sum}" )
done
test "$sUnit" = "1" && sumhr=$( ph.toUnit "$sum" "1" 0 )
test "$sUnit" = "M" && sumhr=$( ph.toUnit "$sum" "M" 3 )
# --- checks with IN and OUT
if [ "$f2" = "1" ]; then
sum2=0
# split 2 value field - get the 2nd one. example: "81.1kB / 1.66MB" --> "1.66M"
for myval in $( echo "$data" | tr -d '%' | jq -r ".${sumkey}" | cut -f 3 -d " " | tr -d "B" )
do
valuniq=$( ph.toUnit "$myval" "" )
sum2=$( bc <<< "$valuniq + ${sum2}" )
done
ph.perfadd "${sMode}-1" "$sum" "" ""
ph.perfadd "${sMode}-2" "$sum2" "" ""
sum2hr=$( ph.toUnit "$sum2" "M" 3 )
sLabel=$( printf "${sLabel}" ${sumhr} ${sum2hr} )
else
ph.perfadd "${sMode}-sum" "$sum" "" ""
sLabel=$( printf "${sLabel}" ${sumhr} )
fi
fi
# --- output
line="------------------------------------------------------------------------------------------"
ph.status "$sLabel"
if [ "$sMode" = "all" ]; then
echo "ID CPU % MEM USAGE / LIMIT MEM % NET I/O PIDs Name"
echo "$line"
jq -r "[$sColumns] | @tsv" <<< "$data"
fi
if [ -n "${maxkey}" ]; then
echo "ID ${maxkey}"
echo "$line"
jq -r "[$sColumns] | @tsv" <<< "$data" | sort -k +2 -h -r
fi
if [ -n "${sumkey}" ]; then
echo "ID ${sumkey}"
echo "$line"
jq -r "[$sColumns] | @tsv" <<< "$data" | sort -k +2 -h -r
fi
echo
# if -d was given then show debug infos too
test $bOptDebug -eq 1 && ( echo; echo "DEBUG: full docker system infos as json"; echo "$data" | jq ; _debugInfos)
ph.exit
# ----------------------------------------------------------------------
docs/20_Checks/check_docker_info.md
+ 1
− 0
View file @ 6627bf5a
......@@ -9,6 +9,7 @@ This check sends performance data.
## Requirements
* `docker` Docker must be installed
* `jq` Docker must be installed
* sudo permissions on docker command
```txt
......
docs/20_Checks/check_docker_stats.md 0 → 100644
+ 243
− 0
View file @ 6627bf5a
# CHECK_DOCKER_INFO
## Introduction
**check_docker_stats** shows the docker status of containers.
This check sends performance data.
## Requirements
* `docker` Docker must be installed
* `jq` must be installed - commandline JSON processor
* `bc` must be installed - an arbitrary precision calculator language
* sudo permissions on docker command
```txt
icingaclient ALL=(ALL) NOPASSWD:SETENV: /usr/bin/docker
```
## Includes
Additional needed files in the current folder that this check can run:
* inc_dockerfunctions.sh
* inc_pluginfunctions
## Rootless docker
It works with docker setups as root and can handle rootless docker instances.
To bring it up and running with an unpriviledged icinga user it must be able to access docker. Copy the docker binary eg. from /home/dockeruser/bin/docker to /usr/bin/. Then use the parameter `-p /usr/bin`.
If DOCKER_HOST is not set then the docker socket is detected from a running docker instance and will be fetched from the process list.
## Syntax
```txt
______________________________________________________________________
CHECK_DOCKER_STATS
v1.0
(c) Institute for Medical Education - University of Bern
Licence: GNU GPL 3
https://os-docs.iml.unibe.ch/icinga-checks/Checks/check_docker_stats.html
______________________________________________________________________
Show resources of running docker instances.
This check provides performance data except on mode "all".
USAGE:
check_docker_stats [OPTIONS]
OPTIONS:
General:
-h, --help this help
-d, --debug Debug; Flag: show all docker system infos
Connect to docker:
-p, --path Custom directory for docker binary
-t, --target Custom docker target; value for DOCKER_HOST
Needed only if Docker does not run on a unix socket or
multiple users run a rootless docker daemon.
mode:
-m, --mode MODE Mode what kind of information to show.
The output of container list is sorted by the maximum
value first.
all default; This view is unsorted.
This check has no performance data.
cpu CPU usage total of all containers
mem Memory max usage of most consuming container
memtotal sum of memory usage
netio sum of netio of all containers IN and OUT
blockio sum of blockio of all containers IN and OUT
EXAMPLES:
check_docker_stats
Show stats of all containers
check_docker_stats -p /usr/bin
Show stats of all containers. The docker binary will be searched in
given path first - then in all other dirs of $PATH
check_docker_stats -d
Show stats of all containers and json with all docker system infos.
check_docker_stats -m cpu
Show cpu usage total of all containers. The container list is sorted
by cpu usage. The most consuming container is on top.
```
### Parameters
None.
## Examples
### Stats
`$ ./check_docker_info` returns
```txt
OK: Docker stats
ID CPU % MEM USAGE / LIMIT MEM % NET I/O PIDs Name
------------------------------------------------------------------------------------------
7cd2e62febe5 0.00% 15.79MiB / 128MiB 12.34% 673kB / 13.9MB 4 portainer_monitoring_node-exporter.qm9ur2pzl7ngw2o6wb2s7fx4i.d6txef4ivx5wizt7ixg7z6rsn
b1733b4d1008 1.07% 40.52MiB / 128MiB 31.66% 915kB / 25.9MB 11 portainer_monitoring_cadvisor.qm9ur2pzl7ngw2o6wb2s7fx4i.vel7o1qcd2ezit82ep7r5qqlt
ad4d32e29aef 0.01% 98.01MiB / 3.828GiB 2.50% 104MB / 4.71MB 8 portainer_monitoring_prometheus.1.rlwvmh5hbo13sujt1rd5szqr6
99a82bf29c82 0.02% 61.07MiB / 3.828GiB 1.56% 4.82MB / 13.1MB 11 portainer_monitoring_grafana.1.1fux1t4q7jy7ja5szun3hbq3p
72023160d7b1 0.00% 2.945MiB / 3.828GiB 0.08% 4.27MB / 7.39MB 3 myapp_viewer.1.76pvzu0srf13nx6vcd99kqjhf
b87843ce6220 0.00% 2.871MiB / 3.828GiB 0.07% 4.08MB / 2.2MB 3 myapp_author.1.0rsdcjt8tjokcvobf9jgqk0el
77e9f55b1113 0.19% 27.79MiB / 3.828GiB 0.71% 88.2MB / 122MB 6 portainer_agent.qm9ur2pzl7ngw2o6wb2s7fx4i.i7cohx1xim0ljmlpaeamd2k9s
```
### CPU
`/check_docker_stats -m cpu`
```txt
OK: Docker CPU usage total of all containers: 1.50 percent
ID CPUPerc
------------------------------------------------------------------------------------------
b1733b4d1008 1.24% portainer_monitoring_cadvisor.qm9ur2pzl7ngw2o6wb2s7fx4i.vel7o1qcd2ezit82ep7r5qqlt
77e9f55b1113 0.25% portainer_agent.qm9ur2pzl7ngw2o6wb2s7fx4i.i7cohx1xim0ljmlpaeamd2k9s
99a82bf29c82 0.01% portainer_monitoring_grafana.1.1fux1t4q7jy7ja5szun3hbq3p
b87843ce6220 0.00% myapp_author.1.0rsdcjt8tjokcvobf9jgqk0el
ad4d32e29aef 0.00% portainer_monitoring_prometheus.1.rlwvmh5hbo13sujt1rd5szqr6
7cd2e62febe5 0.00% portainer_monitoring_node-exporter.qm9ur2pzl7ngw2o6wb2s7fx4i.d6txef4ivx5wizt7ixg7z6rsn
72023160d7b1 0.00% myapp_viewer.1.76pvzu0srf13nx6vcd99kqjhf
|cpu-sum=1.50;;
```
### Memory percent
Show most memory consuming in percent container on top. It checks the memory of the container.
`/check_docker_stats -m mem`
```txt
OK: Docker Memory max usage in most consuming container: 30.38 percent
ID MemPerc
------------------------------------------------------------------------------------------
b1733b4d1008 30.38% 38.89MiB / 128MiB portainer_monitoring_cadvisor.qm9ur2pzl7ngw2o6wb2s7fx4i.vel7o1qcd2ezit82ep7r5qqlt
7cd2e62febe5 12.21% 15.63MiB / 128MiB portainer_monitoring_node-exporter.qm9ur2pzl7ngw2o6wb2s7fx4i.d6txef4ivx5wizt7ixg7z6rsn
ad4d32e29aef 2.21% 86.67MiB / 3.828GiB portainer_monitoring_prometheus.1.rlwvmh5hbo13sujt1rd5szqr6
99a82bf29c82 1.56% 61.07MiB / 3.828GiB portainer_monitoring_grafana.1.1fux1t4q7jy7ja5szun3hbq3p
77e9f55b1113 0.71% 27.8MiB / 3.828GiB portainer_agent.qm9ur2pzl7ngw2o6wb2s7fx4i.i7cohx1xim0ljmlpaeamd2k9s
72023160d7b1 0.08% 2.945MiB / 3.828GiB myapp_viewer.1.76pvzu0srf13nx6vcd99kqjhf
b87843ce6220 0.07% 2.871MiB / 3.828GiB myapp_author.1.0rsdcjt8tjokcvobf9jgqk0el
|mem-max=30.38;;;0;100
```
### Memory total
`/check_docker_stats -m memtotal`
```txt
OK: Docker Memory usage of all containers: 224.977 MB
ID MemUsage
------------------------------------------------------------------------------------------
ad4d32e29aef 86.43MiB / 3.828GiB portainer_monitoring_prometheus.1.rlwvmh5hbo13sujt1rd5szqr6
99a82bf29c82 61.08MiB / 3.828GiB portainer_monitoring_grafana.1.1fux1t4q7jy7ja5szun3hbq3p
b1733b4d1008 39.26MiB / 128MiB portainer_monitoring_cadvisor.qm9ur2pzl7ngw2o6wb2s7fx4i.vel7o1qcd2ezit82ep7r5qqlt
77e9f55b1113 27.67MiB / 3.828GiB portainer_agent.qm9ur2pzl7ngw2o6wb2s7fx4i.i7cohx1xim0ljmlpaeamd2k9s
7cd2e62febe5 15.65MiB / 128MiB portainer_monitoring_node-exporter.qm9ur2pzl7ngw2o6wb2s7fx4i.d6txef4ivx5wizt7ixg7z6rsn
72023160d7b1 2.945MiB / 3.828GiB myapp_viewer.1.76pvzu0srf13nx6vcd99kqjhf
b87843ce6220 2.871MiB / 3.828GiB myapp_author.1.0rsdcjt8tjokcvobf9jgqk0el
|memtotal-sum=235906000.000;;
```
### Netio
`/check_docker_stats -m netio`
```txt
OK: Docker Network I/O of all containers: 209.363 MB IN .. 191.180 MB OUT
ID NetIO
------------------------------------------------------------------------------------------
ad4d32e29aef 106MB / 4.8MB portainer_monitoring_prometheus.1.rlwvmh5hbo13sujt1rd5szqr6
77e9f55b1113 88.4MB / 123MB portainer_agent.qm9ur2pzl7ngw2o6wb2s7fx4i.i7cohx1xim0ljmlpaeamd2k9s
99a82bf29c82 4.91MB / 13.1MB portainer_monitoring_grafana.1.1fux1t4q7jy7ja5szun3hbq3p
72023160d7b1 4.35MB / 7.44MB myapp_viewer.1.76pvzu0srf13nx6vcd99kqjhf
b87843ce6220 4.16MB / 2.24MB myapp_author.1.0rsdcjt8tjokcvobf9jgqk0el
b1733b4d1008 932kB / 26.4MB portainer_monitoring_cadvisor.qm9ur2pzl7ngw2o6wb2s7fx4i.vel7o1qcd2ezit82ep7r5qqlt
7cd2e62febe5 686kB / 14.2MB portainer_monitoring_node-exporter.qm9ur2pzl7ngw2o6wb2s7fx4i.d6txef4ivx5wizt7ixg7z6rsn
|netio-1=219533064.32;; netio-2=200466759.68;;
```
### Blockio
`/check_docker_stats -m bliockio`
```txt
OK: Docker Network I/O of all containers: 5.922 MB IN .. 95.773 MB OUT
ID BlockIO
------------------------------------------------------------------------------------------
b1733b4d1008 2.06MB / 0B portainer_monitoring_cadvisor.qm9ur2pzl7ngw2o6wb2s7fx4i.vel7o1qcd2ezit82ep7r5qqlt
99a82bf29c82 1.99MB / 786kB portainer_monitoring_grafana.1.1fux1t4q7jy7ja5szun3hbq3p
ad4d32e29aef 1.17MB / 95MB portainer_monitoring_prometheus.1.rlwvmh5hbo13sujt1rd5szqr6
77e9f55b1113 479kB / 8.19kB portainer_agent.qm9ur2pzl7ngw2o6wb2s7fx4i.i7cohx1xim0ljmlpaeamd2k9s
7cd2e62febe5 258kB / 0B portainer_monitoring_node-exporter.qm9ur2pzl7ngw2o6wb2s7fx4i.d6txef4ivx5wizt7ixg7z6rsn
b87843ce6220 0B / 8.19kB myapp_author.1.0rsdcjt8tjokcvobf9jgqk0el
72023160d7b1 0B / 8.19kB myapp_viewer.1.76pvzu0srf13nx6vcd99kqjhf
|blockio-1=6210566.72;; blockio-2=100425290.00;;
```
## Troubleshooting
### Missing environment
```txt
sudo: sorry, you are not allowed to preserve the environment
UNKNOWN: No data. Unable to fetch Docker information.
```
Solution: you did not set SETENV in the sudoers config file
icingaclient ALL=(ALL) NOPASSWD:**SETENV:** /usr/bin/docker
### Docker not found
If you get an error message that docker was not found then it is not reachable in ofe of the directories of $PATH.
This can happen especialy in rootless docker instances.
Solution: Use the parameter `-p /usr/bin` to define a target. Copy the docker binary there.
inc_dockerfunctions.sh
+ 4
− 6
View file @ 6627bf5a
......@@ -28,14 +28,14 @@ _is_docker_detected=0
# param string json data
# param string jq filter
function _filterJson(){
echo "$1" | jq "$2"
echo "$1" | jq -r "$2"
}
# filter json data with jq (by expecting a single result) and remove quotes
# param string json data
# param string jq filter
function _getString(){
_filterJson "$1" "$2" | tr -d '"' | sed "s#^null\$##"
_filterJson "$1" "$2" | sed "s#^null\$##g"
}
# if DOCKER_HOST is not set we try to detect the user running "containerd"
......@@ -74,12 +74,10 @@ function _detectDockererror(){
ph.exit
fi
local sSrvErrror;
sSrvErrror=$( _filterJson "$data" ".ServerErrors" )
if [ "$sSrvErrror" != "null" ] ; then
if grep '"ServerErrors"' <<<"$data" >/dev/null; then
ph.setStatus unknown
ph.status "Unable to connect to Docker:"
echo "$sSrvErrror" | grep "^ "
_filterJson "$data" ".ServerErrors" | grep "^ "
ph.exit
fi
......
inc_pluginfunctions
+ 5
− 3
View file @ 6627bf5a
......@@ -299,9 +299,11 @@ function ph._getExp(){
# test "$_unit" = "m" && echo 10^-3
# test "$_unit" = "c" && echo 10^-2
# test "$_unit" = "d" && echo 10^-1
# test "$_unit" = "k" && echo 10^3
# test "$_unit" = "M" && echo 10^6
# test "$_unit" = "G" && echo 10^9
test "$_unit" = "k" && echo 10^3
test "$_unit" = "Mi" && echo 10^6
test "$_unit" = "Gi" && echo 10^9
test "$_unit" = "Ti" && echo 10^9
test "$_unit" = "Pi" && echo 10^9
# echo "ERROR: unsupported value: $_value"
# exit 1
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment