Skip to content
Snippets Groups Projects
Commit f93ecd2c authored by Hahn Axel (hahn)'s avatar Hahn Axel (hahn)
Browse files

update db plugins

parent 17cd658e
Branches
No related tags found
1 merge request!140fix db plugins; Reduce desktop notification
Show whitespace changes
Inline Side-by-side
plugins/localdump/couchdb2.sh
+ 40
− 51
View file @ f93ecd2c
......@@ -30,6 +30,7 @@
# 2023-06-26 ah v1.11 speed up detection of changed databases
# 2023-06-27 ah v1.12 enable tmp file for dblist again (faster); speedup loops in backup
# 2023-06-28 ah v1.13 optimize backup move OUTFILE; measure time; cache backed up sequence ids
# 2024-10-02 ah v1.14 rename backup and restore function
# ================================================================================
if [ -z "$BACKUP_TARGETDIR" ]; then
......@@ -38,19 +39,6 @@ if [ -z "$BACKUP_TARGETDIR" ]; then
exit 1
fi
# --------------------------------------------------------------------------------
# CONFIG
# --------------------------------------------------------------------------------
# contains *.config files for each instance
CFGDIR=~/.iml_backup/couchdb2
# UNUSED
# dirPythonPackages=/usr/lib/python2.7/site-packages
# now set in localdump.sh
# ARCHIVE_DIR=$(_j_getvar "${JOBFILE}" dir-dbarchive)/couchdb2
# --------------------------------------------------------------------------------
# FUNCTIONS
# --------------------------------------------------------------------------------
......@@ -142,7 +130,7 @@ function reqCombined(){
# backup with loop over instances
# param 1 string globbing filter to config files
function doBackup(){
function couchdb2.backup(){
echo "--- instance: $PROFILENAME"
if curl --head -X GET "$COUCH_URL" 2>/dev/null | grep "^HTTP.* 200 "; then
......@@ -357,7 +345,7 @@ function _doBackupOfSingleInstance(){
# restore a single backup file; the instance and db name will be detected from file
# param string filename of db dump (full path or relative to BACKUP_TARGETDIR)
# param string optional: target database; default: detect name from import database
function restoreByFile(){
function couchdb2.restore(){
sMyfile=$1
dbname=$2
......@@ -374,7 +362,8 @@ function restoreByFile(){
# exit 1
# fi
local _sourceDB="$( guessDB $sMyfile | sed 's#.couchdbdump.gz$##' )"
local _sourceDB
_sourceDB="$( guessDB $sMyfile | sed 's#.couchdbdump.gz$##' )"
echo "detected source database : [${_sourceDB}]"
if [ -z "$dbname" ]; then
......@@ -387,7 +376,6 @@ function restoreByFile(){
echo
if [ $bFastMode -eq 0 ]; then
echo connect $couchdbhost on port $couchdbport with user $couchdbuser
curl --head -X GET $COUCH_URL 2>/dev/null | grep "^HTTP.* 200 " >/dev/null
if [ $? -ne 0 ]; then
color error
......@@ -464,30 +452,31 @@ j_requireBinary "curl" 1
j_requireBinary "couchbackup" 1
j_requireBinary "couchrestore" 1
#ls ${dirPythonPackages}/couchdb/tools/dump.py ${dirPythonPackages}/couchdb/tools/load.py >/dev/null && echo "OK: python couchdb tools were found"
#rc=$rc+$?
if [ $rc -eq 0 ]; then
if [ $rc -ne 0 ]; then
rc=1
color.echo error "ERROR: Missing a binary. Your Couchdb data cannot be dumped."
else
echo
if [ "$1" = "restore" ]; then
echo
test "$1" = "ALL" && shift 1
action=$1
shift 1
restoreByFile $*
"${SERVICENAME}.$action" $*
else
shift 1
# if [ "$1" = "restore" ]; then
# echo
# shift 1
# restoreByFile $*
# remove keyword ALL which is used for localdump.sh to loop over all db types
test "$1" = "ALL" && shift 1
# else
# shift 1
doBackup $*
fi
# # remove keyword ALL which is used for localdump.sh to loop over all db types
# test "$1" = "ALL" && shift 1
# doBackup $*
# fi
else
rc=1
color.echo error "ERROR: Your Couchdb data cannot be dumped."
fi
echo "__DB__$SERVICENAME INFO: $0 $* [$SERVICENAME] final returncode rc=$rc"
......
plugins/localdump/ldap.sh
+ 10
− 11
View file @ f93ecd2c
......@@ -14,6 +14,7 @@
# 2021-12-14 ah v1.2 detect sbin path to execute slapcat without path
# 2022-03-17 v1.3 WIP: add lines with prefix __DB__
# 2022-10-07 ah v1.4 unescape regex with space to prevent "grep: warning: stray \ before white space"
# 2024-10-02 ah v1.5 rename backup and restore function
# ================================================================================
......@@ -50,7 +51,7 @@ done
# param string DN
# param string name of output file
# ----------------------------------------------------------------------
function dump_ldap(){
function ldap.dump(){
DN=$1
DUMPFILE=$2
......@@ -68,7 +69,7 @@ function dump_ldap(){
# ----------------------------------------------------------------------
# run ldap backups
# ----------------------------------------------------------------------
function doLdapBackup(){
function ldap.backup(){
create_targetdir
......@@ -80,7 +81,7 @@ function doLdapBackup(){
cfg2=$(echo $cfgname | sed "s#[ =,]#_#g")
outfile=$(hostname)_ldap_olc_config__$(get_outfile ${cfg2}).ldif
dump_ldap "$cfgname" "$BACKUP_TARGETDIR/$outfile"
ldap.dump "$cfgname" "$BACKUP_TARGETDIR/$outfile"
done
......@@ -92,7 +93,7 @@ function doLdapBackup(){
cfg2=`echo $cfgname | sed "s#[ =,]#_#g"`
outfile=$(hostname)_ldap_data__$(get_outfile ${cfg2}).ldif
dump_ldap "$cfgname" "$BACKUP_TARGETDIR/$outfile"
ldap.dump "$cfgname" "$BACKUP_TARGETDIR/$outfile"
done
echo
......@@ -101,7 +102,7 @@ function doLdapBackup(){
}
function restoreByFile(){
function ldap.restore(){
echo "TODO :-/"
rc=$rc+1
}
......@@ -124,13 +125,11 @@ if [ $rc -ne 0 ]; then
rc=0
echo "__DB__$SERVICENAME SKIP: LDAP seems not to be here"
else
if [ "$1" = "restore" ]; then
echo
restoreByFile "${2}"
else
doLdapBackup
fi
action=$1
shift 1
"${SERVICENAME}.$action" $*
fi
echo "__DB__$SERVICENAME INFO: $0 $* [ldap] final returncode rc=$rc"
......
plugins/localdump/mysql.sh
+ 50
− 129
View file @ f93ecd2c
#!/bin/bash
# ================================================================================
#
# LOCALDUMP :: MYSQL / MARIADB
......@@ -16,6 +17,7 @@
# 2022-03-21 v2.3 Check if the created dump contains "insert"
# 2022-03-24 v2.4 On empty dumps: detect if source db is empty too
# 2023-10-06 v2.5 mysqldump params can be customized in jobs/env
# 2023-10-02 v2.6 Detect a connect error before backup; remove unneeded code
# ================================================================================
if [ -z "$LOCALDUMP_LOADED" ]; then
......@@ -28,41 +30,12 @@ fi
# CONFIG
# --------------------------------------------------------------------------------
# flag: service was foound locally? One of 0|1
mysql_FOUND=0
# counters
typeset -i mysql_COUNT_CREATE=0
typeset -i mysql_COUNT_DUMPS=0
typeset -i mysql_COUNT_DB=0
typeset -i mysql_COUNT_ERRORS=0
SOURCE_DIR=/var/lib/mysql
# --------------------------------------------------------------------------------
# PRIVATE FUNCTIONS
# --------------------------------------------------------------------------------
# make checks if a service is available on this machine
# it sets mysql_FOUND as flag
function mysql._check(){
j_requireBinary "mysql" 1
j_requireBinary "mysqldump" 1
# j_requireProcess "mysqld|mariadb" 1
# if [ ! -d $SOURCE_DIR ]; then
# echo "INFO: directory $SOURCE_DIR doees not exist."
# rc=$rc+1
# fi
# set flag and reset return code
test $rc -eq 0 && mysql_FOUND=1
test $rc -eq 0 || mysql_COUNT_ERRORS+=1
rc=0
}
# --------------------------------------------------------------------------------
# PUBLIC FUNCTIONS I :: DB SCHEME - METHODS LOW LEVEL
# --------------------------------------------------------------------------------
......@@ -89,23 +62,25 @@ function mysql.db.dump(){
local _dbname=$1
local _dumpfile=$2
mysqldump ${BACKUP_PARAMS} $LD_MYSQL_DUMP_PARAMS --result-file="$_dumpfile" "$_dbname" 2>&1
if mysqldump ${BACKUP_PARAMS} $LD_MYSQL_DUMP_PARAMS --result-file="$_dumpfile" "$_dbname" 2>&1; then
fetchrc >/dev/null
if [ $myrc -eq 0 ]; then
if ! zgrep -iE "(CREATE|INSERT)" "$_dumpfile" >/dev/null
if ! grep -q -iE "(CREATE|INSERT)" "$_dumpfile"
then
typeset -i local _iTables
local _iTables
typeset -i _iTables
_iTables=$( mysql ${BACKUP_PARAMS} --skip-column-names --batch -e "use $_dbname; show tables ;" | wc -l )
if [ $_iTables -eq 0 ];
then
echo -n "EMPTY DATABASE ... "
echo "EMPTY DATABASE ... "
else
echo "ERROR: no data - the dump doesn't contain any CREATE or INSERT statement."
# force an error
false; fetchrc >/dev/null
fi
fi
else
fetchrc
fi
test $myrc -eq 0 && mysql_COUNT_DUMPS+=1
......@@ -129,7 +104,7 @@ function mysql.db.import(){
# show a list of existing databases
function mysql.db.list(){
# mysql -Ee "show databases ;" | grep "^Database:" | awk '{ print $2 }'
local _result=$( mysql ${BACKUP_PARAMS} -Ee "show databases ;" $BACKUP_PARAMS )
local _result=$( mysql ${BACKUP_PARAMS} -Ee "show databases ;" )
fetchrc >/dev/null
test $myrc -eq 0 && mysql_COUNT_DB=$( echo "$_result" | grep -c "^Database:" )
test $myrc -eq 0 && echo "$_result" | grep "^Database:" | awk '{ print $2 }'
......@@ -140,40 +115,24 @@ function mysql.db.list(){
# PUBLIC FUNCTIONS II :: HIGH LEVEL
# --------------------------------------------------------------------------------
# return result is the current service available
# USAGE: to abort a function if not available:
# mysql.available || return
function mysql.available(){
local _rc;
typeset -i _rc; _rc=(1-$mysql_FOUND)
return $_rc
}
# make checks if the current service is available on this machine
# param bool flag: silence; if any parameter is set it shows no output
function mysql.check(){
if [ -n "$1" ]; then
mysql._check >/dev/null 2>&1
else
echo
echo Details:
mysql._check
echo
fi
}
# start database backup of all schemes of this service
# no prameters
function mysql.backup(){
# abort if service is not available
mysql.available || return
local _dbname
local _outfile
create_targetdir
# it sets the variable "mysql_COUNT_DB"
mysql.db.list >/dev/null 2>&1
if [ $mysql_COUNT_DB -eq 0 ]; then
rc=1
echo "ERROR: no databases found or an connect error occured."
else
echo Found databases: $mysql_COUNT_DB
for _dbname in $( mysql.db.list )
do
echo -n "__DB__${SERVICENAME} backup $_dbname ... "
......@@ -183,48 +142,14 @@ function mysql.backup(){
db._compressDumpfile "$_outfile"
done
fi
}
# show help
function mysql.help(){
# local _bShow=false
# tac "$0" | while read line
# do
# if echo $line | grep "^function mysql\.[a-z]" >/dev/null; then
# _bShow = true
# fi
# if echo $line | grep "^# " >/dev/null; then
# _bShow = true
# fi
# done
cat <<EOHELP
Help for MYSQL-DOT functions
(1) high level functions
mysql.available silent; exitcode is 0 if mysql is available
mysql.check [0|1] check if mysql is available; shows missing checks
mysql.backup backup all databases
mysql.restore [FILE [DBNAME]] restore database
(2) functions on database level
mysql.db.create DBNAME create database
mysql.db.dump DBNAME OUTFILE dump given database to output file
mysql.db.import FILE DBNAME import file into existing database
mysql.db.list list existing databases
EOHELP
}
# restore database dump file into database
# param string database dump file (gzipped)
# param string optional: database to import; default: database is parsed from file
function mysql.restore(){
# abort if service is not available
mysql.available || return
local _infile=$1
local _dbname=$2
......@@ -258,30 +183,26 @@ function mysql.restore(){
}
# WIP: show status
function mysql.status(){
h2 "WIP: Status"
h3 "Databases (max. 15)"
mysql.db.list | head -15
h3 "Counters"
cat <<EOSTATUS
# function mysql.status(){
# h2 "WIP: Status"
# h3 "Databases (max. 15)"
# mysql.db.list | head -15
# h3 "Counters"
# cat <<EOSTATUS
found Dbs: $mysql_COUNT_DB
created : $mysql_COUNT_CREATE
dumped : $mysql_COUNT_DUMPS
# found Dbs: $mysql_COUNT_DB
# created : $mysql_COUNT_CREATE
# dumped : $mysql_COUNT_DUMPS
ERRORS : $mysql_COUNT_ERRORS
# ERRORS : $mysql_COUNT_ERRORS
# EOSTATUS
# }
EOSTATUS
}
# --------------------------------------------------------------------------------
# MAIN
# --------------------------------------------------------------------------------
"${SERVICENAME}".check 1
if ! "${SERVICENAME}".available; then
echo "__DB__$SERVICENAME SKIP: service [$SERVICENAME] is not avilable on this machine."
fi
action=$1
shift 1
"${SERVICENAME}.$action" $*
......
plugins/localdump/pgsql.sh
+ 33
− 38
View file @ f93ecd2c
......@@ -13,6 +13,7 @@
# 2022-01-20 v1.2 fixes with shellcheck
# 2022-03-17 v1.3 WIP: add lines with prefix __DB__
# 2023-09-20 v1.4 FIX could not change directory to "/root": Permission denied
# 2024-10-02 ah v1.5 rename backup and restore function
# ================================================================================
if [ -z "$BACKUP_TARGETDIR" ]; then
......@@ -25,7 +26,6 @@ fi
# CONFIG
# --------------------------------------------------------------------------------
SERVICEFOUND=0
# unix user of postgres database
PGUSER=postgres
......@@ -34,16 +34,8 @@ fi
# FUNCTION
# --------------------------------------------------------------------------------
function checkRequirements(){
j_requireBinary "pg_dump" 1
j_requireProcess "postgres|postmaster" 1
# set flag and reset return code
test $rc -eq 0 && SERVICEFOUND=1
rc=0
}
function doPgsqlBackup(){
function pgsql.backup(){
create_targetdir
......@@ -59,10 +51,12 @@ function doPgsqlBackup(){
do
echo -n "__DB__${SERVICENAME} backup $DATABASE ... "
OUTFILE="${BACKUP_TARGETDIR}/$(get_outfile ${DATABASE}).sql"
su ${PGUSER} -c "pg_dump ${BACKUP_PARAMS} -Fp ${DATABASE} >$OUTFILE"
if su ${PGUSER} -c "pg_dump ${BACKUP_PARAMS} -Fp ${DATABASE} >$OUTFILE"; then
fetchrc >/dev/null
db._compressDumpfile "$OUTFILE"
else
fetchrc
fi
done
cd -
......@@ -72,7 +66,7 @@ function doPgsqlBackup(){
# restore database dump file into database
# param string database dump file (gzipped)
# param string optional: database to import; default: database is parsed from file
function restoreByFile(){
function pgsql.restore(){
sMyfile=$1
sMyDb=$2
......@@ -116,31 +110,32 @@ function restoreByFile(){
# ----- requirements
checkRequirements >/dev/null 2>&1
if [ $SERVICEFOUND -eq 0 ]; then
echo "__DB__$SERVICENAME SKIP: service [$SERVICENAME] is not avilable on this machine."
fi
j_requireBinary "psql" 1
case $1 in
check)
# repeat check ... but show output
echo
echo Details:
checkRequirements
echo
;;
backup)
test $SERVICEFOUND -eq 1 && doPgsqlBackup
;;
restore)
if [ $rc -ne 0 ]; then
color.echo error "ERROR: Missing psql binary. Your Sqlite data cannot be dumped."
else
action=$1
shift 1
test $SERVICEFOUND -eq 1 && restoreByFile $*
;;
*)
echo ERROR: wrong syntax:
echo $0 $*
exit 1
esac
"${SERVICENAME}.$action" $*
fi
# case $1 in
# backup)
# test $SERVICEFOUND -eq 1 && pgsql.backup
# ;;
# restore)
# shift 1
# test $SERVICEFOUND -eq 1 && pgsql.restore $*
# ;;
# *)
# echo ERROR: wrong syntax:
# echo $0 $*
# exit 1
# esac
echo "__DB__$SERVICENAME INFO: $0 $* [$SERVICENAME] final returncode rc=$rc"
......
plugins/localdump/profiles/sqlite.ini.example
+ 1
− 0
View file @ f93ecd2c
......@@ -11,6 +11,7 @@ type = "sqlite"
# list of files to backup
# file[] = "/var/www/database/logs.db"
# file[] = "/var/www/database/another.db"
[set]
......
plugins/localdump/readme.md
+ 12
− 4
View file @ f93ecd2c
# Help plugins/localdump/
## 📝 Scripts per databse type
Here are database plugins that can dump and restore types of databases.
They will be sourced by [APPDIR]/localdump.sh and cannot started directly.
......@@ -10,8 +12,7 @@ They will be sourced by [APPDIR]/localdump.sh and cannot started directly.
* pgsql.sh (*) - PostgreSql - using pg_dump
* sqlite.sh - Sqlite sqlite3
(*) If ever possible we use a zero config method. This plugin detects locally installed binaries
and running processes of a given name to run a database backup or not.
## ⚙️ Settings
See settings in `[APPDIR]/jobs/backup.job`:
......@@ -25,9 +26,16 @@ In those are the dumps containing name of database scheme and a timestamp. All d
keep-days contains an integer for the days to keep database dumps locally. Older dumps will be removed.
## Global vars in plugin scripts
## 📑 Profiles
```
There are a few ini files in plugins/localdump/profiles/ that autodetect local databases using standard ports,
You can create your own ini files to detect a remote database or container.
See <https://os-docs.iml.unibe.ch/iml-backup/Configuration/Database.html>
## ✏️ Global vars in plugin scripts
```text
BACKUP_BASEDIR {string} base directory for db dumps
BACKUP_DATE {string} string with current timestamp; will be part of filename for backups
BACKUP_KEEP_DAYS {int} count of days how long to keep db dumps below $BACKUP_BASEDIR
......
plugins/localdump/sqlite.sh
+ 16
− 21
View file @ f93ecd2c
......@@ -12,6 +12,7 @@
# 2018-02-09 ah,ds v1.1 write a .meta file after successful backup
# 2022-03-17 v1.2 WIP: add lines with prefix __DB__
# 2024-02-20 ah v1.3 Handle profiles; restore into existing db file; restore file owner and perms
# 2024-10-02 ah v1.4 rename backup and restore function
# ================================================================================
if [ -z "$BACKUP_TARGETDIR" ]; then
......@@ -19,9 +20,6 @@ if [ -z "$BACKUP_TARGETDIR" ]; then
rc=$rc+1
exit 1
fi
# --------------------------------------------------------------------------------
# CONFIG
# --------------------------------------------------------------------------------
# --------------------------------------------------------------------------------
# FUNCTIONS
......@@ -29,7 +27,7 @@ fi
# start multiple sqlite3 backups
# files are taken from loaded profiles/sqlite*.ini - section [detect] -> files[]
function doSqliteBackup(){
function sqlite.backup(){
create_targetdir
......@@ -46,13 +44,14 @@ function doSqliteBackup(){
TARGET=$(get_outfile ${DATABASE_FILE})
TARGET=${BACKUP_TARGETDIR}/$(echo ${TARGET} | sed "s#/#_#g").sql
META=${TARGET}.gz.meta
# echo -n " to $TARGET "
sqlite3 "$DATABASE_FILE" .dump >"${TARGET}"
fetchrc >/dev/null
if sqlite3 "$DATABASE_FILE" .dump >"${TARGET}"; then
fetchrc
db._compressDumpfile "${TARGET}" && stat "$DATABASE_FILE" >"${META}"
ls -l ${TARGET}*
else
fetchrc
rm -f "${TARGET}"
fi
fi
done
}
......@@ -60,7 +59,7 @@ function doSqliteBackup(){
# restore database dump file into database
# param string database dump file (gzipped)
# param string optional: database to import; default: database is parsed from file
function restoreByFile(){
function sqlite.restore(){
local sMyfile=$1
local sMyDb=$2
......@@ -95,17 +94,13 @@ function restoreByFile(){
j_requireBinary "sqlite3" 1
if [ $rc -ne 0 ]; then
rc=1
color.echo error "ERROR: Your Sqlite data cannot be dumped."
color.echo error "ERROR: Missing sqlite3 binary. Your Sqlite data cannot be dumped."
else
if [ "$1" = "restore" ]; then
echo
action=$1
shift 1
restoreByFile $*
"${SERVICENAME}.$action" $*
else
doSqliteBackup
fi
fi
echo "__DB__$SERVICENAME INFO: $0 $* [$SERVICENAME] final returncode rc=$rc"
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment