From 1008ddf928b93fd1a853ab23b42286b4fa139f79 Mon Sep 17 00:00:00 2001
From: "Hahn Axel (hahn)" <axel.hahn@unibe.ch>
Date: Tue, 20 Feb 2024 17:20:56 +0100
Subject: [PATCH] Sqlite Handle profiles; restore into existing db file;
restore file owner and perms
---
localdump.sh | 168 ++++++++++++++++++++++++------------
plugins/localdump/sqlite.sh | 83 +++++++-----------
2 files changed, 145 insertions(+), 106 deletions(-)
diff --git a/localdump.sh b/localdump.sh
index f5e72fb..09691c3 100755
--- a/localdump.sh
+++ b/localdump.sh
@@ -66,22 +66,32 @@
# ---------- GETTER
+# write debug output if DBD_DEBUG is enabled
+# param string text to show
function dbdetect._wd(){
test "$DBD_DEBUG" -eq "1" && echo "DEBUG: $*" >&2
}
+# get a list of all config files
+# param string full path of ini file
function dbdetect.getConfigs(){
find ${DBD_BASEDIR} -type f -name "*.ini"
}
+# get type from given ini file
+# param string full path of ini file
function dbdetect.getType(){
basename "$1" | cut -d "_" -f 1 | sed "s,\.ini$,,"
}
+
+# get profile from given ini file
+# param string full path of ini file
function dbdetect.getProfile(){
basename "$1" | cut -d "_" -f 1- | sed "s,\.ini$,,"
}
-
+# check if the requirements for a database match
+# param string full path of ini file to check
function dbdetect.exists(){
local _config="$1"
@@ -93,14 +103,17 @@ function dbdetect.exists(){
ini.set "$_config" "detect"
# --- check tcp
- local tcpport; tcpport=$( ini.value "tcp" )
- local tcptarget; tcptarget=$( ini.value "tcp-target" )
- if { ! >/dev/tcp/$tcptarget/$tcpport; } > /dev/null 2>&1; then
- # echo "No port tcp $tcpport available"
- dbdetect._wd "... No port tcp $tcpport available on $tcptarget"
- return 1
+ local tcpport; tcpport=$( ini.value "tcp" )
+ if [ -n "$tcpport" ]; then
+ local tcptarget; tcptarget=$( ini.value "tcp-target" )
+ tcptarget=${tcptarget:-localhost}
+ if { ! >/dev/tcp/$tcptarget/$tcpport; } > /dev/null 2>&1; then
+ # echo "No port tcp $tcpport available"
+ dbdetect._wd "... No port tcp $tcpport available on $tcptarget"
+ return 1
+ fi
+ dbdetect._wd "... Found tcp $tcpport on $tcptarget."
fi
- dbdetect._wd "... Found tcp $tcpport on $tcptarget."
# --- check tcp process
local tcpprocess; tcpprocess=$( ini.value "tcp-process" )
@@ -139,21 +152,45 @@ function dbdetect.exists(){
dbdetect._wd "... Process ${process} was found"
fi
+ # --- check db files
+ local filetype; filetype=$( ini.value "type" )
+
+ if [ -n "${filetype}" ]; then
+ local myfiles; declare -a myfiles
+ for myfile in $( ini.value "file[]" )
+ do
+ if ! file -b "${myfile}" | grep -i "$filetype" >/dev/null; then
+ dbdetect._wd "... File ${myfile} is no type $filetype"
+ return 1
+ fi
+ dbdetect._wd "... File ${myfile} is type $filetype"
+ myfiles+="${myfile}|"
+ done
+ fi
+
+ # --- OK, everything was found ... we initialize it
+ dbdetect._wd "OK, match: $_config"
+
ini.set "$_config" "set"
local value
local dbuser=$( ini.value "dbuser" )
local dbpassword=$( ini.value "dbpassword" )
- for mykey in $( ini.keys )
+
+ for mykey in su env params
do
- # params = '--port={{tcp}} --password={{dbpassword}} --user={{dbuser}} --host={{tcp-target}}'
value="$( ini.value "$mykey" )"
value="${value//\{\{tcp\}\}/$tcpport}"
value="${value//\{\{tcp-target\}\}/$tcptarget}"
value="${value//\{\{dbuser\}\}/$dbuser}"
value="${value//\{\{dbpassword\}\}/$dbpassword}"
+ value="${value//\{\{file\[\]\}\}/$myfiles}"
+
DBD_PARAMS[$mykey]="$value"
+ dbdetect._wd ">>> $mykey = $value"
done
+ dbdetect._wd ">>> files = $myfiles"
+ DBD_PARAMS[files]="$myfiles"
return 0
}
@@ -167,6 +204,13 @@ function dbdetect.getParams(){
echo "${DBD_PARAMS['params']}"
}
+
+# for backup scripts: get checked files from [detect] -> file[]
+#
+function dbdetect.getFiles(){
+ echo "${DBD_PARAMS['files']}" | tr "|" "\n"
+}
+
# set variables in [set] -> env = ...
# USAGE:
# eval $( dbdetect.setenv )
@@ -189,26 +233,6 @@ function dbdetect.runas(){
# FUNCTIONS 4 DB-WRAPPER
# ----------------------------------------------------------------------
- function db.init(){
-
- BACKUP_BASEDIR=$(_j_getvar ${JOBFILE} "dir-localdumps")
-
- # check
- if [ -z "$BACKUP_BASEDIR" ]; then
- color error
- echo ERROR: missing config for backup target.
- echo There must be an entry dir-localdumps in ${JOBFILE}
- color reset
- exit 1
- fi
- BACKUP_PLUGINDIR=$(dirname $0)/plugins/localdump
- BACKUP_KEEP_DAYS=$(_j_getvar ${JOBFILE} "keep-days")
-
- if [ $BACKUP_KEEP_DAYS -eq 0 ]; then
- BACKUP_KEEP_DAYS=7
- fi
- BACKUP_DATE=$(/bin/date +%Y%m%d-%H%M)
- }
# helpfer function for SERVICENAME.backup
# it is called after the service specific dump was done.
@@ -346,7 +370,7 @@ function dbdetect.runas(){
if [ -z $1 ]; then
find -type f | sed "s#__[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]-[0-9][0-9].*##g" | grep -v "\.meta" | sort -ud| sed "s#^\./##g"
else
- ls -ltr "$*__"* | sed "s#^\./##g"
+ ls -ltr "$*__"* | sed "s#^\./##g" | grep -v "\.meta"
fi
cd - >/dev/null
else
@@ -376,7 +400,7 @@ function dbdetect.runas(){
# the metafile is written in sqlite backup to store full path
metafile=${BACKUP_TARGETDIR}/${dumpfile}.meta
if [ -f $metafile ]; then
- cat $metafile
+ grep "^/" "$metafile" || grep "^ File: " "$metafile" | cut -c 9-
else
sBasename=$(basename $1)
sDb=$(echo ${sBasename} | sed "s#__[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]-[0-9][0-9].*##g")
@@ -391,6 +415,29 @@ function dbdetect.runas(){
}
+ # read .meta file (that contains output of stats) and restore last owner and file permissions
+ # param string filename of db dump
+ # param string restored database file
+ function restorePermissions(){
+ local sMyMeta="${1}.meta"
+ local sTargetfile="$2"
+ if [ -f "${sMyMeta}" ]; then
+
+ # Access: (0674/-rw-rwxr--) Uid: ( 1000/ axel) Gid: ( 1000/ axel)
+ # ^ ^ ^
+ # _sPerm _sUser _sGroup
+ local _sPerm=$( grep "^Access: (" "${sMyMeta}" | cut -f 2 -d '(' | cut -f 1 -d '/')
+ if [ -n "$_sPerm" ]; then
+ local _sUser=$( grep "^Access: (" "${sMyMeta}" | cut -f 3 -d '/' | cut -f 1 -d ')' | tr -d ' ')
+ local _sGroup=$( grep "^Access: (" "${sMyMeta}" | cut -f 4 -d '/' | cut -f 1 -d ')' | tr -d ' ')
+
+ echo -n "Restoring file owner $_sUser:$_sGroup and permissions $_sPerm ... "
+ chown "$_sUser:$_sGroup" "${sTargetfile}" && chmod "$_sPerm" "${sTargetfile}"
+ fetchrc
+ fi
+ fi
+ }
+
# ------------------------------------------------------------
# show help
# ------------------------------------------------------------
@@ -419,7 +466,26 @@ function dbdetect.runas(){
# INIT
# ----------------------------------------------------------------------
- db.init
+ # ----- db init
+ BACKUP_BASEDIR=$(_j_getvar ${JOBFILE} "dir-localdumps")
+
+ # check
+ if [ -z "$BACKUP_BASEDIR" ]; then
+ color error
+ echo ERROR: missing config for backup target.
+ echo There must be an entry dir-localdumps in ${JOBFILE}
+ color reset
+ exit 1
+ fi
+ BACKUP_PLUGINDIR=$(dirname $0)/plugins/localdump
+ DBD_BASEDIR=$BACKUP_PLUGINDIR/profiles
+
+ BACKUP_KEEP_DAYS=$(_j_getvar ${JOBFILE} "keep-days")
+
+ if [ $BACKUP_KEEP_DAYS -eq 0 ]; then
+ BACKUP_KEEP_DAYS=7
+ fi
+ BACKUP_DATE=$(/bin/date +%Y%m%d-%H%M)
# ----- checks
@@ -455,7 +521,14 @@ function dbdetect.runas(){
case "$mode" in
# ------------------------------------------------------------
check)
- . $BACKUP_SCRIPT $mode
+ DBD_DEBUG=1
+ for PROFILENAME in $(dbdetect.getConfigs)
+ do
+ echo "----- $PROFILENAME"
+ dbdetect.exists "${PROFILENAME}"
+ echo
+ done
+ # . $BACKUP_SCRIPT $mode
;;
# ------------------------------------------------------------
backup)
@@ -469,23 +542,6 @@ function dbdetect.runas(){
services=$*
fi
- # ----- check all params
- for SERVICENAME in $services
- do
- BACKUP_SCRIPT=$( get_service_script ${SERVICENAME} )
- if [ ! -f $BACKUP_SCRIPT ]; then
- color error
- echo ERROR: parameter $SERVICENAME seems to be wrong.
- echo The backup script does not exist: $BACKUP_SCRIPT
- color reset
- echo
- echo services in this folder are:
- get_services
- exit 2
- fi
- done
-
-
# ----- GO
# PROFILENAME mysql_localhost_13306
# SERVICENAME mysql
@@ -505,23 +561,25 @@ function dbdetect.runas(){
h2 "START SCRIPT FOR [${PROFILENAME}] -> ${SERVICENAME}"
# ------ set env
+ # echo "BACKUP_PARAMS = $BACKUP_PARAMS"
# dbdetect.setenv
eval $( dbdetect.setenv )
- # echo "BACKUP_PARAMS = $BACKUP_PARAMS"
_j_runHooks "200-before-db-service"
- . $BACKUP_SCRIPT $mode
+ h3 "BACKUP [${PROFILENAME}] -> ${SERVICENAME}"
+ . $BACKUP_SCRIPT $mode
test $rc -gt 0 && j_notify "db ${SERVICENAME}" "$BACKUP_SCRIPT $mode was finished with rc=$rc" $rc
_j_runHooks "230-after-db-service" "$rc"
+ # ------ unset env
+ eval $( dbdetect.unssetenv )
+
# ----- post jobs: cleanup
cleanup_backup_target
show_info_backup_target
- # ------ unset env
- eval $( dbdetect.unssetenv )
else
diff --git a/plugins/localdump/sqlite.sh b/plugins/localdump/sqlite.sh
index b6bbb85..02de27a 100755
--- a/plugins/localdump/sqlite.sh
+++ b/plugins/localdump/sqlite.sh
@@ -12,6 +12,7 @@
# 2018-02-02 ah,ds v1.0 first lines
# 2018-02-09 ah,ds v1.1 write a .meta file after successful backup
# 2022-03-17 v1.2 WIP: add lines with prefix __DB__
+# 2024-02-20 ah v1.3 Handle profiles; restore into existing db file; restore file owner and perms
# ================================================================================
if [ -z "$BACKUP_TARGETDIR" ]; then
@@ -23,22 +24,17 @@ fi
# CONFIG
# --------------------------------------------------------------------------------
-FILEDEFS=${DIR_JOBS}/backup-dbfiles.job
-
# --------------------------------------------------------------------------------
# FUNCTIONS
# --------------------------------------------------------------------------------
-# make sqlite3 backups of all sqlite = ... in backup-dbfiles.job
+# start multiple sqlite3 backups
+# files are taken from loaded profiles/sqlite*.ini - section [detect] -> files[]
function doSqliteBackup(){
- if ! _j_getvar ${FILEDEFS} "sqlite" | grep . ; then
- echo "__DB__$SERVICENAME SKIP: no entries found for sqlite."
- return 0
- fi
create_targetdir
- for DATABASE_FILE in $(_j_getvar ${FILEDEFS} "sqlite")
+ for DATABASE_FILE in $( dbdetect.getFiles )
do
echo -n "__DB__${SERVICENAME} backup $DATABASE_FILE "
if [ ! -f "$DATABASE_FILE" ]; then
@@ -47,24 +43,17 @@ function doSqliteBackup(){
color reset
rc=$rc+1
else
- file "$DATABASE_FILE" | cut -f 2 -d ":" | grep -i "sqlite" >/dev/null
- if [ $? -ne 0 ]; then
- color error
- echo "ERROR: given database file is not a sqlite database"
- color reset
- rc=$rc+1
- else
- TARGET=$(get_outfile ${DATABASE_FILE})
- TARGET=${BACKUP_TARGETDIR}/$(echo ${TARGET} | sed "s#/#_#g").sql
- META=${TARGET}.gz.meta
- # echo -n " to $TARGET "
- sqlite3 "$DATABASE_FILE" .dump >"${TARGET}"
- fetchrc >/dev/null
- db._compressDumpfile "${TARGET}" && echo "$DATABASE_FILE" >"${META}"
-
- ls -l ${TARGET}*
-
- fi
+
+ TARGET=$(get_outfile ${DATABASE_FILE})
+ TARGET=${BACKUP_TARGETDIR}/$(echo ${TARGET} | sed "s#/#_#g").sql
+ META=${TARGET}.gz.meta
+ # echo -n " to $TARGET "
+ sqlite3 "$DATABASE_FILE" .dump >"${TARGET}"
+ fetchrc >/dev/null
+ db._compressDumpfile "${TARGET}" && stat "$DATABASE_FILE" >"${META}"
+
+ ls -l ${TARGET}*
+
fi
done
}
@@ -73,29 +62,27 @@ function doSqliteBackup(){
# param string database dump file (gzipped)
# param string optional: database to import; default: database is parsed from file
function restoreByFile(){
- sMyfile=$1
- sMyDb=$2
+ local sMyfile=$1
+ local sMyDb=$2
if [ -f "${sMyDb}" ]; then
- color error
- echo ERROR: target file already exists. Remove or rename it first.
+ # echo -n "" > "${sMyDb}"
+ echo "Deleting ${sMyDb} before restore..."
+ rm -f "${sMyDb}"
+ fi
+
+ echo -n "Restore ${sMyfile} ... "
+ color cmd
+ zcat "${sMyfile}" | sqlite3 "${sMyDb}"
+ fetchrc
+ if [ $myrc -eq 0 ]; then
+ restorePermissions "${sMyfile}" "${sMyDb}"
ls -l "${sMyDb}"
- color reset
- rc=$rc+1
+ echo
else
- color cmd
- zcat "${sMyfile}" | sqlite3 "${sMyDb}"
- fetchrc
- if [ $myrc -eq 0 ]; then
- color ok
- echo OK, restore was successful
- color reset
- ls -l "${sMyDb}"
- else
- color error
- echo ERROR while restoring backup.
- color reset
- fi
+ color error
+ echo ERROR while restoring backup.
+ color reset
fi
}
@@ -108,12 +95,6 @@ function restoreByFile(){
# ----- check requirements
j_requireBinary "sqlite3" 1
-if [ ! -f "$FILEDEFS" ]; then
- echo "INFO: file definitions $FILEDEFS do not exist."
- rc=$rc+1
-fi
-
-
if [ $rc -ne 0 ]; then
rc=0
echo "SKIP: sqlite seems not to be here"
--
GitLab