#!/bin/bash
# ======================================================================
#
# BACKUP SCRIPTS - CREATE LOCAL DUMPS
#
# ----------------------------------------------------------------------
# ah - Axel Hahn <axel.hahn@iml.unibe.ch>
# ds - Daniel Schueler <daniel.schueler@iml.unibe.ch>
#
# 2016-11-04 ah,ds created
# 2016-11-10 ..... v1.0
# 2017-03-27 ..... added first param for mode backup|restore
# 2018-02-09 ..... fix: restore-selection of target uses default on return only
# 2021-05-18 ..... move supported backup types to plugins/localdump/[service].sh
# 2021-07-13 ..... remove leading ./ in localdump.sh restore
# 2022-02-18 ..... WIP: use class like functions
# 2022-03-17 ..... WIP: add lines with prefix __DB__
# 2022-11-04 ah rename hooks
# 2024-03-14 ah v2.0: use profiles for local and remote databases
# 2024-03-18 ah fix for db detection from file and cli restore
# 2024-10-02 ah reset $rc before calling db plugin
# 2024-12-13 ah set gzip compression from 9 to 4
# ======================================================================
# --- variables:
# ARCHIVE_BASEDIR {string} base directory for db archive (couchdb2 only)
# BACKUP_BASEDIR {string} base directory for db dumps
# BACKUP_DATE {string} string with current timestamp; will be part of filename for backups
# BACKUP_KEEP_DAYS {int} count of days how long to keep db dumps below $BACKUP_BASEDIR
# BACKUP_PLUGINDIR {string} scripts for supported databases; [APP]/plugins/localdump
# BACKUP_SCRIPT {string} script name of db service
# BACKUP_TARGETDIR {string} target directory db dumps of current service
# SERVICENAME {string} name of db service (one of mysql|pgsql|...)
# ----------------------------------------------------------------------
# CONFIG VARS
# ----------------------------------------------------------------------
. $(dirname $0)/vendor/ini.class.sh || exit 1
. $(dirname $0)/vendor/color.class.sh || exit 1
. $(dirname $0)/includes/jobhelper.sh || exit 1
. $(dirname $0)/includes/inc_bash.sh || exit 1
. $(dirname $0)/includes/dbdetect.class.sh || exit 1
if [ ! -r "${JOBFILE}" ]; then
color.echo error "ERROR: missing config file ${JOBFILE}."
exit 1
fi
LOCALDUMP_LOADED=1
ARCHIVE_BASEDIR=
BACKUP_BASEDIR=
BACKUP_PLUGINDIR=
DBD_DEBUG=0
# Cleanup local dumps older N days
typeset -i BACKUP_KEEP_DAYS=0
BACKUP_DATE=
LASTINPUT=
# ----------------------------------------------------------------------
# FUNCTIONS 4 DB-WRAPPER
# ----------------------------------------------------------------------
# helpfer function for SERVICENAME.backup
# it is called after the service specific dump was done.
# param {string} filename of created dump file
function db._compressDumpfile(){
local _outfile=$1
# $myrc is last returncode - set in fetchrc
if [ $myrc -eq 0 ]; then
echo -n "gzip $_outfile ... "
gzip -4 -f "${_outfile}"
fetchrc
else
color.echo error "ERROR occured while dumping - no gzip of $_outfile"
fi
# echo -n "__DB__$SERVICENAME INFO: backup to "
# ls -l "$_outfile"* 2>&1
# echo
}
# ----------------------------------------------------------------------
# FUNCTIONS 4 BACKUP
# ----------------------------------------------------------------------
# ------------------------------------------------------------
# cleanup a backup dir: remove old files and delete empty dirs
function cleanup_backup_target(){
if [ -d "${BACKUP_TARGETDIR}" ]; then
h3 "CLEANUP ${BACKUP_TARGETDIR} older $BACKUP_KEEP_DAYS days ..."
echo find "${BACKUP_TARGETDIR}" -mtime +$BACKUP_KEEP_DAYS -delete -print
color.preset cmd
find "${BACKUP_TARGETDIR}" -mtime +$BACKUP_KEEP_DAYS -delete -print
color.reset
if [ $(find "${BACKUP_TARGETDIR}" -type f | wc -l) -eq 0 ]; then
echo "INFO: the directory is empty - deleting it"
rm -rf "${BACKUP_TARGETDIR}"
fi
fi
}
# ------------------------------------------------------------
# compress a file
# shared function in localdump_*
# param string filename of uncompressed output file
function compress_file(){
echo -n compressing $1 ...
gzip -9 -f "${1}"
fetchrc
}
# ------------------------------------------------------------
# create a backup directory with name of service
# shared function in localdump_*
function create_targetdir(){
mkdir -p "${BACKUP_TARGETDIR}" 2>/dev/null
if [ ! -d "${BACKUP_TARGETDIR}" ]; then
color.echo "error" "FATAL ERROR: directory ${BACKUP_TARGETDIR} was not created"
exit 1
fi
if ! chmod 0700 "${BACKUP_TARGETDIR}"; then
color.echo "error" "FATAL ERROR: permissions could not be set for ${BACKUP_TARGETDIR}."
exit 1
fi
}
# ------------------------------------------------------------
# generate a base filename for backup dump based on on db name
# ... and added timestamp
# param string name of database schema
# --> see listBackupedDBs() and guessDB() - these function must be able to split this
function get_outfile(){
echo $*__${BACKUP_DATE}
}
# ------------------------------------------------------------
# get name of a service script
# param string name of a service
function get_service_script(){
local _service=$1
local _type; _type=$( dbdetect.getType "$_service" )
ls -1 ${BACKUP_PLUGINDIR}/${_type}.sh 2>/dev/null
}
# ------------------------------------------------------------
# get a list of existing database profiles
function get_database_profiles(){
for config in $(dbdetect.getConfigs); do
if dbdetect.exists "$config"; then
echo "$( dbdetect.getProfile $config )"
fi
done
}
# ------------------------------------------------------------
# show directory infos with count of files and used space
# show used space and count of files and dirs
function show_info_backup_target(){
if [ -d "${BACKUP_TARGETDIR}" ]; then
h3 "INFO about backup target ${BACKUP_TARGETDIR}"
echo -n "used space: "
du -hs "${BACKUP_TARGETDIR}"
echo -n "subdirs : "
find "${BACKUP_TARGETDIR}" -type d | wc -l
echo -n "files : "
find "${BACKUP_TARGETDIR}" -type f | wc -l
echo -n "free space: "
df -h "${BACKUP_TARGETDIR}" | tail -1 | awk '{ print $4 }'
echo
fi
}
# ----------------------------------------------------------------------
# FUNCTIONS 4 RESTORE
# ----------------------------------------------------------------------
# ------------------------------------------------------------
# restore: show profiles from that exist backups
# global string BACKUP_BASEDIR base directory of all backups
function listBackupedServices(){
(
test -n "${BACKUP_BASEDIR}" && test -d "${BACKUP_BASEDIR}" \
&& find "${BACKUP_BASEDIR}" -mindepth 1 -maxdepth 1 -type d -exec basename {} \;
test -n "${ARCHIVE_BASEDIR}" && test -d "${ARCHIVE_BASEDIR}" \
&& find "${ARCHIVE_BASEDIR}" -mindepth 1 -maxdepth 1 -type d -exec basename {} \;
) | sort -u
}
# ------------------------------------------------------------
# restore: show databases or dumps of a given database that can be restored
# global string BACKUP_BASEDIR base directory of all backups of selected dbprofile
# param string optional: DB-Name for file filter to select from existing dumps;
function listBackupedDBs(){
local _filter="$1"
if [ -d "${BACKUP_TARGETDIR}" ]; then
if [ -z "$_filter" ]; then
# list all databases
find "${BACKUP_TARGETDIR}" -mindepth 1 -maxdepth 1 -type f -exec basename {} \; \
| sed -e "s#__[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]-[0-9][0-9].*##g" \
-e "s#\.couchdb\.*##g" \
-e "s#\.gz.meta\$##g" \
-e "s#\.gz\$##g" \
| sort -ud \
| sed "s#^\./##g"
else
# list dumps of a database
ls -ltr ${BACKUP_TARGETDIR}/${_filter}*gz | sed "s,${BACKUP_TARGETDIR}/,,g"
fi
else
color.echo error "ERROR: ${BACKUP_TARGETDIR} does not exist - here are no backups to restore."
echo
echo "You can try to restore dumps:"
echo "1) Restore dump files from a backup set"
echo " $(dirname $0)/restore.sh $BACKUP_BASEDIR"
echo "2) Copy restored dumps into $BACKUP_TARGETDIR"
echo "3) Start database restore again"
echo " $(dirname $0)/localdump.sh restore [profile]"
echo
exit 1
fi
}
# ------------------------------------------------------------
# guess name of the database file
# param string filename of db dump; can be full path or not
function guessDB(){
dumpfile=$1
# the metafile is written in sqlite backup to store full path
metafile=${BACKUP_TARGETDIR}/${dumpfile}.meta
if [ -f $metafile ]; then
grep "^/" "$metafile" || grep "^ File: " "$metafile" | cut -c 9-
else
sBasename=$(basename $1)
sDb=$(echo ${sBasename} | sed "s#__[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]-[0-9][0-9].*##g" | sed "s#\.couchdbdump\.gz##g" )
# ^ ^
# timestamp in backup file __/ for couchdb2 restore from archive __/
if [ -z $sDb ]; then
color.echo error "ERROR: db name was not detected from file $1"
exit 1
fi
echo $sDb
fi
}
# ------------------------------------------------------------
# show a selection + a prompt and read the input
# - If the selection is just 1 line it will be returned
# - If the user presses just return the script will exit
# param string selection of items to select from
# param string prompt to show
function showSelectAndInput(){
local _selection="$1"
local _prompt="$2"
local _lines
typeset -i _lines; _lines=$( grep -c "." <<< "$_selection" )
case $_lines in
0)
color.echo error "ERROR: No data found for a selection. Aborting."
echo
exit 1
;;
1)
echo "INFO: No interaction on a single choice. Using"
color.echo "cmd" " $_selection"
LASTINPUT="$_selection"
return 0
;;
*)
echo "$_selection"
color.print input "${_prompt} >"
read -r LASTINPUT
if [ -z "$LASTINPUT" ]; then
echo "No input given. Aborting."
exit 1
fi
;;
esac
}
# ------------------------------------------------------------
# read .meta file (that contains output of stats) and restore last owner and file permissions
# param string filename of db dump
# param string restored database file
function restorePermissions(){
local sMyMeta="${1}.meta"
local sTargetfile="$2"
if [ -f "${sMyMeta}" ]; then
# Access: (0674/-rw-rwxr--) Uid: ( 1000/ axel) Gid: ( 1000/ axel)
# ^ ^ ^
# _sPerm _sUser _sGroup
local _sPerm=$( grep "^Access: (" "${sMyMeta}" | cut -f 2 -d '(' | cut -f 1 -d '/')
if [ -n "$_sPerm" ]; then
local _sUser=$( grep "^Access: (" "${sMyMeta}" | cut -f 3 -d '(' | cut -f 1 -d '/' | tr -d ' ')
local _sGroup=$( grep "^Access: (" "${sMyMeta}" | cut -f 4 -d '(' | cut -f 1 -d '/' | tr -d ' ')
echo -n "Restoring file owner $_sUser:$_sGroup and permissions $_sPerm ... "
chown "$_sUser:$_sGroup" "${sTargetfile}" && chmod "$_sPerm" "${sTargetfile}"
fetchrc
fi
fi
}
# ------------------------------------------------------------
# show help
# ------------------------------------------------------------
function showhelp(){
local _self
_self=$( basename "$0" )
cat <<EOH
LOCALDUMP detects existing local databases and dumps them locally.
It is included in the backup.sh to dump all before a file backup will store
them. It can be started seperately for manual database backups or for restore.
SYNTAX:
$_self [OPTIONS] <operation> <profile [more_profiles]>
OPTIONS:
-h|--help show this help
PARAMETERS:"
operation - one of check|backup|restore; optional parameter
backup dump all databases/ schemes of a given service
check show info only if the service is available
restore import a dump into same or new database
Without a filename it starts an interactive mode
profile - name of database profiles
You get a list of all available services without parameter
Use ALL for bulk command
file - filename of db dump to restore to origin database scheme
EXAMPLES:
$_self backup
$_self backup ALL
Backup all databases of all found services
$_self backup mysql
Backup all Mysql databases.
$_self restore
Start interactive restore of a database of any service.
$_self restore sqlite
Start interactive restore of an sqlite database.
$_self restore <file-to-restore> [<database-name>]
Restore a given dump file to the origin database scheme or
to a new/ other database with the given name.
EOH
}
# ----------------------------------------------------------------------
# INIT
# ----------------------------------------------------------------------
while [[ "$#" -gt 0 ]]; do case $1 in
-h|--help) showhelp; exit 0;;
*) if grep "^-" <<< "$1" >/dev/null ; then
echo; color.echo error "ERROR: Unknown parameter: $1"; echo; showhelp; exit 2
fi
break;
;;
esac; done
mode=""
case "$1" in
backup|check|restore|shell)
mode=$1
shift 1
;;
esac
if [ -z "$mode" ]; then
color.echo error "ERROR: missing parameter for operation."
echo
showhelp
echo
echo "Hint: On this machine working profiles:"
get_database_profiles | nl
echo
exit 1
fi
# ----- init vars
BACKUP_BASEDIR=$(_j_getvar "${JOBFILE}" "dir-localdumps")
# check
if [ -z "$BACKUP_BASEDIR" ]; then
color.echo error "ERROR: missing config for backup target."
echo There must be an entry dir-localdumps in ${JOBFILE}
exit 1
fi
ARCHIVE_BASEDIR=$(_j_getvar "${JOBFILE}" dir-dbarchive)
BACKUP_PLUGINDIR=$(dirname $0)/plugins/localdump
DBD_BASEDIR=$BACKUP_PLUGINDIR/profiles
BACKUP_KEEP_DAYS=$(_j_getvar ${JOBFILE} "keep-days")
if [ $BACKUP_KEEP_DAYS -eq 0 ]; then
BACKUP_KEEP_DAYS=7
fi
BACKUP_DATE=$(/bin/date +%Y%m%d-%H%M)
# ----- checks
# . /usr/local/bin/inc_cronfunctions.sh
j_requireUser "root"
h1 $(date) IML BACKUP :: LOCALDUMP
export SERVICENAME=$1
# BACKUP_TARGETDIR=${BACKUP_BASEDIR}/${SERVICENAME}
# BACKUP_SCRIPT=$( get_service_script ${SERVICENAME} )
case "$mode" in
# ------------------------------------------------------------
check)
DBD_DEBUG=1
for PROFILENAME in $(dbdetect.getConfigs)
do
echo "----- $PROFILENAME"
dbdetect.exists "${PROFILENAME}"
echo
done
# . $BACKUP_SCRIPT $mode
;;
# ------------------------------------------------------------
backup)
if [ "$1" = "ALL" ] || [ -z "$1" ]; then
profiles2run=$(get_database_profiles)
echo "INFO: Calling local backup scripts for all active profiles"
echo "$profiles2run" | nl
else
profiles2run=$*
echo "INFO: I try to dump the profiles you gave as parameter: $profiles2run"
echo
fi
iProfilesFound=$( grep -c . <<< "$profiles2run" )
test "$iProfilesFound" -eq "0" && echo "INFO: No match - no database dumps needed."
typeset -i iProfileCounter=0
# ----- GO
# PROFILENAME mysql_localhost_13306
# SERVICENAME mysql
#
for PROFILENAME in $profiles2run
do
iProfileCounter+=1
if dbdetect.setProfile "${PROFILENAME}"; then
h2 "START PROFILE $iProfileCounter of $iProfilesFound [${PROFILENAME}]"
SERVICENAME=$( dbdetect.getType "$PROFILENAME" )
BACKUP_PARAMS=$( dbdetect.getParams )
BACKUP_TARGETDIR=${BACKUP_BASEDIR}/${PROFILENAME}
ARCHIVE_DIR=${ARCHIVE_BASEDIR}/${PROFILENAME}
BACKUP_SCRIPT=$( get_service_script ${SERVICENAME} )
# ------ set env
# echo "BACKUP_PARAMS = $BACKUP_PARAMS"
# dbdetect.setenv
eval $( dbdetect.setenv )
_j_runHooks "200-before-db-service"
h3 "BACKUP [${PROFILENAME}] -> ${SERVICENAME}"
rcbak=$rc
rc=0
. $BACKUP_SCRIPT $mode
test $rc -gt 0 && j_notify "db ${SERVICENAME}" "$BACKUP_SCRIPT $mode was finished with rc=$rc" $rc
_j_runHooks "230-after-db-service" "$rc"
# ------ unset env
eval $( dbdetect.unssetenv )
# ----- post jobs: cleanup
cleanup_backup_target
show_info_backup_target
rc=$rc+$rcbak
else
echo "SKIP: profile $iProfileCounter of $iProfilesFound '$PROFILENAME' "
# see why it is not active
DBD_DEBUG=1; dbdetect.setProfile "${PROFILENAME}"; echo; DBD_DEBUG=0
fi
# just to have it in the output
dbdetect.validate
done
;;
# ------------------------------------------------------------
restore)
h1 "RESTORE DATABASE"
if ! listBackupedServices | grep -q . ; then
color.echo error "ERROR: No database dump was found in [${BACKUP_BASEDIR}] nor [${ARCHIVE_BASEDIR}]."
exit 1
fi
if [ -z $1 ] || [ ! -f "$1" ]; then
parService="$1"
# ----- interactive selections
h2 "Select profile that has a dump"
if [ -z "${parService}" ]; then
showSelectAndInput "$( listBackupedServices )" "Restore for profile name"
parService="$LASTINPUT"
else
echo "Taken from command line: $parService"
fi
# ----- check if profile exists
if ! dbdetect.setProfile "${parService}"; then
color.echo error "ERROR: profile [${parService}] is not known here (or database service is stopped)."
echo
exit 1
fi
# ----- check if dump exists in archive and in backup
if [ -d "${BACKUP_BASEDIR}/${parService}" ] && [ -d "${ARCHIVE_BASEDIR}/${parService}" ]; then
echo
showSelectAndInput "$(echo "${BACKUP_BASEDIR}"; echo "${ARCHIVE_BASEDIR}")" "Select a source directory"
BACKUP_BASEDIR="$LASTINPUT"
else
# just one test needed because BACKUP_BASEDIR is BACKUP_BASEDIR
test -d "${ARCHIVE_BASEDIR}/${parService}" && BACKUP_BASEDIR="${ARCHIVE_BASEDIR}"
fi
# ----- check if target dir with profile exists
if [ ! -d "${BACKUP_BASEDIR}/${parService}" ]; then
color.echo error "ERROR: Directory does not exist '${BACKUP_BASEDIR}/${parService}'."
exit 1
fi
BACKUP_TARGETDIR="${BACKUP_BASEDIR}/${parService}"
h2 "Select a database schema"
showSelectAndInput "$(listBackupedDBs)" "Name of database to restore"
fileprefix="$LASTINPUT"
echo
h2 "Select a specific dump for that database"
showSelectAndInput "$(listBackupedDBs $fileprefix)" "Backupset to import"
dbfile="$LASTINPUT"
# if there is a single dump in backup folder:
# '-rw-r--r-- 1 root root 481 Mar 13 12:27 ahcrawler__20240313-1227.sql.gz'
# --> take the last part behind the last space to get a filename
echo "$LASTINPUT" | grep -qE "^[rwxsSt\-]{10}" \
&& dbfile="$(rev <<< \'"$LASTINPUT"\' | cut -d ' ' -f 1 | rev | sed "s#'\$##" )"
echo
sTargetDb=$(guessDB ${dbfile})
color.print input "New database name [$sTargetDb] >"
read -r sTargetDb
if [ -z "$sTargetDb" ]; then
sTargetDb=$(guessDB ${dbfile})
fi
echo
sDumpfile="${BACKUP_TARGETDIR}/${dbfile}"
else
sDumpfile=$1
sTargetDb=$2
fi
shift 2
# ----- start restore
if [ ! -f "${sDumpfile}" ]; then
color.echo error "ERROR: [${sDumpfile}] is not a file"
rc=$rc+1
else
# We expect a structure: /somedir/whatever/<PROFILENAME>/<database>.gz
dumpDirname=$( dirname "${sDumpfile}" )
PROFILENAME=$( basename "${dumpDirname}" )
echo "INFO: detected profile: $PROFILENAME"
echo ""
if dbdetect.setProfile "${PROFILENAME}"; then
SERVICENAME=$( dbdetect.getType "$PROFILENAME" )
BACKUP_TARGETDIR=${BACKUP_BASEDIR}/${PROFILENAME}
BACKUP_SCRIPT=$( get_service_script ${SERVICENAME} )
ARCHIVE_DIR=${ARCHIVE_BASEDIR}/${PROFILENAME}
BACKUP_PARAMS=$( dbdetect.getParams )
eval $( dbdetect.setenv )
. $BACKUP_SCRIPT $mode "${sDumpfile}" "${sTargetDb}"
if [ $? -ne 0 -o $rc -ne 0 ]; then
color.echo error "ERROR: $mode failed. See ouput above. :-/"
else
color.echo ok "OK, $mode was successful."
fi
# ------ unset env
eval $( dbdetect.unssetenv )
else
color.echo error "ERROR: Profile $PROFILENAME was detected but its database service is not available."
fi
fi
;;
# ------------------------------------------------------------
# shell)
# export BACKUP_TARGETDIR
# . $BACKUP_SCRIPT
# (
# mycmd=
# echo
# echo "Starting interactive shell..."
# echo
# echo "STATUS: STILL ALPHA as long existing db plugins are not rewritten."
# echo
# echo "INFO: Try ${SERVICENAME}.help to see database specific commands."
# echo "INFO: Type exit and return to leave the shell."
# echo
# while [ ! "$mycmd" = "exit" ]; do
# echo -n "[${SERVICENAME}]"
# color.print input " $( pwd )"
# echo -n " % "
# read -r mycmd
# if [ ! "$mycmd" = "exit" ];then
# color.preset cmd
# eval $mycmd
# color.reset
# fi
# done
# )
# ;;
# ----- start restore
*)
color.echo error "ERROR: unknown command [$mode]"
;;
esac
echo _______________________________________________________________________________
echo STATUS $0 exit with final returncode rc=$rc
exit $rc
# ----------------------------------------------------------------------