Skip to content
Snippets Groups Projects
Commit 4407a486 authored by Hahn Axel (hahn)'s avatar Hahn Axel (hahn)
Browse files

remove bak file

parent 5bfa355f
No related branches found
No related tags found
1 merge request!141remove bak file
Hide whitespace changes
Inline Side-by-side
plugins/localdump/couchdb2.sh_bak deleted 100755 → 0
+ 0
− 428
View file @ 5bfa355f
#!/bin/bash
# ================================================================================
#
# LOCALDUMP :: COUCHDB2 - using nodejs tools couchbackup and couchrestore
# https://github.com/cloudant/couchbackup
#
# Backup:
# - creates gzipped plain text backups (JSON) from each scheme
# - write sequence id into a text file
# - store extra file with security infos
# - latest backup set is written to archive
#
# --------------------------------------------------------------------------------
# ah - Axel Hahn <axel.hahn@iml.unibe.ch>
# ds - Daniel Schueler <daniel.schueler@iml.unibe.ch>
#
# 2019-11-13 ..... v1.0 initial version with backup and restore (single DB)
# 2020-05-19 ..... v1.1 backup a single or multiple couchdb instances by globbing param
# ./localdump.sh backup couchdb2 demo
# 2021-10-11 ..... v1.2 added fastmode in restore: no test connect, do not
# delete DB before create request
# 2022-01-20 v1.3 fixes with shellcheck
# 2022-03-17 v1.4 WIP: add lines with prefix __DB__
# 2022-04-07 v1.5 check archive file, not only seq file
# 2022-04-14 v1.6 backup security infos (no restore yet)
# 2022-04-21 v1.7 restore security infos
# 2022-10-07 ah v1.8 unescape regex with space to prevent "grep: warning: stray \ before white space"
# 2023-06-06 ah v1.9 show a warning if the sequence id was not fetched
# 2023-06-12 ah v1.10 skip couchdb dump if no sequence id was detected (=db deleted since fetching list of all dbs)
# ================================================================================
if [ -z "$BACKUP_TARGETDIR" ]; then
echo "ERROR: you cannot start $(basename "$0") directly"
rc=$rc+1
exit 1
fi
# --------------------------------------------------------------------------------
# CONFIG
# --------------------------------------------------------------------------------
# contains *.config files for each instance
CFGDIR=~/.iml_backup/couchdb2
# UNUSED
# dirPythonPackages=/usr/lib/python2.7/site-packages
ARCHIVE_DIR=$(_j_getvar "${JOBFILE}" dir-dbarchive)/couchdb2
# --------------------------------------------------------------------------------
# FUNCTIONS
# --------------------------------------------------------------------------------
# make an couch api request
# param string method ... one of GET|POST|DELETE
# param string relative url, i.e. _all_dbs or _stats
# param string optional: data for POST|PUT requests
function _couchapi(){
local method=$1
local apiurl=$2
# local outfile=$3
local data=$3
sParams=
# sParams="$sParams -u ${couchdbuser}:${couchdbpw}"
sParams="$sParams -X ${method}"
sParams="$sParams ${COUCH_URL}${apiurl}"
# if [ ! -z "$outfile" ]; then
# sParams="$sParams -o ${outfile}"
# fi
if [ -n "$data" ]; then
sParams="$sParams -d ${data}"
fi
curl $sParams 2>/dev/null
}
function _getDblist(){
_couchapi GET _all_dbs | sed 's#\"#\n#g' | grep -Ev "^(\[|\,|\])$" | grep -v _replicator | grep -v _global_changes
}
# get value update_seq of given couchdb name
function _getDbSeq(){
# _couchapi GET $1 | sed 's#,\"#\n"#g' | egrep -v "^(\[|\,|\])$" | grep update_seq | cut -f 4 -d '"'
_couchapi GET "$1" | sed 's#,\"#\n"#g' | grep -Ev "^(\[|\,|\])$" | grep update_seq | cut -f 4 -d '"' | cut -f 1 -d '-'
}
# ---------- CONFIG/ INSTANCES
# get valid configured instances
function getInstances(){
for mycfg in $(ls -1 ${CFGDIR}/*${1}*.config)
do
if . "$mycfg"; then
echo $(basename "${mycfg}" | cut -f 1 -d ".")
fi
done
}
# load the config of an existing instance
# see getInstances to get valid names
# param string name of the instance to load
function loadInstance(){
COUCH_URL=
if ! . "${CFGDIR}/${1}.config"; then
color error
echo ERROR: invalid instance: $1 - the config file cannot be sourced
color reset
exit 1
fi
if [ -z "${COUCH_URL}" ]; then
color error
echo "ERROR: invalid instance: $1 - the config file has no COUCH_URL"
color reset
exit 1
fi
}
# ---------- BACKUP
# backup with loop over instances
# param 1 string globbing filter to config files
function doBackup(){
# for mycfg in `ls -1 ~/.iml_backup/couchdb/*.config`
for COUCHDB_INSTANCE in $(getInstances $1)
do
loadInstance "$COUCHDB_INSTANCE"
echo "--- instance: $COUCHDB_INSTANCE"
if curl --head -X GET "$COUCH_URL" 2>/dev/null | grep "^HTTP.* 200 "; then
echo OK, connected.
sleep 2
_doBackupOfSingleInstance
else
rc=$rc+1
color error
echo "ERROR: couch DB instance is not available or canot be accessed with these credentials in config file"
# repeat curl to show the error message
curl -X GET "$COUCH_URL"
color reset
fi
echo
echo "--- $(date) done."
echo
done
}
# make backup of all databases in a couchdb instance
# global: COUCH_URL
# global: COUCHDB_INSTANCE
function _doBackupOfSingleInstance(){
create_targetdir
local ARCHIVE_DIR2="${ARCHIVE_DIR}/${COUCHDB_INSTANCE}/deleted_databases"
for _dir in "${BACKUP_TARGETDIR}/${COUCHDB_INSTANCE}" "${ARCHIVE_DIR}/${COUCHDB_INSTANCE}/seq" "${ARCHIVE_DIR}/${COUCHDB_INSTANCE}/security" \
"${ARCHIVE_DIR2}" "${ARCHIVE_DIR2}/seq" "${ARCHIVE_DIR2}/security"
do
test -d "$_dir" || (echo "creating $_dir" ; mkdir -p "$_dir" )
done
echo
echo " MOVE deleted databases into ${ARCHIVE_DIR2}"
echo
# get a list of current databases
dblist=/tmp/couch_list_${COUCHDB_INSTANCE}.txt
_getDblist > "$dblist"
ls -l "$dblist"
# detect deleted databases:
for dumpfile in $( find "${ARCHIVE_DIR}/${COUCHDB_INSTANCE}/" -maxdepth 1 -type f -name "*.couchdbdump.gz" )
do
dbname=$( basename $dumpfile | sed "s#\.couchdbdump\.gz##g" )
if ! grep "^${dbname}" "$dblist" >/dev/null; then
SEQFILE=${ARCHIVE_DIR}/${COUCHDB_INSTANCE}/seq/__seq__${dbname}
SECURITYFILE=${ARCHIVE_DIR}/${COUCHDB_INSTANCE}/security/__security__${dbname}.json
echo "DELETED $dbname ... $( ls -l ${dumpfile} | cut -f 5- -d ' ' )"
mv "${dumpfile}" "${ARCHIVE_DIR2}"
mv "${SEQFILE}" "${ARCHIVE_DIR2}/seq/"
mv "${SECURITYFILE}" "${ARCHIVE_DIR2}/security/"
fi
done
# done | tee /tmp/couch_archive_${COUCHDB_INSTANCE}.txt
echo
typeset -i iDbTotal=$( cat "$dblist" | wc -l )
typeset -i iDb=0
typeset -i iDbCount=0
echo
echo " DUMP databases of instance ${COUCHDB_INSTANCE}: $iDbTotal databases"
echo " TO BACKUP ${BACKUP_TARGETDIR}/${COUCHDB_INSTANCE}"
echo " ARCHIVE ${ARCHIVE_DIR}/${COUCHDB_INSTANCE}"
echo
for dbname in $( cat "$dblist" )
do
iDb=$iDb+1
echo -n "----- $(date) ${COUCHDB_INSTANCE} -- $iDb of $iDbTotal - ${dbname} - "
OUTFILE=${BACKUP_TARGETDIR}/${COUCHDB_INSTANCE}/$(get_outfile "${dbname}").couchdbdump
ARCHIVFILE=${ARCHIVE_DIR}/${COUCHDB_INSTANCE}/${dbname}.couchdbdump.gz
SEQFILE=${ARCHIVE_DIR}/${COUCHDB_INSTANCE}/seq/__seq__${dbname}
SECURITYFILE=${ARCHIVE_DIR}/${COUCHDB_INSTANCE}/security/__security__${dbname}.json
sSequenceCurrent=$(_getDbSeq "${dbname}")
sSequenceLast=$(cat "${SEQFILE}" 2>/dev/null | cut -f 1 -d '-')
# sSequenceLast=`cat ${SEQFILE} 2>/dev/null | tr -d '\n'`
# echo
# echo "update_seq --+-- current [${sSequenceCurrent}]"
# echo " +-- backup [${sSequenceLast}]"
if [ "${sSequenceCurrent}" = "${sSequenceLast}" ] && [ -f "$ARCHIVFILE" ]; then
echo "SKIP: still on sequence ${sSequenceLast}"
# add security file for already existing databases
test -f "${SECURITYFILE}" || (
echo "INFO: creating missing security file ${SECURITYFILE}"
_couchapi GET "${dbname}/_security" > "${SECURITYFILE}"
)
else
if [ -z "$sSequenceCurrent" ]; then
echo "WARNING: unable to fetch current sequence ID - maybe the database was deleted."
else
echo
echo "update_seq --+-- current [${sSequenceCurrent}]"
echo " +-- backup [${sSequenceLast}]"
echo -n "Need to backup ... "
couchbackup --db "${dbname}" >"${OUTFILE}".progress 2>/dev/null && mv "${OUTFILE}".progress "${OUTFILE}"
fetchrc
# $myrc is last returncode - set in fetchrc
if [ $myrc -eq 0 ]; then
echo -n "gzip ... "
compress_file "$OUTFILE"
fetchrc
if [ $myrc -eq 0 ]; then
iDbCount+=1
cp "${OUTFILE}"* "${ARCHIVFILE}" \
&& echo "${sSequenceCurrent}">"${SEQFILE}" \
&& _couchapi GET "${dbname}/_security" > "${SECURITYFILE}"
ls -l "${ARCHIVFILE}" "${SEQFILE}" "${SECURITYFILE}"
fi
else
echo "ERROR occured while dumping - abort"
fi
ls -l "$OUTFILE"*
echo
fi # if [ -z "$sSequenceCurrent" ]; then
fi # if [ "${sSequenceCurrent}" = "${sSequenceLast}" ] ...
done
rm -f "$dblist"
echo "__DB__$SERVICENAME backup INFO: ${COUCHDB_INSTANCE} - backed up $iDbCount dbs of $iDbTotal total"
}
# ---------- RESTORE
#
# example:
#
# (1)
# cd /var/iml-archive/couchdb2
# or
# cd /var/iml-backup/couchdb2
#
# (2)
# /opt/imlbackup/client/localdump.sh restore couchdb2 measured-preview-couchdbcluster/mydb.couchdbdump.gz axel-01
# ^ ^ ^ ^
# | | | |
# action: restore ---------------+ | | |
# database service: couchdb2 ------------+ | |
# filename with instance as relative path --------+ |
# optional: target database --------------------------------------------------------------------------+
#
# restore a single backup file; the instance and db name will be detected from file
# param string filename of db dump (full path or relative to BACKUP_TARGETDIR)
# param string optional: target database; default: detect name from import database
function restoreByFile(){
sMyfile=$1
dbname=$2
bFastMode=0 # 0 = delete db first and import | 1 = create and import (on empty instance only)
echo
h2 "analyze dump $sMyfile"
COUCHDB_INSTANCE=$(echo $sMyfile | sed "s#${BACKUP_TARGETDIR}##g" | sed "s#\./##g" | sed "s#^/##g" | cut -f 1 -d "/")
echo "detected COUCHDB_INSTANCE : [${COUCHDB_INSTANCE}]"
if [ -z "$COUCHDB_INSTANCE" ]; then
echo "ERROR: Name of the instance was not detected."
echo " For couchdb restore you should cd to the ${BACKUP_TARGETDIR} or ${ARCHIVE_DIR}"
exit 1
fi
local _sourceDB="$( guessDB $sMyfile | sed 's#.couchdbdump.gz$##' )"
echo "detected source database : [${_sourceDB}]"
if [ -z "$dbname" ]; then
dbname="$_sourceDB"
echo "using the same as target : [${dbname}]"
else
echo "using db schema from param 2: [${dbname}]"
fi
echo
loadInstance $COUCHDB_INSTANCE
if [ $bFastMode -eq 0 ]; then
echo connect $couchdbhost on port $couchdbport with user $couchdbuser
curl --head -X GET $COUCH_URL 2>/dev/null | grep "^HTTP.* 200 " >/dev/null
if [ $? -ne 0 ]; then
color error
echo ERROR: couch DB instance is not available
curl -X GET $COUCH_URL
color reset
exit 1
fi
color ok
echo OK
color reset
fi
echo
# _getDblist | grep "^${dbname}$"
# if [ $? -eq 0 ]; then
# echo DB exists ... need to drop it first
# fi
if [ $bFastMode -eq 0 ]; then
h2 deleting database [$dbname] ...
color cmd
_couchapi DELETE $dbname
fetchrc
color reset
fi
h2 creating database [$dbname] ...
color cmd
_couchapi PUT $dbname
fetchrc
color reset
h2 import file ...
color cmd
zcat ${sMyfile} | couchrestore --db $dbname
fetchrc
color reset
h2 add security infos ...
# todo: this will fail when restoring from "deleted_databases" folder
SECURITYFILE=${ARCHIVE_DIR}/${COUCHDB_INSTANCE}/security/__security__${_sourceDB}.json
SECDATA="$( cat $SECURITYFILE )"
color cmd
echo "add security data: $SECDATA"
_couchapi PUT "${dbname}/_security" "$SECDATA"
fetchrc
color reset
echo
}
# --------------------------------------------------------------------------------
# MAIN
# --------------------------------------------------------------------------------
# ----- check requirements
# --- is a couchd here
# j_requireProcess "couchdb" 1
# --- very specific :-/ ... check available config files
ls -1 ${CFGDIR}/* >/dev/null 2>&1
rc=$rc+$?
if [ $rc -eq 0 ]; then
echo OK: couchdb2 config was found on this system ... checking requirements for backup ...
j_requireBinary "curl" 1
j_requireBinary "couchbackup" 1
j_requireBinary "couchrestore" 1
#ls ${dirPythonPackages}/couchdb/tools/dump.py ${dirPythonPackages}/couchdb/tools/load.py >/dev/null && echo "OK: python couchdb tools were found"
#rc=$rc+$?
if [ $rc -eq 0 ]; then
echo
if [ "$1" = "restore" ]; then
echo
shift 1
restoreByFile $*
else
shift 1
# remove keyword ALL which is used for localdump.sh to loop over all db types
test "$1" = "ALL" && shift 1
doBackup $*
fi
else
color error
echo ERROR: Couchdb is here but I am missing things for the backup :-/
color reset
fi
else
rc=0
echo "__DB__$SERVICENAME SKIP: couchdb2 config does not seem to be here"
fi
echo "__DB__$SERVICENAME INFO: $0 $* [$SERVICENAME] final returncode rc=$rc"
# --------------------------------------------------------------------------------
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment