Select Git revision
couchdb2.sh
couchdb2.sh 9.91 KiB
#!/bin/bash
# ================================================================================
#
# LOCALDUMP :: COUCHDB2 - using nodejs tools couchbackup and couchrestore
# https://github.com/cloudant/couchbackup
#
# creates gzipped plain text backups (JSON) from each scheme
#
# --------------------------------------------------------------------------------
# ah - Axel Hahn <axel.hahn@iml.unibe.ch>
# ds - Daniel Schueler <daniel.schueler@iml.unibe.ch>
#
# 2019-11-13 ..... v1.0 initial version with backup and restore (single DB)
# 2020-05-19 ..... v1.1 backup a single or multiple couchdb instances by globbing param
# ./localdump.sh backup couchdb2 demo
# 2021-10-11 ..... v1.2 added fastmode in restore: no test connect, do not
# delete DB before create request
# 2022-01-20 v1.3 fixes with shellcheck
# ================================================================================
if [ -z "$BACKUP_TARGETDIR" ]; then
echo "ERROR: you cannot start $(basename "$0") directly"
rc=$rc+1
exit 1
fi
# --------------------------------------------------------------------------------
# CONFIG
# --------------------------------------------------------------------------------
# contains *.config files for each instance
CFGDIR=~/.iml_backup/couchdb2
# UNUSED
# dirPythonPackages=/usr/lib/python2.7/site-packages
ARCHIVE_DIR=$(_j_getvar "${JOBFILE}" dir-dbarchive)/couchdb2
# --------------------------------------------------------------------------------
# FUNCTIONS
# --------------------------------------------------------------------------------
# make an couch api request
# param string method ... one of GET|POST|DELETE
# param string relative url, i.e. _all_dbs or _stats
function _couchapi(){
method=$1
apiurl=$2
outfile=$3
sParams=
# sParams="$sParams -u ${couchdbuser}:${couchdbpw}"
sParams="$sParams -X ${method}"
sParams="$sParams ${COUCH_URL}${apiurl}"
if [ ! -z "$outfile" ]; then
sParams="$sParams -o ${outfile}"
fi
curl $sParams 2>/dev/null
}
function _getDblist(){
_couchapi GET _all_dbs | sed 's#\"#\n#g' | grep -Ev "^(\[|\,|\])$" | grep -v _replicator | grep -v _global_changes
}
# get value update_seq of given couchdb name
function _getDbSeq(){
# _couchapi GET $1 | sed 's#,\"#\n"#g' | egrep -v "^(\[|\,|\])$" | grep update_seq | cut -f 4 -d '"'
_couchapi GET "$1" | sed 's#,\"#\n"#g' | grep -Ev "^(\[|\,|\])$" | grep update_seq | cut -f 4 -d '"' | cut -f 1 -d '-'
}
# ---------- CONFIG/ INSTANCES
# get valid configured instances
function getInstances(){
for mycfg in $(ls -1 ${CFGDIR}/*${1}*.config)
do
if . "$mycfg"; then
echo $(basename "${mycfg}" | cut -f 1 -d ".")
fi
done
}
# load the config of an existing instance
# see getInstances to get valid names
# param string name of the instance to load
function loadInstance(){
COUCH_URL=
if ! . "${CFGDIR}/${1}.config"; then
color error
echo ERROR: invalid instance: $1 - the config file cannot be sourced
color reset
exit 1
fi
if [ -z "${COUCH_URL}" ]; then
color error
echo "ERROR: invalid instance: $1 - the config file has no COUCH_URL"
color reset
exit 1
fi
}
# ---------- BACKUP
# backup with loop over instances
# param 1 string globbing filter to config files
function doBackup(){
# for mycfg in `ls -1 ~/.iml_backup/couchdb/*.config`
for COUCHDB_INSTANCE in $(getInstances $1)
do
loadInstance "$COUCHDB_INSTANCE"
echo "--- instance: $COUCHDB_INSTANCE"
if curl --head -X GET "$COUCH_URL" 2>/dev/null | grep "^HTTP.*\ 200\ "; then
echo OK, connected.
sleep 2
_doBackupOfSingleInstance
else
rc=$rc+1
color error
echo "ERROR: couch DB instance is not available or canot be accessed with these credentials in config file"
# repeat curl to show the error message
curl -X GET "$COUCH_URL"
color reset
fi
echo
echo "--- $(date) done."
echo
done
}
# make backup of all databases in a couchdb instance
# global: COUCH_URL
# global: COUCHDB_INSTANCE
function _doBackupOfSingleInstance(){
create_targetdir
mkdir -p "${BACKUP_TARGETDIR}/${COUCHDB_INSTANCE}" 2>/dev/null
mkdir -p "${ARCHIVE_DIR}/${COUCHDB_INSTANCE}/seq" 2>/dev/null
local ARCHIVE_DIR2="${ARCHIVE_DIR}/${COUCHDB_INSTANCE}/deleted_databases"
test -d "${ARCHIVE_DIR2}" || mkdir -p "${ARCHIVE_DIR2}" 2>/dev/null
echo
echo " MOVE deleted databases into ${ARCHIVE_DIR2}"
echo
# get a list of current databases
dblist=/tmp/couch_list_${COUCHDB_INSTANCE}.txt
_getDblist > "$dblist"
ls -l "$dblist"
# detect deleted databases:
for dumpfile in $( find "${ARCHIVE_DIR}/${COUCHDB_INSTANCE}/" -maxdepth 1 -type f -name "*.couchdbdump.gz" )
do
dbname=$( basename $dumpfile | sed "s#\.couchdbdump\.gz##g" )
if ! grep "^${dbname}" "$dblist" >/dev/null; then
SEQFILE=${ARCHIVE_DIR}/${COUCHDB_INSTANCE}/seq/__seq__${dbname}
echo "DELETED $dbname ... $( ls -l ${dumpfile} | cut -f 5- -d ' ' )"
mv "${dumpfile}" "${ARCHIVE_DIR2}"
rm -f "${SEQFILE}"
fi
done
# done | tee /tmp/couch_archive_${COUCHDB_INSTANCE}.txt
echo
typeset -i iDbTotal=$( cat "$dblist" | wc -l )
typeset -i iDb=0
echo
echo " DUMP databases of instance ${COUCHDB_INSTANCE}: $iDbTotal databases"
echo " TO BACKUP ${BACKUP_TARGETDIR}/${COUCHDB_INSTANCE}"
echo " ARCHIVE ${ARCHIVE_DIR}/${COUCHDB_INSTANCE}"
echo
for dbname in $( cat "$dblist" )
do
iDb=$iDb+1
echo -n "----- $(date) ${COUCHDB_INSTANCE} -- $iDb of $iDbTotal - ${dbname} - "
OUTFILE=${BACKUP_TARGETDIR}/${COUCHDB_INSTANCE}/$(get_outfile "${dbname}").couchdbdump
ARCHIVFILE=${ARCHIVE_DIR}/${COUCHDB_INSTANCE}/${dbname}.couchdbdump.gz
SEQFILE=${ARCHIVE_DIR}/${COUCHDB_INSTANCE}/seq/__seq__${dbname}
sSequenceCurrent=$(_getDbSeq "${dbname}")
sSequenceLast=$(cat "${SEQFILE}" 2>/dev/null | cut -f 1 -d '-')
# sSequenceLast=`cat ${SEQFILE} 2>/dev/null | tr -d '\n'`
# echo
# echo "update_seq --+-- current [${sSequenceCurrent}]"
# echo " +-- backup [${sSequenceLast}]"
if [ "${sSequenceCurrent}" = "${sSequenceLast}" ]; then
echo SKIP: still on sequence "${sSequenceLast}"
else
echo
echo "update_seq --+-- current [${sSequenceCurrent}]"
echo " +-- backup [${sSequenceLast}]"
echo -n "Need to backup ... "
couchbackup --db "${dbname}" >"${OUTFILE}".progress 2>/dev/null && mv "${OUTFILE}".progress "${OUTFILE}"
fetchrc
# $myrc is last returncode - set in fetchrc
if [ $myrc -eq 0 ]; then
echo -n "gzip ... "
compress_file "$OUTFILE"
fetchrc
if [ $myrc -eq 0 ]; then
cp "${OUTFILE}"* "${ARCHIVFILE}" && echo "${sSequenceCurrent}">"${SEQFILE}"
ls -l "${ARCHIVFILE}" "${SEQFILE}"
fi
else
echo "ERROR occured while dumping - abort"
fi
ls -l "$OUTFILE"*
echo
fi
done
rm -f "$dblist"
}
# ---------- RESTORE
# restore a single backup file; the instance and db name will be detected from file
# param string filename of db dump (full path or relative to BACKUP_TARGETDIR)
function restoreByFile(){
sMyfile=$1
sMyDb=$2
bFastMode=1
echo
h2 "analyze dump $sMyfile"
COUCHDB_INSTANCE=`echo $sMyfile | sed "s#${BACKUP_TARGETDIR}##g" | sed "s#\./##g" | sed "s#^/##g" | cut -f 1 -d "/"`
echo "detected COUCHDB_INSTANCE : [${COUCHDB_INSTANCE}]"
if [ -z $sMyDb ]; then
sMyDb=`guessDB $sMyfile`
echo "detected db schema from file: [${sMyDb}]"
else
echo "db schema from param 2: [${sMyDb}]"
fi
echo
loadInstance $COUCHDB_INSTANCE
if [ $bFastMode -eq 0 ]; then
echo connect $couchdbhost on port $couchdbport with user $couchdbuser
curl --head -X GET $COUCH_URL 2>/dev/null | grep "^HTTP.*\ 200\ " >/dev/null
if [ $? -ne 0 ]; then
color error
echo ERROR: couch DB instance is not available
curl -X GET $COUCH_URL
color reset
exit 1
fi
color ok
echo OK
color reset
fi
echo
# _getDblist | grep "^${sMyDb}$"
# if [ $? -eq 0 ]; then
# echo DB exists ... need to drop it first
# fi
if [ $bFastMode -eq 0 ]; then
h2 deleting database [$sMyDb] ...
color cmd
_couchapi DELETE $sMyDb
fetchrc
color reset
fi
h2 creating database [$sMyDb] ...
color cmd
_couchapi PUT $sMyDb
fetchrc
color reset
h2 import file ...
color cmd
# zcat ${sMyfile} | python ${dirPythonPackages}/couchdb/tools/load.py $COUCH_URL/$sMyDb
zcat ${sMyfile} | couchrestore --db $sMyDb
fetchrc
color reset
echo
}
# --------------------------------------------------------------------------------
# MAIN
# --------------------------------------------------------------------------------
# ----- check requirements
# --- is a couchd here
# j_requireProcess "couchdb" 1
# --- very specific :-/ ... check available config files
ls -1 ${CFGDIR}/* >/dev/null 2>&1
rc=$rc+$?
if [ $rc -eq 0 ]; then
echo OK: couchdb2 config was found on this system ... checking requirements for backup ...
j_requireBinary "curl" 1
j_requireBinary "couchbackup" 1
j_requireBinary "couchrestore" 1
#ls ${dirPythonPackages}/couchdb/tools/dump.py ${dirPythonPackages}/couchdb/tools/load.py >/dev/null && echo "OK: python couchdb tools were found"
#rc=$rc+$?
if [ $rc -eq 0 ]; then
echo
if [ "$1" = "restore" ]; then
echo
shift 1
restoreByFile $*
else
shift 1
# remove keyword ALL which is used for localdump.sh to loop over all db types
test "$1" = "ALL" && shift 1
doBackup $*
fi
else
color error
echo ERROR: Couchdb is here but I am missing things for the backup :-/
color reset
fi
else
rc=0
echo "SKIP: couchdb2 config does not seem to be here"
fi
echo $0 $* [couchdb2] final returncode rc=$rc
# --------------------------------------------------------------------------------