diff --git a/localdump_couchdb2.sh b/localdump_couchdb2.sh new file mode 100755 index 0000000000000000000000000000000000000000..cb50ff76cc111a3d232512e278fe3ecf66b49353 --- /dev/null +++ b/localdump_couchdb2.sh @@ -0,0 +1,320 @@ +#!/bin/bash +# ================================================================================ +# +# LOCALDUMP :: COUCHDB2 - using nodejs tools couchbackup and couchrestore +# https://github.com/cloudant/couchbackup +# +# creates gzipped plain text backups (JSON) from each scheme +# +# -------------------------------------------------------------------------------- +# ah - Axel Hahn <axel.hahn@iml.unibe.ch> +# ds - Daniel Schueler <daniel.schueler@iml.unibe.ch> +# +# 2019-11-13 ..... v1.0 initial version with backup and restore (single DB) +# 2020-05-19 ..... v1.1 backup a single or multiple couchdb instances by globbing param +# ./localdump.sh backup couchdb2 demo +# ================================================================================ + +if [ -z $BACKUP_TARGETDIR ]; then + echo ERROR: you cannot start `basename $0` directly + rc=$rc+1 + exit 1 +fi + +# -------------------------------------------------------------------------------- +# CONFIG +# -------------------------------------------------------------------------------- + +# contains *.config files for each instance +CFGDIR=~/.iml_backup/couchdb2 + +# UNUSED +# dirPythonPackages=/usr/lib/python2.7/site-packages + +ARCHIVE_DIR=`_j_getvar ${DIRFILE} dir-dbarchive`/couchdb2 + +# -------------------------------------------------------------------------------- +# FUNCTIONS +# -------------------------------------------------------------------------------- + +# make an couch api request +# param string method ... one of GET|POST|DELETE +# param string relative url, i.e. _all_dbs or _stats +function _couchapi(){ + method=$1 + apiurl=$2 + outfile=$3 + + sParams= + # sParams="$sParams -u ${couchdbuser}:${couchdbpw}" + sParams="$sParams -X ${method}" + sParams="$sParams ${COUCH_URL}${apiurl}" + if [ ! -z $outfile ]; then + sParams="$sParams -o ${outfile}" + fi + curl $sParams 2>/dev/null +} + +function _getDblist(){ + _couchapi GET _all_dbs | sed 's#\"#\n#g' | egrep -v "^(\[|\,|\])$" | grep -v _replicator | grep -v _global_changes +} + +# get value update_seq of given couchdb name +function _getDbSeq(){ + # _couchapi GET $1 | sed 's#,\"#\n"#g' | egrep -v "^(\[|\,|\])$" | grep update_seq | cut -f 4 -d '"' + _couchapi GET $1 | sed 's#,\"#\n"#g' | egrep -v "^(\[|\,|\])$" | grep update_seq | cut -f 4 -d '"' | cut -f 1 -d '-' +} + + +# ---------- CONFIG/ INSTANCES + +# get valid configured instances +function getInstances(){ + for mycfg in `ls -1 ${CFGDIR}/*${1}*.config` + do + . $mycfg + if [ $? -eq 0 ]; then + echo `basename "${mycfg}" | cut -f 1 -d "."` + fi + done +} + + +# load the config of an existing instance +# see getInstances to get valid names +# param string name of the instance to load +function loadInstance(){ + COUCH_URL= + . ${CFGDIR}/${1}.config + if [ $? -ne 0 ]; then + color error + echo ERROR: invalid instance: $1 - the config file cannot be sourced + color reset + exit 1 + fi + if [ -z ${COUCH_URL} ]; then + color error + echo ERROR: invalid instance: $1 - the config file has no COUCH_URL + color reset + exit 1 + fi + +} + + +# ---------- BACKUP + +# backup with loop over instances +# param 1 string globbing filter to config files +function doBackup(){ + # for mycfg in `ls -1 ~/.iml_backup/couchdb/*.config` + for COUCHDB_INSTANCE in `getInstances $1` + do + loadInstance $COUCHDB_INSTANCE + + echo --- instance: $COUCHDB_INSTANCE + curl --head -X GET $COUCH_URL 2>/dev/null | grep "^HTTP.*\ 200\ " + + if [ $? -eq 0 ]; then + echo OK, connected. + sleep 2 + _doBackupOfSingleInstance + + else + rc=$rc+1 + color error + echo ERROR: couch DB instance is not available or canot be accessed with these credentials in config file + # repeat curl to show the error message + curl -X GET $COUCH_URL + color reset + fi + + echo + echo --- `date` done. + echo + done +} + +# make backup of all databases in a couchdb instance +# global: COUCH_URL +# global: COUCHDB_INSTANCE +function _doBackupOfSingleInstance(){ + + create_targetdir + mkdir -p ${BACKUP_TARGETDIR}/${COUCHDB_INSTANCE} 2>/dev/null + mkdir -p ${ARCHIVE_DIR}/${COUCHDB_INSTANCE}/seq 2>/dev/null + typeset -i iDbTotal=`_getDblist | wc -l` + typeset -i iDb=0 + + echo + echo " DUMP databases of instance ${COUCHDB_INSTANCE}: $iDbTotal databases" + echo + + for dbname in `_getDblist` + do + iDb=$iDb+1 + echo -n "----- `date` ${COUCHDB_INSTANCE} -- $iDb of $iDbTotal - ${dbname} - " + OUTFILE=${BACKUP_TARGETDIR}/${COUCHDB_INSTANCE}/`get_outfile ${dbname}`.couchdbdump + ARCHIVFILE=${ARCHIVE_DIR}/${COUCHDB_INSTANCE}/${dbname}.couchdbdump.gz + SEQFILE=${ARCHIVE_DIR}/${COUCHDB_INSTANCE}/seq/__seq__${dbname} + + sSequenceCurrent=`_getDbSeq ${dbname}` + sSequenceLast=`cat ${SEQFILE} 2>/dev/null | cut -f 1 -d '-'` +# sSequenceLast=`cat ${SEQFILE} 2>/dev/null | tr -d '\n'` + + # echo + # echo "update_seq --+-- current [${sSequenceCurrent}]" + # echo " +-- backup [${sSequenceLast}]" + if [ ! -z ${sSequenceLast} ] || [ "${sSequenceCurrent}" = "${sSequenceLast}" ]; then + echo SKIP: still on sequence ${sSequenceLast} + else + echo + echo "update_seq --+-- current [${sSequenceCurrent}]" + echo " +-- backup [${sSequenceLast}]" + echo -n "Need to backup ... " + couchbackup --db ${dbname} >${OUTFILE}.progress 2>/dev/null && mv ${OUTFILE}.progress ${OUTFILE} + fetchrc + + # $myrc is last returncode - set in fetchrc + if [ $myrc -eq 0 ]; then + echo -n "gzip ... " + compress_file $OUTFILE + fetchrc + if [ $myrc -eq 0 ]; then + cp ${OUTFILE}* ${ARCHIVFILE} && echo ${sSequenceCurrent}>${SEQFILE} + ls -l ${ARCHIVFILE} ${SEQFILE} + fi + else + echo "ERROR occured while dumping - abort" + fi + ls -l $OUTFILE* + echo + fi + done + +} + +# ---------- RESTORE + +# restore a single backup file; the instance and db name will be detected from file +# param string filename of db dump (full path or relative to BACKUP_TARGETDIR) +function restoreByFile(){ + sMyfile=$1 + sMyDb=$2 + echo + h2 "analyze dump $sMyfile" + + COUCHDB_INSTANCE=`echo $sMyfile | sed "s#${BACKUP_TARGETDIR}##g" | sed "s#\./##g" | sed "s#^/##g" | cut -f 1 -d "/"` + echo "detected COUCHDB_INSTANCE : [${COUCHDB_INSTANCE}]" + if [ -z $sMyDb ]; then + sMyDb=`guessDB $sMyfile` + echo "detected db schema from file: [${sMyDb}]" + else + echo "db schema from param 2: [${sMyDb}]" + fi + + echo + + loadInstance $COUCHDB_INSTANCE + + echo connect $couchdbhost on port $couchdbport with user $couchdbuser + curl --head -X GET $COUCH_URL 2>/dev/null | grep "^HTTP.*\ 200\ " >/dev/null + if [ $? -ne 0 ]; then + color error + echo ERROR: couch DB instance is not available + curl -X GET $COUCH_URL + color reset + exit 1 + fi + color ok + echo OK + color reset + + + echo + + # _getDblist | grep "^${sMyDb}$" + # if [ $? -eq 0 ]; then + # echo DB exists ... need to drop it first + # fi + + h2 deleting database [$sMyDb] ... + color cmd + _couchapi DELETE $sMyDb + fetchrc + color reset + + h2 creating database [$sMyDb] ... + color cmd + _couchapi PUT $sMyDb + fetchrc + color reset + + h2 import file ... + color cmd + # zcat ${sMyfile} | python ${dirPythonPackages}/couchdb/tools/load.py $COUCH_URL/$sMyDb + zcat ${sMyfile} | couchrestore --db $sMyDb + fetchrc + color reset + echo + +} + +# -------------------------------------------------------------------------------- +# MAIN +# -------------------------------------------------------------------------------- + + +# ----- check requirements + +# --- is a couchd here +# j_requireProcess "couchdb" 1 + +# --- very specific :-/ ... check available config files +ls -1 ${CFGDIR}/* >/dev/null 2>&1 +rc=$rc+$? + + +if [ $rc -eq 0 ]; then + echo OK: couchdb2 config was found on this system ... checking requirements for backup ... + + j_requireBinary "curl" 1 + j_requireBinary "couchbackup" 1 + j_requireBinary "couchrestore" 1 + + #ls ${dirPythonPackages}/couchdb/tools/dump.py ${dirPythonPackages}/couchdb/tools/load.py >/dev/null && echo "OK: python couchdb tools were found" + #rc=$rc+$? + + + if [ $rc -eq 0 ]; then + echo + + if [ "$1" = "restore" ]; then + echo + shift 1 + restoreByFile $* + + else + shift 1 + + # remove keyword ALL which is used for localdump.sh to loop over all db types + test "$1" = "ALL" && shift 1 + + doBackup $* + fi + + else + color error + echo ERROR: Couchdb is here but I am missing things for the backup :-/ + color reset + fi + +else + rc=0 + echo "SKIP: couchdb2 config does not seem to be here" +fi + + +echo $0 $* [couchdb2] final returncode rc=$rc + +# --------------------------------------------------------------------------------