add localdump_couchdb2.sh for couchdb 2 + 3 backups

localdump_couchdb2.sh

+#!/bin/bash
+# ================================================================================
+#
+# LOCALDUMP :: COUCHDB2 - using nodejs tools couchbackup and couchrestore
+# https://github.com/cloudant/couchbackup
+#
+# creates gzipped plain text backups (JSON) from each scheme
+#
+# --------------------------------------------------------------------------------
+# ah - Axel Hahn <axel.hahn@iml.unibe.ch>
+# ds - Daniel Schueler <daniel.schueler@iml.unibe.ch>
+#
+# 2019-11-13  .....  v1.0  initial version with backup and restore (single DB)
+# 2020-05-19  .....  v1.1  backup a single or multiple couchdb instances by globbing param
+#                          ./localdump.sh backup couchdb2 demo
+# ================================================================================
+
+if [ -z $BACKUP_TARGETDIR ]; then
+  echo ERROR: you cannot start `basename $0` directly
+  rc=$rc+1
+  exit 1
+fi
+
+# --------------------------------------------------------------------------------
+# CONFIG
+# --------------------------------------------------------------------------------
+
+# contains *.config files for each instance
+CFGDIR=~/.iml_backup/couchdb2
+
+# UNUSED
+# dirPythonPackages=/usr/lib/python2.7/site-packages
+
+ARCHIVE_DIR=`_j_getvar ${DIRFILE} dir-dbarchive`/couchdb2
+
+# --------------------------------------------------------------------------------
+# FUNCTIONS
+# --------------------------------------------------------------------------------
+
+# make an couch api request
+# param  string  method ... one of GET|POST|DELETE
+# param  string  relative url, i.e. _all_dbs or _stats
+function _couchapi(){
+  method=$1
+  apiurl=$2
+  outfile=$3
+
+  sParams=
+  # sParams="$sParams -u ${couchdbuser}:${couchdbpw}"
+  sParams="$sParams -X ${method}"
+  sParams="$sParams ${COUCH_URL}${apiurl}"
+  if [ ! -z $outfile ]; then
+    sParams="$sParams -o ${outfile}"
+  fi
+  curl $sParams 2>/dev/null
+}
+
+function _getDblist(){
+   _couchapi GET _all_dbs | sed 's#\"#\n#g' | egrep -v "^(\[|\,|\])$" | grep -v _replicator | grep -v _global_changes
+}
+
+# get value update_seq of given couchdb name
+function _getDbSeq(){
+  # _couchapi GET $1 | sed 's#,\"#\n"#g' | egrep -v "^(\[|\,|\])$" | grep update_seq | cut -f 4 -d '"'
+  _couchapi GET $1 | sed 's#,\"#\n"#g' | egrep -v "^(\[|\,|\])$" | grep update_seq | cut -f 4 -d '"' | cut -f 1 -d '-'
+}
+
+
+# ---------- CONFIG/ INSTANCES
+
+# get valid configured instances
+function getInstances(){
+ for mycfg in `ls -1 ${CFGDIR}/*${1}*.config`
+ do
+   . $mycfg
+   if [ $? -eq 0 ]; then
+     echo `basename "${mycfg}" | cut -f 1 -d "."`
+   fi
+ done
+}
+
+
+# load the config of an existing instance
+# see getInstances to get valid names
+# param  string  name of the instance to load
+function loadInstance(){
+  COUCH_URL=
+  . ${CFGDIR}/${1}.config
+  if [ $? -ne 0 ]; then
+    color error
+    echo ERROR: invalid instance: $1 - the config file cannot be sourced
+    color reset
+    exit 1
+  fi
+  if [ -z ${COUCH_URL} ]; then
+    color error
+    echo ERROR: invalid instance: $1 - the config file has no COUCH_URL
+    color reset
+    exit 1
+  fi
+
+}
+
+
+# ---------- BACKUP
+
+# backup with loop over instances
+# param 1  string  globbing filter to config files
+function doBackup(){
+  # for mycfg in `ls -1 ~/.iml_backup/couchdb/*.config`
+  for COUCHDB_INSTANCE in `getInstances $1`
+  do
+    loadInstance $COUCHDB_INSTANCE
+
+      echo --- instance: $COUCHDB_INSTANCE
+      curl --head -X GET $COUCH_URL 2>/dev/null | grep "^HTTP.*\ 200\ "
+
+      if [ $? -eq 0 ]; then
+        echo OK, connected.
+        sleep 2
+        _doBackupOfSingleInstance
+
+      else
+        rc=$rc+1
+        color error
+        echo ERROR: couch DB instance is not available or canot be accessed with these credentials in config file
+        # repeat curl to show the error message
+        curl -X GET $COUCH_URL
+        color reset
+      fi
+
+    echo
+    echo --- `date` done.
+    echo
+  done
+}
+
+# make backup of all databases in a couchdb instance
+# global: COUCH_URL
+# global: COUCHDB_INSTANCE
+function _doBackupOfSingleInstance(){
+
+  create_targetdir
+  mkdir -p ${BACKUP_TARGETDIR}/${COUCHDB_INSTANCE} 2>/dev/null
+  mkdir -p ${ARCHIVE_DIR}/${COUCHDB_INSTANCE}/seq 2>/dev/null
+  typeset -i iDbTotal=`_getDblist | wc -l`
+  typeset -i iDb=0
+
+  echo
+  echo "    DUMP databases of instance ${COUCHDB_INSTANCE}: $iDbTotal databases"
+  echo
+
+  for dbname in `_getDblist`
+  do
+    iDb=$iDb+1
+    echo -n "----- `date` ${COUCHDB_INSTANCE} -- $iDb of $iDbTotal - ${dbname} - "
+    OUTFILE=${BACKUP_TARGETDIR}/${COUCHDB_INSTANCE}/`get_outfile ${dbname}`.couchdbdump
+    ARCHIVFILE=${ARCHIVE_DIR}/${COUCHDB_INSTANCE}/${dbname}.couchdbdump.gz
+    SEQFILE=${ARCHIVE_DIR}/${COUCHDB_INSTANCE}/seq/__seq__${dbname}
+
+    sSequenceCurrent=`_getDbSeq ${dbname}`
+    sSequenceLast=`cat ${SEQFILE} 2>/dev/null | cut -f 1 -d '-'`
+#    sSequenceLast=`cat ${SEQFILE} 2>/dev/null | tr -d '\n'`
+
+    # echo
+    # echo "update_seq --+-- current [${sSequenceCurrent}]" 
+    # echo "             +-- backup  [${sSequenceLast}]"
+    if [ ! -z ${sSequenceLast} ] || [ "${sSequenceCurrent}" = "${sSequenceLast}" ]; then
+      echo SKIP: still on sequence ${sSequenceLast}
+    else
+      echo
+      echo "update_seq --+-- current [${sSequenceCurrent}]" 
+      echo "             +-- backup  [${sSequenceLast}]"
+      echo -n "Need to backup ... "
+      couchbackup --db ${dbname} >${OUTFILE}.progress 2>/dev/null && mv ${OUTFILE}.progress ${OUTFILE}
+      fetchrc
+
+      # $myrc is last returncode - set in fetchrc
+      if [ $myrc -eq 0 ]; then
+        echo -n "gzip ... "
+        compress_file $OUTFILE
+        fetchrc
+        if [ $myrc -eq 0 ]; then
+          cp ${OUTFILE}* ${ARCHIVFILE} && echo ${sSequenceCurrent}>${SEQFILE}
+          ls -l ${ARCHIVFILE} ${SEQFILE}
+        fi
+      else
+        echo "ERROR occured while dumping - abort"
+      fi
+      ls -l $OUTFILE*
+      echo
+    fi
+  done
+
+}
+
+# ---------- RESTORE
+
+# restore a single backup file; the instance and db name will be detected from file
+# param  string  filename of db dump (full path or relative to BACKUP_TARGETDIR)
+function restoreByFile(){
+  sMyfile=$1
+  sMyDb=$2
+  echo
+  h2 "analyze dump $sMyfile"
+
+  COUCHDB_INSTANCE=`echo $sMyfile | sed "s#${BACKUP_TARGETDIR}##g" | sed "s#\./##g" | sed "s#^/##g" | cut -f 1 -d "/"`
+  echo "detected COUCHDB_INSTANCE   : [${COUCHDB_INSTANCE}]"
+  if [ -z $sMyDb ]; then
+    sMyDb=`guessDB $sMyfile`
+    echo "detected db schema from file: [${sMyDb}]"
+  else
+    echo "db schema from param 2: [${sMyDb}]"
+  fi
+
+  echo
+
+  loadInstance $COUCHDB_INSTANCE
+
+  echo connect $couchdbhost on port $couchdbport with user $couchdbuser
+  curl --head -X GET $COUCH_URL 2>/dev/null | grep "^HTTP.*\ 200\ " >/dev/null
+  if [ $? -ne 0 ]; then
+    color error
+    echo ERROR: couch DB instance is not available
+    curl -X GET $COUCH_URL
+    color reset
+    exit 1
+  fi
+  color ok
+  echo OK
+  color reset
+
+
+  echo
+
+  # _getDblist | grep "^${sMyDb}$"
+  # if [ $? -eq 0 ]; then
+  #   echo DB exists ... need to drop it first
+  # fi
+
+  h2 deleting database [$sMyDb] ...
+  color cmd
+  _couchapi DELETE $sMyDb
+  fetchrc
+  color reset
+
+  h2 creating database [$sMyDb] ...
+  color cmd
+  _couchapi PUT $sMyDb
+  fetchrc
+  color reset
+
+  h2 import file ...
+  color cmd
+  # zcat ${sMyfile} | python ${dirPythonPackages}/couchdb/tools/load.py $COUCH_URL/$sMyDb
+  zcat ${sMyfile} | couchrestore --db $sMyDb
+  fetchrc
+  color reset
+  echo
+
+}
+
+# --------------------------------------------------------------------------------
+# MAIN
+# --------------------------------------------------------------------------------
+
+
+# ----- check requirements
+
+# --- is a couchd here
+# j_requireProcess "couchdb"   1
+
+# --- very specific :-/ ... check available config files
+ls -1 ${CFGDIR}/* >/dev/null 2>&1
+rc=$rc+$?
+
+
+if [ $rc -eq 0 ]; then
+  echo OK: couchdb2 config was found on this system ... checking requirements for backup ...
+
+  j_requireBinary  "curl"         1
+  j_requireBinary  "couchbackup"  1
+  j_requireBinary  "couchrestore" 1
+
+  #ls ${dirPythonPackages}/couchdb/tools/dump.py ${dirPythonPackages}/couchdb/tools/load.py >/dev/null && echo "OK: python couchdb tools were found"
+  #rc=$rc+$?
+
+
+  if [ $rc -eq 0 ]; then
+    echo
+
+    if [ "$1" = "restore" ]; then
+      echo
+      shift 1
+      restoreByFile $*
+
+    else
+      shift 1
+
+      # remove keyword ALL which is used for localdump.sh to loop over all db types
+      test "$1" = "ALL" && shift 1
+
+      doBackup $*
+    fi
+
+  else
+    color error
+    echo ERROR: Couchdb is here but I am missing things for the backup :-/
+    color reset
+  fi
+
+else
+  rc=0
+  echo "SKIP: couchdb2 config does not seem to be here"
+fi
+
+
+echo $0 $* [couchdb2] final returncode rc=$rc
+
+# --------------------------------------------------------------------------------