couchdb.sh

#!/bin/bash
# ================================================================================
#
# LOCALDUMP :: COUCHDB
# create gzipped plain text backups from each scheme
#
# --------------------------------------------------------------------------------
# ah - Axel Hahn <axel.hahn@iml.unibe.ch>
# ds - Daniel Schueler <daniel.schueler@iml.unibe.ch>
#
# 2017-03-24  ah,ds  v0.9  backup
# 2017-03-27  .....  v1.0  restore
# 2022-01-20         v1.1  fixes with shellcheck
# 2022-03-17         v1.2  WIP: add lines with prefix __DB__
# 2022-10-07  ah     v1.3  unescape regex with space to prevent "grep: warning: stray \ before white space"
# ================================================================================

if [ -z "$BACKUP_TARGETDIR" ]; then
  echo ERROR: you cannot start $(basename $0) directly
  rc=$rc+1
  exit 1
fi

# --------------------------------------------------------------------------------
# CONFIG
# --------------------------------------------------------------------------------

dirPythonPackages=/usr/lib/python2.7/site-packages


# --------------------------------------------------------------------------------
# FUNCTIONS
# --------------------------------------------------------------------------------

# make an couch api request
# param  string  method ... one of GET|POST|DELETE
# param  string  relative url, i.e. _all_dbs or _stats
function _couchapi(){
  method=$1
  apiurl=$2
  outfile=$3

  sParams=
  # sParams="$sParams -u ${couchdbuser}:${couchdbpw}"
  sParams="$sParams -X ${method}"
  sParams="$sParams ${COUCHDB_URL}${apiurl}"
  if [ ! -z "$outfile" ]; then
    sParams="$sParams -o ${outfile}"
  fi
  curl $sParams 2>/dev/null
}

function _couchGet(){
  _couchapi GET $*
}

function _getDblist(){
   _couchGet _all_dbs | sed 's#\"#\n#g' | grep -Ev "^(\[|\,|\])$"
}


# ---------- CONFIG/ INSTANCES

# get valid configured instances
function getInstances(){
 for mycfg in $(ls -1 "~/.iml_backup/couchdb/*.config" )
 do
   if . "$mycfg"; then
     echo $(basename "${mycfg}" | cut -f 1 -d ".")
   fi
 done
}


# load the config of an existing instance
# see getInstances to get valid names
# param  string  name of the instance to load
function loadInstance(){
  COUCHDB_URL=
  if ! . "~/.iml_backup/couchdb/${1}.config" ; then
    color error
    echo ERROR: invalid instance: $1 - the config file cannot be sourced
    color reset
    exit 1
  fi
  if [ -z "${COUCHDB_URL}" ]; then
    color error
    echo "ERROR: invalid instance: $1 - the config file has no COUCHDB_URL"
    color reset
    exit 1
  fi

  # parse ${COUCHDB_URL} ...
  couchdbhost=$(echo "${COUCHDB_URL}" | cut -f 3 -d "/" | cut -f 2 -d "@" | cut -f 1 -d ":")
  couchdbport=$(echo "${COUCHDB_URL}" | cut -f 3 -d "/" | cut -f 2 -d "@" | cut -f 2 -d ":")
  couchdbuser=$(echo "${COUCHDB_URL}" | cut -f 3 -d "/" | cut -f 1 -d "@" | cut -f 1 -d ":")
  couchdbpw=$(  echo "${COUCHDB_URL}" | cut -f 3 -d "/" | cut -f 1 -d "@" | cut -f 2 -d ":")

}


# ---------- BACKUP

# backup with loop over instances
function doBackup(){
  # for mycfg in `ls -1 ~/.iml_backup/couchdb/*.config`
  for COUCHDB_INSTANCE in $(getInstances)
  do
    loadInstance "$COUCHDB_INSTANCE"

      echo "--- instance: $COUCHDB_INSTANCE"
      if curl --head -X GET "$COUCHDB_URL" 2>/dev/null | grep "^HTTP.* 200 "; then

        _doBackupOfSingleInstance

      else
        rc=$rc+1
        color error
        echo ERROR: couch DB instance is not available or canot be accessed with these credentials in config file
        # repeat curl to show the error message
        curl -X GET "$COUCHDB_URL"
        color reset
      fi

    echo
    echo "--- $(date) done."
    echo
  done
}

# make backup of all databases in a couchdb instance
# global: COUCHDB_URL
# global: COUCHDB_INSTANCE
function _doBackupOfSingleInstance(){

  create_targetdir
  mkdir -p "${BACKUP_TARGETDIR}/${COUCHDB_INSTANCE}" 2>/dev/null

  echo
  echo "    DUMP databases of instance ${COUCHDB_INSTANCE}"
  echo "    couchdbhost $couchdbhost on port $couchdbport with user $couchdbuser"
  echo

  for dbname in $(_getDblist)
  do
    echo -n "__DB__${SERVICENAME} ${COUCHDB_INSTANCE} $_dbname backup "
    OUTFILE=${BACKUP_TARGETDIR}/${COUCHDB_INSTANCE}/$(get_outfile "${dbname}").couchdbdump
    python ${dirPythonPackages}/couchdb/tools/dump.py "${COUCHDB_URL}/${dbname}" >"${OUTFILE}"

    fetchrc
    db._compressDumpfile "${OUTFILE}"

    # $myrc is last returncode - set in fetchrc
    # if [ $myrc -eq 0 ]; then
    #   echo -n "gzip ... "
    #   compress_file "$OUTFILE"
    # else
    #   echo "ERROR occured - no gzip"
    # fi
    # ls -l "$OUTFILE"*
    # echo
  done

}

# ---------- RESTORE

# restore a single backup file; the instance and db name will be detected from file
# param  string  filename of db dump (full path or relative to BACKUP_TARGETDIR)
function restoreByFile(){
  sMyfile=$1
  sMyDb=$2
  echo
  h2 "analyze dump $sMyfile"

  COUCHDB_INSTANCE=$(echo "$sMyfile" | sed "s#${BACKUP_TARGETDIR}##g" | sed "s#\./##g" | sed "s#^/##g" | cut -f 1 -d "/")
  echo "detected COUCHDB_INSTANCE   : [${COUCHDB_INSTANCE}]"
  if [ -z "$sMyDb" ]; then
    sMyDb=$(guessDB "$sMyfile")
    echo "detected db schema from file: [${sMyDb}]"
  else
    echo "db schema from param 2: [${sMyDb}]"
  fi

  echo

  loadInstance "$COUCHDB_INSTANCE"

  echo "connect $couchdbhost on port $couchdbport with user $couchdbuser"
  if ! curl --head -X GET "$COUCHDB_URL" 2>/dev/null | grep "^HTTP.* 200 " >/dev/null; then
    color error
    echo ERROR: couch DB instance is not available
    curl -X GET "$COUCHDB_URL"
    color reset
    exit 1
  fi
  color ok
  echo OK
  color reset


  echo

  # _getDblist | grep "^${sMyDb}$"
  # if [ $? -eq 0 ]; then
  #   echo DB exists ... need to drop it first
  # fi

  h2 "deleting database [$sMyDb] ..."
  color cmd
  _couchapi DELETE "$sMyDb"
  fetchrc
  color reset

  h2 "creating database [$sMyDb] ..."
  color cmd
  _couchapi PUT "$sMyDb"
  fetchrc
  color reset

  h2 import file ...
  color cmd
  zcat "${sMyfile}" | python ${dirPythonPackages}/couchdb/tools/load.py "$COUCHDB_URL/$sMyDb"
  fetchrc
  color reset
  echo

}

# --------------------------------------------------------------------------------
# MAIN
# --------------------------------------------------------------------------------


# ----- check requirements

# --- is a couchd here
j_requireProcess "couchdb"   1

# --- very specific :-/ ... check available config files
ls -1 ~/.iml_backup/couchdb/* >/dev/null 2>&1
rc=$rc+$?


if [ $rc -eq 0 ]; then
  echo OK: couchdb was found on this system ... checking requirements for backup ...

  j_requireBinary  "curl"     1

  ls ${dirPythonPackages}/couchdb/tools/dump.py ${dirPythonPackages}/couchdb/tools/load.py >/dev/null && echo "OK: python couchdb tools were found"
  rc=$rc+$?


  if [ $rc -eq 0 ]; then
    echo

    if [ "$1" = "restore" ]; then
      echo
      shift 1
      restoreByFile $*

    else
      doBackup
    fi

  else
    color error
    echo ERROR: Couchdb is here but I am missing things for the backup :-/
    color reset
  fi

else
  rc=0
  echo "__DB__$SERVICENAME SKIP: couchdb seems not to be here"
fi


echo "__DB__$SERVICENAME INFO: $0 $* [couchdb] final returncode rc=$rc"

# --------------------------------------------------------------------------------