Skip to content
Snippets Groups Projects
Commit 0c7f844b authored by Hahn Axel (hahn)'s avatar Hahn Axel (hahn)
Browse files

rewrite couchdb2-dumper

parent 60f99056
Branches
No related tags found
1 merge request!129Db Profiles
Hide whitespace changes
Inline Side-by-side
plugins/localdump/couchdb2.sh
+ 47
− 43
View file @ 0c7f844b
......@@ -48,7 +48,8 @@ CFGDIR=~/.iml_backup/couchdb2
# UNUSED
# dirPythonPackages=/usr/lib/python2.7/site-packages
ARCHIVE_DIR=$(_j_getvar "${JOBFILE}" dir-dbarchive)/couchdb2
# now set in localdump.sh
# ARCHIVE_DIR=$(_j_getvar "${JOBFILE}" dir-dbarchive)/couchdb2
# --------------------------------------------------------------------------------
# FUNCTIONS
......@@ -112,9 +113,9 @@ function wait4curlcount(){
# param integer iParallel count of curl processes
# param string dblistfile path+file to list of database
function reqCombined(){
typeset -i local iChunksize; iChunksize=$1
typeset -i local iParallel; iParallel=$2
local dblistfile; dblistfile="$3"
local iChunksize; typeset -i iChunksize; iChunksize=$1
local iParallel; typeset -i iParallel; iParallel=$2
local dblistfile; dblistfile="$3"
typeset -i iCounter=0
cmdline=
......@@ -184,12 +185,12 @@ function loadInstance(){
# backup with loop over instances
# param 1 string globbing filter to config files
function doBackup(){
# for mycfg in `ls -1 ~/.iml_backup/couchdb/*.config`
for COUCHDB_INSTANCE in $(getInstances $1)
do
loadInstance "$COUCHDB_INSTANCE"
# # for mycfg in `ls -1 ~/.iml_backup/couchdb/*.config`
# for COUCHDB_INSTANCE in $(getInstances $1)
# do
# loadInstance "$COUCHDB_INSTANCE"
echo "--- instance: $COUCHDB_INSTANCE"
echo "--- instance: $PROFILENAME"
if curl --head -X GET "$COUCH_URL" 2>/dev/null | grep "^HTTP.* 200 "; then
echo OK, connected.
sleep 1
......@@ -207,18 +208,18 @@ function doBackup(){
echo
echo "--- $(date) done."
echo
done
# done
}
# make backup of all databases in a couchdb instance
# global: COUCH_URL
# global: COUCHDB_INSTANCE
# global: PROFILENAME
function _doBackupOfSingleInstance(){
create_targetdir
local ARCHIVE_DIR2="${ARCHIVE_DIR}/${COUCHDB_INSTANCE}/deleted_databases"
for _dir in "${BACKUP_TARGETDIR}/${COUCHDB_INSTANCE}" "${ARCHIVE_DIR}/${COUCHDB_INSTANCE}/seq" "${ARCHIVE_DIR}/${COUCHDB_INSTANCE}/security" \
"${ARCHIVE_DIR2}" "${ARCHIVE_DIR2}/seq" "${ARCHIVE_DIR2}/security"
local ARCHIVE_DIR2="${ARCHIVE_DIR}/deleted_databases"
for _dir in "${ARCHIVE_DIR}" "${ARCHIVE_DIR}/seq" "${ARCHIVE_DIR}/security" \
"${ARCHIVE_DIR2}" "${ARCHIVE_DIR2}/seq" "${ARCHIVE_DIR2}/security"
do
test -d "$_dir" || (echo "creating $_dir" ; mkdir -p "$_dir" )
done
......@@ -233,29 +234,29 @@ function _doBackupOfSingleInstance(){
local ARCHIVFILE
local SEQFILE
local SECURITYFILE
typeset -i local iTsStart
typeset -i local iTsTotal
typeset -i local iDbPerSec
local iTsStart; typeset -i iTsStart
local iTsTotal; typeset -i iTsTotal
local iDbPerSec; typeset -i iDbPerSec
dblistfile="/tmp/dblist_${COUCHDB_INSTANCE}"
dblistfile="/tmp/dblist_${PROFILENAME}.txt"
# this is just a caching file of the sequence id of the last backup and can be safely deleted.
seqfile="${ARCHIVE_DIR}/${COUCHDB_INSTANCE}/seq/all_seqids_of_last_backups_cache.txt"
seqfile="${ARCHIVE_DIR}/seq/all_seqids_of_last_backups_cache.txt"
echo "--- $( date ) Get list of all databases"
_getDblist >"${dblistfile}"
typeset -i iDbTotal=$( wc -l < "$dblistfile")
typeset -i iDbTotal; iDbTotal=$( wc -l < "$dblistfile")
typeset -i iDb=0 # counter for number of database in the loop
typeset -i iDbCount=0 # counter for backed up databases
echo "${COUCHDB_INSTANCE} has $iDbTotal databases"
echo "${PROFILENAME} has $iDbTotal databases"
# detect deleted databases:
echo
echo "--- $( date ) MOVE deleted databases "
echo "... into ${ARCHIVE_DIR2}"
echo
for dumpfile in $( find "${ARCHIVE_DIR}/${COUCHDB_INSTANCE}/" -maxdepth 1 -type f -name "*.couchdbdump.gz" )
for dumpfile in $( find "${ARCHIVE_DIR}/" -maxdepth 1 -type f -name "*.couchdbdump.gz" )
do
# extract database name: get basename and cut extension
# dbname=$( basename $dumpfile | sed "s#\.couchdbdump\.gz##g" )
......@@ -264,8 +265,8 @@ function _doBackupOfSingleInstance(){
dbname=${dbname/\/}
if ! grep "^${dbname}" "${dblistfile}" >/dev/null; then
SEQFILE=${ARCHIVE_DIR}/${COUCHDB_INSTANCE}/seq/__seq__${dbname}
SECURITYFILE=${ARCHIVE_DIR}/${COUCHDB_INSTANCE}/security/__security__${dbname}.json
SEQFILE=${ARCHIVE_DIR}/seq/__seq__${dbname}
SECURITYFILE=${ARCHIVE_DIR}/security/__security__${dbname}.json
echo "DELETED $dbname ... $( ls -l ${dumpfile} | cut -f 5- -d ' ' )"
mv "${dumpfile}" "${ARCHIVE_DIR2}"
mv "${SEQFILE}" "${ARCHIVE_DIR2}/seq/"
......@@ -275,9 +276,9 @@ function _doBackupOfSingleInstance(){
echo
echo "--- $( date ) DUMP databases"
echo " of instance ${COUCHDB_INSTANCE}: $iDbTotal databases"
echo " TO BACKUP ${BACKUP_TARGETDIR}/${COUCHDB_INSTANCE}"
echo " ARCHIVE ${ARCHIVE_DIR}/${COUCHDB_INSTANCE}"
echo " of instance ${PROFILENAME}: $iDbTotal databases"
echo " TO BACKUP ${BACKUP_TARGETDIR}"
echo " ARCHIVE ${ARCHIVE_DIR}"
echo
echo "----- $( date ) - Get database meta infos ... max $iParallel parralel curl requests sending $iChunksize database urls per process"
......@@ -311,12 +312,12 @@ function _doBackupOfSingleInstance(){
for dbname in $( cat "$dblistfile" )
do
iDb+=1
echo -n "----- $(date) ${COUCHDB_INSTANCE} -- $iDb of $iDbTotal - ${dbname} - "
echo -n "----- $(date) ${PROFILENAME} -- $iDb of $iDbTotal - ${dbname} - "
# set later .. OUTFILE=${BACKUP_TARGETDIR}/${COUCHDB_INSTANCE}/$(get_outfile "${dbname}").couchdbdump
ARCHIVFILE=${ARCHIVE_DIR}/${COUCHDB_INSTANCE}/${dbname}.couchdbdump.gz
SEQFILE=${ARCHIVE_DIR}/${COUCHDB_INSTANCE}/seq/__seq__${dbname}
SECURITYFILE=${ARCHIVE_DIR}/${COUCHDB_INSTANCE}/security/__security__${dbname}.json
ARCHIVFILE=${ARCHIVE_DIR}/${dbname}.couchdbdump.gz
SEQFILE=${ARCHIVE_DIR}/seq/__seq__${dbname}
SECURITYFILE=${ARCHIVE_DIR}/security/__security__${dbname}.json
# sSequenceCurrent=$(_getDbSeq "${dbname}")
sSequenceCurrent="${aSeq[$dbname]}"
......@@ -335,7 +336,7 @@ function _doBackupOfSingleInstance(){
)
else
OUTFILE=${BACKUP_TARGETDIR}/${COUCHDB_INSTANCE}/$(get_outfile "${dbname}").couchdbdump
OUTFILE=${BACKUP_TARGETDIR}/$(get_outfile "${dbname}").couchdbdump
if [ -z "$sSequenceCurrent" ]; then
echo "WARNING: unable to fetch current sequence ID - maybe the database was deleted."
else
......@@ -344,7 +345,10 @@ function _doBackupOfSingleInstance(){
echo " +-- backup [${sSequenceLast}]"
echo -n "Need to backup ... "
couchbackup --db "${dbname}" >"${OUTFILE}".progress 2>/dev/null && mv "${OUTFILE}".progress "${OUTFILE}"
# TODO
# check command line
echo couchbackup --db "${dbname}" >"${OUTFILE}".progress 2>/dev/null && mv "${OUTFILE}".progress "${OUTFILE}"
# exit 1;
fetchrc
# $myrc is last returncode - set in fetchrc
......@@ -385,7 +389,7 @@ function _doBackupOfSingleInstance(){
rm -f "$dblistfile"
echo "__DB__$SERVICENAME backup INFO: ${COUCHDB_INSTANCE} - backed up $iDbCount dbs of $iDbTotal total ... in $iTsTotal sec ($iDbPerSec databases per sec)"
echo "__DB__$SERVICENAME backup INFO: ${PROFILENAME} - backed up $iDbCount dbs of $iDbTotal total ... in $iTsTotal sec ($iDbPerSec databases per sec)"
}
......@@ -420,13 +424,13 @@ function restoreByFile(){
echo
h2 "analyze dump $sMyfile"
COUCHDB_INSTANCE=$(echo $sMyfile | sed "s#${BACKUP_TARGETDIR}##g" | sed "s#\./##g" | sed "s#^/##g" | cut -f 1 -d "/")
echo "detected COUCHDB_INSTANCE : [${COUCHDB_INSTANCE}]"
if [ -z "$COUCHDB_INSTANCE" ]; then
echo "ERROR: Name of the instance was not detected."
echo " For couchdb restore you should cd to the ${BACKUP_TARGETDIR} or ${ARCHIVE_DIR}"
exit 1
fi
# COUCHDB_INSTANCE=$(echo $sMyfile | sed "s#${BACKUP_TARGETDIR}##g" | sed "s#\./##g" | sed "s#^/##g" | cut -f 1 -d "/")
# echo "detected COUCHDB_INSTANCE : [${COUCHDB_INSTANCE}]"
# if [ -z "$COUCHDB_INSTANCE" ]; then
# echo "ERROR: Name of the instance was not detected."
# echo " For couchdb restore you should cd to the ${BACKUP_TARGETDIR} or ${ARCHIVE_DIR}"
# exit 1
# fi
local _sourceDB="$( guessDB $sMyfile | sed 's#.couchdbdump.gz$##' )"
echo "detected source database : [${_sourceDB}]"
......@@ -440,7 +444,7 @@ function restoreByFile(){
echo
loadInstance $COUCHDB_INSTANCE
# loadInstance $COUCHDB_INSTANCE
if [ $bFastMode -eq 0 ]; then
echo connect $couchdbhost on port $couchdbport with user $couchdbuser
......@@ -486,7 +490,7 @@ function restoreByFile(){
h2 add security infos ...
# todo: this will fail when restoring from "deleted_databases" folder
SECURITYFILE=${ARCHIVE_DIR}/${COUCHDB_INSTANCE}/security/__security__${_sourceDB}.json
SECURITYFILE="${ARCHIVE_DIR}/security/__security__${_sourceDB}.json"
SECDATA="$( cat $SECURITYFILE )"
color cmd
echo "add security data: $SECDATA"
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment