Skip to content
Snippets Groups Projects
Commit 0c7f844b authored by Hahn Axel (hahn)'s avatar Hahn Axel (hahn)
Browse files

rewrite couchdb2-dumper

parent 60f99056
Branches
No related tags found
1 merge request!129Db Profiles
Hide whitespace changes
Inline Side-by-side
plugins/localdump/couchdb2.sh
+ 47
− 43
View file @ 0c7f844b
...@@ -48,7 +48,8 @@ CFGDIR=~/.iml_backup/couchdb2 ...@@ -48,7 +48,8 @@ CFGDIR=~/.iml_backup/couchdb2
# UNUSED # UNUSED
# dirPythonPackages=/usr/lib/python2.7/site-packages # dirPythonPackages=/usr/lib/python2.7/site-packages
ARCHIVE_DIR=$(_j_getvar "${JOBFILE}" dir-dbarchive)/couchdb2 # now set in localdump.sh
# ARCHIVE_DIR=$(_j_getvar "${JOBFILE}" dir-dbarchive)/couchdb2
# -------------------------------------------------------------------------------- # --------------------------------------------------------------------------------
# FUNCTIONS # FUNCTIONS
...@@ -112,9 +113,9 @@ function wait4curlcount(){ ...@@ -112,9 +113,9 @@ function wait4curlcount(){
# param integer iParallel count of curl processes # param integer iParallel count of curl processes
# param string dblistfile path+file to list of database # param string dblistfile path+file to list of database
function reqCombined(){ function reqCombined(){
typeset -i local iChunksize; iChunksize=$1 local iChunksize; typeset -i iChunksize; iChunksize=$1
typeset -i local iParallel; iParallel=$2 local iParallel; typeset -i iParallel; iParallel=$2
local dblistfile; dblistfile="$3" local dblistfile; dblistfile="$3"
typeset -i iCounter=0 typeset -i iCounter=0
cmdline= cmdline=
...@@ -184,12 +185,12 @@ function loadInstance(){ ...@@ -184,12 +185,12 @@ function loadInstance(){
# backup with loop over instances # backup with loop over instances
# param 1 string globbing filter to config files # param 1 string globbing filter to config files
function doBackup(){ function doBackup(){
# for mycfg in `ls -1 ~/.iml_backup/couchdb/*.config` # # for mycfg in `ls -1 ~/.iml_backup/couchdb/*.config`
for COUCHDB_INSTANCE in $(getInstances $1) # for COUCHDB_INSTANCE in $(getInstances $1)
do # do
loadInstance "$COUCHDB_INSTANCE" # loadInstance "$COUCHDB_INSTANCE"
echo "--- instance: $COUCHDB_INSTANCE" echo "--- instance: $PROFILENAME"
if curl --head -X GET "$COUCH_URL" 2>/dev/null | grep "^HTTP.* 200 "; then if curl --head -X GET "$COUCH_URL" 2>/dev/null | grep "^HTTP.* 200 "; then
echo OK, connected. echo OK, connected.
sleep 1 sleep 1
...@@ -207,18 +208,18 @@ function doBackup(){ ...@@ -207,18 +208,18 @@ function doBackup(){
echo echo
echo "--- $(date) done." echo "--- $(date) done."
echo echo
done # done
} }
# make backup of all databases in a couchdb instance # make backup of all databases in a couchdb instance
# global: COUCH_URL # global: COUCH_URL
# global: COUCHDB_INSTANCE # global: PROFILENAME
function _doBackupOfSingleInstance(){ function _doBackupOfSingleInstance(){
create_targetdir create_targetdir
local ARCHIVE_DIR2="${ARCHIVE_DIR}/${COUCHDB_INSTANCE}/deleted_databases" local ARCHIVE_DIR2="${ARCHIVE_DIR}/deleted_databases"
for _dir in "${BACKUP_TARGETDIR}/${COUCHDB_INSTANCE}" "${ARCHIVE_DIR}/${COUCHDB_INSTANCE}/seq" "${ARCHIVE_DIR}/${COUCHDB_INSTANCE}/security" \ for _dir in "${ARCHIVE_DIR}" "${ARCHIVE_DIR}/seq" "${ARCHIVE_DIR}/security" \
"${ARCHIVE_DIR2}" "${ARCHIVE_DIR2}/seq" "${ARCHIVE_DIR2}/security" "${ARCHIVE_DIR2}" "${ARCHIVE_DIR2}/seq" "${ARCHIVE_DIR2}/security"
do do
test -d "$_dir" || (echo "creating $_dir" ; mkdir -p "$_dir" ) test -d "$_dir" || (echo "creating $_dir" ; mkdir -p "$_dir" )
done done
...@@ -233,29 +234,29 @@ function _doBackupOfSingleInstance(){ ...@@ -233,29 +234,29 @@ function _doBackupOfSingleInstance(){
local ARCHIVFILE local ARCHIVFILE
local SEQFILE local SEQFILE
local SECURITYFILE local SECURITYFILE
typeset -i local iTsStart local iTsStart; typeset -i iTsStart
typeset -i local iTsTotal local iTsTotal; typeset -i iTsTotal
typeset -i local iDbPerSec local iDbPerSec; typeset -i iDbPerSec
dblistfile="/tmp/dblist_${COUCHDB_INSTANCE}" dblistfile="/tmp/dblist_${PROFILENAME}.txt"
# this is just a caching file of the sequence id of the last backup and can be safely deleted. # this is just a caching file of the sequence id of the last backup and can be safely deleted.
seqfile="${ARCHIVE_DIR}/${COUCHDB_INSTANCE}/seq/all_seqids_of_last_backups_cache.txt" seqfile="${ARCHIVE_DIR}/seq/all_seqids_of_last_backups_cache.txt"
echo "--- $( date ) Get list of all databases" echo "--- $( date ) Get list of all databases"
_getDblist >"${dblistfile}" _getDblist >"${dblistfile}"
typeset -i iDbTotal=$( wc -l < "$dblistfile") typeset -i iDbTotal; iDbTotal=$( wc -l < "$dblistfile")
typeset -i iDb=0 # counter for number of database in the loop typeset -i iDb=0 # counter for number of database in the loop
typeset -i iDbCount=0 # counter for backed up databases typeset -i iDbCount=0 # counter for backed up databases
echo "${COUCHDB_INSTANCE} has $iDbTotal databases" echo "${PROFILENAME} has $iDbTotal databases"
# detect deleted databases: # detect deleted databases:
echo echo
echo "--- $( date ) MOVE deleted databases " echo "--- $( date ) MOVE deleted databases "
echo "... into ${ARCHIVE_DIR2}" echo "... into ${ARCHIVE_DIR2}"
echo echo
for dumpfile in $( find "${ARCHIVE_DIR}/${COUCHDB_INSTANCE}/" -maxdepth 1 -type f -name "*.couchdbdump.gz" ) for dumpfile in $( find "${ARCHIVE_DIR}/" -maxdepth 1 -type f -name "*.couchdbdump.gz" )
do do
# extract database name: get basename and cut extension # extract database name: get basename and cut extension
# dbname=$( basename $dumpfile | sed "s#\.couchdbdump\.gz##g" ) # dbname=$( basename $dumpfile | sed "s#\.couchdbdump\.gz##g" )
...@@ -264,8 +265,8 @@ function _doBackupOfSingleInstance(){ ...@@ -264,8 +265,8 @@ function _doBackupOfSingleInstance(){
dbname=${dbname/\/} dbname=${dbname/\/}
if ! grep "^${dbname}" "${dblistfile}" >/dev/null; then if ! grep "^${dbname}" "${dblistfile}" >/dev/null; then
SEQFILE=${ARCHIVE_DIR}/${COUCHDB_INSTANCE}/seq/__seq__${dbname} SEQFILE=${ARCHIVE_DIR}/seq/__seq__${dbname}
SECURITYFILE=${ARCHIVE_DIR}/${COUCHDB_INSTANCE}/security/__security__${dbname}.json SECURITYFILE=${ARCHIVE_DIR}/security/__security__${dbname}.json
echo "DELETED $dbname ... $( ls -l ${dumpfile} | cut -f 5- -d ' ' )" echo "DELETED $dbname ... $( ls -l ${dumpfile} | cut -f 5- -d ' ' )"
mv "${dumpfile}" "${ARCHIVE_DIR2}" mv "${dumpfile}" "${ARCHIVE_DIR2}"
mv "${SEQFILE}" "${ARCHIVE_DIR2}/seq/" mv "${SEQFILE}" "${ARCHIVE_DIR2}/seq/"
...@@ -275,9 +276,9 @@ function _doBackupOfSingleInstance(){ ...@@ -275,9 +276,9 @@ function _doBackupOfSingleInstance(){
echo echo
echo "--- $( date ) DUMP databases" echo "--- $( date ) DUMP databases"
echo " of instance ${COUCHDB_INSTANCE}: $iDbTotal databases" echo " of instance ${PROFILENAME}: $iDbTotal databases"
echo " TO BACKUP ${BACKUP_TARGETDIR}/${COUCHDB_INSTANCE}" echo " TO BACKUP ${BACKUP_TARGETDIR}"
echo " ARCHIVE ${ARCHIVE_DIR}/${COUCHDB_INSTANCE}" echo " ARCHIVE ${ARCHIVE_DIR}"
echo echo
echo "----- $( date ) - Get database meta infos ... max $iParallel parralel curl requests sending $iChunksize database urls per process" echo "----- $( date ) - Get database meta infos ... max $iParallel parralel curl requests sending $iChunksize database urls per process"
...@@ -311,12 +312,12 @@ function _doBackupOfSingleInstance(){ ...@@ -311,12 +312,12 @@ function _doBackupOfSingleInstance(){
for dbname in $( cat "$dblistfile" ) for dbname in $( cat "$dblistfile" )
do do
iDb+=1 iDb+=1
echo -n "----- $(date) ${COUCHDB_INSTANCE} -- $iDb of $iDbTotal - ${dbname} - " echo -n "----- $(date) ${PROFILENAME} -- $iDb of $iDbTotal - ${dbname} - "
# set later .. OUTFILE=${BACKUP_TARGETDIR}/${COUCHDB_INSTANCE}/$(get_outfile "${dbname}").couchdbdump # set later .. OUTFILE=${BACKUP_TARGETDIR}/${COUCHDB_INSTANCE}/$(get_outfile "${dbname}").couchdbdump
ARCHIVFILE=${ARCHIVE_DIR}/${COUCHDB_INSTANCE}/${dbname}.couchdbdump.gz ARCHIVFILE=${ARCHIVE_DIR}/${dbname}.couchdbdump.gz
SEQFILE=${ARCHIVE_DIR}/${COUCHDB_INSTANCE}/seq/__seq__${dbname} SEQFILE=${ARCHIVE_DIR}/seq/__seq__${dbname}
SECURITYFILE=${ARCHIVE_DIR}/${COUCHDB_INSTANCE}/security/__security__${dbname}.json SECURITYFILE=${ARCHIVE_DIR}/security/__security__${dbname}.json
# sSequenceCurrent=$(_getDbSeq "${dbname}") # sSequenceCurrent=$(_getDbSeq "${dbname}")
sSequenceCurrent="${aSeq[$dbname]}" sSequenceCurrent="${aSeq[$dbname]}"
...@@ -335,7 +336,7 @@ function _doBackupOfSingleInstance(){ ...@@ -335,7 +336,7 @@ function _doBackupOfSingleInstance(){
) )
else else
OUTFILE=${BACKUP_TARGETDIR}/${COUCHDB_INSTANCE}/$(get_outfile "${dbname}").couchdbdump OUTFILE=${BACKUP_TARGETDIR}/$(get_outfile "${dbname}").couchdbdump
if [ -z "$sSequenceCurrent" ]; then if [ -z "$sSequenceCurrent" ]; then
echo "WARNING: unable to fetch current sequence ID - maybe the database was deleted." echo "WARNING: unable to fetch current sequence ID - maybe the database was deleted."
else else
...@@ -344,7 +345,10 @@ function _doBackupOfSingleInstance(){ ...@@ -344,7 +345,10 @@ function _doBackupOfSingleInstance(){
echo " +-- backup [${sSequenceLast}]" echo " +-- backup [${sSequenceLast}]"
echo -n "Need to backup ... " echo -n "Need to backup ... "
couchbackup --db "${dbname}" >"${OUTFILE}".progress 2>/dev/null && mv "${OUTFILE}".progress "${OUTFILE}" # TODO
# check command line
echo couchbackup --db "${dbname}" >"${OUTFILE}".progress 2>/dev/null && mv "${OUTFILE}".progress "${OUTFILE}"
# exit 1;
fetchrc fetchrc
# $myrc is last returncode - set in fetchrc # $myrc is last returncode - set in fetchrc
...@@ -385,7 +389,7 @@ function _doBackupOfSingleInstance(){ ...@@ -385,7 +389,7 @@ function _doBackupOfSingleInstance(){
rm -f "$dblistfile" rm -f "$dblistfile"
echo "__DB__$SERVICENAME backup INFO: ${COUCHDB_INSTANCE} - backed up $iDbCount dbs of $iDbTotal total ... in $iTsTotal sec ($iDbPerSec databases per sec)" echo "__DB__$SERVICENAME backup INFO: ${PROFILENAME} - backed up $iDbCount dbs of $iDbTotal total ... in $iTsTotal sec ($iDbPerSec databases per sec)"
} }
...@@ -420,13 +424,13 @@ function restoreByFile(){ ...@@ -420,13 +424,13 @@ function restoreByFile(){
echo echo
h2 "analyze dump $sMyfile" h2 "analyze dump $sMyfile"
COUCHDB_INSTANCE=$(echo $sMyfile | sed "s#${BACKUP_TARGETDIR}##g" | sed "s#\./##g" | sed "s#^/##g" | cut -f 1 -d "/") # COUCHDB_INSTANCE=$(echo $sMyfile | sed "s#${BACKUP_TARGETDIR}##g" | sed "s#\./##g" | sed "s#^/##g" | cut -f 1 -d "/")
echo "detected COUCHDB_INSTANCE : [${COUCHDB_INSTANCE}]" # echo "detected COUCHDB_INSTANCE : [${COUCHDB_INSTANCE}]"
if [ -z "$COUCHDB_INSTANCE" ]; then # if [ -z "$COUCHDB_INSTANCE" ]; then
echo "ERROR: Name of the instance was not detected." # echo "ERROR: Name of the instance was not detected."
echo " For couchdb restore you should cd to the ${BACKUP_TARGETDIR} or ${ARCHIVE_DIR}" # echo " For couchdb restore you should cd to the ${BACKUP_TARGETDIR} or ${ARCHIVE_DIR}"
exit 1 # exit 1
fi # fi
local _sourceDB="$( guessDB $sMyfile | sed 's#.couchdbdump.gz$##' )" local _sourceDB="$( guessDB $sMyfile | sed 's#.couchdbdump.gz$##' )"
echo "detected source database : [${_sourceDB}]" echo "detected source database : [${_sourceDB}]"
...@@ -440,7 +444,7 @@ function restoreByFile(){ ...@@ -440,7 +444,7 @@ function restoreByFile(){
echo echo
loadInstance $COUCHDB_INSTANCE # loadInstance $COUCHDB_INSTANCE
if [ $bFastMode -eq 0 ]; then if [ $bFastMode -eq 0 ]; then
echo connect $couchdbhost on port $couchdbport with user $couchdbuser echo connect $couchdbhost on port $couchdbport with user $couchdbuser
...@@ -486,7 +490,7 @@ function restoreByFile(){ ...@@ -486,7 +490,7 @@ function restoreByFile(){
h2 add security infos ... h2 add security infos ...
# todo: this will fail when restoring from "deleted_databases" folder # todo: this will fail when restoring from "deleted_databases" folder
SECURITYFILE=${ARCHIVE_DIR}/${COUCHDB_INSTANCE}/security/__security__${_sourceDB}.json SECURITYFILE="${ARCHIVE_DIR}/security/__security__${_sourceDB}.json"
SECDATA="$( cat $SECURITYFILE )" SECDATA="$( cat $SECURITYFILE )"
color cmd color cmd
echo "add security data: $SECDATA" echo "add security data: $SECDATA"
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment