diff --git a/plugins/localdump/couchdb2.sh b/plugins/localdump/couchdb2.sh index 1daa1098d569c0301318787dbb24cef19e5ef73e..a41f12b9d8e0a86c1a765896183c2e9178d91640 100755 --- a/plugins/localdump/couchdb2.sh +++ b/plugins/localdump/couchdb2.sh @@ -29,6 +29,7 @@ # 2023-06-12 ah v1.10 skip couchdb dump if no sequence id was detected (=db deleted since fetching list of all dbs) # 2023-06-26 ah v1.11 speed up detection of changed databases # 2023-06-27 ah v1.12 enable tmp file for dblist again (faster); speedup loops in backup +# 2023-06-28 ah v1.13 optimize backup move OUTFILE; measure time; cache backed up sequence ids # ================================================================================ if [ -z "$BACKUP_TARGETDIR" ]; then @@ -225,12 +226,23 @@ function _doBackupOfSingleInstance(){ local iChunksize=100 local iParallel=6 + local dblistfile local sSequenceCurrent local sSequenceLast + local OUTFILE + local ARCHIVFILE + local SEQFILE + local SECURITYFILE + typeset -i local iTsStart + typeset -i local iTsTotal + typeset -i local iDbPerSec - echo "--- $( date ) Get list of all databases" - local dblistfile dblistfile="/tmp/dblist_${COUCHDB_INSTANCE}" + + # this is just a caching file of the sequence id of the last backup and can be safely deleted. + seqfile="${ARCHIVE_DIR}/${COUCHDB_INSTANCE}/seq/all_seqids_of_last_backups_cache.txt" + + echo "--- $( date ) Get list of all databases" _getDblist >"${dblistfile}" typeset -i iDbTotal=$( wc -l < "$dblistfile") @@ -279,7 +291,7 @@ function _doBackupOfSingleInstance(){ # candidate-00649860284626638ac6fd12bf000df5,40 # candidate-04561cddbd0fa305714b48a57929d8b4,3 - echo "----- $( date ) - reading sequence ids..." + echo "----- $( date ) - reading current sequence ids..." declare -A aSeq for line in $( echo "$seq" ) do @@ -287,19 +299,32 @@ function _doBackupOfSingleInstance(){ aSeq+=([$db]=$seqid) done + echo "----- $( date ) - reading sequence ids of last backup..." + declare -A aSeqBackup + for line in $( cat "${seqfile}" 2>/dev/null ) + do + IFS="," read -r db seqid <<< "$line" + aSeqBackup+=([$db]=$seqid) + done + + iTsStart=$( date +%s) for dbname in $( cat "$dblistfile" ) do iDb+=1 echo -n "----- $(date) ${COUCHDB_INSTANCE} -- $iDb of $iDbTotal - ${dbname} - " - OUTFILE=${BACKUP_TARGETDIR}/${COUCHDB_INSTANCE}/$(get_outfile "${dbname}").couchdbdump + + # set later .. OUTFILE=${BACKUP_TARGETDIR}/${COUCHDB_INSTANCE}/$(get_outfile "${dbname}").couchdbdump ARCHIVFILE=${ARCHIVE_DIR}/${COUCHDB_INSTANCE}/${dbname}.couchdbdump.gz SEQFILE=${ARCHIVE_DIR}/${COUCHDB_INSTANCE}/seq/__seq__${dbname} SECURITYFILE=${ARCHIVE_DIR}/${COUCHDB_INSTANCE}/security/__security__${dbname}.json # sSequenceCurrent=$(_getDbSeq "${dbname}") sSequenceCurrent="${aSeq[$dbname]}" - sSequenceLast=$(cat "${SEQFILE}" 2>/dev/null | cut -f 1 -d '-') + + # sSequenceLast=$(cat "${SEQFILE}" 2>/dev/null | cut -f 1 -d '-') + sSequenceLast="${aSeqBackup[$dbname]:-$(cat ${SEQFILE} 2>/dev/null | cut -f 1 -d '-')}" + aSeqBackup[${dbname}]=$sSequenceLast if [ "${sSequenceCurrent}" = "${sSequenceLast}" ] && [ -f "$ARCHIVFILE" ]; then echo "SKIP: still on sequence ${sSequenceLast}" @@ -310,6 +335,7 @@ function _doBackupOfSingleInstance(){ ) else + OUTFILE=${BACKUP_TARGETDIR}/${COUCHDB_INSTANCE}/$(get_outfile "${dbname}").couchdbdump if [ -z "$sSequenceCurrent" ]; then echo "WARNING: unable to fetch current sequence ID - maybe the database was deleted." else @@ -328,6 +354,11 @@ function _doBackupOfSingleInstance(){ fetchrc if [ $myrc -eq 0 ]; then iDbCount+=1 + + aSeqBackup[${dbname}]=${sSequenceCurrent} + # flushing cached information + rm -f "${seqfile}" 2>/dev/null + cp "${OUTFILE}"* "${ARCHIVFILE}" \ && echo "${sSequenceCurrent}">"${SEQFILE}" \ && _couchapi GET "${dbname}/_security" > "${SECURITYFILE}" @@ -341,10 +372,20 @@ function _doBackupOfSingleInstance(){ fi # if [ -z "$sSequenceCurrent" ]; then fi # if [ "${sSequenceCurrent}" = "${sSequenceLast}" ] ... done + iTsTotal=$( date +%s)-$iTsStart + iDbPerSec=$iDbTotal/$iTsTotal + + echo "----- $( date ) - writing sequence ids ..." + rm -f "${seqfile}" 2>/dev/null + for key in "${!aSeqBackup[@]}"; do + echo "$key,${aSeqBackup[$key]}" >> "${seqfile}" + done + ls -l "${seqfile}" rm -f "$dblistfile" - echo "__DB__$SERVICENAME backup INFO: ${COUCHDB_INSTANCE} - backed up $iDbCount dbs of $iDbTotal total" + echo "__DB__$SERVICENAME backup INFO: ${COUCHDB_INSTANCE} - backed up $iDbCount dbs of $iDbTotal total ... in $iTsTotal sec ($iDbPerSec databases per sec)" + exit }