diff --git a/plugins/localdump/couchdb2.sh b/plugins/localdump/couchdb2.sh index 47a803c1c5e5fb9b84372dba86398d8e55d59b0e..3ee91e6a2187af78e8f28e4af4017a3d16e3c14d 100755 --- a/plugins/localdump/couchdb2.sh +++ b/plugins/localdump/couchdb2.sh @@ -27,6 +27,7 @@ # 2022-10-07 ah v1.8 unescape regex with space to prevent "grep: warning: stray \ before white space" # 2023-06-06 ah v1.9 show a warning if the sequence id was not fetched # 2023-06-12 ah v1.10 skip couchdb dump if no sequence id was detected (=db deleted since fetching list of all dbs) +# 2023-06-26 ah v1.11 speed up detection of changed databases # ================================================================================ if [ -z "$BACKUP_TARGETDIR" ]; then @@ -85,6 +86,55 @@ function _getDbSeq(){ } + +# active curl prozesses +function curlCount(){ + ps -ef | grep -v grep | grep "curl" | wc -l +} + +# wait until min N curl prozesses exist +function wait4curlcount(){ + typeset -i local iContinue + typeset -i local iCount + + iContinue=${1:-0} + iCount=$( curlCount ) + + test $iCount -gt $iContinue && wait4curlcount $iContinue +} + +function reqCombined(){ + typeset -i local iChunksize; iChunksize=$1 + typeset -i local iParallel; iParallel=$2 + + typeset -i iCounter=0 + cmdline= + + for mydb in $dblist + do + + iCounter+=1 + test -n "$cmdline" && cmdline+=" -: " + cmdline+="${COUCH_URL}${mydb} " + + if [ $iCounter -ge $iChunksize ]; then + + curl -s $cmdline & + # warte, solange Zahl curl Prozesse kleiner Maximum + wait4curlcount $iParallel + + iCounter=0 + cmdline= + fi + + done + test -n "${cmdline}" && curl -s $cmdline & + + wait4curlcount 0 +} + + + # ---------- CONFIG/ INSTANCES # get valid configured instances @@ -168,15 +218,13 @@ function _doBackupOfSingleInstance(){ echo # get a list of current databases - dblist=/tmp/couch_list_${COUCHDB_INSTANCE}.txt - _getDblist > "$dblist" - ls -l "$dblist" + dblist=$( _getDblist ) # detect deleted databases: for dumpfile in $( find "${ARCHIVE_DIR}/${COUCHDB_INSTANCE}/" -maxdepth 1 -type f -name "*.couchdbdump.gz" ) do dbname=$( basename $dumpfile | sed "s#\.couchdbdump\.gz##g" ) - if ! grep "^${dbname}" "$dblist" >/dev/null; then + if ! grep "^${dbname}" <<< "$dblist" >/dev/null; then SEQFILE=${ARCHIVE_DIR}/${COUCHDB_INSTANCE}/seq/__seq__${dbname} SECURITYFILE=${ARCHIVE_DIR}/${COUCHDB_INSTANCE}/security/__security__${dbname}.json echo "DELETED $dbname ... $( ls -l ${dumpfile} | cut -f 5- -d ' ' )" @@ -188,7 +236,7 @@ function _doBackupOfSingleInstance(){ # done | tee /tmp/couch_archive_${COUCHDB_INSTANCE}.txt echo - typeset -i iDbTotal=$( cat "$dblist" | wc -l ) + typeset -i iDbTotal=$( echo "$dblist" | wc -l ) typeset -i iDb=0 typeset -i iDbCount=0 @@ -198,16 +246,36 @@ function _doBackupOfSingleInstance(){ echo " ARCHIVE ${ARCHIVE_DIR}/${COUCHDB_INSTANCE}" echo - for dbname in $( cat "$dblist" ) + local iChunksize=100 + local iParallel=10 + echo "--- $( date ) - Get database meta infos ... max $iParallel parralel curl requests sending $iChunksize database urls per connection" + seq=$( reqCombined 100 6 | jq -r ' [ .db_name, .update_seq ] | @csv ' | tr -d '"') + + declare -A aSeq + + echo "--- $( date ) - reading sequence ids..." + for line in $( echo "$seq" ) do - iDb=$iDb+1 + IFS="," read -r db seqvalue <<< "$line" + IFS="-" read -r seqid dummy <<< "$seqvalue" + # seqid=$( echo $seqvalue | cut -f 1 -d '-') + aSeq+=([$db]=$seqid) + done + + + + for dbname in $( echo "$dblist" ) + do + iDb+=1 echo -n "----- $(date) ${COUCHDB_INSTANCE} -- $iDb of $iDbTotal - ${dbname} - " OUTFILE=${BACKUP_TARGETDIR}/${COUCHDB_INSTANCE}/$(get_outfile "${dbname}").couchdbdump ARCHIVFILE=${ARCHIVE_DIR}/${COUCHDB_INSTANCE}/${dbname}.couchdbdump.gz SEQFILE=${ARCHIVE_DIR}/${COUCHDB_INSTANCE}/seq/__seq__${dbname} SECURITYFILE=${ARCHIVE_DIR}/${COUCHDB_INSTANCE}/security/__security__${dbname}.json - sSequenceCurrent=$(_getDbSeq "${dbname}") + # sSequenceCurrent=$(_getDbSeq "${dbname}") + sSequenceCurrent="${aSeq[$dbname]}" + sSequenceLast=$(cat "${SEQFILE}" 2>/dev/null | cut -f 1 -d '-') # sSequenceLast=`cat ${SEQFILE} 2>/dev/null | tr -d '\n'` @@ -231,6 +299,7 @@ function _doBackupOfSingleInstance(){ echo "update_seq --+-- current [${sSequenceCurrent}]" echo " +-- backup [${sSequenceLast}]" echo -n "Need to backup ... " + couchbackup --db "${dbname}" >"${OUTFILE}".progress 2>/dev/null && mv "${OUTFILE}".progress "${OUTFILE}" fetchrc @@ -254,7 +323,6 @@ function _doBackupOfSingleInstance(){ fi # if [ -z "$sSequenceCurrent" ]; then fi # if [ "${sSequenceCurrent}" = "${sSequenceLast}" ] ... done - rm -f "$dblist" echo "__DB__$SERVICENAME backup INFO: ${COUCHDB_INSTANCE} - backed up $iDbCount dbs of $iDbTotal total" }