Speedup couchdb backup

plugins/localdump/couchdb2.sh

@@ -29,6 +29,7 @@
 # 2023-06-12  ah     v1.10 skip couchdb dump if no sequence id was detected (=db deleted since fetching list of all dbs)
 # 2023-06-26  ah     v1.11 speed up detection of changed databases
 # 2023-06-27  ah     v1.12 enable tmp file for dblist again (faster); speedup loops in backup
+# 2023-06-28  ah     v1.13 optimize backup move OUTFILE; measure time; cache backed up sequence ids
 # ================================================================================
 
 if [ -z "$BACKUP_TARGETDIR" ]; then
@@ -225,12 +226,23 @@ function _doBackupOfSingleInstance(){
   local iChunksize=100
   local iParallel=6
 
+  local dblistfile
   local sSequenceCurrent
   local sSequenceLast
+  local OUTFILE
+  local ARCHIVFILE
+  local SEQFILE
+  local SECURITYFILE
+  typeset -i local iTsStart
+  typeset -i local iTsTotal
+  typeset -i local iDbPerSec
 
-  echo "--- $( date ) Get list of all databases"
-  local dblistfile 
   dblistfile="/tmp/dblist_${COUCHDB_INSTANCE}"
+
+  # this is just a caching file of the sequence id of the last backup and can be safely deleted.
+  seqfile="${ARCHIVE_DIR}/${COUCHDB_INSTANCE}/seq/all_seqids_of_last_backups_cache.txt"
+
+  echo "--- $( date ) Get list of all databases"
   _getDblist >"${dblistfile}"
 
   typeset -i iDbTotal=$( wc -l < "$dblistfile")
@@ -279,7 +291,7 @@ function _doBackupOfSingleInstance(){
   # candidate-00649860284626638ac6fd12bf000df5,40
   # candidate-04561cddbd0fa305714b48a57929d8b4,3
 
-  echo "----- $( date ) - reading sequence ids..."
+  echo "----- $( date ) - reading current sequence ids..."
   declare -A aSeq
   for line in $( echo "$seq" )
   do
@@ -287,19 +299,32 @@ function _doBackupOfSingleInstance(){
     aSeq+=([$db]=$seqid)
   done
 
+  echo "----- $( date ) - reading sequence ids of last backup..."
+  declare -A aSeqBackup
+  for line in $( cat "${seqfile}" 2>/dev/null )
+  do
+    IFS="," read -r db seqid <<< "$line"
+    aSeqBackup+=([$db]=$seqid)
+  done
+
+  iTsStart=$( date +%s)
   for dbname in $( cat "$dblistfile" )
   do
     iDb+=1
     echo -n "----- $(date) ${COUCHDB_INSTANCE} -- $iDb of $iDbTotal - ${dbname} - "
-    OUTFILE=${BACKUP_TARGETDIR}/${COUCHDB_INSTANCE}/$(get_outfile "${dbname}").couchdbdump
+
+    # set later .. OUTFILE=${BACKUP_TARGETDIR}/${COUCHDB_INSTANCE}/$(get_outfile "${dbname}").couchdbdump
     ARCHIVFILE=${ARCHIVE_DIR}/${COUCHDB_INSTANCE}/${dbname}.couchdbdump.gz
     SEQFILE=${ARCHIVE_DIR}/${COUCHDB_INSTANCE}/seq/__seq__${dbname}
     SECURITYFILE=${ARCHIVE_DIR}/${COUCHDB_INSTANCE}/security/__security__${dbname}.json
 
     # sSequenceCurrent=$(_getDbSeq "${dbname}")
     sSequenceCurrent="${aSeq[$dbname]}"
-    sSequenceLast=$(cat "${SEQFILE}" 2>/dev/null | cut -f 1 -d '-')
+    
+    # sSequenceLast=$(cat "${SEQFILE}" 2>/dev/null | cut -f 1 -d '-')
+    sSequenceLast="${aSeqBackup[$dbname]:-$(cat ${SEQFILE} 2>/dev/null | cut -f 1 -d '-')}"
 
+    aSeqBackup[${dbname}]=$sSequenceLast
     if [ "${sSequenceCurrent}" = "${sSequenceLast}" ] && [ -f "$ARCHIVFILE" ]; then
       echo "SKIP: still on sequence ${sSequenceLast}"
 
@@ -310,6 +335,7 @@ function _doBackupOfSingleInstance(){
       )
       
     else
+      OUTFILE=${BACKUP_TARGETDIR}/${COUCHDB_INSTANCE}/$(get_outfile "${dbname}").couchdbdump
       if [ -z "$sSequenceCurrent" ]; then
         echo "WARNING: unable to fetch current sequence ID - maybe the database was deleted."
       else
@@ -328,6 +354,11 @@ function _doBackupOfSingleInstance(){
           fetchrc
           if [ $myrc -eq 0 ]; then
             iDbCount+=1
+
+            aSeqBackup[${dbname}]=${sSequenceCurrent}
+            # flushing cached information
+            rm -f "${seqfile}" 2>/dev/null
+
             cp "${OUTFILE}"* "${ARCHIVFILE}"                             \
               && echo "${sSequenceCurrent}">"${SEQFILE}"                 \
               && _couchapi GET "${dbname}/_security" > "${SECURITYFILE}"
@@ -341,10 +372,19 @@ function _doBackupOfSingleInstance(){
       fi # if [ -z "$sSequenceCurrent" ]; then
     fi # if [ "${sSequenceCurrent}" = "${sSequenceLast}" ] ...
   done
+  iTsTotal=$( date +%s)-$iTsStart
+  iDbPerSec=$iDbTotal/$iTsTotal
+
+  echo "----- $( date ) - writing sequence ids ..."
+  rm -f "${seqfile}" 2>/dev/null
+  for key in "${!aSeqBackup[@]}"; do
+    echo "$key,${aSeqBackup[$key]}" >> "${seqfile}"
+  done
+  ls -l "${seqfile}"
 
   rm -f "$dblistfile"
 
-  echo "__DB__$SERVICENAME backup INFO: ${COUCHDB_INSTANCE} - backed up $iDbCount dbs of $iDbTotal total"
+  echo "__DB__$SERVICENAME backup INFO: ${COUCHDB_INSTANCE} - backed up $iDbCount dbs of $iDbTotal total ... in $iTsTotal sec ($iDbPerSec databases per sec)"
 
 }