Merge branch 'speedup-couchdb-backup' into 'master'

enable tmp file (faster); speedup loops in backup

See merge request !109

plugins/localdump/couchdb2.sh

@@ -28,6 +28,7 @@
 # 2023-06-06  ah     v1.9  show a warning if the sequence id was not fetched
 # 2023-06-12  ah     v1.10 skip couchdb dump if no sequence id was detected (=db deleted since fetching list of all dbs)
 # 2023-06-26  ah     v1.11 speed up detection of changed databases
+# 2023-06-27  ah     v1.12 enable tmp file for dblist again (faster); speedup loops in backup
 # ================================================================================
 
 if [ -z "$BACKUP_TARGETDIR" ]; then
@@ -106,11 +107,13 @@ function wait4curlcount(){
 function reqCombined(){
   typeset -i local iChunksize; iChunksize=$1
   typeset -i local iParallel;  iParallel=$2
+  local dblistfile;            dblistfile="$3"
 
   typeset -i iCounter=0
   cmdline=
 
-  for mydb in $dblist
+  # for mydb in $dblist
+  for mydb in $( cat $dblistfile )
   do 
 
       iCounter+=1
@@ -213,18 +216,34 @@ function _doBackupOfSingleInstance(){
     test -d "$_dir" || (echo "creating $_dir" ; mkdir -p "$_dir" )
   done
 
-  echo
-  echo "    MOVE deleted databases into ${ARCHIVE_DIR2}"
-  echo
-
+  echo "--- $( date ) Get list of all databases"
   # get a list of current databases
-  dblist=$( _getDblist )
+  # dblist=$( _getDblist )
+  local dblistfile 
+  dblistfile="/tmp/dblist_${COUCHDB_INSTANCE}"
+  _getDblist >"${dblistfile}"
+
+  # typeset -i iDbTotal=$( echo "$dblist" | wc -l )
+  typeset -i iDbTotal=$( cat "$dblistfile" | wc -l )
+  typeset -i iDb=0        # counter for number of database in the loop
+  typeset -i iDbCount=0   # counter for backed up databases
+  echo "${COUCHDB_INSTANCE} has $iDbTotal databases"
 
   # detect deleted databases: 
+  echo
+  echo "--- $( date ) MOVE deleted databases "
+  echo "... into ${ARCHIVE_DIR2}"
+  echo
   for dumpfile in $( find "${ARCHIVE_DIR}/${COUCHDB_INSTANCE}/" -maxdepth 1 -type f -name "*.couchdbdump.gz" )
   do
-      dbname=$( basename $dumpfile | sed "s#\.couchdbdump\.gz##g" )
-      if ! grep "^${dbname}" <<< "$dblist" >/dev/null; then
+      # extract database name: get basename and cut extension
+      # dbname=$( basename $dumpfile | sed "s#\.couchdbdump\.gz##g" )
+      dbname=${dumpfile##*/}
+      dbname=${dbname/%.couchdbdump.gz//}
+      dbname=${dbname/\/}
+
+      # if ! grep "^${dbname}" <<< "$dblist" >/dev/null; then
+      if ! grep "^${dbname}" "${dblistfile}"  >/dev/null; then
               SEQFILE=${ARCHIVE_DIR}/${COUCHDB_INSTANCE}/seq/__seq__${dbname}
               SECURITYFILE=${ARCHIVE_DIR}/${COUCHDB_INSTANCE}/security/__security__${dbname}.json
               echo "DELETED $dbname ... $( ls -l ${dumpfile} | cut -f 5- -d ' ' )"
@@ -234,22 +253,21 @@ function _doBackupOfSingleInstance(){
       fi
   done
   # done | tee /tmp/couch_archive_${COUCHDB_INSTANCE}.txt
-  echo
-
-  typeset -i iDbTotal=$( echo "$dblist" | wc -l )
-  typeset -i iDb=0
-  typeset -i iDbCount=0
 
   echo
-  echo "    DUMP databases of instance ${COUCHDB_INSTANCE}: $iDbTotal databases"
+  echo "--- $( date ) DUMP databases"
+  echo "    of instance ${COUCHDB_INSTANCE}: $iDbTotal databases"
   echo "    TO BACKUP ${BACKUP_TARGETDIR}/${COUCHDB_INSTANCE}"
   echo "      ARCHIVE ${ARCHIVE_DIR}/${COUCHDB_INSTANCE}"
   echo
 
+  # TODO: optimze creation of hash with database and sequence id
   local iChunksize=100
-  local iParallel=10
+  local iParallel=6
   echo "--- $( date ) - Get database meta infos ... max $iParallel parralel curl requests sending $iChunksize database urls per connection"
-  seq=$( reqCombined 100 6 | jq -r ' [ .db_name, .update_seq ] | @csv ' | tr -d '"')
+  seq=$( reqCombined $iChunksize $iParallel "$dblistfile" | jq -r ' [ .db_name, .update_seq ] | @csv ' | tr -d '"')
+  # seq=$( reqCombined $iChunksize $iParallel "$dblistfile" | jq -r ' [ .db_name, .update_seq ] | @csv ' | tr -d '"' | tr ',' ' '  | awk '{ sub(/-.*/, "\1", $2 ); print $1 " " $2  }' )
+  echo "$seq" | head
 
   declare -A aSeq
 
@@ -261,10 +279,20 @@ function _doBackupOfSingleInstance(){
     # seqid=$( echo $seqvalue | cut -f 1 -d '-')
     aSeq+=([$db]=$seqid)
   done
+  # for line in $( echo "$seq" )
+  # do
+  #   IFS=" " read -r db seqid <<< "$line"
+  #   echo "... $line ... $db - $seqid"
+  #   aSeq+=([$db]=$seqid)
+  # done
 
+  dbname="_users"
 
+  # echo "Seq ID von $dbname: ${aSeq[$dbname]}"
+  # echo
+  # exit
 
-  for dbname in $( echo "$dblist" )
+  for dbname in $( cat "$dblistfile" )
   do
     iDb+=1
     echo -n "----- $(date) ${COUCHDB_INSTANCE} -- $iDb of $iDbTotal - ${dbname} - "
@@ -323,6 +351,9 @@ function _doBackupOfSingleInstance(){
       fi # if [ -z "$sSequenceCurrent" ]; then
     fi # if [ "${sSequenceCurrent}" = "${sSequenceLast}" ] ...
   done
+
+  rm -f "$dblistfile"
+
   echo "__DB__$SERVICENAME backup INFO: ${COUCHDB_INSTANCE} - backed up $iDbCount dbs of $iDbTotal total"
 
 }