diff --git a/jobhelper.sh b/jobhelper.sh index 02519c458938d0e9be33385cf8954192f8930c86..d2c247c4fecff8aa2580451c5aeac766ecdbee89 100755 --- a/jobhelper.sh +++ b/jobhelper.sh @@ -434,18 +434,21 @@ function j_notify(){ local _body="$( date +%H:%M:%S ) $2" local _rc="$3" - local _urgency="normal" - - if [ -n "$DBUS_SESSION_BUS_ADDRESS" ]; then - if [ -n "$_rc" ]; then - if [ "$_rc" = "0" ]; then - _summary="OK: ${_summary}" - else - _summary="ERROR: ${_summary}" - _urgency="critical" + if which notify-send >/dev/null 2>&1; then + if [ -n "$DBUS_SESSION_BUS_ADDRESS" ]; then + + local _urgency="normal" + if [ -n "$_rc" ]; then + if [ "$_rc" = "0" ]; then + _summary="OK: ${_summary}" + else + _summary="ERROR: ${_summary}" + _urgency="critical" + fi fi + + su "$SUDO_USER" -c "notify-send --urgency=${_urgency} '${_summary}' '${_body}'" fi - su "$SUDO_USER" -c "notify-send --urgency=${_urgency} '${_summary}' '${_body}'" fi } diff --git a/plugins/localdump/couchdb2.sh b/plugins/localdump/couchdb2.sh index 47a803c1c5e5fb9b84372dba86398d8e55d59b0e..3ee91e6a2187af78e8f28e4af4017a3d16e3c14d 100755 --- a/plugins/localdump/couchdb2.sh +++ b/plugins/localdump/couchdb2.sh @@ -27,6 +27,7 @@ # 2022-10-07 ah v1.8 unescape regex with space to prevent "grep: warning: stray \ before white space" # 2023-06-06 ah v1.9 show a warning if the sequence id was not fetched # 2023-06-12 ah v1.10 skip couchdb dump if no sequence id was detected (=db deleted since fetching list of all dbs) +# 2023-06-26 ah v1.11 speed up detection of changed databases # ================================================================================ if [ -z "$BACKUP_TARGETDIR" ]; then @@ -85,6 +86,55 @@ function _getDbSeq(){ } + +# active curl prozesses +function curlCount(){ + ps -ef | grep -v grep | grep "curl" | wc -l +} + +# wait until min N curl prozesses exist +function wait4curlcount(){ + typeset -i local iContinue + typeset -i local iCount + + iContinue=${1:-0} + iCount=$( curlCount ) + + test $iCount -gt $iContinue && wait4curlcount $iContinue +} + +function reqCombined(){ + typeset -i local iChunksize; iChunksize=$1 + typeset -i local iParallel; iParallel=$2 + + typeset -i iCounter=0 + cmdline= + + for mydb in $dblist + do + + iCounter+=1 + test -n "$cmdline" && cmdline+=" -: " + cmdline+="${COUCH_URL}${mydb} " + + if [ $iCounter -ge $iChunksize ]; then + + curl -s $cmdline & + # warte, solange Zahl curl Prozesse kleiner Maximum + wait4curlcount $iParallel + + iCounter=0 + cmdline= + fi + + done + test -n "${cmdline}" && curl -s $cmdline & + + wait4curlcount 0 +} + + + # ---------- CONFIG/ INSTANCES # get valid configured instances @@ -168,15 +218,13 @@ function _doBackupOfSingleInstance(){ echo # get a list of current databases - dblist=/tmp/couch_list_${COUCHDB_INSTANCE}.txt - _getDblist > "$dblist" - ls -l "$dblist" + dblist=$( _getDblist ) # detect deleted databases: for dumpfile in $( find "${ARCHIVE_DIR}/${COUCHDB_INSTANCE}/" -maxdepth 1 -type f -name "*.couchdbdump.gz" ) do dbname=$( basename $dumpfile | sed "s#\.couchdbdump\.gz##g" ) - if ! grep "^${dbname}" "$dblist" >/dev/null; then + if ! grep "^${dbname}" <<< "$dblist" >/dev/null; then SEQFILE=${ARCHIVE_DIR}/${COUCHDB_INSTANCE}/seq/__seq__${dbname} SECURITYFILE=${ARCHIVE_DIR}/${COUCHDB_INSTANCE}/security/__security__${dbname}.json echo "DELETED $dbname ... $( ls -l ${dumpfile} | cut -f 5- -d ' ' )" @@ -188,7 +236,7 @@ function _doBackupOfSingleInstance(){ # done | tee /tmp/couch_archive_${COUCHDB_INSTANCE}.txt echo - typeset -i iDbTotal=$( cat "$dblist" | wc -l ) + typeset -i iDbTotal=$( echo "$dblist" | wc -l ) typeset -i iDb=0 typeset -i iDbCount=0 @@ -198,16 +246,36 @@ function _doBackupOfSingleInstance(){ echo " ARCHIVE ${ARCHIVE_DIR}/${COUCHDB_INSTANCE}" echo - for dbname in $( cat "$dblist" ) + local iChunksize=100 + local iParallel=10 + echo "--- $( date ) - Get database meta infos ... max $iParallel parralel curl requests sending $iChunksize database urls per connection" + seq=$( reqCombined 100 6 | jq -r ' [ .db_name, .update_seq ] | @csv ' | tr -d '"') + + declare -A aSeq + + echo "--- $( date ) - reading sequence ids..." + for line in $( echo "$seq" ) do - iDb=$iDb+1 + IFS="," read -r db seqvalue <<< "$line" + IFS="-" read -r seqid dummy <<< "$seqvalue" + # seqid=$( echo $seqvalue | cut -f 1 -d '-') + aSeq+=([$db]=$seqid) + done + + + + for dbname in $( echo "$dblist" ) + do + iDb+=1 echo -n "----- $(date) ${COUCHDB_INSTANCE} -- $iDb of $iDbTotal - ${dbname} - " OUTFILE=${BACKUP_TARGETDIR}/${COUCHDB_INSTANCE}/$(get_outfile "${dbname}").couchdbdump ARCHIVFILE=${ARCHIVE_DIR}/${COUCHDB_INSTANCE}/${dbname}.couchdbdump.gz SEQFILE=${ARCHIVE_DIR}/${COUCHDB_INSTANCE}/seq/__seq__${dbname} SECURITYFILE=${ARCHIVE_DIR}/${COUCHDB_INSTANCE}/security/__security__${dbname}.json - sSequenceCurrent=$(_getDbSeq "${dbname}") + # sSequenceCurrent=$(_getDbSeq "${dbname}") + sSequenceCurrent="${aSeq[$dbname]}" + sSequenceLast=$(cat "${SEQFILE}" 2>/dev/null | cut -f 1 -d '-') # sSequenceLast=`cat ${SEQFILE} 2>/dev/null | tr -d '\n'` @@ -231,6 +299,7 @@ function _doBackupOfSingleInstance(){ echo "update_seq --+-- current [${sSequenceCurrent}]" echo " +-- backup [${sSequenceLast}]" echo -n "Need to backup ... " + couchbackup --db "${dbname}" >"${OUTFILE}".progress 2>/dev/null && mv "${OUTFILE}".progress "${OUTFILE}" fetchrc @@ -254,7 +323,6 @@ function _doBackupOfSingleInstance(){ fi # if [ -z "$sSequenceCurrent" ]; then fi # if [ "${sSequenceCurrent}" = "${sSequenceLast}" ] ... done - rm -f "$dblist" echo "__DB__$SERVICENAME backup INFO: ${COUCHDB_INSTANCE} - backed up $iDbCount dbs of $iDbTotal total" }