Skip to content
GitLab
Explore
Sign in
Primary navigation
Search or go to…
Project
I
iml-backup
Manage
Activity
Members
Labels
Plan
Issues
Issue boards
Milestones
Wiki
Code
Merge requests
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Snippets
Build
Pipelines
Jobs
Pipeline schedules
Artifacts
Deploy
Releases
Container registry
Model registry
Operate
Environments
Monitor
Incidents
Analyze
Value stream analytics
Contributor analytics
CI/CD analytics
Repository analytics
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
Community forum
Contribute to GitLab
Provide feedback
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
IML Open Source
iml-backup
Commits
acbce830
Commit
acbce830
authored
3 years ago
by
Hahn Axel (hahn)
Browse files
Options
Downloads
Patches
Plain Diff
couchdb2: fixes with shellcheck
parent
c4868db7
Branches
Branches containing commit
No related tags found
1 merge request
!26
Eliminate scheduler
Changes
1
Hide whitespace changes
Inline
Side-by-side
Showing
1 changed file
plugins/localdump/couchdb2.sh
+44
-48
44 additions, 48 deletions
plugins/localdump/couchdb2.sh
with
44 additions
and
48 deletions
plugins/localdump/couchdb2.sh
+
44
−
48
View file @
acbce830
...
@@ -15,10 +15,11 @@
...
@@ -15,10 +15,11 @@
# ./localdump.sh backup couchdb2 demo
# ./localdump.sh backup couchdb2 demo
# 2021-10-11 ..... v1.2 added fastmode in restore: no test connect, do not
# 2021-10-11 ..... v1.2 added fastmode in restore: no test connect, do not
# delete DB before create request
# delete DB before create request
# 2022-01-20 v1.3 fixes with shellcheck
# ================================================================================
# ================================================================================
if
[
-z
$BACKUP_TARGETDIR
]
;
then
if
[
-z
"
$BACKUP_TARGETDIR
"
]
;
then
echo
ERROR: you cannot start
`
basename
$0
`
directly
echo
"
ERROR: you cannot start
$(
basename
"
$0
"
)
directly
"
rc
=
$rc
+1
rc
=
$rc
+1
exit
1
exit
1
fi
fi
...
@@ -33,7 +34,7 @@ CFGDIR=~/.iml_backup/couchdb2
...
@@ -33,7 +34,7 @@ CFGDIR=~/.iml_backup/couchdb2
# UNUSED
# UNUSED
# dirPythonPackages=/usr/lib/python2.7/site-packages
# dirPythonPackages=/usr/lib/python2.7/site-packages
ARCHIVE_DIR
=
`
_j_getvar
${
JOBFILE
}
dir-dbarchive
`
/couchdb2
ARCHIVE_DIR
=
$(
_j_getvar
"
${
JOBFILE
}
"
dir-dbarchive
)
/couchdb2
# --------------------------------------------------------------------------------
# --------------------------------------------------------------------------------
# FUNCTIONS
# FUNCTIONS
...
@@ -51,20 +52,20 @@ function _couchapi(){
...
@@ -51,20 +52,20 @@ function _couchapi(){
# sParams="$sParams -u ${couchdbuser}:${couchdbpw}"
# sParams="$sParams -u ${couchdbuser}:${couchdbpw}"
sParams
=
"
$sParams
-X
${
method
}
"
sParams
=
"
$sParams
-X
${
method
}
"
sParams
=
"
$sParams
${
COUCH_URL
}${
apiurl
}
"
sParams
=
"
$sParams
${
COUCH_URL
}${
apiurl
}
"
if
[
!
-z
$outfile
]
;
then
if
[
!
-z
"
$outfile
"
]
;
then
sParams
=
"
$sParams
-o
${
outfile
}
"
sParams
=
"
$sParams
-o
${
outfile
}
"
fi
fi
curl
$sParams
2>/dev/null
curl
"
$sParams
"
2>/dev/null
}
}
function
_getDblist
(){
function
_getDblist
(){
_couchapi GET _all_dbs |
sed
's#\"#\n#g'
|
e
grep
-v
"^(
\[
|
\,
|
\]
)$"
|
grep
-v
_replicator |
grep
-v
_global_changes
_couchapi GET _all_dbs |
sed
's#\"#\n#g'
|
grep
-
E
v
"^(
\[
|
\,
|
\]
)$"
|
grep
-v
_replicator |
grep
-v
_global_changes
}
}
# get value update_seq of given couchdb name
# get value update_seq of given couchdb name
function
_getDbSeq
(){
function
_getDbSeq
(){
# _couchapi GET $1 | sed 's#,\"#\n"#g' | egrep -v "^(\[|\,|\])$" | grep update_seq | cut -f 4 -d '"'
# _couchapi GET $1 | sed 's#,\"#\n"#g' | egrep -v "^(\[|\,|\])$" | grep update_seq | cut -f 4 -d '"'
_couchapi GET
$1
|
sed
's#,\"#\n"#g'
|
e
grep
-v
"^(
\[
|
\,
|
\]
)$"
|
grep
update_seq |
cut
-f
4
-d
'"'
|
cut
-f
1
-d
'-'
_couchapi GET
"
$1
"
|
sed
's#,\"#\n"#g'
|
grep
-
E
v
"^(
\[
|
\,
|
\]
)$"
|
grep
update_seq |
cut
-f
4
-d
'"'
|
cut
-f
1
-d
'-'
}
}
...
@@ -72,11 +73,10 @@ function _getDbSeq(){
...
@@ -72,11 +73,10 @@ function _getDbSeq(){
# get valid configured instances
# get valid configured instances
function
getInstances
(){
function
getInstances
(){
for
mycfg
in
`
ls
-1
${
CFGDIR
}
/
*${
1
}*
.config
`
for
mycfg
in
$(
ls
-1
${
CFGDIR
}
/
*${
1
}*
.config
)
do
do
.
$mycfg
if
.
"
$mycfg
"
;
then
if
[
$?
-eq
0
]
;
then
echo
$(
basename
"
${
mycfg
}
"
|
cut
-f
1
-d
"."
)
echo
`
basename
"
${
mycfg
}
"
|
cut
-f
1
-d
"."
`
fi
fi
done
done
}
}
...
@@ -87,16 +87,15 @@ function getInstances(){
...
@@ -87,16 +87,15 @@ function getInstances(){
# param string name of the instance to load
# param string name of the instance to load
function
loadInstance
(){
function
loadInstance
(){
COUCH_URL
=
COUCH_URL
=
.
${
CFGDIR
}
/
${
1
}
.config
if
!
.
"
${
CFGDIR
}
/
${
1
}
.config"
;
then
if
[
$?
-ne
0
]
;
then
color error
color error
echo
ERROR: invalid instance:
$1
- the config file cannot be sourced
echo
ERROR: invalid instance:
$1
- the config file cannot be sourced
color reset
color reset
exit
1
exit
1
fi
fi
if
[
-z
${
COUCH_URL
}
]
;
then
if
[
-z
"
${
COUCH_URL
}
"
]
;
then
color error
color error
echo
ERROR: invalid instance:
$1
- the config file has no COUCH_URL
echo
"
ERROR: invalid instance:
$1
- the config file has no COUCH_URL
"
color reset
color reset
exit
1
exit
1
fi
fi
...
@@ -110,14 +109,12 @@ function loadInstance(){
...
@@ -110,14 +109,12 @@ function loadInstance(){
# param 1 string globbing filter to config files
# param 1 string globbing filter to config files
function
doBackup
(){
function
doBackup
(){
# for mycfg in `ls -1 ~/.iml_backup/couchdb/*.config`
# for mycfg in `ls -1 ~/.iml_backup/couchdb/*.config`
for
COUCHDB_INSTANCE
in
`
getInstances
$1
`
for
COUCHDB_INSTANCE
in
$(
getInstances
$1
)
do
do
loadInstance
$COUCHDB_INSTANCE
loadInstance
"
$COUCHDB_INSTANCE
"
echo
---
instance:
$COUCHDB_INSTANCE
echo
"--- instance:
$COUCHDB_INSTANCE
"
curl
--head
-X
GET
$COUCH_URL
2>/dev/null |
grep
"^HTTP.*
\
200
\
"
if
curl
--head
-X
GET
"
$COUCH_URL
"
2>/dev/null |
grep
"^HTTP.*
\
200
\
"
;
then
if
[
$?
-eq
0
]
;
then
echo
OK, connected.
echo
OK, connected.
sleep
2
sleep
2
_doBackupOfSingleInstance
_doBackupOfSingleInstance
...
@@ -125,14 +122,14 @@ function doBackup(){
...
@@ -125,14 +122,14 @@ function doBackup(){
else
else
rc
=
$rc
+1
rc
=
$rc
+1
color error
color error
echo
ERROR: couch DB instance is not available or canot be accessed with these credentials
in
config file
echo
"
ERROR: couch DB instance is not available or canot be accessed with these credentials in config file
"
# repeat curl to show the error message
# repeat curl to show the error message
curl
-X
GET
$COUCH_URL
curl
-X
GET
"
$COUCH_URL
"
color reset
color reset
fi
fi
echo
echo
echo
---
`
date
`
done
.
echo
"
---
$(
date
)
done.
"
echo
echo
done
done
}
}
...
@@ -143,11 +140,11 @@ function doBackup(){
...
@@ -143,11 +140,11 @@ function doBackup(){
function
_doBackupOfSingleInstance
(){
function
_doBackupOfSingleInstance
(){
create_targetdir
create_targetdir
mkdir
-p
${
BACKUP_TARGETDIR
}
/
${
COUCHDB_INSTANCE
}
2>/dev/null
mkdir
-p
"
${
BACKUP_TARGETDIR
}
/
${
COUCHDB_INSTANCE
}
"
2>/dev/null
mkdir
-p
${
ARCHIVE_DIR
}
/
${
COUCHDB_INSTANCE
}
/seq 2>/dev/null
mkdir
-p
"
${
ARCHIVE_DIR
}
/
${
COUCHDB_INSTANCE
}
/seq
"
2>/dev/null
local
ARCHIVE_DIR2
=
${
ARCHIVE_DIR
}
/
${
COUCHDB_INSTANCE
}
/deleted_databases
local
ARCHIVE_DIR2
=
"
${
ARCHIVE_DIR
}
/
${
COUCHDB_INSTANCE
}
/deleted_databases
"
test
-d
"
${
ARCHIVE_DIR2
}
"
||
mkdir
-p
${
ARCHIVE_DIR2
}
2>/dev/null
test
-d
"
${
ARCHIVE_DIR2
}
"
||
mkdir
-p
"
${
ARCHIVE_DIR2
}
"
2>/dev/null
echo
echo
echo
" MOVE deleted databases into
${
ARCHIVE_DIR2
}
"
echo
" MOVE deleted databases into
${
ARCHIVE_DIR2
}
"
...
@@ -155,25 +152,24 @@ function _doBackupOfSingleInstance(){
...
@@ -155,25 +152,24 @@ function _doBackupOfSingleInstance(){
# get a list of current databases
# get a list of current databases
dblist
=
/tmp/couch_list_
${
COUCHDB_INSTANCE
}
.txt
dblist
=
/tmp/couch_list_
${
COUCHDB_INSTANCE
}
.txt
_getDblist
>
$dblist
_getDblist
>
"
$dblist
"
ls
-l
$dblist
ls
-l
"
$dblist
"
# detect deleted databases:
# detect deleted databases:
for
dumpfile
in
$(
find
${
ARCHIVE_DIR
}
/
${
COUCHDB_INSTANCE
}
/
-maxdepth
1
-type
f
-name
"*.couchdbdump.gz"
)
for
dumpfile
in
$(
find
"
${
ARCHIVE_DIR
}
/
${
COUCHDB_INSTANCE
}
/
"
-maxdepth
1
-type
f
-name
"*.couchdbdump.gz"
)
do
do
dbname
=
$(
basename
$dumpfile
|
sed
"s#
\.
couchdbdump
\.
gz##g"
)
dbname
=
$(
basename
$dumpfile
|
sed
"s#
\.
couchdbdump
\.
gz##g"
)
grep
"^
${
dbname
}
"
$dblist
>
/dev/null
if
!
grep
"^
${
dbname
}
"
"
$dblist
"
>
/dev/null
;
then
if
[
$?
-ne
0
]
;
then
SEQFILE
=
${
ARCHIVE_DIR
}
/
${
COUCHDB_INSTANCE
}
/seq/__seq__
${
dbname
}
SEQFILE
=
${
ARCHIVE_DIR
}
/
${
COUCHDB_INSTANCE
}
/seq/__seq__
${
dbname
}
echo
"DELETED
$dbname
...
$(
ls
-l
${
dumpfile
}
|
cut
-f
5-
-d
' '
)
"
echo
"DELETED
$dbname
...
$(
ls
-l
${
dumpfile
}
|
cut
-f
5-
-d
' '
)
"
mv
${
dumpfile
}
${
ARCHIVE_DIR2
}
mv
"
${
dumpfile
}
"
"
${
ARCHIVE_DIR2
}
"
rm
-f
${
SEQFILE
}
rm
-f
"
${
SEQFILE
}
"
fi
fi
done
done
# done | tee /tmp/couch_archive_${COUCHDB_INSTANCE}.txt
# done | tee /tmp/couch_archive_${COUCHDB_INSTANCE}.txt
echo
echo
typeset
-i
iDbTotal
=
$(
cat
$dblist
|
wc
-l
)
typeset
-i
iDbTotal
=
$(
cat
"
$dblist
"
|
wc
-l
)
typeset
-i
iDb
=
0
typeset
-i
iDb
=
0
echo
echo
...
@@ -182,48 +178,48 @@ function _doBackupOfSingleInstance(){
...
@@ -182,48 +178,48 @@ function _doBackupOfSingleInstance(){
echo
" ARCHIVE
${
ARCHIVE_DIR
}
/
${
COUCHDB_INSTANCE
}
"
echo
" ARCHIVE
${
ARCHIVE_DIR
}
/
${
COUCHDB_INSTANCE
}
"
echo
echo
for
dbname
in
$(
cat
$dblist
)
for
dbname
in
$(
cat
"
$dblist
"
)
do
do
iDb
=
$iDb
+1
iDb
=
$iDb
+1
echo
-n
"-----
`
date
`
${
COUCHDB_INSTANCE
}
--
$iDb
of
$iDbTotal
-
${
dbname
}
- "
echo
-n
"-----
$(
date
)
${
COUCHDB_INSTANCE
}
--
$iDb
of
$iDbTotal
-
${
dbname
}
- "
OUTFILE
=
${
BACKUP_TARGETDIR
}
/
${
COUCHDB_INSTANCE
}
/
`
get_outfile
${
dbname
}
`
.couchdbdump
OUTFILE
=
${
BACKUP_TARGETDIR
}
/
${
COUCHDB_INSTANCE
}
/
$(
get_outfile
"
${
dbname
}
"
)
.couchdbdump
ARCHIVFILE
=
${
ARCHIVE_DIR
}
/
${
COUCHDB_INSTANCE
}
/
${
dbname
}
.couchdbdump.gz
ARCHIVFILE
=
${
ARCHIVE_DIR
}
/
${
COUCHDB_INSTANCE
}
/
${
dbname
}
.couchdbdump.gz
SEQFILE
=
${
ARCHIVE_DIR
}
/
${
COUCHDB_INSTANCE
}
/seq/__seq__
${
dbname
}
SEQFILE
=
${
ARCHIVE_DIR
}
/
${
COUCHDB_INSTANCE
}
/seq/__seq__
${
dbname
}
sSequenceCurrent
=
`
_getDbSeq
${
dbname
}
`
sSequenceCurrent
=
$(
_getDbSeq
"
${
dbname
}
"
)
sSequenceLast
=
`
cat
${
SEQFILE
}
2>/dev/null |
cut
-f
1
-d
'-'
`
sSequenceLast
=
$(
cat
"
${
SEQFILE
}
"
2>/dev/null |
cut
-f
1
-d
'-'
)
# sSequenceLast=`cat ${SEQFILE} 2>/dev/null | tr -d '\n'`
# sSequenceLast=`cat ${SEQFILE} 2>/dev/null | tr -d '\n'`
# echo
# echo
# echo "update_seq --+-- current [${sSequenceCurrent}]"
# echo "update_seq --+-- current [${sSequenceCurrent}]"
# echo " +-- backup [${sSequenceLast}]"
# echo " +-- backup [${sSequenceLast}]"
if
[
"
${
sSequenceCurrent
}
"
=
"
${
sSequenceLast
}
"
]
;
then
if
[
"
${
sSequenceCurrent
}
"
=
"
${
sSequenceLast
}
"
]
;
then
echo
SKIP: still on sequence
${
sSequenceLast
}
echo
SKIP: still on sequence
"
${
sSequenceLast
}
"
else
else
echo
echo
echo
"update_seq --+-- current [
${
sSequenceCurrent
}
]"
echo
"update_seq --+-- current [
${
sSequenceCurrent
}
]"
echo
" +-- backup [
${
sSequenceLast
}
]"
echo
" +-- backup [
${
sSequenceLast
}
]"
echo
-n
"Need to backup ... "
echo
-n
"Need to backup ... "
couchbackup
--db
${
dbname
}
>
${
OUTFILE
}
.progress 2>/dev/null
&&
mv
${
OUTFILE
}
.progress
${
OUTFILE
}
couchbackup
--db
"
${
dbname
}
"
>
"
${
OUTFILE
}
"
.progress 2>/dev/null
&&
mv
"
${
OUTFILE
}
"
.progress
"
${
OUTFILE
}
"
fetchrc
fetchrc
# $myrc is last returncode - set in fetchrc
# $myrc is last returncode - set in fetchrc
if
[
$myrc
-eq
0
]
;
then
if
[
$myrc
-eq
0
]
;
then
echo
-n
"gzip ... "
echo
-n
"gzip ... "
compress_file
$OUTFILE
compress_file
"
$OUTFILE
"
fetchrc
fetchrc
if
[
$myrc
-eq
0
]
;
then
if
[
$myrc
-eq
0
]
;
then
cp
${
OUTFILE
}*
${
ARCHIVFILE
}
&&
echo
${
sSequenceCurrent
}
>
${
SEQFILE
}
cp
"
${
OUTFILE
}
"
*
"
${
ARCHIVFILE
}
"
&&
echo
"
${
sSequenceCurrent
}
"
>
"
${
SEQFILE
}
"
ls
-l
${
ARCHIVFILE
}
${
SEQFILE
}
ls
-l
"
${
ARCHIVFILE
}
"
"
${
SEQFILE
}
"
fi
fi
else
else
echo
"ERROR occured while dumping - abort"
echo
"ERROR occured while dumping - abort"
fi
fi
ls
-l
$OUTFILE
*
ls
-l
"
$OUTFILE
"
*
echo
echo
fi
fi
done
done
rm
-f
$dblist
rm
-f
"
$dblist
"
}
}
# ---------- RESTORE
# ---------- RESTORE
...
...
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment