diff --git a/backup.sh b/backup.sh
index e4bcfabdf07088b08d95c4d46b979e7aa01ab959..86f1ebda632769447c3f1a85c173c06f38381d60 100755
--- a/backup.sh
+++ b/backup.sh
@@ -91,7 +91,7 @@ do
echo "Param: setting to AUTO on duplicity"
echo "$2" | grep "^[1-9][0-9]*[a-zA-Z]$" 2>/dev/null
if [ $? -ne 0 ]; then
- echo "ERROR: the value after $1 must be a number and a letter (for unit); i.e. 1M for 1 month"
+ >&2 echo "ERROR: the value after $1 must be a number and a letter (for unit); i.e. 1M for 1 month"
exit 1
else
JOBDEF_TYPE=auto
@@ -112,7 +112,7 @@ do
bStart=1
;;
*)
- echo "ERROR: parameter [$1] is unknown"
+ >&2 echo "ERROR: parameter [$1] is unknown"
showHelp
exit 1
esac
diff --git a/docs/30_Configuration/50_File_backup.job.md b/docs/30_Configuration/50_File_backup.job.md
new file mode 100644
index 0000000000000000000000000000000000000000..8a0aa2e0c89f893b9a3e400af762e00e1de474d1
--- /dev/null
+++ b/docs/30_Configuration/50_File_backup.job.md
@@ -0,0 +1,59 @@
+# dist file #
+
+```
+# ----------------------------------------------------------------------
+# jobfile backup.job for scheduling local dumps
+#
+# SYNTAX:
+# [variable] = [value]
+#
+# - variable must start in first column
+# - char "=" must be surrounded by space
+# - value - any string; no " needed
+#
+# ----------------------------------------------------------------------
+
+lang = en_us
+
+# ----------------------------------------------------------------------
+# local dumps; each service has its own subdir there
+# this entry is mapped to BACKUP_TARGETDIR in "localdump.sh"
+# ----------------------------------------------------------------------
+dir-localdumps = /var/localdumps/backup
+keep-days = 7
+
+# archive databases (to keep deleted schemes or databases that
+# are not backupped anymore; couchdb2 feature)
+dir-dbarchive = /var/localdumps/archive
+
+# ----------------------------------------------------------------------
+```
+
+# Description #
+
+## dir-dbarchive ##
+
+Target dir for database archives (to keep deleted schemes or databases that are not backupped anymore; couchdb2 feature)
+
+`dir-dbarchive = /var/localdumps/archive`
+
+## dir-localdumps ##
+
+Target dir for database backups of all types.
+Below that directory a subdir for the database type will be created (i.e. mysql or pgsql).
+
+`dir-localdumps = /var/localdumps/backup`
+
+see also **keep-days**
+
+## keep-days ##
+
+How many days to keep database dumps locally in `dir-localdumps = ...`
+
+`keep-days = 7`
+
+see also **dir-localdumps**
+
+## lang ##
+
+Language of the shell. Do not change to keep documented keywords for date and time.
diff --git a/docs/30_Configuration/50_File_dirs.job.md b/docs/30_Configuration/50_File_dirs.job.md
new file mode 100644
index 0000000000000000000000000000000000000000..a31ac0448b7bb207b5642ae9e997485a40b1ca1b
--- /dev/null
+++ b/docs/30_Configuration/50_File_dirs.job.md
@@ -0,0 +1,83 @@
+# dist file #
+
+```
+# ----------------------------------------------------------------------
+#
+# jobfile for directories:
+# - which directories you want to backup
+#
+# ----------------------------------------------------------------------
+#
+# SYNTAX:
+# [variable] = [value]
+#
+# - variable must start in first column
+# - char "=" must be surrounded by space
+# - value - any string; no " needed
+#
+# ----------------------------------------------------------------------
+
+
+# ----------------------------------------------------------------------
+# directory list to transfer
+# without ending "/"
+# missing directories on a system will be ignored
+# ----------------------------------------------------------------------
+
+# --- /etc ... just in case
+include = /etc
+
+# --- system logs
+include = /var/log
+
+# --- user homes
+include = /home
+
+# --- web folder
+include = /var/www
+
+
+# ----------------------------------------------------------------------
+# excludes
+# see duplicity ... added as -exclude-regex parameter
+# ----------------------------------------------------------------------
+
+# exclude = .*\.(swp|tmp)
+
+# mac file
+# exclude = \.DS_Store
+
+# all files below a dir named "cache"
+# exclude = /cache/.*
+# exclude = /.cache/.*
+# exclude = local/share/Trash/*
+# exclude = /tmp/.*
+
+
+# ----------------------------------------------------------------------
+```
+
+# Description #
+
+## include ##
+
+Add a directory to backup with a single line per directory.
+
+`include = /etc`
+
+## exclude ##
+
+Add a exclude pattern. You can use multiple lines.
+If it is a lobbing ore a regex depends on the used backup tool.
+
+**Hint:**
+
+Finally do not spend time with defining exact exludes... better to backup all files
+of defined include rules and add some "trash" to the backup instead of missing files
+because of a wrong exclude pattern.
+
+`exclude = /tmp/.*`
+
+## TODO: custom single dirs with includes and exludes ##
+
+TODO.
diff --git a/docs/30_Configuration/10_backup.job.md b/docs/30_Configuration/50_File_transfer.job.md
similarity index 50%
rename from docs/30_Configuration/10_backup.job.md
rename to docs/30_Configuration/50_File_transfer.job.md
index dbbb2d4bb371a313fe23d064c88eff9f63b741d2..c02b4b2186f80c4978a255eec8d5dd566e892c8c 100644
--- a/docs/30_Configuration/10_backup.job.md
+++ b/docs/30_Configuration/50_File_transfer.job.md
@@ -1,8 +1,11 @@
-# dist file #
+# dist file
```
# ----------------------------------------------------------------------
-# jobfile backup.job for scheduling local dumps
+#
+# jobfile for transfer of local data to backup-target
+#
+# ----------------------------------------------------------------------
#
# SYNTAX:
# [variable] = [value]
@@ -13,7 +16,59 @@
#
# ----------------------------------------------------------------------
-lang = en_us
+
+
+# ......................................................................
+# backend:
+# ......................................................................
+#
+# ===== Duplicity:
+#
+# file based / Samba share
+# storage = file:///some/where
+#
+# ssh connection:
+# storage = rsync://bacup@storage.example.com//netshare/backup
+# if rsync does not work try scp (is slower)
+# storage = scp://backup@storage.example.com//netshare/backup
+#
+# ===== Restic
+# DOCS: https://restic.readthedocs.io/
+#
+# ssh connection:
+# storage = sftp://backup@storage.example.com//netshare/backup
+#
+# https connection (requires a Http Rest server - i.e. Restic Http or Rclone http server)
+# storage = rest:https://user:pass@host:8000/my_backup_repo/
+#
+#
+#
+# ===== for scp/ sftp/ rsync transfer you need to add an ssh private key
+# ssh-privatekey = /opt/imlbackup/client/keys/id_imlbackup@storage-connector.iml.unibe.ch
+#
+# HINT: if using scp:// on Debian 8 enable ssh-backend; default: no value
+# duplicity_ssh-backend = pexpect
+
+# for local or locally mounted targets - you can add a test file that must be found
+# to detect that a backup volume is mounted
+# storage-file = /run/media/backup/I_am_mounted.txt
+
+# OPTIONAL: register for a backup slot
+# storage-register = ssh
+# storage-registercmd = ./storage_helper.sh
+
+
+# ----------------------------------------------------------------------
+# BASIC settings
+# ----------------------------------------------------------------------
+
+# what tool to use? one of duplicity|restic ... see plugins/transfer/*.sh
+bin = restic
+# bin = duplicity
+
+# encrypt backup sets with a password
+# remark: "gnupg-passphrase = ..." is deprecated
+passphrase = EnterYourSecretHere
# ----------------------------------------------------------------------
# backup type; one of auto|inc
@@ -47,25 +102,65 @@ type = auto
# auto = <time> i.e. 1M
# auto = 1M
+# automatic backup of samba shares (only if samba shares were detected)
+sambashares = 1
+
+restore-path = /restore
+
# ----------------------------------------------------------------------
-# local dumps; each service has its own subdir there
-# this entry is mapped to BACKUP_TARGETDIR in "localdump.sh"
+#
+# DUPLICITY
+# values without prefix "duplicity_" are deprecated
+#
# ----------------------------------------------------------------------
-dir-localdumps = /var/localdumps/backup
-keep-days = 7
-# archive databases (to keep deleted schemes or databases that
-# are not backupped anymore; couchdb2 feature)
-dir-dbarchive = /var/localdumps/archive
+# for fallback on Debian 8:
+# ssh-backend = pexpect
+# duplicity_ssh-backend =
+
+# time to keep backup sets on storage
+# M - monthes
+# Y - years
+# keep = 6M
+duplicity_keep = 6M
+
+# volume size on backup target im MB
+# volsize = 350
+duplicity_volsize = 350
+
+# use another cache directory than ~/.cache/duplicity/
+# cachedir =
+duplicity_cachedir =
+
# ----------------------------------------------------------------------
+#
+# RESTIC
+#
+# ----------------------------------------------------------------------
+
+# use another cache directory than /root/.cache/restic
+restic_cachedir =
+
+restic_verbose = 2
+restic_tag = imlbackup
+restic_mountpoint = /mnt/restore
+
+# prune
+restic_keep-hourly = 100
+restic_keep-daily = 90
+restic_keep-weekly = 12
+restic_keep-monthly = 12
+restic_keep-yearly = 10
+
+# ----------------------------------------------------------------------
```
-# Description #
+# Description
-## auto ##
+## auto
Define when to make full backups for `type = auto`. It starts incremental backups
only as long the last full backup is not older than the given range.
@@ -77,30 +172,40 @@ s, m, h, D, W, M, or Y (indicating seconds, minutes, hours, days, weeks, months,
`auto = 1M`
-## dir-dbarchive ##
-
-Target dir for database archives (to keep deleted schemes or databases that are not backupped anymore; couchdb2 feature)
-
-`dir-dbarchive = /var/localdumps/archive`
-
-## dir-localdumps ##
-
-Target dir for database backups of all types.
-Below that directory a subdir for the database type will be created (i.e. mysql or pgsql).
+## bin
-`dir-localdumps = /var/localdumps/backup`
+Name of the tool to use for file transfer.
-see also **keep-days**
+The value is one of restic|duplicity
-## keep-days ##
+`bin = restic`
-How many days to keep database dumps locally in `dir-localdumps = ...`
+## duplicity_\* ###
-`keep-days = 7`
+Variables that are specific for duplicity. They will be used when you set
+bin = duplicity.
-see also **dir-localdumps**
+```text
+# for fallback on Debian 8:
+# ssh-backend = pexpect
+# duplicity_ssh-backend =
+
+# time to keep backup sets on storage
+# M - monthes
+# Y - years
+# keep = 6M
+duplicity_keep = 6M
+
+# volume size on backup target im MB
+# volsize = 350
+duplicity_volsize = 350
+
+# use another cache directory than ~/.cache/duplicity/
+# cachedir =
+duplicity_cachedir =
+```
-## full ##
+## full
Scheduling days for full backup
You can set
@@ -125,11 +230,54 @@ Examples:
See also: **start-time-full**
-## lang ##
+## passphrase
+
+Passphrase to encrypt data locally before sending backup data to a target.
+
+Keep your password on an additional (safe) place: without it you cannot
+restore any data anymore.
+
+`passphrase = EnterYourSecretHere`
+
+## restic_*
+
+Variables that are specific for restic. They will be used when you set
+bin = restic.
+
+```text
+# use another cache directory than /root/.cache/restic
+restic_cachedir =
+
+restic_verbose = 2
+restic_tag = imlbackup
+
+restic_mountpoint = /mnt/restore
+
+# prune
+restic_keep-hourly = 100
+restic_keep-daily = 90
+restic_keep-weekly = 12
+restic_keep-monthly = 12
+restic_keep-yearly = 10
+```
+
+## restore-path
-Language of the shell. Do not change to keep documented keywords for date and time.
+Default path where to put restored file when running *restore.sh*.
-## start-time-full ##
+After restore action you need to move/ copy the needed files to the wanted place.
+
+`restore-path = /restore`
+
+## sambashares
+
+If the backup is running on a server with installed samba all shares
+will be detected from /etc/samba/smb.conf and handled like a
+added directory in dirs.job
+
+`sambashares = 1`
+
+## start-time-full
Time when to start an incrmental backup. This value will be handled
if a day matches **full** only.
@@ -143,7 +291,7 @@ This will start it if backup ist started between 23:00:00 and 23:09:59:
see also **full**
-## type ##
+## type
Type of backup. The value is one of auto|inc
@@ -152,110 +300,23 @@ Type of backup. The value is one of auto|inc
This setting depends on the possibilities of the used backup tool.
-Restic:
+### Restic
* set to "auto" only
-Duplicity:
+### Duplicity
Duplicity supports 2 modes.
* "inc" - make full and incremental backups defined times. You need more values:
+
* full = WDM:3rd Fri
* start-time-full = 23:0.
-
* "auto" - starts incremental backups as long last full backup is not older value in
+
* auto = 1M
`type = auto`
see also **bin**
-## Example: Restic ##
-
-```text
-lang = en_us
-type = auto
-
-
-# ----------------------------------------------------------------------
-# local dumps; each service has its own subdir there
-# this entry is mapped to BACKUP_TARGETDIR in "localdump.sh"
-# ----------------------------------------------------------------------
-dir-localdumps = /var/localdumps/backup
-keep-days = 7
-
-# archive databases (to keep deleted schemes or databases that
-# are not backupped anymore; couchdb2 feature)
-dir-dbarchive = /var/localdumps/archive
-
-```
-
-## Example: Duplicity ##
-
-### Full backups on a specified time ###
-
-This can be used on servers where you want to define when the full
-backup must be started.
-
-```text
-lang = en_us
-type = inc
-
-# ----------------------------------------------------------------------
-# scheduling for full backup
-# type = inc only
-# for inc and full you can set
-# DOM:06,22 - days of month
-# DOW:Mon,Fri - weekdays
-# WDM:WDM:1st Fri - nth weekday in month
-# ----------------------------------------------------------------------
-
-full = WDM:3rd Fri
-
-# when to start a full backup?
-# regex that must match a time HH:MM somewhen
-# This will start it if backup ist started between 23:00:00 and 23:09:59
-start-time-full = 23:0.
-
-
-# ----------------------------------------------------------------------
-# local dumps; each service has its own subdir there
-# this entry is mapped to BACKUP_TARGETDIR in "localdump.sh"
-# ----------------------------------------------------------------------
-dir-localdumps = /var/localdumps/backup
-keep-days = 7
-
-# archive databases (to keep deleted schemes or databases that
-# are not backupped anymore; couchdb2 feature)
-dir-dbarchive = /var/localdumps/archive
-
-```
-
-### Full backups after a month ###
-
-This can be used on client backups. A full backup automatically starts
-if the last backup is older a given time.
-
-```text
-lang = en_us
-type = auto
-
-# ----------------------------------------------------------------------
-# full backups for type = auto AND when using duplicity
-# ----------------------------------------------------------------------
-# auto = <time> i.e. 1M
-auto = 1M
-
-
-# ----------------------------------------------------------------------
-# local dumps; each service has its own subdir there
-# this entry is mapped to BACKUP_TARGETDIR in "localdump.sh"
-# ----------------------------------------------------------------------
-dir-localdumps = /var/localdumps/backup
-keep-days = 7
-
-# archive databases (to keep deleted schemes or databases that
-# are not backupped anymore; couchdb2 feature)
-dir-dbarchive = /var/localdumps/archive
-```
\ No newline at end of file
diff --git a/jobhelper.sh b/jobhelper.sh
index 130925715778b5817f529b28ba9c361648546d6f..1158e2c128b78724bb9def2a68a1d34d4db5e203 100755
--- a/jobhelper.sh
+++ b/jobhelper.sh
@@ -47,7 +47,7 @@ function j_init(){
if [ ! -d "${DIR_JOBS}" ]; then
# mkdir -p ${DIR_JOBS} && echo "INFO: dir created ${DIR_JOBS}"
- echo "ERROR: missing jobs directory. Aborting."
+ >&2 echo "ERROR: missing jobs directory. Aborting."
exit 1
fi
@@ -169,7 +169,7 @@ function j_getSafename(){
# ------------------------------------------------------------
function _j_getvar(){
if [ ! -r "${1}" ]; then
- echo "ERROR: cannot read file: ${1}. Abort."
+ >&2 echo "ERROR: cannot read file: ${1}. Abort."
exit 100
fi
grep "^${2}\ =\ " < "${1}"| cut -f 3- -d " "
@@ -373,7 +373,7 @@ function j_requireProcess(){
function j_requireUser(){
sUser=$(id | cut -f 2 -d "(" | cut -f 1 -d ")")
if [[ "$sUser" != "$1" ]]; then
- echo "ERROR: user $1 is reqired."
+ >&2 echo "ERROR: user $1 is reqired."
exit 5
fi
}
diff --git a/jobs/backup.job.dist b/jobs/backup.job.dist
index 81f253b0d1973114bfb236146588bb85b68bff42..b7b09c241ea29ca66f72c1e713befa7a95fbdda8 100644
--- a/jobs/backup.job.dist
+++ b/jobs/backup.job.dist
@@ -12,7 +12,6 @@
lang = en_us
-
# ----------------------------------------------------------------------
# local dumps; each service has its own subdir there
# this entry is mapped to BACKUP_TARGETDIR in "localdump.sh"
diff --git a/jobs/dirs.job.dist b/jobs/dirs.job.dist
index c527e3e709499b2bad58c42d0512af4eeb0ba94a..50e49eda92859f3b62899823bfc5f3daf86b4d9d 100644
--- a/jobs/dirs.job.dist
+++ b/jobs/dirs.job.dist
@@ -1,8 +1,8 @@
# ----------------------------------------------------------------------
#
# jobfile for directories:
-# - where to put local dumps
# - which directories you want to backup
+# - what to exclude
#
# ----------------------------------------------------------------------
#