jobhelper.sh

#!/bin/bash
# ================================================================================
#
# JOBHELPER
# helper script to share functions for parsing and handlinmg backup jobs
#
# --------------------------------------------------------------------------------
# ah - Axel Hahn <axel.hahn@iml.unibe.ch>
# ds - Daniel Schueler <daniel.schueler@iml.unibe.ch>
#
# 2016-11-10  ah,ds  v1.0
# 2017-01-23  ah,ds  v1.1  added j_getLastBackupAge
# 2017-02-16  ah,ds  v1.2  added storage helper function
# 2018-02-13  ah,ds  v1.3  detect samba shares based on a flag
# ================================================================================


# ----------------------------------------------------------------------
# CONFIG
# ----------------------------------------------------------------------

DIR_SELF=`dirname $0`

DIR_JOBS=${DIR_SELF}/jobs
DIR_LOGS=${DIR_SELF}/logs

JOBFILE=${DIR_JOBS}/backup.job
DIRFILE=${DIR_JOBS}/dirs.job
STORAGEFILE=${DIR_JOBS}/transfer.job

# RMQ_SERVER=



# ----------------------------------------------------------------------
# METHODS
# ----------------------------------------------------------------------


# ------------------------------------------------------------
# IDEA ONLY; UNUSED
# fetch current job from queue and store it to job directory
# ------------------------------------------------------------
function j_fetch(){
  echo TODO: fetch from RMQ_SERVER
}


# ------------------------------------------------------------
# init function
# ------------------------------------------------------------
function j_init(){

  j_banner

  if [ ! -d ${DIR_JOBS} ]; then
    # mkdir -p ${DIR_JOBS} && echo "INFO: dir created ${DIR_JOBS}"
    echo "ERROR: missing jobs directory. Aborting."
    exit 1
  fi

  # if transfer.sh exists, then a transfer.job must exist too
  if [ -x "${DIR_SELF}/transfer.sh" ]; then
    ls ${STORAGEFILE} >/dev/null
    if [ $? -ne 0 ]; then
      echo "ERROR: missing a config file in ${DIR_JOBS}"
      exit 1
    fi
  fi

  if [ ! -d ${DIR_LOGS} ]; then
    mkdir -p ${DIR_LOGS} && echo "INFO: dir created ${DIR_LOGS}"
  fi

  ls ${JOBFILE} ${DIRFILE} >/dev/null
  if [ $? -ne 0 ]; then
    echo "ERROR: missing a config file in ${DIR_JOBS}"
    exit 1
  fi

  # for date definitions like weekdays
  JOBDEF_LANG=`_j_getvar ${JOBFILE} "lang"`
  if [ -z $JOBDEF_LANG ]; then
    JOBDEF_LANG="en_us"
  fi
  export LANG=$JOBDEF_LANG

  j_requireBinary "cut"
  j_requireBinary "date"
  j_requireBinary "grep"
  j_requireBinary "sleep"
  j_requireBinary "tee"
  j_requireBinary "touch"

  # j_read
}

# ------------------------------------------------------------
# draw an ascii banner
# ------------------------------------------------------------
function j_banner(){
  cat << eofbanner

  ___ ___ ___ ___         _______            __
 |   |   Y   |   |       |   _   .---.-.----|  |--.--.--.-----.
 |.  |.      |.  |       |.  1   |  _  |  __|    <|  |  |  _  |
 |.  |. \_/  |.  |___    |.  _   |___._|____|__|__|_____|   __|
 |:  |:  |   |:  1   |   |:  1    \                     |__|
 |::.|::.|:. |::.. . |   |::.. .  /
 \`---\`--- ---\`-------'   \`-------'

eofbanner
  sed "s#^#   #g" "$0.banner" 2>/dev/null || echo $0
  echo '_______________________________________________________________________________  ___  __   _'
  echo
}

# ------------------------------------------------------------
# get list of all directories to backup / restore
# ------------------------------------------------------------
function j_getDirs2Backup(){
  STORAGE_SAMBASHARES=`_j_getvar ${STORAGEFILE} "sambashares"`

  _j_getvar ${DIRFILE} dir-localdumps
  _j_getvar ${DIRFILE} dir-dbarchive
  _j_getvar ${DIRFILE} include

  # get dirs of filesets, i.e.
  # set-custom-[key]--dir = /home/ladmin
  _j_getvar ${DIRFILE} "set.*--dir"

  # detect Samba shares (set sambashares = 1 for it)
  if [ -z "${STORAGE_SAMBASHARES}" -o ${STORAGE_SAMBASHARES} -eq 0 ]; then
    echo NO >/dev/null
  else
    if [ -f /etc/samba/smb.conf ]; then
      for dirOfShare in `cat /etc/samba/smb.conf  | grep "path.*="  | grep -v "#.*path" | cut -f 2 -d "=" `
      do
        echo $dirOfShare
      done
    fi
  fi

}

# ------------------------------------------------------------
# get the id of a given backup dir (to find includes and
# excludes for it)
# param  string  path
# ------------------------------------------------------------
function j_getSetnameOfPath(){
  cat ${DIRFILE} | grep "^set.*dir\ =\ $*$" | cut -f 1 -d "=" | sed "s#\-\-dir\ ##g"
}

# ------------------------------------------------------------
# get full target of the current host with a given
# backed up directory
# param  string  name of directory
# param  string  name of host (for future releases)
# ------------------------------------------------------------
function j_getFullTarget(){
  sTmpSafeName=`j_getSafename "$1"`
  sTmpHostname=$2
  if [ -z $sTmpHostname ]; then
    sTmpHostname=`hostname -f`
  fi
  if [ -z ${STORAGE_BASEDIR} ]; then
    STORAGE_BASEDIR=`_j_getvar ${STORAGEFILE} "storage"`
  fi
  echo ${STORAGE_BASEDIR}/${sTmpHostname}/${sTmpSafeName}
}

# ------------------------------------------------------------
# get minimal Age of last backup that had to run in hours
# it returns a value between 24 and 96
# ------------------------------------------------------------
function j_getLastBackupAge(){
  typeset -i sCmpDate
  typeset -i iWasInc
  typeset -i iWasFull
  j_read

  JOBDEF_INC=`_j_getvar ${JOBFILE} "inc"`
  JOBDEF_FULL=`_j_getvar ${JOBFILE} "full"`

  for iDeltaH in {24..96}
  do
    sCmpDate=`date +%s`-iDeltaH*60*60
    iWasInc=`_j_wasThisDay @${sCmpDate} $JOBDEF_INC`
    iWasFull=`_j_wasThisDay @${sCmpDate} $JOBDEF_FULL`
    if [ ${iWasInc} -gt 0  -o  ${iWasFull} -gt 0 ]; then
      echo $iDeltaH
      exit
    fi
  done
  echo $iDeltaH
}

# ------------------------------------------------------------
# replace / to _ to get a save filename for a directory to
# backup
# param  string  name of directory
# ------------------------------------------------------------
function j_getSafename(){
  echo $* | sed 's#/#_#g'
}

# ------------------------------------------------------------
# get a value from a job config file
# param  string  name of jobfile
# param  string  name of variable to fetch value from
# ------------------------------------------------------------
function _j_getvar(){
  if [ ! -r "${1}" ]; then
    echo "ERROR: cannot read file: ${1}. Abort."
    exit 100
  fi
  cat "${1}" | grep "^${2}\ =\ " | cut -f 3- -d " "
}

# ------------------------------------------------------------
# read local jobdescription and set as variables
# ------------------------------------------------------------

# ------------------------------------------------------------
# parse day of week and day of month and echo 0 or 1
#
#   full = DOW:Mon,Tue,Wed
#   full = DOM:07,08,09,
#
# It is used for inc = ... and full = ...
#
# param  string  date to compare
# param  string  value of full|inc in backup.job
# ------------------------------------------------------------
function _j_wasThisDay(){
  typeset -i bToday=0
  sCompDate="$1"
  shift 1
  value="$*"

  # grep weekday
  echo $value | grep "^DOW:" | grep `date +%a -d $sCompDate` >/dev/null && bToday=1

  # grep day of month
  echo $value | grep "^DOM:" | grep `date +%d -d $sCompDate` >/dev/null && bToday=1

  # grep nth weekday of a month
  echo $value | grep "^WDM:" >/dev/null
  if [ $? -eq 0 ]; then

    typeset -i iDayOfMonth=`date +%e -d $sCompDate`
    typeset -i iWeekday=`date +%u -d $sCompDate`
    # `date +%u` - weekday as int; Sun = 0
    # `date +%e` - day in date
    typeset -i iWeekInMonth=$(echo $(( ( ${iDayOfMonth} - ${iWeekday} + 6 ) / 7 )) )

    typeset -i n=`echo $value | grep "^WDM:" | cut -f 2- -d ":" | cut -c 1`
    sDay=`echo $value | grep "^WDM:" | cut -f 2- -d ":" | cut -f 2 -d " "`

    if [ ${n} -eq ${iWeekInMonth} -a ${sDay} = `date +%a  -d $sCompDate` ]; then
        bToday=1
    fi
  fi

  echo $bToday
}

# ------------------------------------------------------------
# parse day of week and day of month and echo 0 or 1
#
#   full = DOW:Mon,Tue,Wed
#   full = DOM:07,08,09,
#
# It is used for inc = ... and full = ...
#
# param  string  value of full|inc in backup.job
# ------------------------------------------------------------
function _j_isToday(){
  sCmpDate=`date +%s`
  _j_wasThisDay "@$sCmpDate" $*
}

# ------------------------------------------------------------
# read local jobdescription and set as variables
# ------------------------------------------------------------
function j_read(){

  mytime=`date +%H%M`

  # --- parse something

  BACKUP_TARGETDIR=`_j_getvar ${JOBFILE} "dir-local-dumps"`

  JOBDEF_TYPE=`_j_getvar ${JOBFILE} "type"`
  if [ -z "$JOBDEF_TYPE" ]; then
    JOBDEF_TYPE=auto
  fi


  JOBDEF_INC=`_j_getvar ${JOBFILE} "inc"`
  JOBDEF_FULL=`_j_getvar ${JOBFILE} "full"`
  JOBDEF_AUTO=`_j_getvar ${JOBFILE} "auto"`
  if [ "$JOBDEF_TYPE" = "auto" ]; then
    if [ -z "$JOBDEF_AUTO" ]; then
      JOBDEF_AUTO=1W
    fi
  else
    JOBDEF_AUTO=
  fi

  bIsTodayInc=`_j_isToday $JOBDEF_INC`
  bIsTodayFull=`_j_isToday $JOBDEF_FULL`


  JOB_DOTODAY=1
  if [ $bIsTodayFull -eq 0 -a $bIsTodayInc -eq 0 ]; then
    JOB_DOTODAY=0
  fi


  sStartInc=`_j_fetchLatestStarttime "start-time-inc"`
  JOBDEF_STARTTIME=$sStartInc

  if [ $bIsTodayFull -eq 1 ]; then
    sStartFull=`_j_fetchLatestStarttime "start-time-full"`

    if [ $bIsTodayInc -eq 1  -a  $sStartFull -ne $sStartInc ]; then
      echo INFO: full backup today $sStartFull - but incremental is at $sStartInc
      echo -n ""
    else
      # echo INFO: full backup today $sStartFull
      JOBDEF_TYPE="full"
      JOBDEF_STARTTIME=$sStartFull
    fi
  fi

  JOB_DONEFILE=${DIR_LOGS}/${JOBDEF_TYPE}-`date +%Y%m%d`-${JOBDEF_STARTTIME}
  JOB_LOGFILE="${JOB_DONEFILE}.log"
}

# ------------------------------------------------------------
# date helper for job entries in start-time-inc/ start-time-full
# * get first the first entry or - if many - the latest entry
# * if empty: take value from start-time
# param  string  one of start-time-inc|start-time-full
# ------------------------------------------------------------
function _j_fetchLatestStarttime(){
  sLatest=
  sStart=`_j_getvar ${JOBFILE} "$1" | sed "s#[\ \:\-]##g"`
  for sTime in `echo $sStart | sed "s#,# #g"`
  do
    if [ -z $sLatest ]; then
      sLatest=$sTime
    fi
    if [ $sTime -le `date +%H%M` ]; then
      sLatest=$sTime
    fi
  done
  if [ -z $sLatest ]; then
    $sLatest=`_j_getvar ${JOBFILE} "start-time" | sed "s#[\ \:\-]##g"`
  fi
  if [ -z $sLatest ]; then
    color error
    echo ERROR: missing start time info for $1
    color reset
    exit 1
  fi
  echo $sLatest
}

# ------------------------------------------------------------
# check if a binary exists - and abort if not
# param  string  name of file
# param  bool    flag to skip; default: none = abort on miss
# ------------------------------------------------------------
function j_requireBinary(){
  # echo "CHECK binary $1"
  which "$1" >/dev/null
  rcself=$?
  if [ $rcself -ne 0 ]; then
    rc=$rc+$rcself
    echo "INFO: missing binary $1"
    if [ -z $2 ]; then
      exit 3
    fi
  fi
}

# ------------------------------------------------------------
# check if a process name exisats
# param  string  regex to find in ps -ef output
# param  bool    flag to skip; default: none = abort on miss
# ------------------------------------------------------------
function j_requireProcess(){
  # echo "CHECK process $1"
  ps -ef | grep -v grep | grep -E "$1" >/dev/null
  rcself=$?
  if [ $rcself -ne 0 ]; then
    rc=$rc+$rcself
    echo "INFO: missing process $1"
    if [ -z $2 ]; then
      exit 4
    fi
  fi
}

# ------------------------------------------------------------
# check if it was startet with a given user
# param  string  username, i.e. root
# ------------------------------------------------------------
function j_requireUser(){
  sUser=`id | cut -f 2 -d "(" | cut -f 1 -d ")"`
  if [[ $sUser != "$1" ]]; then
    echo "ERROR: user $1 is reqired."
    exit 5
  fi
}

# ----------------------------------------------------------------------
# INIT
# ----------------------------------------------------------------------

j_init