#!/bin/bash
#------------------------------------------------------------------------------#
# vi: set sw=4 ts=4 ai:                            ("set modeline" in ~/.exrc) #
#------------------------------------------------------------------------------#
# Program      : makebackup                                                    #
#                                                                              #
# Author       : Ton Kersten                        Groesbeek, The Netherlands #
#                                                                              #
# Date         : 26-03-2008                                       Time : 11:35 #
#                                                                              #
# Description  : Create a backup of all specified directories                  #
#                                                                              #
# Parameters   : None                                                          #
#                                                                              #
# Pre reqs     : Rsync, the backupit tool and Perl                             #
#                                                                              #
# Exit codes   :    0 -> OK                                                    #
#                <> 0 -> !OK                                                   #
#                                                                              #
# Updates      : None (yet)                                                    #
#------------------------------------------------------------------------------#

#------------------------------------------------------------------------------#
#                    V e r s i o n   i n f o r m a t i o n                     #
#------------------------------------------------------------------------------#
# $Id:: makebackup 32 2011-05-24 09:08:57 tonk                              $: #
# $Revision:: 32                                                            $: #
# $Author:: Ton Kersten <tonk@tonkersten.com>                               $: #
# $Date:: 2011-05-24 09:13:28 +0200 (Tue, 24 May 2011)                      $: #
# $Hash:: 3e91776659ef7b65488e9f17d1fc1f6b73591687 (tonk)                   $: #
#------------------------------------------------------------------------------#
#             E n d   o f   v e r s i o n   i n f o r m a t i o n              #
#------------------------------------------------------------------------------#

#------------------------------------------------------------------------------#
# Determine the program name and the 'running directory'                       #
#------------------------------------------------------------------------------#
IAM="${0##*/}"
CRD="$( [[ "${0:0:2}" = "./" ]] &&
	{	printf "${PWD}/${0#./}"
	} || {
		printf "${0}"
	})"
CRD="${CRD%/*}"
CUR="${PWD}"

#------------------------------------------------------------------------------#
# Save the shell settings                                                      #
#------------------------------------------------------------------------------#
SETA=0; [[ ${-} = *a* ]] && SETA=1
SETE=0; [[ ${-} = *e* ]] && SETE=1
SETU=0; [[ ${-} = *u* ]] && SETU=1
SETX=0; [[ ${-} = *x* ]] && SETX=1

#------------------------------------------------------------------------------#
# Set and unset the needed shell settings                                      #
#------------------------------------------------------------------------------#
set +o noclobber            # Overwrite existing files, if needed              #
set -o nounset              # Don't allow uninitialized variables              #
set +o errexit              # No returncode checking                           #

#------------------------------------------------------------------------------#
# Define the date/time function                                                #
#------------------------------------------------------------------------------#
Now()
{
	date '+%Y-%m-%d %H:%M:%S'

	return 0
}

#------------------------------------------------------------------------------#
# Calculate h:m:s from seconds                                                 #
#------------------------------------------------------------------------------#
hms()
{
	s="${1}"

	h=$(( ${s}/3600 ))
	s=$(( ${s} - ( ${h} * 3600) ))
	m=$(( ${s} / 60))
	s=$(( ${s} -( ${m} * 60 ) ))

	printf "%d:%02d:%02d" ${h} ${m} ${s}
}

#------------------------------------------------------------------------------#
# Send a line to syslog (if $1 = '-f' the line is forced to syslog)            #
#------------------------------------------------------------------------------#
syslog()
{	if [[ x"${1:-}" = x"-f" ]]
	then
		shift
		echo "${*:-}" | logger -t "${IAM}" -p "${SYSLOGFAC:-local4.info}"
		return
	fi

	if [[ x"${SYSLOG:-0}" = x"1" ]]
	then
		echo "${*:-}" | logger -t "${IAM}" -p "${SYSLOGFAC:-local4.info}"
	fi
}

#------------------------------------------------------------------------------#
# Check the configfile                                                         #
#------------------------------------------------------------------------------#
CONFIGFILE="${1:-${CRD}/${IAM}.conf}"
if [[ ${#} = 0 ]]
then
	LOGFILE="/var/log/${IAM}.log"
else
	LOGFILE="/var/log/${IAM}-$(basename ${CONFIGFILE} | sed 's!\.conf.*$!!').log"
fi

#------------------------------------------------------------------------------#
# Start the logging                                                            #
#------------------------------------------------------------------------------#
exec > ${LOGFILE}
exec 2>&1

#------------------------------------------------------------------------------#
# Define constants and variables                                               #
#------------------------------------------------------------------------------#
VER="1.$(awk '/^# \$Revision::/ { print $3 }' ${0})"
PATH="${PATH}:/bin:/sbin"
PATH="${PATH}:/usr/bin:/usr/sbin"
PATH="${PATH}:/usr/local/bin:/usr/local/sbin"
PATH="${PATH}:${CRD}"
BCK="${CRD}/backupit"
HOSTNAME="$(hostname)"
WD="80"                             # Screen width                             #
s="$(printf "%${WD}s" "")"; s="${s// /-}"			# Single dash line         #
d="$(printf "%${WD}s" "")"; d="${d// /=}"			# Double dash line         #
typeset -i RC=0
typeset -i BCKS=0

#------------------------------------------------------------------------------#
# Announce the start of the program                                            #
#------------------------------------------------------------------------------#
BCKSTART=$(date '+%s')
BCKSTARTTIME=$(Now)
echo "${d}"
echo
echo "$(Now) -> Starting ${IAM} version ${VER}"
echo
echo "${d}"
echo
echo "     System backup at ${HOSTNAME} ended BACKUPSTATUS"
echo
echo "${d}"
echo
syslog -f "Starting ${IAM}"

#------------------------------------------------------------------------------#
# Read the config file                                                         #
#------------------------------------------------------------------------------#
if [[ -r "${CONFIGFILE}" ]]
then
	. "${CONFIGFILE}" || {
		echo "Error processing config '${CONFIGFILE}'!"
		syslog -f "Error processing config '${CONFIGFILE}'!"
		exit 1
	}
else
	echo "Could not find config '${CONFIGFILE}'!" >&2
	syslog -f "Could not find config '${CONFIGFILE}'!"
	exit 1
fi

#------------------------------------------------------------------------------#
# Set verbose and debug                                                        #
#------------------------------------------------------------------------------#
[[ "${VERBOSE:-0}" = "1" ]] && VERBOSE="-v" || VERBOSE=""
[[ "${DEBUG:-0}"   = "1" ]] && DEBUG="-d"   || DEBUG=""
[[ x"${DEBUG}"    != x"" ]] && set -x

#------------------------------------------------------------------------------#
# Set defaults for unset stuff                                                 #
#------------------------------------------------------------------------------#
FAILS=0
MAX="${MAX:-99}"
BACKUPMYSQL="${BACKUPMYSQL:-no}"
SYSLOG="${SYSLOG:-1}"
SYSLOGFAC="${SYSLOGFAC:-local4.info}"
MAILTO="${MAILTO:-root@localhost}"
PREBACKUP="${PREBACKUP:-}"
POSTBACKUP="${POSTBACKUP:-}"

#------------------------------------------------------------------------------#
# Define the language settings (not allways needed)                            #
#------------------------------------------------------------------------------#
export LANG=${LANG:-C}
export LC_ALL=${LC_ALL:-C}

#------------------------------------------------------------------------------#
# Run a command before we begin                                                #
#------------------------------------------------------------------------------#
if [[ x"${PREBACKUP:-}" != x"" ]]
then
	syslog "Starting pre backup command"
	echo "${d}"
	echo "Prebackup command output."
	echo
	eval ${PREBACKUP} 2>&1 > /tmp/${IAM}_${$}
	THISRC=${?}

	sed 's/^/    /' /tmp/${IAM}_${$}

	if [[ ( ${THISRC} != 0 ) && ( ${SYSLOG:-0} = 1 ) ]]
	then
		sed "s/^/${SHRT[${CNT}]:-} :   /" /tmp/${IAM}_${$} 	| \
			logger -t "${IAM}" -p "${SYSLOGFAC:-local4.info}"
	fi

	echo
	echo "Return code for the prebackup command: ${THISRC}"
	echo
	echo "${d}"
	echo
	syslog "Prebackup command ended with returncode ${THISRC}"
	rm -f /tmp/${IAM}_${$}
fi

#------------------------------------------------------------------------------#
# Backup MySQL if this is requested                                            #
#------------------------------------------------------------------------------#
if [[ x"${BACKUPMYSQL:-no}" = x"yes" ]]
then
	syslog "Starting MyQL backup"
	if [[ -x ${CRD}/mysqlbackup ]]
	then
		echo "$(Now) -> ${IAM}: Starting MySQL backup on host '$(hostname)'"
		syslog "Starting MySQL backup on host '$(hostname)'"
		${CRD}/mysqlbackup
		echo "$(Now) -> ${IAM}: Finished MySQL backup on host '$(hostname)'"
		syslog "Finished MySQL backup on host '$(hostname)'"
	else
		echo "Cannot find MySQL backup tool '${CRD}/mysqlbackup'" >&2
		echo "MySQL backup will be skipped!!" >&2
		syslog "Cannot find MySQL backup tool '${CRD}/mysqlbackup'"
		syslog "MySQL backup will be skipped!!"
	fi
	syslog "Finished MyQL backup"
fi

#------------------------------------------------------------------------------#
# Check if the 'backupit' program exists. If not, there is no way to backup    #
#------------------------------------------------------------------------------#
if [[ ! -x "${BCK}" ]]
then
	echo "The backup program '${BCK}' cannot be found!" >&2
	echo "There is no way to create a backup." >&2
	echo "The program will be stopped." >&2

	syslog "The backup program '${BCK}' cannot be found!"
	syslog "There is no way to create a backup."
	syslog "The program will be stopped. (RC=1)"
	exit 1
fi

#------------------------------------------------------------------------------#
# Create the backups for all defined sources and destinations                  #
#------------------------------------------------------------------------------#
for CNT in $(seq 1 ${BCKS})
do
	for DST in ${DSTS[${CNT}]}
	do
		CYCLESTART=$(date '+%s')
		CYCLESTARTTIME=$(Now)

		[[ ! -d ${DST} ]] &&
		{	mkdir -p ${DST}
			chmod 700 ${DST}
		}

		#----------------------------------------------------------------------#
		# Create the backup source string                                      #
		#----------------------------------------------------------------------#
		SRC=""
		SRCTXT=""
		for dir in ${DIRS[${CNT}]}
		do
			SRC="${SRC} --source='${dir}'"
			SRCTXT="${SRCTXT} ${dir}"
		done

		#----------------------------------------------------------------------#
		# Create the backup exclude string (may be empty)                      #
		#----------------------------------------------------------------------#
		EXC=""
		EXCTXT=""
		for exc in ${EXCL[${CNT}]:-}
		do
			EXC="${EXC} --exclude='${exc}'"
			EXCTXT="${EXCTXT} ${exc}"
		done
		[[ x"${EXCTXT}" = x"" ]] && EXCTXT="None"

		#----------------------------------------------------------------------#
		# Find out the number of cycles to keep                                #
		#----------------------------------------------------------------------#
		CYCLES=${CYCL[${CNT}]:-${MAX}}

		#----------------------------------------------------------------------#
		# Start with a summary of what to do                                   #
		#----------------------------------------------------------------------#
		set ${SRCTXT}; SRC1="${1}"; shift; SRCR="${@:-}"
		set ${EXCTXT}; EXC1="${1}"; shift; EXCR="${@:-}"
		echo
		echo
		echo "     Starting backup cycle"
		echo "    ${s:4}"
		echo "        Description       : "$(echo ${DESC[${CNT}]})
		echo "        Sources           : "${SRC1}
		for f in ${SRCR}
		do
			echo "                            "${f}
		done
		echo "        Excludes          : "${EXC1}
		for f in ${EXCR}
		do
			echo "                            "${f}
		done
		echo "        Destination       : "${DST}
		echo "        Number of cycles  : "${MAX}
		echo "        Cycle start time  : "$(Now)
		echo "    ${s:4}"
		echo
		echo
		syslog "${SHRT[${CNT}]:-} : Starting backup cycle for '${DESC[${CNT}]}'"

		#----------------------------------------------------------------------#
		# Do the backup                                                        #
		#----------------------------------------------------------------------#
		${BCK}					\
			${VERBOSE}			\
			${DEBUG}			\
			${SRC}				\
			${EXC}				\
			--max=${CYCLES}		\
			--target=${DST} 2>&1 > /tmp/${IAM}_${$}
		THISRC=${?}

		#----------------------------------------------------------------------#
		# Parse the logfile and send it to the standard logging                #
		#----------------------------------------------------------------------#
		sed 's/^/    /' /tmp/${IAM}_${$}

		#----------------------------------------------------------------------#
		# Send the logging to syslog if needed and wanted                      #
		#----------------------------------------------------------------------#
		if [[ ( ${THISRC} != 0 ) && ( ${SYSLOG:-0} = 1 ) ]]
		then
			sed "s/^/${SHRT[${CNT}]:-} :   /" /tmp/${IAM}_${$} | \
				logger -t "${IAM}" -p "${SYSLOGFAC:-local4.info}"
		fi

		#----------------------------------------------------------------------#
		# And, if things fail, increase the failure number                     #
		#----------------------------------------------------------------------#
		if [[ ( ${THISRC} != 0 ) ]]
		then
			FAILS=$(( ${FAILS} + 1 ))
			FAILHOST[${FAILS}]="${SHRT[${CNT}]:-}"
			FAILRC[${FAILS}]="${THISRC}"
		fi

		#----------------------------------------------------------------------#
		# Create a nice summary                                                #
		#----------------------------------------------------------------------#
		RC=${RC}+${THISRC}
		CYCLEEND=$(date '+%s')
		CYCLEENDTIME=$(Now)
		if [[ -e ${DST}/latest/ ]]
		then
			USED="$(du -hs ${DST}/latest/ 2>/dev/null | awk '{ print $1 " (" $2 ")" }')"
		else
			USED="Unknown"
		fi

		if [[ -e ${DST} ]]
		then
			FREE="$(df -hP ${DST} 2>/dev/null | awk ' !/^F/ { print $4 }')"
		else
			FREE="Unknown"
		fi

		RUN=$(hms $(( ${CYCLEEND} - ${CYCLESTART} )) )
		echo
		echo
		echo "     Cycle statistics"
		echo "    ${s:4}"
		echo "        Return code       : "${THISRC}
		echo "        Used disk space   : "${USED}
		echo "        Free disk space   : "${FREE}
		echo "        Cycle end time    : "$(Now)
		echo "        Cycle run time    : "${RUN}
		echo "    ${s:4}"
		echo
		echo
		SUM=""
		SUM="${SUM} START=${CYCLESTARTTIME};"
		SUM="${SUM} END=${CYCLEENDTIME};"
		SUM="${SUM} RUN=${RUN};"
		SUM="${SUM} RC=${THISRC};"
		SUM="${SUM} USED=${USED};"
		SUM="${SUM} FREE=${FREE};"
		syslog "${SHRT[${CNT}]:-} :   Backup sources     : ${SRCTXT}"
		syslog "${SHRT[${CNT}]:-} :   Backup destination : ${DST}"
		syslog "${SHRT[${CNT}]:-} :   Return code        : ${THISRC}"
		syslog "${SHRT[${CNT}]:-} :   Used disk space    : "${USED}
		syslog "${SHRT[${CNT}]:-} :   Free disk space    : "${FREE}
		syslog "${SHRT[${CNT}]:-} :   Cycle run time     : "${RUN}
		syslog "${SHRT[${CNT}]:-} : Summary : ${SUM}"
		syslog "${SHRT[${CNT}]:-} : Finished backup cycle for '${DESC[${CNT}]}'"
		rm -f /tmp/${IAM}_${$}
	done
	echo
	echo "${d}"
	echo
done

#------------------------------------------------------------------------------#
# Check if it all went well                                                    #
#------------------------------------------------------------------------------#
if [[ ${RC} = 0 ]]
then
	echo "$(Now) -> Backup ended OK!"
	BCKSTATUS="OK! Returncode = 0"
else
	if [[ ${RC} = 24 ]]
	then
		echo "$(Now) -> Backup ended OK!"
		BCKSTATUS="OK! (Some files vanished). Returncode = 24"
	else
		echo "$(Now) -> Backup ended NOT OK! Returncode = ${RC}"
		BCKSTATUS="NOT OK! Returncode = ${RC}"
	fi
fi

#------------------------------------------------------------------------------#
# Run a command after we are done                                              #
#------------------------------------------------------------------------------#
if [[ x"${POSTBACKUP:-}" != x"" ]]
then
	syslog "Starting pre backup command"
	echo "${d}"
	echo "Postbackup command output."
	echo
	eval ${POSTBACKUP} 2>&1 > /tmp/${IAM}_${$}
	THISRC=${?}

	sed 's/^/    /' /tmp/${IAM}_${$}

	if [[ ( ${THISRC} != 0 ) && ( ${SYSLOG:-0} = 1 ) ]]
	then
		sed "s/^/${SHRT[${CNT}]:-} :   /" /tmp/${IAM}_${$} | \
			logger -t "${IAM}" -p "${SYSLOGFAC:-local4.info}"
	fi
	echo
	echo "Return code for the postbackup command: ${THISRC}"
	echo
	echo "${d}"
	echo
	syslog "Postbackup command ended with returncode ${THISRC}"
	rm -f /tmp/${IAM}_${$}
fi

#------------------------------------------------------------------------------#
# Log and display the summary                                                  #
#------------------------------------------------------------------------------#
BCKEND=$(date '+%s')
BCKENDTIME=$(Now)
RUN=$(hms $(( ${BCKEND} - ${BCKSTART} )) )
SUM=""
SUM="${SUM} START=${BCKSTARTTIME};"
SUM="${SUM} END=${BCKENDTIME};"
SUM="${SUM} RUN=${RUN};"
SUM="${SUM} RC=${RC};"
SUM="${SUM} STATUS=${BCKSTATUS};"
echo
echo "Total run time : "${RUN}
echo "${d}"

syslog "Total run time: "${RUN}
syslog "Ended on '${HOSTNAME}' (RC=${BCKSTATUS})"
syslog "Summary : ${SUM}"
syslog "Ending ${IAM}"

#------------------------------------------------------------------------------#
# And, if things failed, show that as well                                     #
#------------------------------------------------------------------------------#
if [[ ${FAILS} != 0 ]]
then
	typeset -i i=0
	echo
	echo "Failed backups"
	echo "${d}"
	syslog "Failed backups:"
	while [[ ${i} < ${FAILS} ]]
	do
		i=$(( ${i} + 1 ))
		echo "    RC=${FAILRC[${i}]} for host ${FAILHOST[${i}]}"
		syslog "    RC=${FAILRC[${i}]} for host ${FAILHOST[${i}]}"
	done
	echo "${d}"
fi

#------------------------------------------------------------------------------#
# Clear the log redirection and insert the program status into the log         #
#------------------------------------------------------------------------------#
exec 1>&-
exec 2>&-
sed -i.bck "s/BACKUPSTATUS/${BCKSTATUS}/" ${LOGFILE}
rm ${LOGFILE}.bck

#------------------------------------------------------------------------------#
# Mail the logging if requested                                                #
#------------------------------------------------------------------------------#
case "${MAILTO}"
in
	stdout)
		#----------------------------------------------------------------------#
		# Send the logging to syslog                                           #
		#----------------------------------------------------------------------#
		cat ${LOGFILE}
		;;

	syslog)
		#----------------------------------------------------------------------#
		# Send the logging to syslog                                           #
		#----------------------------------------------------------------------#
		cat ${LOGFILE} | logger -t "${IAM}" -p "${SYSLOGFAC:-local4.info}"
		;;

	*)
		SUBJ="Logging of '${IAM}' on '${HOSTNAME}' at $(Now) (RC=${BCKSTATUS})"
		cat ${LOGFILE} | mail -s "${SUBJ}" ${MAILTO}
		;;
esac

#------------------------------------------------------------------------------#
# That's all, folks!                                                           #
#------------------------------------------------------------------------------#
exit ${RC}