#!/usr/bin/env bash

script_name="$(basename $(realpath $0))"
working_dir="$(dirname $(realpath $0))"

conf_file="${working_dir}/conf/${script_name%%.*}.conf"

LOCK_DIR="/tmp/${script_name%%.*}.LOCK"
log_file="${LOCK_DIR}/${script_name%%.*}.log"

backup_date="$(date +%Y-%m-%d-%H%M)"

startTime=$(date +%s)
currentDate=$(date --date @"$startTime" +"%Y%m%d_%H%M%S")
currentDateReadable=$(date --date @"$startTime" +"%d.%m.%Y - %H:%M:%S")


# ----------
# Base Function(s)
# ----------

clean_up() {

   # Perform program exit housekeeping
   rm -f ${DB_BACKUP_DIR}/*-${backup_date}.sql
   rm -rf "$LOCK_DIR"
   blank_line
   exit $1
}

echononl(){
   if $terminal ; then

      echo X\\c > /tmp/shprompt$$
      if [ `wc -c /tmp/shprompt$$ | awk '{print $1}'` -eq 1 ]; then
         echo -e -n "   $*\\c" 1>&2
      else
         echo -e -n "   $*" 1>&2
      fi
      rm /tmp/shprompt$$

   fi

   if [ -d "${BORG_LOG_DIR}" ]; then
      echo "" >> ${BORG_LOG_FILE}
      echo -e "# $*" >> ${BORG_LOG_FILE}
      echo "#" >> ${BORG_LOG_FILE}
   fi

}

fatal(){
   echo ""
   if $terminal ; then
      echo -e "   [ \033[31m\033[1mFatal\033[m ]: $*"
      echo ""
      echo -e "   \033[31m\033[1mScript was interupted\033[m!"
   else
      echo "   [ Fatal ]: $*"
      echo ""
      echo "              Script was terminated...."
   fi
   echo ""
   clean_up 1
}
error (){
   echo ""
   if $terminal ; then
      echo -e "   [ \033[31m\033[1mError\033[m ]: $*"
   else
      echo "[ Error ]: $*"
   fi
   if [ -d "${BORG_LOG_DIR}" ]; then
      echo "[ Error ]: $*" >> ${BORG_LOG_FILE}
   fi
}

warn (){
   echo ""
   if $terminal ; then
      echo -e "   [ \033[33m\033[1mWarning\033[m ]: $*"
   else
      echo "[ Warning ]: $*"
   fi
   echo ""
}

warn_only_terminal () {
   if $terminal ; then
      echo ""
      echo -e "\t[ \033[33m\033[1mWarning\033[m ]: $*"
      echo ""
   fi
}

info (){
   if $terminal ; then
      echo ""
      echo -e "   [ \033[32m\033[1mInfo\033[m ]  $*"
      echo ""
   fi
}

ok (){
   if $terminal ; then
      echo ""
      echo -e "   [ \033[32m\033[1mOk\033[m ]    $*"
      echo ""
   fi
}

echo_done() {
   if $terminal ; then
      echo -e "\033[75G[ \033[32mdone\033[m ]"
   fi
}
echo_ok() {
   if $terminal ; then
      echo -e "\033[75G[ \033[32mok\033[m ]"
   fi
}
echo_warn() {
   if $terminal ; then
      echo -e "\033[75G[ \033[33mwarn\033[m ]"
   fi
}
echo_failed(){
   if $terminal ; then
      echo -e "\033[75G[ \033[1;31mfailed\033[m ]"
   fi
}
echo_skipped() {
   if $terminal ; then
      echo -e "\033[75G[ \033[90m\033[1mskipped\033[m ]"
   fi
}
echo_wait(){
   if $terminal ; then
      echo -en "\033[75G[ \033[5m\033[1m...\033[m ]"
   fi
}

## - Check if a given array (parameter 2) contains a given string (parameter 1)
## -
containsElement () {
   local e
   for e in "${@:2}"; do [[ "$e" == "$1" ]] && return 0; done
   return 1
}

trim() {
    local var="$*"
    var="${var#"${var%%[![:space:]]*}"}"   # remove leading whitespace characters
    var="${var%"${var##*[![:space:]]}"}"   # remove trailing whitespace characters
    echo -n "$var"
}

blank_line() {
   if $terminal ; then
      echo ""
   fi
}

# ----------
# - Some checks ..
# ----------

# - Running in a terminal?
# -
if [[ -t 1 ]] ; then
   terminal=true
else
   terminal=false
fi


# ==========
# - Begin Main Script
# ==========

# ----------
# - Headline
# ----------

if $terminal ; then
   echo ""

fi


# ==========
# - Begin Main Script
# ==========

# ----------
# - Headline
# ----------

if $terminal ; then
   echo ""
   echo -e "\033[1m----------\033[m"
   echo -e "\033[32m\033[1mRunning script \033[m\033[1m$script_name\033[32m .. \033[m"
   echo -e "\033[1m----------\033[m"
   echo ""
fi

# ----------
# Default values
# ----------

DEFAULT_BORG_LOG_DIR="/var/log/borg/"

DEFAULT_SSH_USER="borg"
DEFAULT_SSH_PORT=22
DEFAULT_BORG_RSH='ssh -i /root/.ssh/id_ed25519-borg-backup'

DEFAULT_DB_BACKUP_DIR="/data/backup/sql"

DEFAULT_BACKUP_DIRECTORIES="/etc /home /root /var/spool/cron"

DEFAULT_KEEP_DAILY=10
DEFAULT_KEEP_WEEKLY=5
DEFAULT_KEEP_MONTHLY=6
DEFAULT_KEEP_YEARLY=1

DEFAULT_DB_TYPE="None"
DEFAULT_MYSQL_CREDENTIALS="-u root -S /run/mysqld/mysqld.sock"

DEFAULT_PSQL_SYSTEM_USER="postgres"

DEFAULT_ADMIN_EMAIL="root"
DEFAULT_COMPANY="O.OPEN"
DEFAULT_FROM_ADDRESS="borg-backup@$(hostname -f)"

# ----------
#  Read Configurations from $conf_file
# ----------

if [[ -f "$conf_file" ]]; then
   source "$conf_file"
else
   fatal  "No configuration file '$conf_file' present!"
fi


# -------------
# - Job is already running?
# -------------

[[ -z "${ADMIN_EMAIL}" ]] && ADMIN_EMAIL="${DEFAULT_ADMIN_EMAIL}"
[[ -z "${COMPANY}" ]] && COMPANY="${DEFAULT_COMPANY}"
[[ -z "${FROM_ADDRESS}" ]] && FROM_ADDRESS="${DEFAULT_FROM_ADDRESS}"

# - If job already runs, stop execution..
# -
if mkdir "$LOCK_DIR" 2> /dev/null ; then

   ## - Remove lockdir when the script finishes, or when it receives a signal
   trap "clean_up 1" SIGHUP SIGINT SIGTERM

else

   datum="$(date +"%d.%m.%Y %H:%M")"

   msg="[ Error ]: A previos instance of \"`basename $0`\" seems already be running.\n\n           Exiting now.."

   datum="$(date +"%d.%m.%Y")"

   content_type='Content-Type: text/plain;\n charset="utf-8"'

   subject="Error Borg Backup - ${COMPANY} - ${datum}"
   subject_utf8="$(echo "$subject" | iconv -t UTF8)"
   subject_utf8_encoded="=?utf-8?B?$(echo $subject_utf8 | base64  --wrap=0)?="

   echo -e "To:${ADMIN_EMAIL}\n${content_type}\nSubject:${subject_utf8_encoded}\n${msg}\n" | /usr/sbin/sendmail -F "Errors BACKUP ${COMPANY}" -f ${FROM_ADDRESS} ${ADMIN_EMAIL}

   echo ""
   echo "[ Error ]: A previos instance of that script \"`basename $0`\" seems already be running."
   echo ""
   echo -e "           Exiting now.."
   echo ""

   exit 1

fi


# ----------
# Set Parameter values
# ----------

if [[ -z "${BORG_PASSPHRASE}" ]] ; then
   fatal "Borg Passphrase (BORG_PASSPHRASE) NOT found!"
fi

if [[ -z "${BORG_HOST}" ]] ; then
   fatal "Backup Hosth (var BORG_HOST) not given!"
fi

if [[ -z "${BORG_REPO}" ]] ; then
   fatal "Borg Repository Path (var BORG_REPO) not given!"
fi

[[ -z "${SSH_USER}" ]] && SSH_USER="${DEFAULT_SSH_USER}"

[[ -z "${SSH_PORT}" ]] && SSH_PORT="${DEFAULT_SSH_PORT}"

[[ -z "${BORG_RSH}" ]] && BORG_RSH="${DEFAULT_BORG_RSH}"

[[ -z "${DB_BACKUP_DIR}" ]] && DB_BACKUP_DIR="${DEFAULT_DB_BACKUP_DIR}"

[[ -z "${BORG_LOG_DIR}" ]] && BORG_LOG_DIR="${DEFAULT_BORG_LOG_DIR}"
BORG_LOG_FILE="${BORG_LOG_DIR}/borg-backup.log"

[[ -z "${KEEP_DAILY}" ]] && KEEP_DAILY=${DEFAULT_KEEP_DAILY}
[[ -z "${KEEP_WEEKLY}" ]] && KEEP_WEEKLY=${DEFAULT_KEEP_WEEKLY}
[[ -z "${KEEP_MONTHLY}" ]] && KEEP_MONTHLY=${DEFAULT_KEEP_MONTHLY}
[[ -z "${KEEP_YEARLY}" ]] && KEEP_YEARLY=${DEFAULT_KEEP_YEARLY}


if [[ -z "${DB_TYPE}" ]] ; then
   DB_TYPE="${DEFAULT_DB_TYPE}"
else
   # replace comma by blank sign
   DB_TYPE=${DB_TYPE/,/ }
fi

declare -a db_type_arr=()

for _db_type in ${DB_TYPE}; do
   if [[ "${DB_TYPE,,}" = "psql" ]] || [[ "${DB_TYPE,,}" = "postgresql" ]] || [[ "${DB_TYPE,,}" = "postgres" ]] ; then
       db_type_arr+=("postgresql")
   else
      db_type_arr+=("${_db_type,,}")
   fi
done

if containsElement "mysql" "${db_type_arr[@]}" ; then
   [[ -z "${MYSQL_CREDENTIALS}" ]] && MYSQL_CREDENTIALS="${DEFAULT_MYSQL_CREDENTIALS}"
fi

if containsElement "postgresql" "${db_type_arr[@]}" ; then
   [[ -z "${PSQL_SYSTEM_USER}" ]] && PSQL_SYSTEM_USER="${DEFAULT_PSQL_SYSTEM_USER}"
fi


[[ -z "${BACKUP_DIRECTORIES}" ]] && BACKUP_DIRECTORIES="${DEFAULT_BACKUP_DIRECTORIES}"

# By default: Borg won't follow symlinks
#
BORG_BACKUP_DIRS=""
for _dir in $BACKUP_DIRECTORIES ; do
   if [[ -h "${_dir}" ]] ; then
      BORG_BACKUP_DIRS="$BORG_BACKUP_DIRS $(realpath "${_dir}")"
   fi
   BORG_BACKUP_DIRS="$BORG_BACKUP_DIRS ${_dir}"
done

# remove leading whitespace characters
#
BORG_BACKUP_DIRS="${BORG_BACKUP_DIRS#"${BORG_BACKUP_DIRS%%[![:space:]]*}"}"


BORG_URL="ssh://${SSH_USER}@${BORG_HOST}:${SSH_PORT}${BORG_REPO}"

export BORG_PASSPHRASE
export BORG_REPO

# ---
# Some automatic 'answerers' (if set, they automatically answer confirmation questions):
# ---

# For 'Warning: The repository at location … was previously located at ..'
#

# This has to be set when the repository has been created by user and
# the script is called by cron
#
export BORG_RELOCATED_REPO_ACCESS_IS_OK=yes

# For "Warning: The repository at location ... was previously located at ..."
#
export BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK=yes


if [ ! -d "${BORG_LOG_DIR}" ]; then
   echononl "Create borg log directory.."
   mkdir -p "${BORG_LOG_DIR}" > ${log_file} 2>&1
   if [[ $? -gt 0 ]] ; then
      echo_failed
      error "$(cat "${log_file}")"

      fatal "No log directory '${BORG_LOG_DIR}' present!"

   else
      echo_ok
      echo "mkdir -p \"${BORG_LOG_DIR}\"" >> ${BORG_LOG_FILE}
      echo ""  >> ${BORG_LOG_FILE}
   fi
fi

if [ ! -d "${DB_BACKUP_DIR}" ]; then
   echononl "Create DB backup directory.."
   mkdir -p "${DB_BACKUP_DIR}" > ${log_file} 2>&1
   if [[ $? -gt 0 ]] ; then
      echo_failed
      error "$(cat "${log_file}")"

      fatal "No backup directory '${DB_BACKUP_DIR}' present!"

   else
      echo_ok
      echo "mkdir -p \"${DB_BACKUP_DIR}\"" >> ${BORG_LOG_FILE}
      echo ""  >> ${BORG_LOG_FILE}
   fi
fi

echo "# ----------" >> ${BORG_LOG_FILE}
echo -e "# Running script '${script_name}' on $(date +%Y.%m.%d) at $(date +%H:%M) h .."  >> ${BORG_LOG_FILE}
echo "# ----------" >> ${BORG_LOG_FILE}


echo "" >> ${BORG_LOG_FILE}
echo "# Script parameter settings:" >> ${BORG_LOG_FILE}
echo "#" >> ${BORG_LOG_FILE}

echo "SSH_USER=\"${SSH_USER}\"" >> ${BORG_LOG_FILE}
echo "SSH_PORT=\"${SSH_PORT}\"" >> ${BORG_LOG_FILE}
echo "BORG_HOST=\"${BORG_HOST}\"" >> ${BORG_LOG_FILE}
echo "BORG_REPO=\"${BORG_REPO}\"" >> ${BORG_LOG_FILE}
echo "BORG_PASSPHRASE='${BORG_PASSPHRASE}'"  >> ${BORG_LOG_FILE}
echo "BORG_RSH=\"${BORG_RSH}\"" >> ${BORG_LOG_FILE}
echo "BORG_LOG_DIR=\"${BORG_LOG_DIR}\"" >> ${BORG_LOG_FILE}
echo "BORG_URL=\"${BORG_URL}\"" >> ${BORG_LOG_FILE}
echo "" >> ${BORG_LOG_FILE}
echo "DB_TYPE=\"${DB_TYPE}\"" >> ${BORG_LOG_FILE}
echo "DB_BACKUP_DIR=\"${DB_BACKUP_DIR}\"" >> ${BORG_LOG_FILE}
echo "" >> ${BORG_LOG_FILE}
echo "ADMIN_EMAIL=\"${ADMIN_EMAIL}\"" >> ${BORG_LOG_FILE}
echo "COMPANY=\"${COMPANY}\"" >> ${BORG_LOG_FILE}
echo "FROM_ADDRESS=\"${FROM_ADDRESS}\"" >> ${BORG_LOG_FILE}
echo "" >> ${BORG_LOG_FILE}

if [[ -n "${DB_TYPE}" ]] && [[ "${DB_TYPE,,}" = "!none" ]]; then
   echo "" >> ${BORG_LOG_FILE}
   echo "# ---" >> ${BORG_LOG_FILE}
   echo "# Create database backup .."  >> ${BORG_LOG_FILE}
   echo "# ---" >> ${BORG_LOG_FILE}

   blank_line
fi

if containsElement "mysql" "${db_type_arr[@]}" ; then

   blank_line

   echononl "Create an array of MySQL databases."

   cat <<EOF >> ${BORG_LOG_FILE}
declare -a mysql_database_arr=();
for _db_name in \$(mysql ${MYSQL_CREDENTIALS} -s -r -N -e 'show databases' 2> /dev/null ) ; do

   [[ "\${_db_name}" = "information_schema" ]] && continue
   [[ "\${_db_name}" = "performance_schema" ]] && continue
   [[ "\${_db_name}" = "mysql" ]] && continue
   [[ "\${_db_name}" = "sys" ]] && continue

   mysql_database_arr+=("\${_db_name}")

done
EOF

   declare -a mysql_database_arr=();
   for _db_name in $(mysql ${MYSQL_CREDENTIALS} -s -r -N -e 'show databases' 2> ${log_file}) ; do

      [[ "${_db_name}" = "information_schema" ]] && continue
      [[ "${_db_name}" = "performance_schema" ]] && continue
      [[ "${_db_name}" = "mysql" ]] && continue
      [[ "${_db_name}" = "sys" ]] && continue

      mysql_database_arr+=("${_db_name}")

   done

   if [[ -s "${log_file}" ]] ; then
      echo_failed
      error "$(cat "${log_file}")"
   else
      echo_ok
   fi

   for _db in ${mysql_database_arr[@]} ; do

      echononl "Dump Database '${_db}'.."

      cat <<EOF >> ${BORG_LOG_FILE}
mysqldump ${MYSQL_CREDENTIALS} \\
   --single-transaction --routines \\
   -h localhost \\
   -e ${_db} > ${DB_BACKUP_DIR}/${_db}-${backup_date}.sql

EOF

      mysqldump ${MYSQL_CREDENTIALS} \
         --single-transaction --routines \
         -h localhost \
         -e ${_db} > ${DB_BACKUP_DIR}/${_db}-${backup_date}.sql 2> ${log_file}

      if [[ -s "${log_file}" ]] ; then
         echo_failed
         error "$(cat "${log_file}")"
      else
         echo_ok
      fi

   done
fi

if containsElement "postgresql" "${db_type_arr[@]}" ; then

   blank_line

   echononl "Create an array of PostgreSQL databases."

   cat <<EOF >> ${BORG_LOG_FILE}
declare -a psql_database_arr=();
for _db_name in \$(su - ${PSQL_SYSTEM_USER} -c "psql  -lt" 2> /dev/null | grep -v -e"^$" | awk '{print\$1}') ; do

   [[ "\${_db_name}" = ""template0 ]] && continue
   [[ "\${_db_name}" = ""template1 ]] && continue
   [[ "\${_db_name}" = ""postgres ]] && continue
   [[ "\${_db_name}" = "|" ]] && continue
   [[ "\${_db_name}" = ":" ]] && continue

   psql_database_arr+=("\${_db_name}")

done
EOF

   declare -a psql_database_arr=();
   for _db_name in $(su - ${PSQL_SYSTEM_USER} -c "psql  -lt" 2> "${log_file}" | grep -v -e"^$" | awk '{print$1}') ; do

      [[ "${_db_name}" = ""template0 ]] && continue
      [[ "${_db_name}" = ""template1 ]] && continue
      [[ "${_db_name}" = ""postgres ]] && continue
      [[ "${_db_name}" = "|" ]] && continue
      [[ "${_db_name}" = ":" ]] && continue

      psql_database_arr+=("${_db_name}")

   done

   if [[ -s "${log_file}" ]] ; then
      echo_failed
      error "$(cat "${log_file}")"
   else
      echo_ok
   fi

   for _db in ${psql_database_arr[@]} ; do

      echononl "Dump Database '${_db}'.."

      cat <<EOF >> ${BORG_LOG_FILE}
su - ${PSQL_SYSTEM_USER} -c "pg_dump -c ${_db}" > ${DB_BACKUP_DIR}/${_db}-${backup_date}.sql
EOF

      su - ${PSQL_SYSTEM_USER} -c "pg_dump -c ${_db}" > ${DB_BACKUP_DIR}/${_db}-${backup_date}.sql 2> ${log_file}

      if [[ -s "${log_file}" ]] ; then
         echo_failed
         error "$(cat "${log_file}")"
      else
         echo_ok
      fi

   done

fi

blank_line


echononl "Create borg backup.."

cat << EOF >> ${BORG_LOG_FILE}
borg create                            \\
    --verbose                          \\
    --filter AME                       \\
    --list                             \\
    --stats                            \\
    --show-rc                          \\
    --compression lz4                  \\
    --exclude-caches                   \\
    --exclude 'home/*/.cache/*'        \\
    --exclude '/root/.cache/*'         \\
    --exclude '/root/.config/*'        \\
    --exclude '/root/.local/*'         \\
    --exclude 'var/tmp/*'              \\
                                       \\
    ${BORG_URL}::${backup_date}        \\
    ${DB_BACKUP_DIR}                   \\
    ${NC_DATA_DIR}                     \\
    ${BORG_BACKUP_DIRS}
EOF

borg create                            \
    --verbose                          \
    --filter AME                       \
    --list                             \
    --stats                            \
    --show-rc                          \
    --compression lz4                  \
    --exclude-caches                   \
    --exclude 'home/*/.cache/*'        \
    --exclude '/root/.cache/*'         \
    --exclude '/root/.config/*'        \
    --exclude '/root/.local/*'         \
    --exclude 'var/tmp/*'              \
                                       \
    ${BORG_URL}::${backup_date}        \
    ${DB_BACKUP_DIR}                   \
    ${NC_DATA_DIR}                     \
    ${BORG_BACKUP_DIRS}                \
    > ${log_file} 2>&1



if [[ $? -gt 0 ]] ; then
   echo_failed
   error "$(cat "${log_file}")"
else
   echo_ok
   cat ${log_file} >> ${BORG_LOG_FILE}
fi


echononl "Pruning repository.."

cat << EOF >> ${BORG_LOG_FILE}
borg prune                          \\
    --list                          \\
    --show-rc                       \\
    --keep-daily    ${KEEP_DAILY}   \\
    --keep-weekly   ${KEEP_WEEKLY}  \\
    --keep-monthly  ${KEEP_MONTHLY} \\
    --keep-yearly   ${KEEP_YEARLY}  \\
    ${BORG_URL}
EOF

borg prune                          \
    --list                          \
    --show-rc                       \
    --keep-daily    ${KEEP_DAILY}   \
    --keep-weekly   ${KEEP_WEEKLY}  \
    --keep-monthly  ${KEEP_MONTHLY} \
    --keep-yearly   ${KEEP_YEARLY}  \
    ${BORG_URL}                     \
    > ${log_file} 2>&1

if [[ $? -gt 0 ]] ; then
   echo_failed
   error "$(cat "${log_file}")"
else
   echo_ok
   cat ${log_file} >> ${BORG_LOG_FILE}
fi

blank_line


# Remove DB local backup
#
echononl "Remove local DB backups.."

echo "rm -f ${DB_BACKUP_DIR}/*-${backup_date}.sql" >> ${BORG_LOG_FILE}

rm -f ${DB_BACKUP_DIR}/*-${backup_date}.sql > ${log_file} 2>&1

if [[ $? -gt 0 ]] ; then
   echo_failed
   error "$(cat "${log_file}")"
else
   echo_ok
fi

# Duration of backup
endTime=$(date +%s)
endDateReadable=$(date --date @"$endTime" +"%d.%m.%Y - %H:%M:%S")
duration=$((endTime-startTime))
durationSec=$((duration % 60))
durationMin=$(((duration / 60) % 60))
durationHour=$((duration / 3600))
durationReadable=$(printf "%02d hours %02d minutes %02d seconds" $durationHour $durationMin $durationSec)

echo -e "\n\n###### Server backup finished on ${endDateReadable} (${durationReadable}) ######\n" >> ${BORG_LOG_FILE}

blank_line


# Error Handling
#
declare -i error=$(grep -i -e"\[\s*ERROR" ${BORG_LOG_FILE} | wc -l)
declare -i warning=$(grep -i -e"\[\s*WARN" ${BORG_LOG_FILE} | wc -l)

   if [ $error -gt 0 -o $warning -gt 0 ] ; then

   # Send Summery
   #
   if $terminal ; then
      echo -e -n "   Send Summary.."
   fi

   datum="$(date +"%d.%m.%Y %H:%M")"

   filesystem_usage="$(df -h)"

   msg_head_line="\n\n# ---\n# Server backup finished on ${endDateReadable} (${durationReadable})\n# ---\n\n"

   msg="$(cat "${BORG_LOG_FILE}")"

   datum="$(date +"%d.%m.%Y")"

   content_type='Content-Type: text/plain;\n charset="utf-8"'

   subject="Borg Backup - ${COMPANY} - ${datum}"
   subject_utf8="$(echo "$subject" | iconv -t UTF8)"
   subject_utf8_encoded="=?utf-8?B?$(echo $subject_utf8 | base64  --wrap=0)?="


   echo -e "To:${ADMIN_EMAIL}\n${content_type}\nSubject:${subject_utf8_encoded}\n${filesystem_usage}\n${msg_head_line}\n${msg}\n" | /usr/sbin/sendmail -F "BACKUP ${COMPANY}" -f ${FROM_ADDRESS} ${ADMIN_EMAIL} > ${log_file} 2>&1


   if [[ $? -gt 0 ]] ; then
      echo_failed
      error "$(cat "${log_file}")"
   else
      echo_ok
   fi

   blank_line

   # Send Error Message
   #
   if $terminal ; then
      echo -e -n "   Send Error Message to ${ADMIN_EMAIL}.."
   fi


   err_msg="$(grep -A 2 -i -e"\[\s*ERROR" ${BORG_LOG_FILE})\n"
   warn_msg="$(grep -A 2  -i -e"\[\s*WARN" ${BORG_LOG_FILE})\n"

   msg_top="Borg Backup Error ${COMPANY} vom ${datum}.\nBorg Backup on '$(hostname -f)' beendet mit:"
   msg_body_01="\t${error} Error(s)\n\t${warning} Warning(s)\n"
   msg_body_02="\n\n--- ERROR(S) ---\n${err_msg}\n\n--- WARNING(S) ...\n${warn_msg}"

   msg="${msg_top}\n${msg_body_01}\n${msg_body_02}"

   subject="Borg Backup Errors at ${host_name} -- ${datum}"
   subject_utf8="$(echo "${subject}" | iconv -t UTF8)"
   subject_utf8_encoded="=?utf-8?B?$(echo ${subject_utf8} | base64 --wrap=0)?="

   echo -e "To:${ADMIN_EMAIL}\n${content_type}\nSubject:${subject_utf8_encoded}\n${msg}\n" | /usr/sbin/sendmail -F "Errors Borg BACKUP ${COMPANY}" -f ${FROM_ADDRESS} ${ADMIN_EMAIL} > ${log_file} 2>&1

   if [[ $? -gt 0 ]] ; then
      echo_failed
      error "$(cat "${log_file}")"
   else
      echo_ok
   fi

   if [[ -n "${ADMIN_EXTRA_EMAIL}" ]] ; then

      echo -e "To:${ADMIN_EXTRA_EMAIL}\n${content_type}\nSubject:${subject_utf8_encoded}\n${msg}\n" | /usr/sbin/sendmail -F "Errors Borg BACKUP ${COMPANY}" -f ${FROM_ADDRESS} ${ADMIN_EXTRA_EMAIL} > ${log_file} 2>&1

      # Send Error Message
      #
      if $terminal ; then
         echo -e -n "   Send Error Message to ${ADMIN_EMAIL}.."
      fi

      if [[ $? -gt 0 ]] ; then
         echo_failed
         error "$(cat "${log_file}")"
      else
         echo_ok
      fi
   fi

   blank_line

fi

blank_line


echononl "Create temporary logorotate configuration file 'logrotate_borg.conf'.."

cat << EOF > ${LOCK_DIR}/logrotate_borg.conf 2> ${log_file}
${BORG_LOG_FILE} {
   daily
   rotate 7
   missingok
   compress
   su root root
}
EOF
if [[ $? -gt 0 ]] ; then
   echo_failed
   error "$(cat "${log_file}")"
else
   echo_ok
fi

echononl "Rotate logfile '${BORG_LOG_FILE}'.."
logrotate ${LOCK_DIR}/logrotate_borg.conf > ${log_file} 2>&1
if [[ $? -gt 0 ]] ; then
   echo_failed
   error "$(cat "${log_file}")"
else
   echo_ok
fi


clean_up 0