diff --git a/borg-backup.sh b/borg-backup.sh new file mode 100755 index 0000000..54bcf9b --- /dev/null +++ b/borg-backup.sh @@ -0,0 +1,640 @@ +#!/usr/bin/env bash + +script_name="$(basename $(realpath $0))" +working_dir="$(dirname $(realpath $0))" + +conf_file="${working_dir}/conf/${script_name%%.*}.conf" + +LOCK_DIR="/tmp/${script_name%%.*}.LOCK" +log_file="${LOCK_DIR}/${script_name%%.*}.log" + +backup_date="$(date +%Y-%m-%d-%H%M)" + +startTime=$(date +%s) +currentDate=$(date --date @"$startTime" +"%Y%m%d_%H%M%S") +currentDateReadable=$(date --date @"$startTime" +"%d.%m.%Y - %H:%M:%S") + + +# ---------- +# Base Function(s) +# ---------- + +clean_up() { + + # Perform program exit housekeeping + rm -rf "$LOCK_DIR" + blank_line + exit $1 +} + +echononl(){ + if $terminal ; then + + echo X\\c > /tmp/shprompt$$ + if [ `wc -c /tmp/shprompt$$ | awk '{print $1}'` -eq 1 ]; then + echo -e -n " $*\\c" 1>&2 + else + echo -e -n " $*" 1>&2 + fi + rm /tmp/shprompt$$ + + fi + + if [ -d "${BORG_LOG_DIR}" ]; then + echo "" >> ${BORG_LOG_FILE} + echo -e "# $*" >> ${BORG_LOG_FILE} + echo "#" >> ${BORG_LOG_FILE} + fi + +} + +fatal(){ + echo "" + if $terminal ; then + echo -e " [ \033[31m\033[1mFatal\033[m ]: $*" + echo "" + echo -e " \033[31m\033[1mScript was interupted\033[m!" + else + echo " [ Fatal ]: $*" + echo "" + echo " Script was terminated...." + fi + echo "" + clean_up 1 +} +error (){ + echo "" + if $terminal ; then + echo -e " [ \033[31m\033[1mError\033[m ]: $*" + else + echo "[ Error ]: $*" + fi + if [ -d "${BORG_LOG_DIR}" ]; then + echo "[ Error ]: $*" >> ${BORG_LOG_FILE} + fi +} + +warn (){ + echo "" + if $terminal ; then + echo -e " [ \033[33m\033[1mWarning\033[m ]: $*" + else + echo "[ Warning ]: $*" + fi + echo "" +} + +warn_only_terminal () { + if $terminal ; then + echo "" + echo -e "\t[ \033[33m\033[1mWarning\033[m ]: $*" + echo "" + fi +} + +info (){ + if $terminal ; then + echo "" + echo -e " [ \033[32m\033[1mInfo\033[m ] $*" + echo "" + fi +} + +ok (){ + if $terminal ; then + echo "" + echo -e " [ \033[32m\033[1mOk\033[m ] $*" + echo "" + fi +} + +echo_done() { + if $terminal ; then + echo -e "\033[75G[ \033[32mdone\033[m ]" + fi +} +echo_ok() { + if $terminal ; then + echo -e "\033[75G[ \033[32mok\033[m ]" + fi +} +echo_warn() { + if $terminal ; then + echo -e "\033[75G[ \033[33mwarn\033[m ]" + fi +} +echo_failed(){ + if $terminal ; then + echo -e "\033[75G[ \033[1;31mfailed\033[m ]" + fi +} +echo_skipped() { + if $terminal ; then + echo -e "\033[75G[ \033[90m\033[1mskipped\033[m ]" + fi +} +echo_wait(){ + if $terminal ; then + echo -en "\033[75G[ \033[5m\033[1m...\033[m ]" + fi +} + +trim() { + local var="$*" + var="${var#"${var%%[![:space:]]*}"}" # remove leading whitespace characters + var="${var%"${var##*[![:space:]]}"}" # remove trailing whitespace characters + echo -n "$var" +} + +blank_line() { + if $terminal ; then + echo "" + fi +} + +# ---------- +# - Some checks .. +# ---------- + +# - Running in a terminal? +# - +if [[ -t 1 ]] ; then + terminal=true +else + terminal=false +fi + + +# ========== +# - Begin Main Script +# ========== + +# ---------- +# - Headline +# ---------- + +if $terminal ; then + echo "" + +fi + + +# ------------- +# - Job is already running? +# ------------- + +# - If job already runs, stop execution.. +# - +if mkdir "$LOCK_DIR" 2> /dev/null ; then + + ## - Remove lockdir when the script finishes, or when it receives a signal + trap "clean_up 1" SIGHUP SIGINT SIGTERM + +else + + datum="$(date +"%d.%m.%Y %H:%M")" + + msg="[ Error ]: A previos instance of \"`basename $0`\" seems already be running.\n\n Exiting now.." + + echo "" + echo "[ Error ]: A previos instance of that script \"`basename $0`\" seems already be running." + echo "" + echo -e " Exiting now.." + echo "" + + for _email in ${alert_email_arr[@]} ; do + echo -e "To:${_email}\n${content_type}\nSubject:Error cronjob `basename $0` -- $datum\n${msg}\n" \ + | sendmail -F "Error `hostname -f`" -f $sender_address $_email + done + + exit 1 + +fi + + +# ========== +# - Begin Main Script +# ========== + +# ---------- +# - Headline +# ---------- + +if $terminal ; then + echo "" + echo -e "\033[1m----------\033[m" + echo -e "\033[32m\033[1mRunning script \033[m\033[1m$script_name\033[32m .. \033[m" + echo -e "\033[1m----------\033[m" + echo "" +fi + +# ---------- +# Default values +# ---------- + +DEFAULT_BORG_LOG_DIR="/var/log/borg/" + +DEFAULT_SSH_USER="borg" +DEFAULT_SSH_PORT=22 +DEFAULT_BORG_RSH='ssh -i /root/.ssh/id_ed25519-borg-backup' + +DEFAULT_DB_BACKUP_DIR="/data/backup/sql" + +DEFAULT_BACKUP_DIRECTORIES="/etc /home /root /var/spool/cron" + +DEFAULT_DB_TYPE="None" +DEFAULT_MYSQL_CREDENTIALS="-u root -S /run/mysqld/mysqld.sock" + +DEFAULT_PSQL_SYSTEM_USER="postgres" + +# ---------- +# Read Configurations from $conf_file +# ---------- + +if [[ -f "$conf_file" ]]; then + source "$conf_file" +else + fatal "No configuration file '$conf_file' present!" +fi + + +# ---------- +# Set Parameter values +# ---------- + +if [[ -z "${BORG_PASSPHRASE}" ]] ; then + fatal "Borg Passphrase (BORG_PASSPHRASE) NOT found!" +fi + +if [[ -z "${BORG_HOST}" ]] ; then + fatal "Backup Hosth (var BORG_HOST) not given!" +fi + +if [[ -z "${BORG_REPO}" ]] ; then + fatal "Borg Repository Path (var BORG_REPO) not given!" +fi + +[[ -z "${SSH_USER}" ]] && SSH_USER="${DEFAULT_SSH_USER}" + +[[ -z "${SSH_PORT}" ]] && SSH_PORT="${DEFAULT_SSH_PORT}" + +[[ -z "${BORG_RSH}" ]] && BORG_RSH="${DEFAULT_BORG_RSH}" + +[[ -z "${DB_BACKUP_DIR}" ]] && DB_BACKUP_DIR="${DEFAULT_DB_BACKUP_DIR}" + +[[ -z "${BORG_LOG_DIR}" ]] && BORG_LOG_DIR="${DEFAULT_BORG_LOG_DIR}" +BORG_LOG_FILE="${BORG_LOG_DIR}/${backup_date}.log" + + +[[ -z "${DB_TYPE}" ]] && DB_TYPE="${DEFAULT_DB_TYPE}" +if [[ "${DB_TYPE,,}" = "mysql" ]] ; then + [[ -z "${MYSQL_CREDENTIALS}" ]] && MYSQL_CREDENTIALS="${DEFAULT_MYSQL_CREDENTIALS}" +elif [[ "${DB_TYPE,,}" = "psql" ]] || [[ "${DB_TYPE,,}" = "postgresql" ]] || [[ "${DB_TYPE,,}" = "postgres" ]] ; then + [[ -z "${PSQL_SYSTEM_USER}" ]] && PSQL_SYSTEM_USER="${DEFAULT_PSQL_SYSTEM_USER}" +fi + + + +[[ -z "${BACKUP_DIRECTORIES}" ]] && BACKUP_DIRECTORIES="${DEFAULT_BACKUP_DIRECTORIES}" + +# By default: Borg won't follow symlinks +# +BORG_BACKUP_DIRS="" +for _dir in $BACKUP_DIRECTORIES ; do + if [[ -h "${_dir}" ]] ; then + BORG_BACKUP_DIRS="$BORG_BACKUP_DIRS $(realpath "${_dir}")" + fi + BORG_BACKUP_DIRS="$BORG_BACKUP_DIRS ${_dir}" +done + +# remove leading whitespace characters +# +BORG_BACKUP_DIRS="${BORG_BACKUP_DIRS#"${BORG_BACKUP_DIRS%%[![:space:]]*}"}" + + +BORG_URL="ssh://${SSH_USER}@${BORG_HOST}:${SSH_PORT}${BORG_REPO}" + +export BORG_PASSPHRASE +export BORG_REPO + +# --- +# Some automatic 'answerers' (if set, they automatically answer confirmation questions): +# --- + +# For 'Warning: The repository at location … was previously located at ..' +# + +# This has to be set when the repository has been created by user and +# the script is called by cron +# +export BORG_RELOCATED_REPO_ACCESS_IS_OK=yes + +# For "Warning: The repository at location ... was previously located at ..." +# +export BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK=yes + + +if [ ! -d "${BORG_LOG_DIR}" ]; then + echononl "Create borg log directory.." + mkdir -p "${BORG_LOG_DIR}" > ${log_file} 2>&1 + if [[ $? -gt 0 ]] ; then + echo_failed + error "$(cat "${log_file}")" + + fatal "No log directory '${BORG_LOG_DIR}' present!" + + else + echo_ok + echo "mkdir -p \"${BORG_LOG_DIR}\"" >> ${BORG_LOG_FILE} + echo "" >> ${BORG_LOG_FILE} + fi +fi + +if [ ! -d "${DB_BACKUP_DIR}" ]; then + echononl "Create DB backup directory.." + mkdir -p "${DB_BACKUP_DIR}" > ${log_file} 2>&1 + if [[ $? -gt 0 ]] ; then + echo_failed + error "$(cat "${log_file}")" + + fatal "No backup directory '${DB_BACKUP_DIR}' present!" + + else + echo_ok + echo "mkdir -p \"${DB_BACKUP_DIR}\"" >> ${BORG_LOG_FILE} + echo "" >> ${BORG_LOG_FILE} + fi +fi + +echo "# ----------" >> ${BORG_LOG_FILE} +echo -e "# Running script '${script_name}' on $(date +%Y.%m.%d) at $(date +%H:%M) h .." >> ${BORG_LOG_FILE} +echo "# ----------" >> ${BORG_LOG_FILE} + + +echo "" >> ${BORG_LOG_FILE} +echo "# Script parameter settings:" >> ${BORG_LOG_FILE} +echo "#" >> ${BORG_LOG_FILE} + +echo "SSH_USER=\"${SSH_USER}\"" >> ${BORG_LOG_FILE} +echo "SSH_PORT=\"${SSH_PORT}\"" >> ${BORG_LOG_FILE} +echo "BORG_HOST=\"${BORG_HOST}\"" >> ${BORG_LOG_FILE} +echo "BORG_REPO=\"${BORG_REPO}\"" >> ${BORG_LOG_FILE} +echo "BORG_PASSPHRASE='${BORG_PASSPHRASE}'" >> ${BORG_LOG_FILE} +echo "BORG_RSH=\"${BORG_RSH}\"" >> ${BORG_LOG_FILE} +echo "BORG_LOG_DIR=\"${BORG_LOG_DIR}\"" >> ${BORG_LOG_FILE} +echo "BORG_URL=\"${BORG_URL}\"" >> ${BORG_LOG_FILE} +echo "" >> ${BORG_LOG_FILE} +echo "DB_TYPE=\"${DB_TYPE}\"" >> ${BORG_LOG_FILE} +echo "DB_BACKUP_DIR=\"${DB_BACKUP_DIR}\"" >> ${BORG_LOG_FILE} +echo "" >> ${BORG_LOG_FILE} + +if [[ -n "${DB_TYPE}" ]] && [[ "${DB_TYPE,,}" = "!none" ]]; then + echo "" >> ${BORG_LOG_FILE} + echo "# ---" >> ${BORG_LOG_FILE} + echo "# Create database backup .." >> ${BORG_LOG_FILE} + echo "# ---" >> ${BORG_LOG_FILE} + + blank_line +fi + +if [[ "${DB_TYPE,,}" = "mysql" ]] ; then + + echononl "Create an array of MySQL databases." + + cat <> ${BORG_LOG_FILE} +declare -a mysql_database_arr=(); +for _db_name in \$(mysql -s -r -N -e 'show databases' 2> /dev/null ) ; do + + [[ "\${_db_name}" = "information_schema" ]] && continue + [[ "\${_db_name}" = "performance_schema" ]] && continue + [[ "\${_db_name}" = "mysql" ]] && continue + [[ "\${_db_name}" = "sys" ]] && continue + + mysql_database_arr+=("\${_db_name}") + +done +EOF + + declare -a mysql_database_arr=(); + for _db_name in $(mysql -s -r -N -e 'show databases' 2> ${log_file}) ; do + + [[ "${_db_name}" = "information_schema" ]] && continue + [[ "${_db_name}" = "performance_schema" ]] && continue + [[ "${_db_name}" = "mysql" ]] && continue + [[ "${_db_name}" = "sys" ]] && continue + + mysql_database_arr+=("${_db_name}") + + done + + if [[ -s "${log_file}" ]] ; then + echo_failed + error "$(cat "${log_file}")" + else + echo_ok + fi + + for _db in ${mysql_database_arr[@]} ; do + + echononl "Dump Database '${_db}'.." + + cat <> ${BORG_LOG_FILE} +mysqldump $MYSQL_CREDENTIALS \\ + --single-transaction --routines \\ + -h localhost \\ + -e ${_db} > ${DB_BACKUP_DIR}/${_db}-${backup_date}.sql + +EOF + + mysqldump $MYSQL_CREDENTIALS \ + --single-transaction --routines \ + -h localhost \ + -e ${_db} > ${DB_BACKUP_DIR}/${_db}-${backup_date}.sql 2> ${log_file} + + if [[ -s "${log_file}" ]] ; then + echo_failed + error "$(cat "${log_file}")" + else + echo_ok + fi + + done + +elif [[ "${DB_TYPE,,}" = "psql" ]] || [[ "${DB_TYPE,,}" = "postgresql" ]] || [[ "${DB_TYPE,,}" = "postgres" ]] ; then + + echononl "Create an array of PostgreSQL databases." + + cat <> ${BORG_LOG_FILE} +declare -a psql_database_arr=(); +for _db_name in \$(su - ${PSQL_SYSTEM_USER} -c "psql -lt" 2> /dev/null | grep -v -e"^$" | awk '{print\$1}') ; do + + [[ "\${_db_name}" = ""template0 ]] && continue + [[ "\${_db_name}" = ""template1 ]] && continue + [[ "\${_db_name}" = ""postgres ]] && continue + [[ "\${_db_name}" = "|" ]] && continue + [[ "\${_db_name}" = ":" ]] && continue + + psql_database_arr+=("\${_db_name}") + +done +EOF + + declare -a psql_database_arr=(); + for _db_name in $(su - ${PSQL_SYSTEM_USER} -c "psql -lt" 2> "${log_file}" | grep -v -e"^$" | awk '{print$1}') ; do + + [[ "${_db_name}" = ""template0 ]] && continue + [[ "${_db_name}" = ""template1 ]] && continue + [[ "${_db_name}" = ""postgres ]] && continue + [[ "${_db_name}" = "|" ]] && continue + [[ "${_db_name}" = ":" ]] && continue + + psql_database_arr+=("${_db_name}") + + done + + if [[ -s "${log_file}" ]] ; then + echo_failed + error "$(cat "${log_file}")" + else + echo_ok + fi + + for _db in ${psql_database_arr[@]} ; do + + echononl "Dump Database '${_db}'.." + + cat <> ${BORG_LOG_FILE} +su - ${PSQL_SYSTEM_USER} -c "pg_dump -c ${_db}" > ${DB_BACKUP_DIR}/${_db}-${backup_date}.sql +EOF + + su - ${PSQL_SYSTEM_USER} -c "pg_dump -c ${_db}" > ${DB_BACKUP_DIR}/${_db}-${backup_date}.sql 2> ${log_file} + + if [[ -s "${log_file}" ]] ; then + echo_failed + error "$(cat "${log_file}")" + else + echo_ok + fi + + done + +fi +clean_up 0 + + +echononl "Create borg backup.." + +cat << EOF >> ${BORG_LOG_FILE} +borg create \\ + --verbose \\ + --filter AME \\ + --list \\ + --stats \\ + --show-rc \\ + --compression lz4 \\ + --exclude-caches \\ + --exclude 'home/*/.cache/*' \\ + --exclude '/root/.cache/*' \\ + --exclude '/root/.config/*' \\ + --exclude '/root/.local/*' \\ + --exclude 'var/tmp/*' \\ + \\ + ${BORG_URL}::${backup_date} \\ + ${DB_BACKUP_DIR} \\ + ${NC_DATA_DIR} \\ + ${BORG_BACKUP_DIRS} +EOF + +borg create \ + --verbose \ + --filter AME \ + --list \ + --stats \ + --show-rc \ + --compression lz4 \ + --exclude-caches \ + --exclude 'home/*/.cache/*' \ + --exclude '/root/.cache/*' \ + --exclude '/root/.config/*' \ + --exclude '/root/.local/*' \ + --exclude 'var/tmp/*' \ + \ + ${BORG_URL}::${backup_date} \ + ${DB_BACKUP_DIR} \ + ${NC_DATA_DIR} \ + ${BORG_BACKUP_DIRS} \ + > ${log_file} 2>&1 + + + +if [[ $? -gt 0 ]] ; then + echo_failed + error "$(cat "${log_file}")" +else + echo_ok + cat ${log_file} >> ${BORG_LOG_FILE} +fi + + +echononl "Pruning repository.." + +cat << EOF >> ${BORG_LOG_FILE} +borg prune \\ + --list \\ + --show-rc \\ + --keep-daily 7 \\ + --keep-weekly 4 \\ + --keep-monthly 6 \\ + --keep-yearly 1 \\ + ${BORG_URL} +EOF + +borg prune \ + --list \ + --show-rc \ + --keep-daily 7 \ + --keep-weekly 4 \ + --keep-monthly 6 \ + --keep-yearly 1 \ + ${BORG_URL} \ + > ${log_file} 2>&1 + +if [[ $? -gt 0 ]] ; then + echo_failed + error "$(cat "${log_file}")" +else + echo_ok + cat ${log_file} >> ${BORG_LOG_FILE} +fi + +blank_line + + +# Remove DB local backup +# +echononl "Remove local DB backups.." + +echo "rm -f ${DB_BACKUP_DIR}/*-${backup_date}.sql" >> ${BORG_LOG_FILE} + +rm -f ${DB_BACKUP_DIR}/*-${backup_date}.sql > ${log_file} 2>&1 + +if [[ $? -gt 0 ]] ; then + echo_failed + error "$(cat "${log_file}")" +else + echo_ok +fi + +# Duration of backup +endTime=$(date +%s) +endDateReadable=$(date --date @"$endTime" +"%d.%m.%Y - %H:%M:%S") +duration=$((endTime-startTime)) +durationSec=$((duration % 60)) +durationMin=$(((duration / 60) % 60)) +durationHour=$((duration / 3600)) +durationReadable=$(printf "%02d hours %02d minutes %02d seconds" $durationHour $durationMin $durationSec) + +echo -e "\n\n###### Server backup finished on ${endDateReadable} (${durationReadable}) ######\n" >> ${BORG_LOG_FILE} + + +clean_up 0 diff --git a/conf/borg-backup.conf.sample b/conf/borg-backup.conf.sample new file mode 100644 index 0000000..9230e67 --- /dev/null +++ b/conf/borg-backup.conf.sample @@ -0,0 +1,169 @@ +#----------------------------- +# Settings for script borg-backup-nc.sh +#----------------------------- + +# SSH_USER +# +# The (remote) user under which the SSH session for the +# data transfer is running. +# +# Defaults to: +# +# SSH_USER="borg" +# +#BACKUP_USER="borg" + + +# SSH_PORT +# +# The port under which the SSH service is accessible +# +# Defaults to: +# +# SSH_PORT=22 +# +#SSH_PORT=22 + + +# BORG_HOST +# +# The (remote) host on which the backups are saved. +# +# !! No Defaults - this parameter is mandatory +# +#BORG_HOST="" + + +# BORG_RSH +# +# The command, which is used instead of ssh. +# +# This can be used to specify ssh options, such as a custom identity file +# ssh -i /path/to/private/key. See man ssh for other options. +# +# Using the --rsh CMD commandline option overrides the environment variable. +# +# Defaults to: +# BORG_RSH='ssh -i /root/.ssh/id_ed25519-borg-backup' +# +#BORG_RSH='ssh -i /root/.ssh/id_ed25519-borg-backup' + + +# BORG_PASSPHRASE +# +# Password of the Borg Repository (which was used while initializing) +# +# !! No Defaults - this parameter is mandatory +# +#BORG_PASSPHRASE='borg-password' + + +# BORG_REPO +# +# The repository location on the (remote) backup host. +# +# !! No Defaults - this parameter is mandatory +# +#BORG_REPO= + + +# BORG_LOG_DIR +# +# Log to this directory +# +# Defaults to: +# +# BORG_LOG_DIR="/var/log/borg" +# +#BORG_LOG_DIR="/var/log/borg" + + +# --- +# DB related parameters +# --- + +# DB_TYPE +# +# Which database ist installed? +# +# possible values: +# - MySQL +# - PostgreSQL or postgres or psql +# - None +# +# Defaults to: +# +# DB_TYPE = "None" +# +# DB_TYPE = "None" + +# MYSQL_CREDENTIALS +# +# Giving password on command line is insecure an sind mysql 5.5 +# you will get a warning doing so. +# +# Reading username/password fro file ist also possible, using MySQL/MariaDB +# commandline parameter '--defaults-file'. +# +# Since Version 5.6, that method is considered as insecure. +# To avoid giving the password on command line, we use an +# encrypted option file +# +# Create (encrypted) option file: +# $ mysql_config_editor set --login-path=local --socket=/var/run/mysqld/mysqld.sock --user=backup --password +# $ Password: +# +# Use of option file: +# $ mysql --login-path=local ... +# +# Example +# MYSQL_CREDENTIALS="--login-path=local" +# MYSQL_CREDENTIALS="--defaults-file=/etc/mysql/debian.cnf" (Debian default) +# MYSQL_CREDENTIALS="--defaults-file=/usr/local/mysql/sys-maint.cnf" +# +# Defaults to: +# +# MYSQL_CREDENTIALS="-u root -S /run/mysqld/mysqld.sock" +# +#MYSQL_CREDENTIALS="-u root -S /run/mysqld/mysqld.sock" + + +# PSQL_SYSTEM_USER +# +# User who owns PostgreSQL services +# +# Defaults to: +# +# PSQL_SYSTEM_USER="postgres" +# +#PSQL_SYSTEM_USER="postgres" + + + +# DB_BACKUP_DIR +# +# Directory in which current database dumps are stored (and transferred to the repository) +# +# Defaults to: +# +# DB_BACKUP_DIR="/data/backup/sql" +# +#DB_BACKUP_DIR="/data/backup/sql" + + +# --- +# Backups directories +# --- + +# BACKUP_DIRECTORIES +# +# Directories (or files) that should be backed up +# +# Defaults to: +# +# BACKUP_DIRECTORIES="/etc /home /root /var/spool/cron" +# +# - Blank separated list of directories - +# +#BACKUP_DIRECTORIES="/etc /home /root /var/spool/cron" +