#!/usr/bin/env bash script_name="$(basename $(realpath $0))" working_dir="$(dirname $(realpath $0))" conf_file="${working_dir}/conf/${script_name%%.*}.conf" LOCK_DIR="/tmp/${script_name%%.*}.LOCK" RESTART_CHECK_FILE="/tmp/${script_name%%.*}.NEED-RESTART" declare -a alert_email_arr # ------------- # --- Some functions # ------------- clean_up() { # Perform program exit housekeeping rm -rf "$LOCK_DIR" if $LOGGING ; then echo "" fi exit $1 } echononl(){ if $terminal && $LOGGING ; then echo X\\c > /tmp/shprompt$$ if [ `wc -c /tmp/shprompt$$ | awk '{print $1}'` -eq 1 ]; then echo -e -n "$*\\c" 1>&2 else echo -e -n "$*" 1>&2 fi rm /tmp/shprompt$$ fi } fatal(){ echo "" if $terminal ; then echo -e "[ \033[31m\033[1mError\033[m ]: $*" echo "" echo -e "\t\033[31m\033[1mScript was interupted\033[m!" else echo " [ Fatal ]: $*" echo "" echo " Script was terminated...." fi echo "" clean_up 1 } error (){ echo "" if $terminal ; then echo -e "\t[ \033[31m\033[1mError\033[m ]: $*" else echo "[ Error ]: $*" fi echo "" } warn (){ echo "" if $terminal ; then echo -e "\t[ \033[33m\033[1mWarning\033[m ]: $*" else echo "[ Warning ]: $*" fi echo "" } warn_only_terminal () { if $terminal ; then echo "" echo -e "\t[ \033[33m\033[1mWarning\033[m ]: $*" echo "" fi } echo_done() { if $terminal && $LOGGING ; then echo -e "\033[75G[ \033[32mdone\033[m ]" else if $LOGGING ; then echo " [ done ]" fi fi } echo_ok() { if $terminal && $LOGGING ; then echo -e "\033[75G[ \033[32mok\033[m ]" else if $LOGGING ; then echo " [ ok ]" fi fi } echo_failed(){ if $terminal && $LOGGING ; then echo -e "\033[75G[ \033[1;31mfailed\033[m ]" else if $LOGGING ; then echo " [ failed ]" fi fi } echo_skipped() { if $terminal && $LOGGING ; then echo -e "\033[75G[ \033[33m\033[1mskipped\033[m ]" else if $LOGGING ; then echo " [ skipped ]" fi fi } is_number() { return $(test ! -z "${1##*[!0-9]*}" > /dev/null 2>&1); # - also possible # - #[[ ! -z "${1##*[!0-9]*}" ]] && return 0 || return 1 #return $([[ ! -z "${1##*[!0-9]*}" ]]) } trim() { local var="$*" var="${var#"${var%%[![:space:]]*}"}" # remove leading whitespace characters var="${var%"${var##*[![:space:]]}"}" # remove trailing whitespace characters echo -n "$var" } blank_line() { if $terminal ; then echo "" fi } reboot_system() { # content_type='Content-Type: text/plain;\n charset="utf-8"' # datum="$(date +"%d.%m.%Y")" # from_address="root@$(hostname --long)" # msg="S*" # # # for _email in ${alert_email_arr[@]} ; do # # echo -e "To:${_email}\n${content_type}\nSubject:[Fatal: Local Webservice NOT rsponding] - Reboot System\n${msg}" \ # | sendmail -F "Error `hostname -f`" -f $sender_address $_email # done sleep 10 /sbin/reboot -f > /dev/null 2>&1 } # ------------- # --- Read Configurations from $conf_file # ------------- # Some default values # DEFAULT_NUMBER_LINES=20 DEFAULT_TIME_OUT=20 DEFAULT_CONFLICTING_SCRIPTS=" /root/bin/mysql/optimize_mysql_tables.sh /root/bin/mysql/optimize_mysql_tables-ND.sh /root/bin/monitoring/check_webservice_load.sh " if [[ ! -f "$conf_file" ]]; then echo "" echo -e " [ Fatal ] Configuration file '$(basename ${conf_file})' not found!" echo "" echo -e "\tScript terminated.." echo "" exit 1 else source "$conf_file" fi for _email in $alert_email_addresses ; do alert_email_arr+=("$_email") done [[ -n "$sender_address" ]] || sender_address="check_local_webservice@$(hostname -f)" [[ -n "$content_type" ]] || content_type='Content-Type: text/plain;\n charset="utf-8"' [[ -n "$TIME_OUT" ]] || TIME_OUT=$DEFAULT_TIME_OUT TIME_OUT_MAX="$(expr ${TIME_OUT} + 5)" [[ -n "$CONFLICTING_SCRIPTS" ]] || CONFLICTING_SCRIPTS="$DEFAULT_CONFLICTING_SCRIPTS" # ------------- # --- Check some prerequisites # ------------- # - Running in a terminal? # - if [[ -t 1 ]] ; then terminal=true LOGGING=true else terminal=false LOGGING=false fi # - Stop here, if these give scripts are running # - if [[ ${#CONFLICTING_SCRIPTS} -gt 0 ]] ; then # - Try using a random start delay to prevent (or at least have a small chance) that # - conflicting scripts will both/all abort if they start at the same time. # - # - !! Notice !! # - This only makes sense if a fixed LOCK directory is used, otherwise the process list # - (and NOT the LOCK-directory) is used to look for scripts running in parallel. # - # - Skip delay if running in an terminal (from copnsole) # - if ! $terminal ; then if [[ "$LOCK_DIR" = "/tmp/${script_name%%.*}.LOCK" ]]; then _shift="$(( $RANDOM % 10 + 1 ))" sleep $(( $RANDOM % 25 + $_shift )) fi fi _stop_running=false for _val in $CONFLICTING_SCRIPTS ; do IFS=':' read -a _val_arr <<< "${_val}" _script_name="$(basename ${_val_arr[0]})" if [[ -n "${_val_arr[1]}" ]] ; then if [[ "${_val_arr[1]}" = "CHECK_PROCESS_LIST" ]] ; then check_string_ps="${_val_arr[0]}" if ps -e f | grep -E "\s+${check_string_ps}" | grep -v grep | grep -v -E "\s+vim\s+" > /dev/null ; then _stop_running=true fi elif [[ -d "${_val_arr[1]}" ]] ; then _stop_running=true fi elif [[ -d "/tmp/${_script_name%%.*}.LOCK" ]]; then _stop_running=true fi if $_stop_running ; then warn_only_terminal "\033[1m${_script_name}\033[m is currently running, but it conflicts with this script. Exiting now.." clean_up 1 fi # if $_stop_running ; then done # for _val in $CONFLICTING_SCRIPTS ; do fi # if [[ ${#CONFLICTING_SCRIPTS} -gt 0 ]] ; then # ------------- # - Job is already running? # ------------- # - If job already runs, stop execution.. # - if mkdir "$LOCK_DIR" 2> /dev/null ; then ## - Remove lockdir when the script finishes, or when it receives a signal trap "clean_up 1" SIGHUP SIGINT SIGTERM else datum="$(date +"%d.%m.%Y %H:%M")" msg="[ Error ]: A previos instance of \"`basename $0`\" seems already be running.\n\n Exiting now.." echo "" echo "[ Error ]: A previos instance of that script \"`basename $0`\" seems already be running." echo "" echo -e " Exiting now.." echo "" for _email in ${alert_email_arr[@]} ; do echo -e "To:${_email}\n${content_type}\nSubject:Error cronjob `basename $0` -- $datum\n${msg}\n" \ | sendmail -F "Error `hostname -f`" -f $sender_address $_email done exit 1 fi # ------------- # --- Check some further prerequisites # ------------- # - Systemd supported ? # - systemd=$(which systemd) systemctl=$(which systemctl) systemd_supported=false if [[ -n "$systemd" ]] && [[ -n "$systemctl" ]] ; then systemd_supported=true fi if $LOGGING ; then echo "" fi if [[ -n "$LOCAL_WEBSITES_TO_CHECK" ]] ; then for _site in $LOCAL_WEBSITES_TO_CHECK ; do echononl " Check site \033[1m$_site\033[m .." declare -i i=0 while [[ $i -lt 3 ]] ; do response="$(curl --max-time $TIME_OUT_MAX --connect-timeout $TIME_OUT \ -I -k -L --write-out %{http_code} --silent --output /dev/null $_site 2> ${LOCK_DIR}/err.msg)" # 200 - OK # 401 - Unauthorized (response of roundcube webmailer (since version 1.4) if [[ "$response" -eq 200 ]] || [[ "$response" -eq 401 ]]; then echo_ok if [[ -f "$RESTART_CHECK_FILE" ]] ; then rm -f "$RESTART_CHECK_FILE" fi clean_up 0 fi sleep 2 ((i++)) done echo_failed done else warn "No Website to check given (empty var 'LOCAL_WEBSITES_TO_CHECK')" clean_up 10 fi msg_head="\n==========\nSystem logfiles\n==========\n" msg00="Last entries (20 lines) of \"/var/log/syslog\":" msg01="===============================================" msg02=`tail -n 20 /var/log/syslog` msg03="Last entries (20 lines) of \"/var/log/messages\":" msg04="=================================================" msg05=`tail -n 20 /var/log/messages` msg06="Last entries (20 lines) of \"/var/log/auth.log\":" msg07="=================================================" msg08=`tail -n 20 /var/log/auth.log` msg09="Last entries (20 lines) of \"/var/log/daemon.log\":" msg10="=================================================" msg11=`tail -n 20 /var/log/daemon.log` msg12="Last entries (20 lines) of \"/var/log/kern.log\":" msg13="=================================================" msg14=`tail -n 20 /var/log/kern.log` msg15="Lastlog:" msg16="========" msg17=`lastlog` msg18="dmesg:" msg19="======" msg20=`dmesg -T` msg="${msg_head}\n${msg00}\n${msg01}\n${msg02}\n\n${msg03}\n${msg04}\n${msg05}\n\n${msg06}\n${msg07}\n${msg08}\n\n${msg09}\n${msg10}\n${msg11}\n\n${msg12}\n${msg13}\n${msg14}\n\n${msg15}\n${msg16}\n${msg17}\n\n${msg18}\n${msg19}\n${msg20}" if [[ ${#LOG_FILES_TO_MONITOR} -gt 0 ]] ; then msg_user_defined="\n==========\nUser defined logfiles\n==========\n" for _val in $LOG_FILES_TO_MONITOR ; do IFS=':' read -a _val_arr <<< "${_val}" _log_file="${_val_arr[0]}" if [[ -n "${_val_arr[0]}" ]] && is_number "${_val_arr[1]}" ; then _number_lines=${_val_arr[1]} else _number_lines=$DEFAULT_NUMBER_LINES fi if [[ -s "${_log_file}" ]] ; then msg_user_defined="${msg_user_defined}\n---\nLast entries (${_number_lines} lines) of \"${_log_file}\":\n---\n$(tail -n ${_number_lines} ${_log_file})\n" else msg_user_defined="${msg_user_defined}\n---\nLast entries (${_number_lines} lines) of \"${_log_file}\":\n---\n-- FILE IS EMPTY --\n" fi done msg_user_defined="${msg_user_defined}\n\n" else msg_user_defined="" fi if [[ ! -f "$RESTART_CHECK_FILE" ]]; then touch "$RESTART_CHECK_FILE" error "The local webservice seems to be down. Some Websites do not respond as expected:" err_msg="\n[ Warning ]: The local webservice seems to be down.\n\n Some Websites do not respond as expected:\n" for _site in $LOCAL_WEBSITES_TO_CHECK ; do if $LOGGING ; then echo -e "\t \033[1m$_site\033[m" else echo " $_site" fi err_msg="${err_msg} ${_site}\n" done err_msg="${err_msg}\nFor now nothing is to. because its the first time..\n" datum="$(date +"%d.%m.%Y %H:%M")" for _email in ${alert_email_arr[@]} ; do echo -e "To:${_email}\n${content_type}\nSubject:[Warn: Local Webservice NOT rsponding] - Do nothing for now\n$err_msg\nFilesystem usage:\n=================\n$(df -h)\n\n${msg_user_defined}\n${msg}" \ | sendmail -F "Error `hostname -f`" -f $sender_address $_email done clean_up 20 else error "The local webservice is still down. Some Websites still not responding:" err_msg="\n[ Error ]: The local webservice seems to be down.\n\n Some Websites still not responding:\n" for _site in $LOCAL_WEBSITES_TO_CHECK ; do if $LOGGING ; then echo -e "\t \033[1m$_site\033[m" else echo " $_site" fi err_msg="${err_msg} ${_site}\n" done if $LOGGING ; then echo -e "\n\033[1mGoing to restart the system NOW..\033[m" else echo "" echo "Going to restart the system NOW.." fi err_msg="${err_msg}\nGoing to restart the system..\n" datum="$(date +"%d.%m.%Y %H:%M")" for _email in ${alert_email_arr[@]} ; do echo -e "To:${_email}\n${content_type}\nSubject:[ REBOOT System ]: Local Webservice IS STILL NOT rsponding.\n$err_msg\nFilesystem usage:\n=================\n$(df -h)\n\n${msg_user_defined}\n${msg}" \ | sendmail -F "Error `hostname -f`" -f $sender_address $_email done rm -f "$RESTART_CHECK_FILE" reboot_system fi if $LOGGING ; then echo "" fi clean_up 0