#!/usr/bin/env bash script_name="$(basename $(realpath $0))" working_dir="$(dirname $(realpath $0))" conf_file="${working_dir}/conf/check_webservice_load.conf" # - Lock directory exists, until the script ends. So # - we can check, if a previos instanze is already running. # - LOCK_DIR="/tmp/${script_name%%.*}.LOCK" #LOCK_DIR=`mktemp -d` #--------------------------------------- #----------------------------- # Read Configurations from $conf_file #----------------------------- #--------------------------------------- # - Defaults # - check_load=false check_mysql=false check_postgresql=false check_apache=false check_nginx=false check_php_fpm=false check_redis=false check_website=false ommit_curl_check_nginx=false vserver_guest=false #--------------------------------------- #----------------------------- # Base Function(s) #----------------------------- #--------------------------------------- clean_up() { # Perform program exit housekeeping rm -rf "$LOCK_DIR" if $LOGGING ; then echo "" fi exit $1 } fatal(){ echo "" if $terminal ; then echo -e " [ \033[31m\033[1mFatal\033[m ] $*" else echo -e " [ Fatal ] $*" fi echo "" rm -rf $LOCK_DIR exit 1 } error (){ echo "" if $terminal ; then echo -e " [ \033[31m\033[1mError\033[m ] $*" else echo " [ Error ] $*" fi echo "" } warn_only_terminal () { if $terminal ; then echo "" echo -e "\t[ \033[33m\033[1mWarning\033[m ]: $*" echo "" fi } warn (){ echo "" if $terminal ; then echo -e " [ \033[33m\033[1mWarn\033[m ] $*" else echo " [ Warn ] $*" fi echo "" } info (){ echo "" if $terminal ; then echo -e " [ \033[32m\033[1mInfo\033[m ] $*" else echo " [ Info ] $*" fi echo "" } ok (){ echo "" if $terminal ; then echo -e " [ \033[32m\033[1mOk\033[m ] $*" else echo " [ Ok ] $*" fi echo "" } detect_os () { if $(which lsb_release > /dev/null 2>&1) ; then os_dist="$(lsb_release -i | awk '{print tolower($3)}')" os_version="$(lsb_release -r | awk '{print tolower($2)}')" os_codename="$(lsb_release -c | awk '{print tolower($2)}')" if [[ "$os_dist" = "debian" ]]; then if $(echo "$os_version" | grep -q '\.') ; then os_version=$(echo "$os_version" | cut --delimiter='.' -f1) fi fi elif [[ -e "/etc/os-release" ]]; then . /etc/os-release os_dist=$ID os_version=${VERSION_ID} fi # remove whitespace from os_dist and os_version os_dist="${os_dist// /}" os_version="${os_version// /}" } trim() { local var="$*" var="${var#"${var%%[![:space:]]*}"}" # remove leading whitespace characters var="${var%"${var##*[![:space:]]}"}" # remove trailing whitespace characters echo -n "$var" } blank_line() { if $terminal ; then echo "" fi } #--------------------------------------- #----------------------------- # Check some prerequisites #----------------------------- #--------------------------------------- if [[ -t 1 ]] ; then terminal=true LOGGING=true else terminal=false LOGGING=false fi # - Detect linux distribution # - detect_os # ------------- # --- Read Configurations from $conf_file # ------------- # Some default values # DEFAULT_CONFLICTING_SCRIPTS="/root/bin/monitoring/check_local_webservice.sh" if [[ ! -f "$conf_file" ]]; then echo "" echo -e " [ Fatal ] Configuration file '$(basename ${conf_file})' not found!" echo "" echo -e "\tScript terminated.." echo "" clean_up 1 else source "$conf_file" fi [[ -n "$CONFLICTING_SCRIPTS" ]] || CONFLICTING_SCRIPTS="$DEFAULT_CONFLICTING_SCRIPTS" # - Stop here, if these give scripts are running # - if [[ ${#CONFLICTING_SCRIPTS} -gt 0 ]] ; then # - Try using a random start delay to prevent (or at least have a small chance) that # - conflicting scripts will both/all abort if they start at the same time. # - # - !! Notice !! # - This only makes sense if a fixed LOCK directory is used, otherwise the process list # - (and NOT the LOCK-directory) is used to look for scripts running in parallel. # - # - Skip delay if running in an terminal (from copnsole) # - if ! $terminal ; then if [[ "$LOCK_DIR" = "/tmp/${script_name%%.*}.LOCK" ]]; then _shift="$(( $RANDOM % 10 + 1 ))" sleep $(( $RANDOM % 25 + $_shift )) fi fi _stop_running=false for _val in $CONFLICTING_SCRIPTS ; do IFS=':' read -a _val_arr <<< "${_val}" _script_name="$(basename ${_val_arr[0]})" if [[ -n "${_val_arr[1]}" ]] ; then if [[ "${_val_arr[1]}" = "CHECK_PROCESS_LIST" ]] ; then check_string_ps="${_val_arr[0]}" if ps -e f | grep -E "\s+${check_string_ps}" | grep -v grep | grep -v -E "\s+vim\s+" > /dev/null ; then _stop_running=true fi elif [[ -d "${_val_arr[1]}" ]] ; then _stop_running=true fi elif [[ -d "/tmp/${_script_name%%.*}.LOCK" ]]; then _stop_running=true fi if $_stop_running ; then warn_only_terminal "\033[1m${_script_name}\033[m is currently running, but it conflicts with this script. Exiting now.." clean_up 1 fi # if $_stop_running ; then done # for _val in $CONFLICTING_SCRIPTS ; do fi # if [[ ${#CONFLICTING_SCRIPTS} -gt 0 ]] ; then #--------------------------------------- #----------------------------- # Job is already running? #----------------------------- #--------------------------------------- ## - If job already runs, stop execution.. ## - if mkdir "$LOCK_DIR" 2> /dev/null ; then ## - Remove lockdir when the script finishes, or when it receives a signal trap "clean_up 1" SIGHUP SIGINT SIGTERM else datum=`date +"%d.%m.%Y"` msg="[ Error ]: A previos instance of \"`basename $0`\" seems already be running.\n\tExiting now.." echo "" echo "[ Error ]: A previos instance of that script \"`basename $0`\" seems already be running." echo "" echo -e "\tExiting now.." echo "" for _to_address in $to_addresses ; do echo -e "To:${_to_address}\n${content_type}\nSubject:Error cronjob `basename $0` -- $datum\n${msg}\n" \ | sendmail -F "Error `hostname -f`" -f $from_address $_to_address done exit 1 fi ## - Remove lockdir when the script finishes, or when it receives a signal ## - #trap 'rm -rf "$LOCK_DIR"' 0 2 15 #--------------------------------------- #----------------------------- # Check some further prerequisites #----------------------------- #--------------------------------------- if [ -z "`which basename`" ]; then fatal 'It seems "basename" is not installed, but needed!' fi if [ -z "`which sendmail`" ]; then fatal 'It seems "sendmail" is not installed, but needed!' fi if [ -z "`which realpath`" ]; then fatal 'It seems "realpath" is not installed, but needed!' fi if $check_apache || $check_nginxA ; then if [ -z "$(which curl)" ]; then fatal 'It seems "curl" is not installed, but needed!' fi fi if $check_load ; then if [ -z "`which bc`" ]; then fatal 'It seems "bc" is not installed, but needed!' fi fi # - Systemd supported ? ## - systemd_supported=false if ! $vserver_guest ; then systemd=$(which systemd) systemctl=$(which systemctl) if [[ -n "$systemd" ]] && [[ -n "$systemctl" ]] ; then systemd_supported=true fi fi if $check_mysql ; then MYSQLD=`realpath $(which mysqld) 2>/dev/null` if [ -z "$MYSQLD" ]; then if [ -x "/usr/local/mysql/bin/mysqld" ]; then MYSQLD=`realpath "/usr/local/mysql/bin/mysqld"` else fatal 'Command \"mysqld\" not found!' fi fi MYSQL_INIT_SCRIPT="" MYSQLD_SERVICE_FILE="" if $systemd_supported ; then # - Is Service exclusive controlled by systemd # - if systemctl -t service list-unit-files \ | grep -e "^mysql" \ | grep -q -E "(enabled|disabled)" 2> /dev/null ; then MYSQLD_SERVICE_FILE=$(systemctl -t service list-unit-files \ | grep -e "^mysql[^0-9]*\.service" \ | grep -E " enabled" \ | awk '{print$1}'\ | head -1) elif systemctl -t service list-unit-files \ | grep -e "^mariadb" \ | grep -q -E "(enabled|disabled)" 2> /dev/null ; then MYSQLD_SERVICE_FILE=$(systemctl -t service list-unit-files \ | grep -e "^mariadb.*.service" \ | grep -E " enabled" \ | awk '{print$1}'\ | head -1) fi fi if [[ -z "$MYSQLD_SERVICE_FILE" ]] ; then if [ -x "$(realpath /etc/init.d/mysql.server)" ]; then MYSQL_INIT_SCRIPT="/etc/init.d/mysql.server" elif [ -x "$(realpath /etc/init.d/mysql)" ]; then MYSQL_INIT_SCRIPT="/etc/init.d/mysql" fi fi if [[ -z "$MYSQL_INIT_SCRIPT" ]] && [[ -z "$MYSQLD_SERVICE_FILE" ]]; then fatal 'Neither an init-script nor a service file for MySQL found!' elif [[ -n "$MYSQL_INIT_SCRIPT" ]] ; then MYSQL_PS_CHECK_STRING="$(dirname "$MYSQLD")/(mysqld_safe |$(basename "$MYSQLD") )" #MYSQL_PS_CHECK_STRING="(bin/mysqld_safe |bin/mysqld )" else MYSQL_PS_CHECK_STRING="$MYSQLD" fi if [[ -z "$MYSQL_PS_CHECK_STRING" ]]; then fatal "Cannot determin a check string for output of command 'ps ax'" fi if [[ -n "$MYSQL_INIT_SCRIPT" ]] ; then MYSQLD_SAFE=`realpath $(which mysqld_safe) 2>/dev/null` if [ -z "$MYSQLD_SAFE" ]; then if [ -x "/usr/local/mysql/bin/mysqld_safe" ]; then MYSQLD_SAFE=`realpath "/usr/local/mysql/bin/mysqld_safe"` else fatal 'Command \"mysqld_safe\" not found!' fi fi fi MYADMIN=`realpath $(which mysqladmin) 2>/dev/null` if [ -z "$MYADMIN" ]; then if [ -x "/usr/local/mysql/bin/mysqladmin" ]; then MYADMIN=`realpath "/usr/local/mysql/bin/mysqladmin"` else fatal 'Command \"mysqladmin\" not found!' fi fi fi if $check_postgresql ; then PSQL_SERVICE_FILE="" PSQL_INIT_SCRIPT="" if $systemd_supported ; then # - Is Service exclusive controlled by systemd # - if systemctl -t service list-unit-files \ | grep -e "^ postgresql" \ | grep -q -E "(enabled|disabled)" 2> /dev/null ; then PSQL_SERVICE_FILE=$(systemctl -t service list-unit-files \ | grep -e "^postgresql" \ | awk '{print$1}') fi fi if [[ -z "$PSQL_SERVICE_FILE" ]] ; then if [ -x "/etc/init.d/postgresql" ]; then PSQL_INIT_SCRIPT="/etc/init.d/postgresql" fi fi if [[ -z "$PSQL_INIT_SCRIPT" ]] && [[ -z "$PSQL_SERVICE_FILE" ]] ; then fatal "Neither an init-script nor a service file for 'postgesql' databaseservice found!" fi fi if $check_apache ; then HTTPD=`realpath $(which httpd) 2>/dev/null` if [ -z "$HTTPD" ]; then if [ -x "/usr/local/apache2/bin/httpd" ]; then HTTPD=`realpath "/usr/local/apache2/bin/httpd"` else HTTPD=`realpath $(which apache2) 2>/dev/null` if [ -z "$HTTPD" ]; then fatal 'Command \"httpd\"/\"apache2\" not found!' fi fi fi APACHE_SERVICE_FILE="" APACHE_INIT_SCRIPT="" if $systemd_supported ; then # - Is Service exclusive controlled by systemd # - if systemctl -t service list-unit-files \ | grep -e "^apache2.service" \ | grep -q -E "(enabled|disabled)" 2> /dev/null ; then APACHE_SERVICE_FILE="apache2.service" elif systemctl -t service list-unit-files \ | grep -e "^apache" \ | grep -q -E "(enabled|disabled)" 2> /dev/null ; then APACHE_SERVICE_FILE=$(systemctl -t service list-unit-files \ | grep -e "^apache" \ | awk '{print$1}' \ | head -1) fi fi if [[ -z "$APACHE_SERVICE_FILE" ]] ; then if [ -x "/etc/init.d/apache2" ]; then APACHE_INIT_SCRIPT="/etc/init.d/apache2" elif [ -x "/etc/init.d/apachectl" ]; then APACHE_INIT_SCRIPT="/etc/init.d/apachectl" fi fi if [[ -z "$APACHE_INIT_SCRIPT" ]] && [[ -z "$APACHE_SERVICE_FILE" ]] ; then fatal 'Neither an init-script nor a service file for apache2 found!' fi fi if $check_nginx ; then NGINXD="$(realpath $(which nginx))" if [ -z "$NGINXD" ]; then if [ -x "/usr/local/nginx/bin/nginx" ]; then NGINXD="$(realpath "/usr/local/nginx/bin/nginx")" elif [ -x "/usr/local/nginx/sbin/nginx" ]; then NGINXD="$(realpath "/usr/local/nginx/sbin/nginx")" else fatal 'Command \"nginx\" not found!' fi fi NGINX_SERVICE_FILE="" NGINX_INIT_SCRIPT="" if $systemd_supported ; then # - Is Service exclusive controlled by systemd # - if systemctl -t service list-unit-files \ | grep -e "^nginx" \ | grep -q -E "(enabled|disabled)" 2> /dev/null ; then NGINX_SERVICE_FILE=$(systemctl -t service list-unit-files \ | grep -e "^nginx" \ | awk '{print$1}') fi fi if [[ -z "$NGINX_SERVICE_FILE" ]] ; then if [ -x "/etc/init.d/nginx" ]; then NGINX_INIT_SCRIPT="/etc/init.d/nginx" fi fi if [[ -z "$NGINX_INIT_SCRIPT" ]] && [[ -z "$NGINX_SERVICE_FILE" ]] ; then fatal "Neither an init-script nor a service file for 'nginx' webservice found!" fi fi declare -A start_stop_method_php_fpm declare -A start_stop_file_php_fpm declare -A php_fpm_ping_path declare -A php_fpm_pid_search_path if $check_php_fpm ; then if [ -z "`which curl`" ]; then fatal 'It seems "curl" is not installed, but needed!' fi if [[ "$(trim $php_versions)" = "" ]]; then fatal 'No PHP version is given. Setting of \"php_version\" is needed!' fi if [[ -n "$ping_path" ]];then for _path in $ping_path ;do IFS=':' read -a _path_arr <<< "${_path}" # - some checks.. # - if [[ -z "${_path_arr[0]}" ]] || [[ -z "${_path_arr[0]}" ]] ; then fatal "Wrong settings in variable \"ping_path\"!" fi if [[ ! "$php_versions" =~ "${_path_arr[0]}" ]] ; then fatal "Unknown PHP version \"${_path_arr[0]}\" (variable ping_path)!\n\n PHP version must match given versions of var \"php_versions\"" fi php_fpm_ping_path["${_path_arr[0]}"]="${_path_arr[1]}" done fi for _version in $php_versions ; do PHP_FPM_SERVICE_FILE="" if $systemd_supported ; then # - Is Service exclusive controlled by systemd # - if systemctl -t service list-unit-files \ | grep -E "^php-?$_version" \ | grep -q enabled 2> /dev/null ; then PHP_FPM_SERVICE_FILE=$(systemctl -t service list-unit-files \ | grep -E "^php-?$_version" \ | awk '{print$1}') fi fi if [[ -z "${php_fpm_ping_path[$_version]}" ]] ; then php_fpm_ping_path["$_version"]="ping-$_version" fi if [[ -n "$PHP_FPM_SERVICE_FILE" ]]; then start_stop_method_php_fpm["$_version"]="systemd_service" start_stop_file_php_fpm["$_version"]="$PHP_FPM_SERVICE_FILE" #php_fpm_ping_path["$_version"]="ping-$_version" php_fpm_pid_search_path["$_version"]="$_version" elif [[ -x "/etc/init.d/php-${_version}-fpm" ]]; then start_stop_method_php_fpm["$_version"]="init_script" start_stop_file_php_fpm["$_version"]="/etc/init.d/php-${_version}-fpm" #php_fpm_ping_path["$_version"]="ping-$_version" php_fpm_pid_search_path["$_version"]="$_version" elif [[ -x "/etc/init.d/php5-fpm" ]]; then start_stop_method_php_fpm["$_version"]="init_script" start_stop_file_php_fpm["$_version"]="/etc/init.d/php5-fpm" #php_fpm_ping_path["$_version"]="$_ping_path" php_fpm_pid_search_path["$_version"]="/etc/php5" else fatal "Cannot find systemd service or init-script for PHP FPM (v$_version) !" fi done fi if $check_redis ; then REDISD="$(which redis-server)" if [ -z "$REDISD" ]; then if [ -x "/usr/sbin/redis-server" ]; then REDISD="/usr/sbin/redis-server" else fatal 'Command \"redis-server\" not found!' fi fi REDIS_SERVICE_FILE="" REDIS_INIT_SCRIPT="" if $systemd_supported ; then # - Is Service exclusive controlled by systemd # - if systemctl -t service list-unit-files \ | grep -e "^redis-server" \ | grep -q -E "(enabled|disabled)" 2> /dev/null ; then REDIS_SERVICE_FILE=$(systemctl -t service list-unit-files \ | grep -e "^redis-server" \ | awk '{print$1}') fi fi if [[ -z "$REDIS_SERVICE_FILE" ]] ; then if [ -x "/etc/init.d/redis-server" ]; then REDIS_INIT_SCRIPT="/etc/init.d/redis-server" fi fi if [[ -z "$REDIS_INIT_SCRIPT" ]] && [[ -z "$REDIS_SERVICE_FILE" ]] ; then fatal "Neither an init-script nor a service file for 'nginx' webservice found!" fi fi if $check_website ; then if [[ -n "$php_version_of_working_url" ]] && [[ "$php_version_of_working_url" != "None" ]] ; then if [[ ! "$php_versions" =~ $php_version_of_working_url ]]; then fatal "Unknown PHP version \"$php_version_of_working_url\" (variable php_version_of_working_url)!\n\n Var \"php_version_of_working_url\" must match given versions of var \"php_versions\"" fi PHP_FPM_SERVICE_FILE="" if $systemd_supported ; then # - Is Service exclusive controlled by systemd # - if systemctl -t service list-unit-files \ | grep -e "^php-$php_version_of_working_url-" \ | grep -q enabled 2> /dev/null ; then PHP_FPM_SERVICE_FILE=$(systemctl -t service list-unit-files \ | grep -e "^php-$php_version_of_working_url-" \ | awk '{print$1}') fi fi if [[ -z "$PHP_FPM_SERVICE_FILE" ]]; then if [[ ! -x "/etc/init.d/php-${php_version_of_working_url}-fpm" ]] ; then fatal "Cannot find systemd service or init-script for PHP FPM (v$php_version_of_working_url)!\n Check Variable \"php_version_of_working_url\"." fi fi fi if [[ -z "$(trim $is_working_url)" ]];then fatal "Check respond of a certain website is requested, but no url (is_working_url) is given!" fi if [[ -z "$(trim $check_string)" ]];then fatal "Check respond of a certain website is requested, but no check string is given!" fi if $include_cleanup_function && [[ -z $(trim $cleanup_function) ]]; then fatal "include_cleanup_function is enabled, but no cleanup functions are given!\n Check Variable \"cleanup_function\"." else cleanup_functions_file=$(mktemp) printf '%s' "$cleanup_function" > $cleanup_functions_file fi fi #--------------------------------------- #----------------------------- # Start/Stop functions #----------------------------- #--------------------------------------- stop_apache() { send_msg=$1 send_msg=${send_msg:=true} ## - Stop Apache Webservice ## - if [[ -n "$APACHE_SERVICE_FILE" ]] ; then systemctl stop "$APACHE_SERVICE_FILE" > /dev/null 2>&1 else $APACHE_INIT_SCRIPT stop > /dev/null 2>&1 fi sleep 12 declare -i i=0 PIDS=$(ps aux | grep "$HTTPD" | grep -v grep | awk '{print$2}') while [ "X$PIDS" != "X" ]; do if [ $i -eq 0 ]; then echo "" >> $LOCK_DIR/extra_msg.txt echo -e "\t[ Error ]: Stopping Apache Webservice failed !!!" >> $LOCK_DIR/extra_msg.txt echo "" >> $LOCK_DIR/extra_msg.txt echo -e "\t[ Info ]; Going to kill remaining httpd-processes.." >> $LOCK_DIR/extra_msg.txt echo "" >> $LOCK_DIR/extra_msg.txt error "Stopping Apache Webservice failed !!!" info "Going to kill remaining httpd-processes.." fi if [ $i -gt 10 ]; then echo "" >> $LOCK_DIR/extra_msg.txt echo -e "\t[ Error ]: Killing remaining httpd-processes failed !!!" >> $LOCK_DIR/extra_msg.txt echo "" >> $LOCK_DIR/extra_msg.txt error "Killing remaining httpd-processes failed !!!" if $send_msg ; then subject="[ Error ]: Stopping Apache Webserver on `hostname -f` failed -- $datum" msg="\n\n`cat $LOCK_DIR/extra_msg.txt`\n" for _to_address in $to_addresses ; do echo -e "To:${_to_address}\n${content_type}\nSubject:$subject\n\n${msg}\n" \ | /usr/sbin/sendmail -F "Webservice Monitor" -f $from_address $_to_address done fi break fi for _PID in $PIDS ; do kill -9 $_PID > /dev/null 2>&1 done sleep 2 PIDS=$(ps aux | grep "$HTTPD" | grep -v grep | awk '{print$2}') i=i+1 done } stop_nginx() { send_msg=$1 send_msg=${send_msg:=true} ## - Stop Nginx Webservice ## - if [[ -n "$NGINX_SERVICE_FILE" ]] ; then systemctl stop "$NGINX_SERVICE_FILE" > /dev/null 2>&1 else $NGINX_INIT_SCRIPT stop > /dev/null 2>&1 fi sleep 12 declare -i i=0 PIDS=$(ps aux | grep "$NGINXD" | grep -v grep | awk '{print$2}') while [ "X$PIDS" != "X" ]; do if [ $i -eq 0 ]; then echo "" >> $LOCK_DIR/extra_msg.txt echo -e "\t[ Error ]: Stopping Nginx Webservice failed !!!" >> $LOCK_DIR/extra_msg.txt echo "" >> $LOCK_DIR/extra_msg.txt echo -e "\t[ Info ]; Going to kill remaining nginx-processes.." >> $LOCK_DIR/extra_msg.txt echo "" >> $LOCK_DIR/extra_msg.txt error "Stopping Nginx Webservice failed !!!" info "Going to kill remaining nginx-processes.." fi if [ $i -gt 10 ]; then echo "" >> $LOCK_DIR/extra_msg.txt echo -e "\t[ Error ]: Killing remaining nginx-processes failed !!!" >> $LOCK_DIR/extra_msg.txt echo "" >> $LOCK_DIR/extra_msg.txt error "Killing remaining nginx-processes failed !!!" if $send_msg ; then subject="[ Error ]: Stopping Nginx Webservice on `hostname -f` failed -- $datum" msg="\n\n`cat $LOCK_DIR/extra_msg.txt`\n" for _to_address in $to_addresses ; do echo -e "To:${_to_address}\n${content_type}\nSubject:$subject\n\n${msg}\n" \ | /usr/sbin/sendmail -F "Webservice Monitor" -f $from_address $_to_address done fi break fi for _PID in $PIDS ; do kill $_PID > /dev/null 2>&1 done sleep 2 PIDS=$(ps aux | grep "$NGINXD" | grep -v grep | awk '{print$2}') i=i+1 done SUB_PIDS=$(ps aux | grep "$(basename $NGINXD):" | grep -v grep | awk '{print$2}') if [[ "X${SUB_PIDS}X" != 'XX' ]]; then echo "" >> $LOCK_DIR/extra_msg.txt echo -e "\t[ Error ]: Found processes '$(basename $NGINXD):'." >> $LOCK_DIR/extra_msg.txt echo "" >> $LOCK_DIR/extra_msg.txt echo -e "\t[ Info ]; Going to kill remaining nginx-processes.." >> $LOCK_DIR/extra_msg.txt echo "" >> $LOCK_DIR/extra_msg.txt error "Found processes '$(basename $NGINXD):'." info "Going to kill processes '$(basename $NGINXD):'.." for _PID in $SUB_PIDS ; do kill $_PID > /dev/null 2>&1 done fi } stop_mysql() { send_msg=$1 send_msg=${send_msg:=true} # - Don't Start/Stop Services if package management is running (apt,aptitude,dpkg) # - if ( [[ "${os_dist,,}" = "debian" ]] || [[ "${os_dist,,}" = "ubuntu" ]] ) \ && $(lsof /var/lib/dpkg/lock >/dev/null 2>&1) ; then warn "It seems that package management (apt,dpkg) is running. Exiting '${FUNCNAME[0]}' now.." return 1 fi ## - Stop MySQL Service ## - if [[ -n "$MYSQL_INIT_SCRIPT" ]]; then $MYSQL_INIT_SCRIPT stop > /dev/null 2>&1 else systemctl stop "$MYSQLD_SERVICE_FILE" > /dev/null 2>&1 fi _msg=false sleep 5 declare -i i=0 PIDS="$(ps aux | grep -E "${MYSQL_PS_CHECK_STRING}" | grep -v grep | awk '{print$2}')" while [ "X$PIDS" != "X" ]; do if [ $i -eq 0 ]; then echo "" >> $LOCK_DIR/extra_msg.txt echo -e "\t[ Error ]: Stopping MySQL Service failed !!!" >> $LOCK_DIR/extra_msg.txt echo "" >> $LOCK_DIR/extra_msg.txt echo -e "\t[ Info ]; Going to kill MySQL Processes (mysqld_safe).." >> $LOCK_DIR/extra_msg.txt echo "" >> $LOCK_DIR/extra_msg.txt error "Stopping MySQL Service failed !!!" info "Going to kill MySQL Processes (mysqld_safe).." fi if [ $i -gt 10 ]; then echo "" >> $LOCK_DIR/extra_msg.txt echo -e "\t[ Error ]: Killing MySQL Processes failed !!!" >> $LOCK_DIR/extra_msg.txt echo "" >> $LOCK_DIR/extra_msg.txt error "Killing MySQL Processes failed !!!" _msg=true break fi for _PID in $PIDS ; do kill -15 $_PID > /dev/null 2>&1 done sleep 5 PIDS="$(ps aux | grep -E "${MYSQL_PS_CHECK_STRING}" | grep -v grep | awk '{print$2}')" i=i+1 done if $send_msg && $_msg ; then subject="[ Error ]: Stopping MySQL Engine on `hostname -f` failed -- $datum" msg="\n\n`cat $LOCK_DIR/extra_msg.txt`\n" for _to_address in $to_addresses ; do echo -e "To:${_to_address}\n${content_type}\nSubject:$subject\n\n${msg}\n" \ | /usr/sbin/sendmail -F "Webservice Monitor" -f $from_address $_to_address done fi } stop_postgresql() { send_msg=$1 send_msg=${send_msg:=true} # - Don't Start/Stop Services if package management is running (apt,aptitude,dpkg) # - if ( [[ "${os_dist,,}" = "debian" ]] || [[ "${os_dist,,}" = "ubuntu" ]] ) \ && $(lsof /var/lib/dpkg/lock >/dev/null 2>&1) ; then warn "It seems that package management (apt,dpkg) is running. Exiting '${FUNCNAME[0]}' now.." return 1 fi ## - Stop PostgreSQL database service ## - if [[ -n "$PSQL_SERVICE_FILE" ]] ; then systemctl stop "$PSQL_SERVICE_FILE" > /dev/null 2>&1 else $PSQL_INIT_SCRIPT stop > /dev/null 2>&1 fi sleep 12 declare -i i=0 CHILD_PID="$(ps aux | grep -E "\ postgres:\ " | grep -v grep | tail -n 1 | awk '{print$2}')" PSQL_MAIN_PID="$(ps -o ppid= -p $CHILD_PID 2> /dev/null)" while [ "X$PSQL_MAIN_PID" != "X" ]; do if [ $i -eq 0 ]; then echo "" >> $LOCK_DIR/extra_msg.txt echo -e "\t[ Error ]: Stopping PostgreSQL database service failed !!!" >> $LOCK_DIR/extra_msg.txt echo "" >> $LOCK_DIR/extra_msg.txt echo -e "\t[ Info ]; Going to kill remaining postgres-processes.." >> $LOCK_DIR/extra_msg.txt echo "" >> $LOCK_DIR/extra_msg.txt error "Stopping PostgreSQL database service failed !!!" info "Going to kill remaining postgres-processes.." fi if [ $i -gt 10 ]; then echo "" >> $LOCK_DIR/extra_msg.txt echo -e "\t[ Error ]: Killing remaining postgres-processes failed !!!" >> $LOCK_DIR/extra_msg.txt echo "" >> $LOCK_DIR/extra_msg.txt error "Killing remaining postgres-processes failed !!!" if $send_msg ; then subject="[ Error ]: Stopping PostgreSQL database service on `hostname -f` failed -- $datum" msg="\n\n`cat $LOCK_DIR/extra_msg.txt`\n" for _to_address in $to_addresses ; do echo -e "To:${_to_address}\n${content_type}\nSubject:$subject\n\n${msg}\n" \ | /usr/sbin/sendmail -F "Webservice Monitor" -f $from_address $_to_address done fi break fi # - Don't Start/Stop/Kill Services if package management is running (apt,aptitude,dpkg) # - if ( [[ "${os_dist,,}" = "debian" ]] || [[ "${os_dist,,}" = "ubuntu" ]] ) \ && $(lsof /var/lib/dpkg/lock >/dev/null 2>&1) ; then warn "It seems that package management (apt,dpkg) is running. Exiting '${FUNCNAME[0]}' now.." return 1 fi kill $PSQL_MAIN_PID > /dev/null 2>&1 sleep 2 CHILD_PID="$(ps aux | grep -E "\ postgres:\ " | grep -v grep | tail -n 1 | awk '{print$2}')" PSQL_MAIN_PID="$(ps -o ppid= -p $CHILD_PID)" i=i+1 done CHILD_PIDS=$(ps aux | grep -E "postgres:" | grep -v grep | awk '{print$2}') if [[ "X${CHILD_PIDS}X" != 'XX' ]]; then echo "" >> $LOCK_DIR/extra_msg.txt echo -e "\t[ Error ]: Found processes 'postgres:'." >> $LOCK_DIR/extra_msg.txt echo "" >> $LOCK_DIR/extra_msg.txt echo -e "\t[ Info ]; Going to kill remaining postgres-processes.." >> $LOCK_DIR/extra_msg.txt echo "" >> $LOCK_DIR/extra_msg.txt error "Found processes 'postgres:'." info "Going to kill processes 'postgres:'.." # - Don't Start/Stop/Kill Services if package management is running (apt,aptitude,dpkg) # - if ( [[ "${os_dist,,}" = "debian" ]] || [[ "${os_dist,,}" = "ubuntu" ]] ) \ && $(lsof /var/lib/dpkg/lock >/dev/null 2>&1) ; then warn "It seems that package management (apt,dpkg) is running. Exiting '${FUNCNAME[0]}' now.." return 1 fi for _PID in $CHILD_PIDS ; do kill $_PID > /dev/null 2>&1 done fi } stop_php_fpm() { __version=$1 send_msg=$2 send_msg=${send_msg:=true} _msg=false # - Don't Start/Stop Services if package management is running (apt,aptitude,dpkg) # - if ( [[ "${os_dist,,}" = "debian" ]] || [[ "${os_dist,,}" = "ubuntu" ]] ) \ && $(lsof /var/lib/dpkg/lock >/dev/null 2>&1) ; then warn "It seems that package management (apt,dpkg) is running. Exiting '${FUNCNAME[0]}' now.." return 1 fi ## - Stop PHP-FPM Service ## - if [[ "${start_stop_method_php_fpm["$__version"]}" = "systemd_service" ]]; then systemctl stop ${start_stop_file_php_fpm[$__version]} > /dev/null 2>&1 elif [[ "${start_stop_method_php_fpm[$__version]}" = "init_script" ]]; then ${start_stop_file_php_fpm[$__version]} stop > /dev/null 2>&1 fi sleep 2 declare -i i=0 PIDS=$(ps aux | grep "php-fpm: " | grep " master " | grep -E "[-|\(]?${php_fpm_pid_search_path[$__version]}" | grep -v grep | awk '{print$2}') while [ "X$PIDS" != "X" ]; do if [ $i -eq 0 ]; then echo "" >> $LOCK_DIR/extra_msg.txt echo -e "\t[ Error ]: Stopping PHP FPM v$__version failed !!!" >> $LOCK_DIR/extra_msg.txt echo "" >> $LOCK_DIR/extra_msg.txt echo -e "\t[ Info ]; Going to kill PHP FPM v$__version Processes (master).." >> $LOCK_DIR/extra_msg.txt echo "" >> $LOCK_DIR/extra_msg.txt error "Stopping PHP FPM v$__version failed !!!" info "Going to kill PHP FPM v$__version Processes (master).." fi if [ $i -gt 20 ]; then echo "" >> $LOCK_DIR/extra_msg.txt echo -e "\t[ Error ]: Killing PHP FPM Processes v$__version (master) failed !!!" >> $LOCK_DIR/extra_msg.txt echo "" >> $LOCK_DIR/extra_msg.txt error "Killing PHP FPM Processes v$__version (master) failed !!!" _msg=true break fi # - Don't Start/Stop Services if package management is running (apt,aptitude,dpkg) # - if ( [[ "${os_dist,,}" = "debian" ]] || [[ "${os_dist,,}" = "ubuntu" ]] ) \ && $(lsof /var/lib/dpkg/lock >/dev/null 2>&1) ; then warn "It seems that package management (apt,dpkg) is running. Exiting '${FUNCNAME[0]}' now.." return 1 fi for _PID in $PIDS ; do kill -9 $_PID > /dev/null 2>&1 done # - Remove all sockets php-fpm sockets from thid version # - rm -f /tmp/php-${__version}-fpm*.sock sleep 2 PIDS=$(ps aux | grep "php-fpm: " | grep " master " | grep -E "[-|\(]?${php_fpm_pid_search_path[$__version]}" | grep -v grep | awk '{print$2}') i=i+1 done declare -i i=0 PIDS=$(ps aux | grep "php-fpm: " | grep -E "(pool|master)" | grep -E "[-|\(]?${php_fpm_pid_search_path[$__version]}" | grep -v grep | awk '{print$2}') while [ "X$PIDS" != "X" ]; do if [ $i -eq 0 ]; then echo "" >> $LOCK_DIR/extra_msg.txt echo -e "\t[ Error ]: PHP FPM Processes v$__version (pool) still exists !!!" >> $LOCK_DIR/extra_msg.txt echo "" >> $LOCK_DIR/extra_msg.txt echo -e "\t[ Info ]; Going to kill remaining PHP FPM v$__version Processes (pool).." >> $LOCK_DIR/extra_msg.txt echo "" >> $LOCK_DIR/extra_msg.txt error "PHP FPM Processes v$__version (pool) still exists !!!" info "Going to kill remaining PHP FPM Processes v$__version (pool).." fi if [ $i -gt 20 ]; then echo "" >> $LOCK_DIR/extra_msg.txt echo -e "\t[ Error ]: Killing remaining PHP FPM Processes v$__version (pool) failed !!!" >> $LOCK_DIR/extra_msg.txt echo "" >> $LOCK_DIR/extra_msg.txt error "Killing remaining PHP FPM Processes v$__version (pool) failed !!!" _msg=true break fi # - Don't Start/Stop Services if package management is running (apt,aptitude,dpkg) # - if ( [[ "${os_dist,,}" = "debian" ]] || [[ "${os_dist,,}" = "ubuntu" ]] ) \ && $(lsof /var/lib/dpkg/lock >/dev/null 2>&1) ; then warn "It seems that package management (apt,dpkg) is running. Exiting '${FUNCNAME[0]}' now.." return 1 fi for _PID in $PIDS ; do kill -9 $_PID > /dev/null 2>&1 done sleep 2 PIDS=$(ps aux | grep "php-fpm: " | grep -E "(pool|master)" | grep -E "[-|\(]?${php_fpm_pid_search_path[$__version]}" | grep -v grep | awk '{print$2}') i=i+1 done if $send_msg && $_msg ; then subject="[ Error ]: Stopping PHP FPM v$__version on `hostname -f` failed -- $datum" msg="\n\n`cat $LOCK_DIR/extra_msg.txt`\n" for _to_address in $to_addresses ; do echo -e "To:${_to_address}\n${content_type}\nSubject:$subject\n\n${msg}\n" \ | /usr/sbin/sendmail -F "Webservice Monitor" -f $from_address $_to_address done fi } stop_redis() { send_msg=$1 send_msg=${send_msg:=true} # - Don't Start/Stop Services if package management is running (apt,aptitude,dpkg) # - if ( [[ "${os_dist,,}" = "debian" ]] || [[ "${os_dist,,}" = "ubuntu" ]] ) \ && $(lsof /var/lib/dpkg/lock >/dev/null 2>&1) ; then warn "It seems that package management (apt,dpkg) is running. Exiting '${FUNCNAME[0]}' now.." return 1 fi ## - Stop Redis Caching Service ## - if [[ -n "$REDIS_SERVICE_FILE" ]] ; then systemctl stop "$REDIS_SERVICE_FILE" > /dev/null 2>&1 else $REDIS_INIT_SCRIPT stop > /dev/null 2>&1 fi sleep 12 declare -i i=0 PIDS=$(ps aux | grep "$REDISD" | grep -v grep | awk '{print$2}') while [ "X$PIDS" != "X" ]; do if [ $i -eq 0 ]; then echo "" >> $LOCK_DIR/extra_msg.txt echo -e "\t[ Error ]: Stopping Redis Caching Sservice failed !!!" >> $LOCK_DIR/extra_msg.txt echo "" >> $LOCK_DIR/extra_msg.txt echo -e "\t[ Info ]; Going to kill remaining 'redis-server'-processes.." >> $LOCK_DIR/extra_msg.txt echo "" >> $LOCK_DIR/extra_msg.txt error "Stopping Redis Caching Service failed !!!" info "Going to kill remaining 'redis-server'-processes.." fi if [ $i -gt 10 ]; then echo "" >> $LOCK_DIR/extra_msg.txt echo -e "\t[ Error ]: Killing remaining 'redis-server'-processes failed !!!" >> $LOCK_DIR/extra_msg.txt echo "" >> $LOCK_DIR/extra_msg.txt error "Killing remaining 'redis-server'-processes failed !!!" if $send_msg ; then subject="[ Error ]: Stopping Redis Caching Service on `hostname -f` failed -- $datum" msg="\n\n`cat $LOCK_DIR/extra_msg.txt`\n" for _to_address in $to_addresses ; do echo -e "To:${_to_address}\n${content_type}\nSubject:$subject\n\n${msg}\n" \ | /usr/sbin/sendmail -F "Webservice Monitor" -f $from_address $_to_address done fi break fi # - Don't Start/Stop Services if package management is running (apt,aptitude,dpkg) # - if [[ "${os_dist,,}" = "debian" ]] || [[ "${os_dist,,}" = "ubuntu" ]] ; then if $(lsof /var/lib/dpkg/lock >/dev/null 2>&1) ; then warn "It seems that package management (apt,dpkg) is running. Exiting '${FUNCNAME[0]}' now.." return 1 fi fi for _PID in $PIDS ; do kill -9 $_PID > /dev/null 2>&1 done sleep 2 PIDS=$(ps aux | grep "$REDISD" | grep -v grep | awk '{print$2}') i=i+1 done } start_mysql() { send_msg=$1 send_msg=${send_msg:=true} # - Don't Start/Stop Services if package management is running (apt,aptitude,dpkg) # - if ( [[ "${os_dist,,}" = "debian" ]] || [[ "${os_dist,,}" = "ubuntu" ]] ) \ && $(lsof /var/lib/dpkg/lock >/dev/null 2>&1) ; then warn "It seems that package management (apt,dpkg) is running. Exiting '${FUNCNAME[0]}' now.." return 1 fi if [[ -n "$MYSQL_INIT_SCRIPT" ]]; then $MYSQL_INIT_SCRIPT start > /dev/null 2>&1 else systemctl start "$MYSQLD_SERVICE_FILE" > /dev/null 2>&1 fi sleep 5 NEWPID="$(ps aux | grep -E "${MYSQL_PS_CHECK_STRING}" | grep -v grep | awk '{print$2}' | head -1) " if [ "X${NEWPID}X" = "XX" ]; then echo "" >> $LOCK_DIR/extra_msg.txt echo -e "\t[ Error ]: I have started the MySQL server, but the services is not running !!!" >> $LOCK_DIR/extra_msg.txt echo "" >> $LOCK_DIR/extra_msg.txt error "I have started the MySQL server, but the service is not running !!!" datum=`date +"%d.%m.%Y %H:%Mh"` subject="[ Error ]: MySQL Service on `hostname -f` NOT RUNNING -- $datum" msg="\n\n`cat $LOCK_DIR/extra_msg.txt`\n" for _to_address in $to_addresses ; do echo -e "To:${_to_address}\n${content_type}\nSubject:$subject\n\n${msg}\n" \ | /usr/sbin/sendmail -F "Webservice Monitor" -f $from_address $_to_address done else echo "" >> $LOCK_DIR/extra_msg.txt echo -e "\tI have started MySQL service. The new PID is $NEWPID" >> $LOCK_DIR/extra_msg.txt echo "" >> $LOCK_DIR/extra_msg.txt ok "I have started MySQL service. The new PID is $NEWPID" fi if $send_msg ; then datum=`date +"%d.%m.%Y %H:%Mh"` subject="Restarting MySQL on `hostname -f` invoked -- $datum" msg=`cat $LOCK_DIR/extra_msg.txt` for _to_address in $to_addresses ; do echo -e "To:${_to_address}\n${content_type}\nSubject:$subject\n\n${msg}\n" \ | /usr/sbin/sendmail -F "Webservice Monitor" -f $from_address $_to_address done fi } start_postgresql() { send_msg=$1 send_msg=${send_msg:=true} # - Don't Start/Stop Services if package management is running (apt,aptitude,dpkg) # - if ( [[ "${os_dist,,}" = "debian" ]] || [[ "${os_dist,,}" = "ubuntu" ]] ) \ && $(lsof /var/lib/dpkg/lock >/dev/null 2>&1) ; then warn "It seems that package management (apt,dpkg) is running. Exiting '${FUNCNAME[0]}' now.." return 1 fi ## - Start PostgreSQL Webservice ## - if [[ -n "$PSQL_SERVICE_FILE" ]] ; then systemctl start "$PSQL_SERVICE_FILE" > /dev/null 2>&1 else $PSQL_INIT_SCRIPT start > /dev/null 2>&1 fi sleep 2 CHILD_PID="$(ps aux | grep -E "\ postgres:\ " | grep -v grep | tail -n 1 | awk '{print$2}')" NEWPID="$(ps -o ppid= -p $CHILD_PID 2> /dev/null)" if [ "X${NEWPID}X" = "XX" ]; then echo "" >> $LOCK_DIR/extra_msg.txt echo -e "\t[ Error ]: I started the 'PostgreSQL' database server, but service is not running !!!" >> $LOCK_DIR/extra_msg.txt echo "" >> $LOCK_DIR/extra_msg.txt error "I started the 'PostgreSQL' database server, but service is not running !!!" datum=`date +"%d.%m.%Y %H:%Mh"` subject="[ Error ]: PostgreSQL database service on `hostname -f` NOT RUNNING -- $datum" msg="\n\n`cat $LOCK_DIR/extra_msg.txt`\n" for _to_address in $to_addresses ; do echo -e "To:${_to_address}\n${content_type}\nSubject:$subject\n\n${msg}\n" \ | /usr/sbin/sendmail -F "Webservice Monitor" -f $from_address $_to_address done else echo "" >> $LOCK_DIR/extra_msg.txt echo -e "\tI started the 'PostgreSQL' database server. the new PID is $NEWPID" >> $LOCK_DIR/extra_msg.txt echo "" >> $LOCK_DIR/extra_msg.txt ok "I started the 'PostgreSQL' database server. the new PID is $NEWPID" fi if $send_msg ; then datum=`date +"%d.%m.%Y %H:%Mh"` subject="Restarting PostgreSQL on `hostname -f` invoked -- $datum" msg=`cat $LOCK_DIR/extra_msg.txt` for _to_address in $to_addresses ; do echo -e "To:${_to_address}\n${content_type}\nSubject:$subject\n\n${msg}\n" \ | /usr/sbin/sendmail -F "Webservice Monitor" -f $from_address $_to_address done fi } start_apache() { send_msg=$1 send_msg=${send_msg:=true} # - Don't Start/Stop Services if package management is running (apt,aptitude,dpkg) # - if ( [[ "${os_dist,,}" = "debian" ]] || [[ "${os_dist,,}" = "ubuntu" ]] ) \ && $(lsof /var/lib/dpkg/lock >/dev/null 2>&1) ; then warn "It seems that package management (apt,dpkg) is running. Exiting '${FUNCNAME[0]}' now.." return 1 fi ## - Start Apache Webservice ## - if [[ -n "$APACHE_SERVICE_FILE" ]] ; then systemctl start "$APACHE_SERVICE_FILE" > /dev/null 2>&1 else $APACHE_INIT_SCRIPT start > /dev/null 2>&1 fi sleep 2 NEWPID=$(ps aux | grep "$HTTPD" | grep -v grep | grep root | awk '{print$2}') if [ "X${NEWPID}X" = "XX" ]; then echo "" >> $LOCK_DIR/extra_msg.txt echo -e "\t[ Error ]: I have tarted the webserver, but service is not running !!!" >> $LOCK_DIR/extra_msg.txt echo "" >> $LOCK_DIR/extra_msg.txt error "I have started the webserver, but service is not running !!!" datum=`date +"%d.%m.%Y %H:%Mh"` subject="[ Error ]: Webservice on `hostname -f` NOT RUNNING -- $datum" msg="\n\n`cat $LOCK_DIR/extra_msg.txt`\n" for _to_address in $to_addresses ; do echo -e "To:${_to_address}\n${content_type}\nSubject:$subject\n\n${msg}\n" \ | /usr/sbin/sendmail -F "Webservice Monitor" -f $from_address $_to_address done else echo "" >> $LOCK_DIR/extra_msg.txt echo -e "\tI have started the webserver. the new PID is $NEWPID" >> $LOCK_DIR/extra_msg.txt echo "" >> $LOCK_DIR/extra_msg.txt ok "I have started the webserver. the new PID is $NEWPID" fi if $send_msg ; then datum=`date +"%d.%m.%Y %H:%Mh"` subject="Restarting Apache on `hostname -f` invoked -- $datum" msg=`cat $LOCK_DIR/extra_msg.txt` for _to_address in $to_addresses ; do echo -e "To:${_to_address}\n${content_type}\nSubject:$subject\n\n${msg}\n" \ | /usr/sbin/sendmail -F "Webservice Monitor" -f $from_address $_to_address done fi } start_nginx() { send_msg=$1 send_msg=${send_msg:=true} # - Don't Start/Stop Services if package management is running (apt,aptitude,dpkg) # - if ( [[ "${os_dist,,}" = "debian" ]] || [[ "${os_dist,,}" = "ubuntu" ]] ) \ && $(lsof /var/lib/dpkg/lock >/dev/null 2>&1) ; then warn "It seems that package management (apt,dpkg) is running. Exiting '${FUNCNAME[0]}' now.." return 1 fi ## - Start Nginx Webservice ## - if [[ -n "$NGINX_SERVICE_FILE" ]] ; then systemctl start "$NGINX_SERVICE_FILE" > /dev/null 2>&1 else $NGINX_INIT_SCRIPT start > /dev/null 2>&1 fi sleep 2 NEWPID=$(ps aux | grep "$NGINXD" | grep -v grep | grep root | awk '{print$2}') if [ "X${NEWPID}X" = "XX" ]; then echo "" >> $LOCK_DIR/extra_msg.txt echo -e "\t[ Error ]: I started the 'Nginx' webserver, but service is not running !!!" >> $LOCK_DIR/extra_msg.txt echo "" >> $LOCK_DIR/extra_msg.txt error "I started the 'Nginx' webserver, but service is not running !!!" datum=`date +"%d.%m.%Y %H:%Mh"` subject="[ Error ]: Nginx Webservice on `hostname -f` NOT RUNNING -- $datum" msg="\n\n`cat $LOCK_DIR/extra_msg.txt`\n" for _to_address in $to_addresses ; do echo -e "To:${_to_address}\n${content_type}\nSubject:$subject\n\n${msg}\n" \ | /usr/sbin/sendmail -F "Webservice Monitor" -f $from_address $_to_address done else echo "" >> $LOCK_DIR/extra_msg.txt echo -e "\tI started the 'Nginx' webserver. the new PID is $NEWPID" >> $LOCK_DIR/extra_msg.txt echo "" >> $LOCK_DIR/extra_msg.txt ok "I started the 'NGINX' webserver. the new PID is $NEWPID" fi if $send_msg ; then datum=`date +"%d.%m.%Y %H:%Mh"` subject="Restarting Nginx on `hostname -f` invoked -- $datum" msg=`cat $LOCK_DIR/extra_msg.txt` for _to_address in $to_addresses ; do echo -e "To:${_to_address}\n${content_type}\nSubject:$subject\n\n${msg}\n" \ | /usr/sbin/sendmail -F "Webservice Monitor" -f $from_address $_to_address done fi } start_php_fpm() { __version=$1 send_msg=$2 send_msg=${send_msg:=true} start_php_fpm_failed=false # - Don't Start/Stop Services if package management is running (apt,aptitude,dpkg) # - if ( [[ "${os_dist,,}" = "debian" ]] || [[ "${os_dist,,}" = "ubuntu" ]] ) \ && $(lsof /var/lib/dpkg/lock >/dev/null 2>&1) ; then warn "It seems that package management (apt,dpkg) is running. Exiting '${FUNCNAME[0]}' now.." return 1 fi if [[ "${start_stop_method_php_fpm[$__version]}" = "systemd_service" ]]; then systemctl restart ${start_stop_file_php_fpm[$__version]} > /dev/null 2>&1 elif [[ "${start_stop_method_php_fpm["$__version"]}" = "init_script" ]]; then ${start_stop_file_php_fpm["$__version"]} start > /dev/null 2>&1 else error "Neither start/stop script nor systemd service file for PHP FPM v$__version found!" echo "" >> $LOCK_DIR/extra_msg.txt echo -e "\t[ Error ]: Neither start/stop script nor systemd service file for PHP FPM v$__version found !!!" >> $LOCK_DIR/extra_msg.txt echo "" >> $LOCK_DIR/extra_msg.txt start_php_fpm_failed=true fi sleep 2 NEWPID=`ps aux | grep "php-fpm: " | grep " master " | grep -E "[-|\(]?${php_fpm_pid_search_path[$__version]}" | grep -v grep | awk '{print$2}'` if [ "X${NEWPID}X" = "XX" ]; then if ! $start_php_fpm_failed ; then echo "" >> $LOCK_DIR/extra_msg.txt echo -e "\t[ Error ]: I have started PHP FPM v$__version, but service is not running !!!" >> $LOCK_DIR/extra_msg.txt echo "" >> $LOCK_DIR/extra_msg.txt error "I have started PHP FPM v$__version, but service is not running !!!" fi datum=`date +"%d.%m.%Y %H:%Mh"` subject="[ Error ]: PHP-FPM v$__version engine on `hostname -f` NOT RUNNING -- $datum" msg="\n\n`cat $LOCK_DIR/extra_msg.txt`\n" for _to_address in $to_addresses ; do echo -e "To:${_to_address}\n${content_type}\nSubject:$subject\n\n${msg}\n" \ | /usr/sbin/sendmail -F "Webservice Monitor" -f $from_address $_to_address done else echo "" >> $LOCK_DIR/extra_msg.txt echo -e "\tI have started the PHP-FPM v$__version engine. the new PID is $NEWPID" >> $LOCK_DIR/extra_msg.txt echo "" >> $LOCK_DIR/extra_msg.txt ok "I have started the PHP-FPM v$__version engine. the new PID is $NEWPID" fi if $send_msg ; then datum=`date +"%d.%m.%Y %H:%Mh"` subject="Restarting PHP FPM v$__version engine on `hostname -f` invoked -- $datum" msg=`cat $LOCK_DIR/extra_msg.txt` for _to_address in $to_addresses ; do echo -e "To:${_to_address}\n${content_type}\nSubject:$subject\n\n${msg}\n" \ | /usr/sbin/sendmail -F "Webservice Monitor" -f $from_address $_to_address done fi } start_redis() { send_msg=$1 send_msg=${send_msg:=true} # - Don't Start/Stop Services if package management is running (apt,aptitude,dpkg) # - if ( [[ "${os_dist,,}" = "debian" ]] || [[ "${os_dist,,}" = "ubuntu" ]] ) \ && $(lsof /var/lib/dpkg/lock >/dev/null 2>&1) ; then warn "It seems that package management (apt,dpkg) is running. Exiting '${FUNCNAME[0]}' now.." return 1 fi ## - Start Redis Caching Service ## - if [[ -n "$REDIS_SERVICE_FILE" ]] ; then systemctl start "$REDIS_SERVICE_FILE" > /dev/null 2>&1 else $REDIS_INIT_SCRIPT start > /dev/null 2>&1 fi sleep 2 NEWPID=$(ps aux | grep "$REDISD" | grep -v grep | awk '{print$2}') if [ "X${NEWPID}X" = "XX" ]; then echo "" >> $LOCK_DIR/extra_msg.txt echo -e "\t[ Error ]: I started the 'Redis' Server, but service is not running !!!" >> $LOCK_DIR/extra_msg.txt echo "" >> $LOCK_DIR/extra_msg.txt error "I started the 'Redis' Server, but service is not running !!!" datum=`date +"%d.%m.%Y %H:%Mh"` subject="[ Error ]: Redis Caching Service on `hostname -f` NOT RUNNING -- $datum" msg="\n\n`cat $LOCK_DIR/extra_msg.txt`\n" for _to_address in $to_addresses ; do echo -e "To:${_to_address}\n${content_type}\nSubject:$subject\n\n${msg}\n" \ | /usr/sbin/sendmail -F "Webservice Monitor" -f $from_address $_to_address done else echo "" >> $LOCK_DIR/extra_msg.txt echo -e "\tI started the 'Redis' Server. the new PID is $NEWPID" >> $LOCK_DIR/extra_msg.txt echo "" >> $LOCK_DIR/extra_msg.txt ok "I started the 'Redis' Server. the new PID is $NEWPID" fi if $send_msg ; then datum=`date +"%d.%m.%Y %H:%Mh"` subject="Restarting Redis Server on `hostname -f` invoked -- $datum" msg=`cat $LOCK_DIR/extra_msg.txt` for _to_address in $to_addresses ; do echo -e "To:${_to_address}\n${content_type}\nSubject:$subject\n\n${msg}\n" \ | /usr/sbin/sendmail -F "Webservice Monitor" -f $from_address $_to_address done fi } restart_apache() { ## - Stop Apache Webservice ## - if stop_apache ; then sleep 2 ## - Start Apache Webservice ## - start_apache fi } restart_nginx() { ## - Stop Apache Webservice ## - if stop_nginx ; then sleep 2 ## - Start NGINX Webservice ## - start_nginx fi } restart_mysql(){ ## - Stop MySQL Service ## - if stop_mysql ; then sleep 2 ## - Start MySQL Service ## - start_mysql fi } restart_postgresql(){ ## - Stop PostgreSQL Service ## - if stop_postgresql ; then sleep 2 ## - Start MySQL Service ## - start_postgresql fi } restart_php_fpm(){ ## - Stop MySQL Servive ## - if stop_php_fpm $1 ; then sleep 2 ## - Start MySQL Service ## - start_php_fpm $1 fi } restart_redis(){ ## - Stop MySQL Servive ## - if stop_redis ; then sleep 2 ## - Start MySQL Service ## - start_redis fi } graceful_restart_php_fpm() { __version=$1 send_msg=$2 send_msg=${send_msg:=true} start_php_fpm_failed=false # - Don't Start/Stop Services if package management is running (apt,aptitude,dpkg) # - if ( [[ "${os_dist,,}" = "debian" ]] || [[ "${os_dist,,}" = "ubuntu" ]] ) \ && $(lsof /var/lib/dpkg/lock >/dev/null 2>&1) ; then warn "It seems that package management (apt,dpkg) is running. Exiting '${FUNCNAME[0]}' now.." return 1 fi if [[ "$start_stop_method_php_fpm" = "systemd_service" ]]; then systemctl restart $start_stop_method_php_fpm["$__version"] > /dev/null 2>&1 elif [[ "$start_stop_method_php_fpm" = "init_script" ]]; then $start_stop_method_php_fpm["$__version"] restart > /dev/null 2>&1 else error "Neither start/stop script nor systemd service file for PHP FPM v$__version found!" echo "" >> $LOCK_DIR/extra_msg.txt echo -e "\t[ Error ]: Neither start/stop script nor systemd service file for PHP FPM v$__version found !!!" >> $LOCK_DIR/extra_msg.txt echo "" >> $LOCK_DIR/extra_msg.txt start_php_fpm_failed=true fi sleep 2 NEWPID=`ps aux | grep "php-fpm: " | grep " master " | grep -E "[-|\(]?${php_fpm_pid_search_path[\"$__version\"]}" | grep -v grep | awk '{print$2}'` if [ "X${NEWPID}X" = "XX" ]; then if ! $start_php_fpm_failed ; then echo "" >> $LOCK_DIR/extra_msg.txt echo -e "\t[ Error ]: I have restarted PHP-FPM, but service is not running !!!" >> $LOCK_DIR/extra_msg.txt echo "" >> $LOCK_DIR/extra_msg.txt error "I have restarted PHP-FPM, but service is not running !!!" fi datum=`date +"%d.%m.%Y %H:%Mh"` subject="[ Error ]: PHP-FPM engine on `hostname -f` NOT RUNNING -- $datum" msg="\n\n`cat $LOCK_DIR/extra_msg.txt`\n" for _to_address in $to_addresses ; do echo -e "To:${_to_address}\n${content_type}\nSubject:$subject\n\n${msg}\n" \ | /usr/sbin/sendmail -F "Webservice Monitor" -f $from_address $_to_address done else echo "" >> $LOCK_DIR/extra_msg.txt echo -e "\tI have restarted the PHP-FPM engine. the new PID is $NEWPID" >> $LOCK_DIR/extra_msg.txt echo "" >> $LOCK_DIR/extra_msg.txt ok "I have restarted the PHP-FPM engine. the new PID is $NEWPID" fi if $send_msg ; then datum=`date +"%d.%m.%Y %H:%Mh"` subject="Restarting PHP-FPM engine on `hostname -f` invoked -- $datum" msg=`cat $LOCK_DIR/extra_msg.txt` for _to_address in $to_addresses ; do echo -e "To:${_to_address}\n${content_type}\nSubject:$subject\n\n${msg}\n" \ | /usr/sbin/sendmail -F "Webservice Monitor" -f $from_address $_to_address done fi } #--------------------------------------- #----------------------------- # Checking Load.. #----------------------------- #--------------------------------------- if $check_load ; then #echo "" > $LOCK_DIR/extra_msg.txt if $LOGGING ; then echo -e "\nChecking Load on \"`hostname -f`\".." fi warn_msg=${LOCK_DIR}/warn.msg echo "" > $warn_msg ## - Number of processors ## - declare -i number_processors=`grep -E "processor\s+:\s+" /proc/cpuinfo -c` if [ -z "$number_processors" -o $number_processors -eq 0 ]; then fatal "Detecting number of prozessors ($number_processors) failed!" fi ## - Thresholds ## - if [ $number_processors -gt 8 ]; then #threshod_1_min=`echo "scale=2; $number_processors / 1.95" | bc` #threshod_5_min=`echo "scale=2; $number_processors / 2.95" | bc` #threshod_15_min=`echo "scale=2; $number_processors / 3.45" | bc` threshod_1_min=`echo "scale=2; $number_processors / 1.45" | bc` threshod_5_min=`echo "scale=2; $number_processors / 1.95" | bc` threshod_15_min=`echo "scale=2; $number_processors / 2.35" | bc` else threshod_1_min=`echo "scale=2; $number_processors / 1.00" | bc` threshod_5_min=`echo "scale=2; $number_processors / 1.45" | bc` threshod_15_min=`echo "scale=2; $number_processors / 1.95" | bc` fi ## - Determin load avarage values ## - ## - Last minute load_avarage_1m=`cat /proc/loadavg | cut -d ' ' -f1` ## - Last 5 minutes load_avarage_5m=`cat /proc/loadavg | cut -d ' ' -f2` ## -Last 15 minutes load_avarage_15m=`cat /proc/loadavg | cut -d ' ' -f3` ## - Evaluate ## - result_1m=$(echo "scale=2; $load_avarage_1m > $threshod_1_min" | bc) result_5m=$(echo "scale=2; $load_avarage_5m > $threshod_5_min" | bc) result_15m=$(echo "scale=2; $load_avarage_15m > $threshod_15_min" | bc) #if [ "$result_1m" = "1" -o "$result_5m" = "1" -o "$result_15m" = "1" ] ; then if [ "$result_1m" = "1" ] ; then subject="[ Warning ]: High load on `hostname -f`" cat < $warn_msg High load on *`hostname -f`* Server Name...................: `hostname -f` Date..........................: `date +%d.%m.%Y` Time..........................: `date +%H:%M` h Load avarage last minute......: $load_avarage_1m Load avarage last 5 minutes...: $load_avarage_5m Load avarage last 15 minutes..: $load_avarage_15m EOF for _to_address in $to_addresses ; do echo -e "To: ${_to_address}\nSubject: $subject\n\n`cat $warn_msg`" \ | sendmail -F "Load Monitor" -f $from_address $_to_address done echo "" > $warn_msg fi _load_avarage_1m=$(echo "scale=0; $load_avarage_1m/1" | bc -l) _load_avarage_5m=$(echo "scale=0; $load_avarage_5m/1" | bc -l) if [ $_load_avarage_1m -gt 200 -a $_load_avarage_5m -gt 200 ]; then cat<> $warn_msg Server Name...................: `hostname -f` Date..........................: `date +%d.%m.%Y` Time..........................: `date +%H:%M` h Load avarage last minute......: $load_avarage_1m Load avarage last 5 minutes...: $load_avarage_5m Load avarage last 15 minutes..: $load_avarage_15m EOF stop_apache stop_mysql sleep 60 start_mysql start_apache else if $LOGGING ; then ok "Load on \"`hostname -f`\" seems to be fine.." cat< $LOCK_DIR/extra_msg.txt if $LOGGING ; then echo -e "\nChecking Mysql databse service on \"`hostname -f`\".." fi PID="$(ps aux | grep -E "${MYSQL_PS_CHECK_STRING}" | grep -v grep | awk '{print$2}')" if [ "X${PID}X" = "XX" ];then echo -e "\nMySQL is not running! - Try to restart service.." >> $LOCK_DIR/extra_msg.txt echo -e "=================================================" >> $LOCK_DIR/extra_msg.txt error "MySQL is not running! - Try to restart it.." restart_mysql else if [ -z "`$MYADMIN $mysql_credential_args ping 2>/dev/null`" ]; then echo -e "\nMySQL seems to be running, but NOT responding! - Try to restart service.." >> $LOCK_DIR/extra_msg.txt echo -e "=======================================================================" >> $LOCK_DIR/extra_msg.txt error "MySQL seems to be running, but NOT responding! - Try to restart service.." restart_mysql else if $LOGGING ; then ok "MySQL database service on \"`hostname -f`\" seems to be running.." fi fi fi fi #--------------------------------------- #----------------------------- # Check Postgresql database service.. #----------------------------- #--------------------------------------- if $check_postgresql ; then echo "" > $LOCK_DIR/extra_msg.txt if $LOGGING ; then echo -e "\nChecking PostgreSQL database service on \"`hostname -f`\".." fi CHILD_PID="$(ps aux | grep -E "\ postgres:\ " | grep -v grep | tail -n 1 | awk '{print$2}')" PSQL_MAIN_PID="$(ps -o ppid= -p $CHILD_PID 2> /dev/null)" if [ "X${PSQL_MAIN_PID}X" = "XX" ];then echo -e "\nPostgreSQL database service is not running! - Try to restart service.." > $LOCK_DIR/extra_msg.txt echo -e "======================================================================" >> $LOCK_DIR/extra_msg.txt error "PostgeSQL database service is not running! - Try to restart service.." restart_postgresql else if $LOGGING ; then ok "PostgreSQL database server on \"`hostname -f`\" seems to be running.." fi fi fi #--------------------------------------- #----------------------------- # Check Apache webservice.. #----------------------------- #--------------------------------------- if $check_apache ; then echo "" > $LOCK_DIR/extra_msg.txt if $LOGGING ; then echo -e "\nChecking Apache webservice on \"`hostname -f`\".." fi #_PID_FILE=`$HTTPD -t -D DUMP_RUN_CFG | grep -i -e "^PidFile: " | awk '{print$2}' \ # | sed -e "s/^[\"']//" | sed -e "s/[\"']$//"` #PID=`cat $_PID_FILE` #if [ "X`ps ax | grep \"$HTTPD \" | grep -e \"^$PID \"`" = "X" ]; then PID=`ps aux | grep "$HTTPD " | grep -e "^root" | grep -v grep | awk '{print$2}'` if [ "X${PID}X" = "XX" ];then echo -e "\nApache Webservice is not running! - Try to restart service.." > $LOCK_DIR/extra_msg.txt echo -e "============================================================" >> $LOCK_DIR/extra_msg.txt error "Apache webservice is not running! - Try to restart service.." restart_apache else if ! $(curl -Is -m 10 http://$curl_check_host | head -n 1 | grep "200 OK" > /dev/null) ; then echo -e "\nApache Webservice seems to be running, but is NOT responding! - Try to restart service.." >> $LOCK_DIR/extra_msg.txt echo -e "========================================================================================" >> $LOCK_DIR/extra_msg.txt error "Apache Webservice seems to be running, but is NOT responding! - Try to restart service.." restart_apache else if $LOGGING ; then ok "Apache Webserver on \"`hostname -f`\" seems to be running.." fi fi fi fi #--------------------------------------- #----------------------------- # Check Nginx webservice.. #----------------------------- #--------------------------------------- if $check_nginx ; then echo "" > $LOCK_DIR/extra_msg.txt if $LOGGING ; then echo -e "\nChecking Nginx webservice on \"`hostname -f`\".." fi PID=`ps aux | grep "$NGINXD" | grep -e "^root" | grep -v grep | awk '{print$2}'` if [ "X${PID}X" = "XX" ];then echo -e "\nNginx Webservice is not running! - Try to restart service.." > $LOCK_DIR/extra_msg.txt echo -e "===========================================================" >> $LOCK_DIR/extra_msg.txt error "Nginx webservice is not running! - Try to start it.." restart_nginx else if ! $ommit_curl_check_nginx ; then if ! $(curl -Is -m 10 http://$curl_check_host | head -n 1 | grep "200" > /dev/null) ; then # Same but via SSL # if ! $(curl --insecure -Is -m 10 https://$curl_check_host | head -n 1 | grep "200" > /dev/null) ; then echo -e "\nNginx Webservice seems to be running, but is NOT responding! - Try to restart service.." >> $LOCK_DIR/extra_msg.txt echo -e "=======================================================================================" >> $LOCK_DIR/extra_msg.txt error "Nginx Webservice seems to be running, but is NOT responding! - Try to restart service.." restart_nginx else if $LOGGING ; then ok "Nginx Webserver on \"`hostname -f`\" seems to be running.." fi fi else if $LOGGING ; then ok "Nginx Webserver on \"`hostname -f`\" seems to be running.." fi fi else if $LOGGING ; then ok "Nginx Webserver on \"`hostname -f`\" seems to be running.." fi fi fi fi #--------------------------------------- #----------------------------- # Check PHP-FPM engine.. #----------------------------- #--------------------------------------- if $check_php_fpm ; then for _version in $php_versions ; do echo "" > $LOCK_DIR/extra_msg.txt if $LOGGING ; then echo -e "\nChecking PHP-FPM v$_version (FastCGI Process Manager) on \"`hostname -f`\".." fi PID=$(ps aux | grep "php-fpm: " | grep "master" | grep -E "[-|\(]?${php_fpm_pid_search_path[$_version]}" | grep -v grep | awk '{print$2}') if [ "X${PID}X" = "XX" ];then echo -e "\nPHP-FPM v$_version is not running! - Try to restart the service.." > $LOCK_DIR/extra_msg.txt echo -e "======================================================" >> $LOCK_DIR/extra_msg.txt error "PHP-FPM v$_version is not running! - Try to restart the service.." restart_php_fpm $_version elif [[ -n ${php_fpm_ping_path[$_version]} ]] ; then if ! $(curl -Is -m 10 http://${curl_check_host}/${php_fpm_ping_path[$_version]} | head -n 1 | grep "200" > /dev/null) ; then # Same via SSL # if ! $(curl --insecure -Is -m 10 https://${curl_check_host}/${php_fpm_ping_path[$_version]} | head -n 1 | grep "200" > /dev/null) ; then echo -e "\nPHP-FPM v$_version seems to be running, but is NOT responding! - Try to restart service.." >> $LOCK_DIR/extra_msg.txt echo -e "====================================================================================" >> $LOCK_DIR/extra_msg.txt error "PHP-FPM v$_version seems to be running, but is NOT responding! - Try to restart service.." restart_php_fpm $_version else if $LOGGING ; then ok "PHP-FPM v$_version engine on \"`hostname -f`\" seems to be running.." fi fi else if $LOGGING ; then ok "PHP-FPM v$_version engine on \"`hostname -f`\" seems to be running.." fi fi fi done fi #--------------------------------------- #----------------------------- # Check Redis Caching Service.. #----------------------------- #--------------------------------------- if $check_redis ; then echo "" > $LOCK_DIR/extra_msg.txt if $LOGGING ; then echo -e "\nChecking Redis Caching Service on \"`hostname -f`\".." fi PID=`ps aux | grep "$REDISD" | grep -v grep | awk '{print$2}'` if [ "X${PID}X" = "XX" ];then echo -e "\nRedis Caching Service is not running! - Try to restart service.." > $LOCK_DIR/extra_msg.txt echo -e "================================================================" >> $LOCK_DIR/extra_msg.txt error "Redis Caching Service is not running! - Try to restart it.." restart_redis else if $LOGGING ; then ok "Redis Caching Service on \"`hostname -f`\" seems to be running.." fi fi fi #--------------------------------------- #----------------------------- # Check website.. #----------------------------- #--------------------------------------- if $check_website ; then send_msg=false if $LOGGING ; then echo -e "\nDo some extra checks.." fi declare -i rval=$(curl -s $is_working_url 2> /dev/null | grep "$check_string" | wc -l) if [[ $rval -lt 1 ]] ; then echo -e "The website does not respond as expected!" > $LOCK_DIR/extra_msg.txt echo -e "=========================================" >> $LOCK_DIR/extra_msg.txt error "The website $is_working_url does not respond as expected." # - Stop services.. # - stop_apache false if [[ "$php_version_of_working_url" != "None" ]]; then if [[ -n "$php_version_of_working_url" ]];then stop_php_fpm $php_version_of_working_url false else for _version in $php_versions ; do stop_php_fpm $_version false done fi fi # - Clean up cache/session files.. # - if $include_cleanup_function && [[ -n "$(trim $cleanup_function)" ]] ; then info "Execute configured cleanup tasks.." echo -e "\n\t[ Info ]: Execute configured cleanup tasks .." >> $LOCK_DIR/extra_msg.txt source $cleanup_functions_file fi # - Start services # - if [[ "$php_version_of_working_url" != "None" ]]; then if [[ -n "$php_version_of_working_url" ]];then start_php_fpm $php_version_of_working_url false else for _version in $php_versions ; do start_php_fpm $_version false done fi fi start_apache false info "Sleeping for 30 seconds.." sleep 30 declare -i rval=$(curl -s $is_working_url 2> /dev/null | grep "$check_string" | wc -l) if [[ $rval -lt 1 ]] ; then echo -e "\n\t[ Error ]: URL $is_working_url is still NOT responding as expected !!!" >> $LOCK_DIR/extra_msg.txt error "URL $is_working_url is still NOT responding as expected !!!" # - Only send extra message, if cleaning up and restarting wasn't successfully # - send_msg=true else ok "Cleaning up and restarting services was successfully." fi else if $LOGGING ; then ok "URL $is_working_url is responding as expected." fi fi if $send_msg ; then subject="[ Error ]: Shop System at `hostname -f` failed -- $datum" msg="\n\n`cat $LOCK_DIR/extra_msg.txt`\n" if [[ -n "$extra_alert_address" ]]; then to_addresses="$to_addresses $extra_alert_address" fi for _to_address in $to_addresses ; do echo -e "To:${_to_address}\n${content_type}\nSubject:$subject\n\n${msg}\n" \ | /usr/sbin/sendmail -F "Webservice Monitor" -f $from_address $_to_address done fi rm -rf $cleanup_functions_file fi clean_up 0