monitoring/check_webservice_load.sh
2017-02-13 00:24:47 +01:00

1292 lines
38 KiB
Bash
Executable File

#!/usr/bin/env bash
working_dir="$(dirname $(realpath $0))"
conf_file="${working_dir}/conf/check_webservice_load.conf"
#---------------------------------------
#-----------------------------
# Read Configurations from $conf_file
#-----------------------------
#---------------------------------------
if [[ ! -f "$conf_file" ]]; then
echo ""
echo -e " [ Fatal ] Configuration file '$(basename ${conf_file})' not found!"
echo ""
echo -e "\tScript terminated.."
echo ""
exit 1
else
source "$conf_file"
fi
# - Lock directory exists, until the script ends. So
# - we can check, if a previos instanze is already running.
# -
#LOCK_DIR=/tmp/check_webservice.lock
LOCK_DIR=`mktemp -d`
#---------------------------------------
#-----------------------------
# Base Function(s)
#-----------------------------
#---------------------------------------
fatal(){
echo ""
echo -e " [ Fatal ] $*"
echo ""
echo -e "\tScript terminated.."
echo ""
exit 1
}
error (){
echo ""
echo -e " [ Error ] $*"
echo ""
}
warn (){
echo ""
echo -e " [ Warn ] $*"
echo ""
}
info (){
echo ""
echo -e " [ Info ] $*"
echo ""
}
ok (){
echo ""
echo -e " [ Ok ] $*"
echo ""
}
trim() {
local var="$*"
var="${var#"${var%%[![:space:]]*}"}" # remove leading whitespace characters
var="${var%"${var##*[![:space:]]}"}" # remove trailing whitespace characters
echo -n "$var"
}
#---------------------------------------
#-----------------------------
# Check some prerequisites
#-----------------------------
#---------------------------------------
if [ -z "`which basename`" ]; then
fatal 'It seems "basename" is not installed, but needed!'
fi
if [ -z "`which sendmail`" ]; then
fatal 'It seems "sendmail" is not installed, but needed!'
fi
if [ -z "`which realpath`" ]; then
fatal 'It seems "realpath" is not installed, but needed!'
fi
if $check_load ; then
if [ -z "`which bc`" ]; then
fatal 'It seems "bc" is not installed, but needed!'
fi
fi
# - Systemd supported ?
## -
systemd=$(which systemd)
systemctl=$(which systemctl)
systemd_supported=false
if [[ -n "$systemd" ]] && [[ -n "$systemd" ]] ; then
systemd_supported=true
fi
if $check_mysql ; then
MYSQLD=`realpath $(which mysqld) 2>/dev/null`
if [ -z "$MYSQLD" ]; then
if [ -x "/usr/local/mysql/bin/mysqld" ]; then
MYSQLD=`realpath "/usr/local/mysql/bin/mysqld"`
else
fatal 'Command \"mysqld\" not found!'
fi
fi
MYSQLD_SAFE=`realpath $(which mysqld_safe) 2>/dev/null`
if [ -z "$MYSQLD_SAFE" ]; then
if [ -x "/usr/local/mysql/bin/mysqld_safe" ]; then
MYSQLD_SAFE=`realpath "/usr/local/mysql/bin/mysqld_safe"`
else
fatal 'Command \"mysqld_safe\" not found!'
fi
fi
MYADMIN=`realpath $(which mysqladmin) 2>/dev/null`
if [ -z "$MYADMIN" ]; then
if [ -x "/usr/local/mysql/bin/mysqladmin" ]; then
MYADMIN=`realpath "/usr/local/mysql/bin/mysqladmin"`
else
fatal 'Command \"mysqladmin\" not found!'
fi
fi
MYSQL_INIT_SCRIPT=""
if [ -x "/etc/init.d/mysql.server" ]; then
MYSQL_INIT_SCRIPT="/etc/init.d/mysql.server"
elif [ -x "/etc/init.d/mysql" ]; then
MYSQL_INIT_SCRIPT="/etc/init.d/mysql"
else
fatal 'Cannot find init-script for MySQL server!'
fi
fi
if $check_apache ; then
HTTPD=`realpath $(which httpd) 2>/dev/null`
if [ -z "$HTTPD" ]; then
if [ -x "/usr/local/apache2/bin/httpd" ]; then
HTTPD=`realpath "/usr/local/apache2/bin/httpd"`
else
HTTPD=`realpath $(which apache2) 2>/dev/null`
if [ -z "$HTTPD" ]; then
fatal 'Command \"httpd\"/\"apache2\" not found!'
fi
fi
fi
APACHE_INIT_SCRIPT=""
if [ -x "/etc/init.d/apache2" ]; then
APACHE_INIT_SCRIPT="/etc/init.d/apache2"
elif [ -x "/etc/init.d/apachectl" ]; then
APACHE_INIT_SCRIPT="/etc/init.d/apachectl"
else
fatal 'Cannot find init-script for Apache web server!'
fi
fi
declare -A start_stop_method_php_fpm
declare -A start_stop_file_php_fpm
declare -A php_fpm_ping_path
declare -A php_fpm_pid_search_path
if $check_php_fpm ; then
if [ -z "`which curl`" ]; then
fatal 'It seems "curl" is not installed, but needed!'
fi
if [[ "$(trim $php_versions)" = "" ]]; then
fatal 'No PHP version is given. Setting of \"php_version\" is needed!'
fi
if [[ -n "$ping_path" ]];then
for _path in $ping_path ;do
IFS=':' read -a _path_arr <<< "${_path}"
# - some checks..
# -
if [[ -z "${_path_arr[0]}" ]] || [[ -z "${_path_arr[0]}" ]] ; then
fatal "Wrong settings in variable \"ping_path\"!"
fi
if [[ ! "$php_versions" =~ "${_path_arr[0]}" ]] ; then
fatal "Unknown PHP version \"${_path_arr[0]}\" (variable ping_path)!\n\n PHP version must match given versions of var \"php_versions\""
fi
php_fpm_ping_path["${_path_arr[0]}"]="${_path_arr[1]}"
done
fi
for _version in $php_versions ; do
PHP_FPM_SERVICE_FILE=""
if $systemd_supported ; then
# - Is Service exclusive controlled by systemd
# -
if systemctl -t service list-unit-files \
| grep -e "^php-$_version" \
| grep -q enabled 2> /devnull ; then
PHP_FPM_SERVICE_FILE=$(systemctl -t service list-unit-files \
| grep -e "^php-$_version" \
| awk '{print$1}')
fi
fi
if [[ -z "${php_fpm_ping_path[$_version]}" ]] ; then
php_fpm_ping_path["$_version"]="ping-$_version"
fi
if [[ -n "$PHP_FPM_SERVICE_FILE" ]]; then
start_stop_method_php_fpm["$_version"]="systemd_service"
start_stop_file_php_fpm["$_version"]="$PHP_FPM_SERVICE_FILE"
#php_fpm_ping_path["$_version"]="ping-$_version"
php_fpm_pid_search_path["$_version"]="$_version"
elif [[ -x "/etc/init.d/php-${_version}-fpm" ]]; then
start_stop_method_php_fpm["$_version"]="init_script"
start_stop_file_php_fpm["$_version"]="/etc/init.d/php-${_version}-fpm"
#php_fpm_ping_path["$_version"]="ping-$_version"
php_fpm_pid_search_path["$_version"]="$_version"
elif [[ -x "/etc/init.d/php5-fpm" ]]; then
start_stop_method_php_fpm["$_version"]="init_script"
start_stop_file_php_fpm["$_version"]="/etc/init.d/php5-fpm"
#php_fpm_ping_path["$_version"]="$_ping_path"
php_fpm_pid_search_path["$_version"]="/etc/php5"
else
fatal "Cannot find systemd service or init-script for PHP FPM (v$_version) !"
fi
done
fi
if $check_website ; then
if [[ -n "$php_version_of_working_url" ]] && [[ "$php_version_of_working_url" != "None" ]] ; then
if [[ ! "$php_versions" =~ $php_version_of_working_url ]]; then
fatal "Unknown PHP version \"$php_version_of_working_url\" (variable php_version_of_working_url)!\n\n Var \"php_version_of_working_url\" must match given versions of var \"php_versions\""
fi
PHP_FPM_SERVICE_FILE=""
if $systemd_supported ; then
# - Is Service exclusive controlled by systemd
# -
if systemctl -t service list-unit-files \
| grep -e "^php-$php_version_of_working_url-" \
| grep -q enabled 2> /devnull ; then
PHP_FPM_SERVICE_FILE=$(systemctl -t service list-unit-files \
| grep -e "^php-$php_version_of_working_url-" \
| awk '{print$1}')
fi
fi
if [[ -z "$PHP_FPM_SERVICE_FILE" ]]; then
if [[ ! -x "/etc/init.d/php-${php_version_of_working_url}-fpm" ]] ; then
fatal "Cannot find systemd service or init-script for PHP FPM (v$php_version_of_working_url)!\n Check Variable \"php_version_of_working_url\"."
fi
fi
fi
if [[ -z "$(trim $is_working_url)" ]];then
fatal "Check respond of a certain website is requested, but no url (is_working_url) is given!"
fi
if [[ -z "$(trim $check_string)" ]];then
fatal "Check respond of a certain website is requested, but no check string is given!"
fi
if $include_cleanup_function && [[ -z $(trim $cleanup_function) ]]; then
fatal "include_cleanup_function is enabled, but no cleanup functions are given!\n Check Variable \"cleanup_function\"."
else
cleanup_functions_file=$(mktemp)
printf '%s' "$cleanup_function" > $cleanup_functions_file
fi
fi
## - #---------------------------------------
## - #-----------------------------
## - # Job is already running?
## - #-----------------------------
## - #---------------------------------------
## -
## - ## - If job already runs, stop execution..
## - ## -
## - if mkdir "$LOCK_DIR" 2> /dev/null ; then
## -
## - ## - Remove lockdir when the script finishes, or when it receives a signal
## - trap 'rm -rf "$LOCK_DIR"' 0 2 15
## -
## - else
## -
## - datum=`date +"%d.%m.%Y"`
## -
## - msg="[ Error ]: A previos instance of \"`basename $0`\" seems already be running.\n\tExiting now.."
## -
## - echo ""
## - echo "[ Error ]: A previos instance of that script \"`basename $0`\" seems already be running."
## - echo ""
## - echo -e "\tExiting now.."
## - echo ""
## -
## - for _to_address in $to_addresses ; do
## - echo -e "To:${_to_address}\n${content_type}\nSubject:Error cronjob `basename $0` -- $datum\n${msg}\n" \
## - | sendmail -F "Error `hostname -f`" -f $from_address $_to_address
## - done
## -
## - exit 1
## -
## - fi
## - Remove lockdir when the script finishes, or when it receives a signal
trap 'rm -rf "$LOCK_DIR"' 0 2 15
#---------------------------------------
#-----------------------------
# Start/Stop functions
#-----------------------------
#---------------------------------------
stop_apache() {
send_msg=$1
send_msg=${send_msg:=true}
## - Stop Apache Webservice
## -
$APACHE_INIT_SCRIPT stop > /dev/null 2>&1
sleep 12
declare -i i=0
PIDS=$(ps aux | grep "$HTTPD" | grep -v grep | awk '{print$2}')
while [ "X$PIDS" != "X" ]; do
if [ $i -eq 0 ]; then
echo "" >> $LOCK_DIR/extra_msg.txt
echo -e "\t[ Error ]: Stopping Apache Webservice failed !!!" >> $LOCK_DIR/extra_msg.txt
echo "" >> $LOCK_DIR/extra_msg.txt
echo -e "\t[ Info ]; Going to kill remaining httpd-processes.." >> $LOCK_DIR/extra_msg.txt
echo "" >> $LOCK_DIR/extra_msg.txt
error "Stopping Apache Webservice failed !!!"
info "Going to kill remaining httpd-processes.."
fi
if [ $i -gt 10 ]; then
echo "" >> $LOCK_DIR/extra_msg.txt
echo -e "\t[ Error ]: Killing remaining httpd-processes failed !!!" >> $LOCK_DIR/extra_msg.txt
echo "" >> $LOCK_DIR/extra_msg.txt
error "Killing remaining httpd-processes failed !!!"
if $send_msg ; then
subject="[ Error ]: Stopping Apache Webserver on `hostname -f` failed -- $datum"
msg="\n\n`cat $LOCK_DIR/extra_msg.txt`\n"
for _to_address in $to_addresses ; do
echo -e "To:${_to_address}\n${content_type}\nSubject:$subject\n\n${msg}\n" \
| /usr/sbin/sendmail -F "Webservice Monitor" -f $from_address $_to_address
done
fi
break
fi
for _PID in $PIDS ; do
kill -9 $_PID > /dev/null 2>&1
done
sleep 2
PIDS=$(ps aux | grep "$HTTPD" | grep -v grep | awk '{print$2}')
i=i+1
done
}
stop_mysql() {
send_msg=$1
send_msg=${send_msg:=true}
## - Stop MySQL Service
## -
$MYSQL_INIT_SCRIPT stop > /dev/null 2>&1
_msg=false
sleep 2
declare -i i=0
PIDS=$(ps aux | grep "$MYSQLD_SAFE" | grep -v grep | awk '{print$2}')
while [ "X$PIDS" != "X" ]; do
if [ $i -eq 0 ]; then
echo "" >> $LOCK_DIR/extra_msg.txt
echo -e "\t[ Error ]: Stopping MySQL Service failed !!!" >> $LOCK_DIR/extra_msg.txt
echo "" >> $LOCK_DIR/extra_msg.txt
echo -e "\t[ Info ]; Going to kill MySQL Processes (mysqld_safe).." >> $LOCK_DIR/extra_msg.txt
echo "" >> $LOCK_DIR/extra_msg.txt
error "Stopping MySQL Service failed !!!"
info "Going to kill MySQL Processes (mysqld_safe).."
fi
if [ $i -gt 20 ]; then
echo "" >> $LOCK_DIR/extra_msg.txt
echo -e "\t[ Error ]: Killing MySQL Processes (mysqld_safe) failed !!!" >> $LOCK_DIR/extra_msg.txt
echo "" >> $LOCK_DIR/extra_msg.txt
error "Killing MySQL Processes (mysqld_safe) failed !!!"
_msg=true
break
fi
for _PID in $PIDS ; do
kill -9 $_PID > /dev/null 2>&1
done
sleep 2
PIDS=$(ps aux | grep "$MYSQLD_SAFE" | grep -v grep | awk '{print$2}')
i=i+1
done
PIDS=$(ps aux | grep "$MYSQLD" | grep -v grep | awk '{print$2}')
declare -i i=0
while [ "X$PIDS" != "X" ]; do
if [ $i -eq 0 ]; then
echo "" >> $LOCK_DIR/extra_msg.txt
echo -e "\t[ Error ]: MySQL Processes (mysqld) still exists !!!" >> $LOCK_DIR/extra_msg.txt
echo "" >> $LOCK_DIR/extra_msg.txt
echo -e "\t[ Info ]; Going to kill remaining MySQL Processes (mysqld).." >> $LOCK_DIR/extra_msg.txt
echo "" >> $LOCK_DIR/extra_msg.txt
error "MySQL Processes (mysqld) still exists !!!"
info "Going to kill remaining MySQL Processes (mysqld).."
fi
if [ $i -gt 20 ]; then
echo "" >> $LOCK_DIR/extra_msg.txt
echo -e "\t[ Error ]: Killing remaining MySQL Processes (mysqld) failed !!!" >> $LOCK_DIR/extra_msg.txt
echo "" >> $LOCK_DIR/extra_msg.txt
error "Killing remaining MySQL Processes (mysqld) failed !!!"
_msg=true
break
fi
for _PID in $PIDS ; do
kill -9 $_PID > /dev/null 2>&1
done
sleep 2
PIDS=i$(ps aux | grep "$MYSQLD" | grep -v grep | awk '{print$2}')
i=i+1
done
if $send_msg && $_msg ; then
subject="[ Error ]: Stopping MySQL Engine on `hostname -f` failed -- $datum"
msg="\n\n`cat $LOCK_DIR/extra_msg.txt`\n"
for _to_address in $to_addresses ; do
echo -e "To:${_to_address}\n${content_type}\nSubject:$subject\n\n${msg}\n" \
| /usr/sbin/sendmail -F "Webservice Monitor" -f $from_address $_to_address
done
fi
}
stop_php_fpm() {
__version=$1
send_msg=$2
send_msg=${send_msg:=true}
_msg=false
## - Stop PHP-FPM Service
## -
if [[ "${start_stop_method_php_fpm["$__version"]}" = "systemd_service" ]]; then
systemctl stop ${start_stop_file_php_fpm[$__version]} > /dev/null 2>&1
elif [[ "${start_stop_method_php_fpm[$__version]}" = "init_script" ]]; then
${start_stop_file_php_fpm[$__version]} stop > /dev/null 2>&1
fi
sleep 2
declare -i i=0
PIDS=$(ps aux | grep "php-fpm: " | grep " master " | grep -E "[-|\(]${php_fpm_pid_search_path[$__version]}" | grep -v grep | awk '{print$2}')
while [ "X$PIDS" != "X" ]; do
if [ $i -eq 0 ]; then
echo "" >> $LOCK_DIR/extra_msg.txt
echo -e "\t[ Error ]: Stopping PHP FPM v$__version failed !!!" >> $LOCK_DIR/extra_msg.txt
echo "" >> $LOCK_DIR/extra_msg.txt
echo -e "\t[ Info ]; Going to kill PHP FPM v$__version Processes (master).." >> $LOCK_DIR/extra_msg.txt
echo "" >> $LOCK_DIR/extra_msg.txt
error "Stopping PHP FPM v$__version failed !!!"
info "Going to kill PHP FPM v$__version Processes (master).."
fi
if [ $i -gt 20 ]; then
echo "" >> $LOCK_DIR/extra_msg.txt
echo -e "\t[ Error ]: Killing PHP FPM Processes v$__version (master) failed !!!" >> $LOCK_DIR/extra_msg.txt
echo "" >> $LOCK_DIR/extra_msg.txt
error "Killing PHP FPM Processes v$__version (master) failed !!!"
_msg=true
break
fi
for _PID in $PIDS ; do
kill -9 $_PID > /dev/null 2>&1
done
# - Remove all sockets php-fpm sockets from thid version
# -
rm -f /tmp/php-${__version}-fpm*.sock
sleep 2
PIDS=$(ps aux | grep "php-fpm: " | grep " master " | grep -E "[-|\(]${php_fpm_pid_search_path[$__version]}" | grep -v grep | awk '{print$2}')
i=i+1
done
declare -i i=0
PIDS=$(ps aux | grep "php-fpm: " | grep -E "(pool|master)" | grep -E "\-${php_fpm_pid_search_path[$__version]}" | grep -v grep | awk '{print$2}')
while [ "X$PIDS" != "X" ]; do
if [ $i -eq 0 ]; then
echo "" >> $LOCK_DIR/extra_msg.txt
echo -e "\t[ Error ]: PHP FPM Processes v$__version (pool) still exists !!!" >> $LOCK_DIR/extra_msg.txt
echo "" >> $LOCK_DIR/extra_msg.txt
echo -e "\t[ Info ]; Going to kill remaining PHP FPM v$__version Processes (pool).." >> $LOCK_DIR/extra_msg.txt
echo "" >> $LOCK_DIR/extra_msg.txt
error "PHP FPM Processes v$__version (pool) still exists !!!"
info "Going to kill remaining PHP FPM Processes v$__version (pool).."
fi
if [ $i -gt 20 ]; then
echo "" >> $LOCK_DIR/extra_msg.txt
echo -e "\t[ Error ]: Killing remaining PHP FPM Processes v$__version (pool) failed !!!" >> $LOCK_DIR/extra_msg.txt
echo "" >> $LOCK_DIR/extra_msg.txt
error "Killing remaining PHP FPM Processes v$__version (pool) failed !!!"
_msg=true
break
fi
for _PID in $PIDS ; do
kill -9 $_PID > /dev/null 2>&1
done
sleep 2
PIDS=$(ps aux | grep "php-fpm: " | grep -E "(pool|master)" | grep -E "\-${php_fpm_pid_search_path[$__version]}" | grep -v grep | awk '{print$2}')
i=i+1
done
if $send_msg && $_msg ; then
subject="[ Error ]: Stopping PHP FPM v$__version on `hostname -f` failed -- $datum"
msg="\n\n`cat $LOCK_DIR/extra_msg.txt`\n"
for _to_address in $to_addresses ; do
echo -e "To:${_to_address}\n${content_type}\nSubject:$subject\n\n${msg}\n" \
| /usr/sbin/sendmail -F "Webservice Monitor" -f $from_address $_to_address
done
fi
}
start_mysql() {
send_msg=$1
send_msg=${send_msg:=true}
$MYSQL_INIT_SCRIPT start > /dev/null 2>&1
sleep 2
NEWPID=$(ps aux | grep "$MYSQLD_SAFE" | grep -v grep | grep root | awk '{print$2}')
if [ "X${NEWPID}X" = "XX" ]; then
echo "" >> $LOCK_DIR/extra_msg.txt
echo -e "\t[ Error ]: I have started the MySQL server, but the services is not running !!!" >> $LOCK_DIR/extra_msg.txt
echo "" >> $LOCK_DIR/extra_msg.txt
error "I have started the MySQL server, but the service is not running !!!"
datum=`date +"%d.%m.%Y %H:%Mh"`
subject="[ Error ]: MySQL Service on `hostname -f` NOT RUNNING -- $datum"
msg="\n\n`cat $LOCK_DIR/extra_msg.txt`\n"
for _to_address in $to_addresses ; do
echo -e "To:${_to_address}\n${content_type}\nSubject:$subject\n\n${msg}\n" \
| /usr/sbin/sendmail -F "Webservice Monitor" -f $from_address $_to_address
done
else
echo "" >> $LOCK_DIR/extra_msg.txt
echo -e "\tI have started MySQL service. The new PID is $NEWPID" >> $LOCK_DIR/extra_msg.txt
echo "" >> $LOCK_DIR/extra_msg.txt
ok "I have started MySQL service. The new PID is $NEWPID"
fi
if $send_msg ; then
datum=`date +"%d.%m.%Y %H:%Mh"`
subject="Restarting MySQL on `hostname -f` invoked -- $datum"
msg=`cat $LOCK_DIR/extra_msg.txt`
for _to_address in $to_addresses ; do
echo -e "To:${_to_address}\n${content_type}\nSubject:$subject\n\n${msg}\n" \
| /usr/sbin/sendmail -F "Webservice Monitor" -f $from_address $_to_address
done
fi
}
start_apache() {
send_msg=$1
send_msg=${send_msg:=true}
$APACHE_INIT_SCRIPT start > /dev/null 2>&1
sleep 2
NEWPID=$(ps aux | grep "$HTTPD" | grep -v grep | grep root | awk '{print$2}')
if [ "X${NEWPID}X" = "XX" ]; then
echo "" >> $LOCK_DIR/extra_msg.txt
echo -e "\t[ Error ]: I have tarted the webserver, but service is not running !!!" >> $LOCK_DIR/extra_msg.txt
echo "" >> $LOCK_DIR/extra_msg.txt
error "I have started the webserver, but service is not running !!!"
datum=`date +"%d.%m.%Y %H:%Mh"`
subject="[ Error ]: Webservice on `hostname -f` NOT RUNNING -- $datum"
msg="\n\n`cat $LOCK_DIR/extra_msg.txt`\n"
for _to_address in $to_addresses ; do
echo -e "To:${_to_address}\n${content_type}\nSubject:$subject\n\n${msg}\n" \
| /usr/sbin/sendmail -F "Webservice Monitor" -f $from_address $_to_address
done
else
echo "" >> $LOCK_DIR/extra_msg.txt
echo -e "\tI have started the webserver. the new PID is $NEWPID" >> $LOCK_DIR/extra_msg.txt
echo "" >> $LOCK_DIR/extra_msg.txt
ok "I have started the webserver. the new PID is $NEWPID"
fi
if $send_msg ; then
datum=`date +"%d.%m.%Y %H:%Mh"`
subject="Restarting Apache on `hostname -f` invoked -- $datum"
msg=`cat $LOCK_DIR/extra_msg.txt`
for _to_address in $to_addresses ; do
echo -e "To:${_to_address}\n${content_type}\nSubject:$subject\n\n${msg}\n" \
| /usr/sbin/sendmail -F "Webservice Monitor" -f $from_address $_to_address
done
fi
}
start_php_fpm() {
__version=$1
send_msg=$2
send_msg=${send_msg:=true}
start_php_fpm_failed=false
if [[ "${start_stop_method_php_fpm[$__version]}" = "systemd_service" ]]; then
systemctl restart ${start_stop_file_php_fpm[$__version]} > /dev/null 2>&1
elif [[ "${start_stop_method_php_fpm["$__version"]}" = "init_script" ]]; then
${start_stop_file_php_fpm["$__version"]} start > /dev/null 2>&1
else
error "Neither start/stop script nor systemd service file for PHP FPM v$__version found!"
echo "" >> $LOCK_DIR/extra_msg.txt
echo -e "\t[ Error ]: Neither start/stop script nor systemd service file for PHP FPM v$__version found !!!" >> $LOCK_DIR/extra_msg.txt
echo "" >> $LOCK_DIR/extra_msg.txt
start_php_fpm_failed=true
fi
sleep 2
NEWPID=`ps aux | grep "php-fpm: " | grep " master " | grep -E "[-|\(]${php_fpm_pid_search_path[$__version]}" | grep -v grep | awk '{print$2}'`
if [ "X${NEWPID}X" = "XX" ]; then
if ! $start_php_fpm_failed ; then
echo "" >> $LOCK_DIR/extra_msg.txt
echo -e "\t[ Error ]: I have started PHP FPM v$__version, but service is not running !!!" >> $LOCK_DIR/extra_msg.txt
echo "" >> $LOCK_DIR/extra_msg.txt
error "I have started PHP FPM v$__version, but service is not running !!!"
fi
datum=`date +"%d.%m.%Y %H:%Mh"`
subject="[ Error ]: PHP-FPM v$__version engine on `hostname -f` NOT RUNNING -- $datum"
msg="\n\n`cat $LOCK_DIR/extra_msg.txt`\n"
for _to_address in $to_addresses ; do
echo -e "To:${_to_address}\n${content_type}\nSubject:$subject\n\n${msg}\n" \
| /usr/sbin/sendmail -F "Webservice Monitor" -f $from_address $_to_address
done
else
echo "" >> $LOCK_DIR/extra_msg.txt
echo -e "\tI have started the PHP-FPM v$__version engine. the new PID is $NEWPID" >> $LOCK_DIR/extra_msg.txt
echo "" >> $LOCK_DIR/extra_msg.txt
ok "I have started the PHP-FPM v$__version engine. the new PID is $NEWPID"
fi
if $send_msg ; then
datum=`date +"%d.%m.%Y %H:%Mh"`
subject="Restarting PHP FPM v$__version engine on `hostname -f` invoked -- $datum"
msg=`cat $LOCK_DIR/extra_msg.txt`
for _to_address in $to_addresses ; do
echo -e "To:${_to_address}\n${content_type}\nSubject:$subject\n\n${msg}\n" \
| /usr/sbin/sendmail -F "Webservice Monitor" -f $from_address $_to_address
done
fi
}
restart_apache() {
## - Stop Apache Webservice
## -
stop_apache
sleep 2
## - Start Apache Webservice
## -
start_apache
}
restart_mysql(){
## - Stop MySQL Servive
## -
stop_mysql
sleep 2
## - Start MySQL Service
## -
start_mysql
}
restart_php_fpm(){
## - Stop MySQL Servive
## -
stop_php_fpm $1
sleep 2
## - Start MySQL Service
## -
start_php_fpm $1
}
graceful_restart_php_fpm() {
__version=$1
send_msg=$2
send_msg=${send_msg:=true}
start_php_fpm_failed=false
if [[ "$start_stop_method_php_fpm" = "systemd_service" ]]; then
systemctl restart $start_stop_method_php_fpm["$__version"] > /dev/null 2>&1
elif [[ "$start_stop_method_php_fpm" = "init_script" ]]; then
$start_stop_method_php_fpm["$__version"] restart > /dev/null 2>&1
else
error "Neither start/stop script nor systemd service file for PHP FPM v$__version found!"
echo "" >> $LOCK_DIR/extra_msg.txt
echo -e "\t[ Error ]: Neither start/stop script nor systemd service file for PHP FPM v$__version found !!!" >> $LOCK_DIR/extra_msg.txt
echo "" >> $LOCK_DIR/extra_msg.txt
start_php_fpm_failed=true
fi
sleep 2
NEWPID=`ps aux | grep "php-fpm: " | grep " master " | grep "[-|\(]${php_fpm_pid_search_path[\"$__version\"]}" | grep -v grep | awk '{print$2}'`
if [ "X${NEWPID}X" = "XX" ]; then
if ! $start_php_fpm_failed ; then
echo "" >> $LOCK_DIR/extra_msg.txt
echo -e "\t[ Error ]: I have restarted PHP-FPM, but service is not running !!!" >> $LOCK_DIR/extra_msg.txt
echo "" >> $LOCK_DIR/extra_msg.txt
error "I have restarted PHP-FPM, but service is not running !!!"
fi
datum=`date +"%d.%m.%Y %H:%Mh"`
subject="[ Error ]: PHP-FPM engine on `hostname -f` NOT RUNNING -- $datum"
msg="\n\n`cat $LOCK_DIR/extra_msg.txt`\n"
for _to_address in $to_addresses ; do
echo -e "To:${_to_address}\n${content_type}\nSubject:$subject\n\n${msg}\n" \
| /usr/sbin/sendmail -F "Webservice Monitor" -f $from_address $_to_address
done
else
echo "" >> $LOCK_DIR/extra_msg.txt
echo -e "\tI have restarted the PHP-FPM engine. the new PID is $NEWPID" >> $LOCK_DIR/extra_msg.txt
echo "" >> $LOCK_DIR/extra_msg.txt
ok "I have restarted the PHP-FPM engine. the new PID is $NEWPID"
fi
if $send_msg ; then
datum=`date +"%d.%m.%Y %H:%Mh"`
subject="Restarting PHP-FPM engine on `hostname -f` invoked -- $datum"
msg=`cat $LOCK_DIR/extra_msg.txt`
for _to_address in $to_addresses ; do
echo -e "To:${_to_address}\n${content_type}\nSubject:$subject\n\n${msg}\n" \
| /usr/sbin/sendmail -F "Webservice Monitor" -f $from_address $_to_address
done
fi
}
#---------------------------------------
#-----------------------------
# Checking Load..
#-----------------------------
#---------------------------------------
if $check_load ; then
#echo "" > $LOCK_DIR/extra_msg.txt
if $LOGGING ; then
echo -e "\nChecking Load on \"`hostname -f`\".."
fi
warn_msg=${LOCK_DIR}/warn.msg
echo "" > $warn_msg
## - Number of processors
## -
declare -i number_processors=`grep -E "processor\s+:\s+" /proc/cpuinfo -c`
if [ -z "$number_processors" -o $number_processors -eq 0 ]; then
fatal "Detecting number of prozessors ($number_processors) failed!"
fi
## - Thresholds
## -
if [ $number_processors -gt 8 ]; then
#threshod_1_min=`echo "scale=2; $number_processors / 1.95" | bc`
#threshod_5_min=`echo "scale=2; $number_processors / 2.95" | bc`
#threshod_15_min=`echo "scale=2; $number_processors / 3.45" | bc`
threshod_1_min=`echo "scale=2; $number_processors / 1.45" | bc`
threshod_5_min=`echo "scale=2; $number_processors / 1.95" | bc`
threshod_15_min=`echo "scale=2; $number_processors / 2.35" | bc`
else
threshod_1_min=`echo "scale=2; $number_processors / 1.00" | bc`
threshod_5_min=`echo "scale=2; $number_processors / 1.45" | bc`
threshod_15_min=`echo "scale=2; $number_processors / 1.95" | bc`
fi
## - Determin load avarage values
## -
## - Last minute
load_avarage_1m=`cat /proc/loadavg | cut -d ' ' -f1`
## - Last 5 minutes
load_avarage_5m=`cat /proc/loadavg | cut -d ' ' -f2`
## -Last 15 minutes
load_avarage_15m=`cat /proc/loadavg | cut -d ' ' -f3`
## - Evaluate
## -
result_1m=$(echo "scale=2; $load_avarage_1m > $threshod_1_min" | bc)
result_5m=$(echo "scale=2; $load_avarage_5m > $threshod_5_min" | bc)
result_15m=$(echo "scale=2; $load_avarage_15m > $threshod_15_min" | bc)
#if [ "$result_1m" = "1" -o "$result_5m" = "1" -o "$result_15m" = "1" ] ; then
if [ "$result_1m" = "1" ] ; then
subject="[ Warning ]: High load on `hostname -f`"
cat <<EOF > $warn_msg
High load on *`hostname -f`*
Server Name...................: `hostname -f`
Date..........................: `date +%d.%m.%Y`
Time..........................: `date +%H:%M` h
Load avarage last minute......: $load_avarage_1m
Load avarage last 5 minutes...: $load_avarage_5m
Load avarage last 15 minutes..: $load_avarage_15m
EOF
for _to_address in $to_addresses ; do
echo -e "To: ${_to_address}\nSubject: $subject\n\n`cat $warn_msg`" \
| sendmail -F "Load Monitor" -f $from_address $_to_address
done
echo "" > $warn_msg
fi
_load_avarage_1m=$(echo "scale=0; $load_avarage_1m/1" | bc -l)
_load_avarage_5m=$(echo "scale=0; $load_avarage_5m/1" | bc -l)
if [ $_load_avarage_1m -gt 200 -a $_load_avarage_5m -gt 200 ]; then
cat<<EOF
Server Name...................: `hostname -f`
Date..........................: `date +%d.%m.%Y`
Time..........................: `date +%H:%M` h
Load avarage last minute......: $load_avarage_1m
Load avarage last 5 minutes...: $load_avarage_5m
Load avarage last 15 minutes..: $load_avarage_15m
EOF
cat<<EOF >> $warn_msg
Server Name...................: `hostname -f`
Date..........................: `date +%d.%m.%Y`
Time..........................: `date +%H:%M` h
Load avarage last minute......: $load_avarage_1m
Load avarage last 5 minutes...: $load_avarage_5m
Load avarage last 15 minutes..: $load_avarage_15m
EOF
stop_apache
stop_mysql
sleep 60
start_mysql
start_apache
else
if $LOGGING ; then
ok "Load on \"`hostname -f`\" seems to be fine.."
cat<<EOF
Server Name...................: `hostname -f`
Date..........................: `date +%d.%m.%Y`
Time..........................: `date +%H:%M` h
Load avarage last minute......: $load_avarage_1m
Load avarage last 5 minutes...: $load_avarage_5m
Load avarage last 15 minutes..: $load_avarage_15m
EOF
fi
fi
fi # End: check_load
#---------------------------------------
#-----------------------------
# Checking MySQL Database server..
#-----------------------------
#---------------------------------------
if $check_mysql ; then
echo "" > $LOCK_DIR/extra_msg.txt
if $LOGGING ; then
echo -e "\nChecking Mysql databse service on \"`hostname -f`\".."
fi
PID=`ps aux | grep "$MYSQLD_SAFE" | grep -v grep | awk '{print$2}'`
if [ "X${PID}X" = "XX" ];then
echo -e "\nMySQL is not running! - Try to start it.." >> $LOCK_DIR/extra_msg.txt
echo -e "=========================================" >> $LOCK_DIR/extra_msg.txt
error "MySQL is not running! - Try to start it.."
restart_mysql
else
if [ -z "`$MYADMIN $mysql_credential_args ping 2>/dev/null`" ]; then
echo -e "\nMySQL seems to be running, but NOT responding! - Try to restart service.." >> $LOCK_DIR/extra_msg.txt
echo -e "=======================================================================" >> $LOCK_DIR/extra_msg.txt
error "MySQL seems to be running, but NOT responding! - Try to restart service.."
restart_mysql
else
if $LOGGING ; then
ok "MySQL database service on \"`hostname -f`\" seems to be running.."
fi
fi
fi
fi
#---------------------------------------
#-----------------------------
# Check Apache webservice..
#-----------------------------
#---------------------------------------
if $check_apache ; then
echo "" > $LOCK_DIR/extra_msg.txt
if $LOGGING ; then
echo -e "\nChecking Apache webservice on \"`hostname -f`\".."
fi
#_PID_FILE=`$HTTPD -t -D DUMP_RUN_CFG | grep -i -e "^PidFile: " | awk '{print$2}' \
# | sed -e "s/^[\"']//" | sed -e "s/[\"']$//"`
#PID=`cat $_PID_FILE`
#if [ "X`ps ax | grep \"$HTTPD \" | grep -e \"^$PID \"`" = "X" ]; then
PID=`ps aux | grep "$HTTPD " | grep -e "^root" | grep -v grep | awk '{print$2}'`
if [ "X${PID}X" = "XX" ];then
echo -e "\nApache Webservice is not running! - Try to start it.." > $LOCK_DIR/extra_msg.txt
echo -e "=====================================================" >> $LOCK_DIR/extra_msg.txt
error "Apache webservice is not running! - Try to start it.."
restart_apache
else
if ! $(curl -Is -m 10 http://$curl_check_host | head -n 1 | grep "200 OK" > /dev/null) ; then
echo -e "\nApache Webservice seems to be running, but is NOT responding! - Try to restart service.." >> $LOCK_DIR/extra_msg.txt
echo -e "========================================================================================" >> $LOCK_DIR/extra_msg.txt
error "Apache Webservice seems to be running, but is NOT responding! - Try to restart service.."
restart_apache
else
if $LOGGING ; then
ok "Apache Webserver on \"`hostname -f`\" seems to be running.."
fi
fi
fi
fi
#---------------------------------------
#-----------------------------
# Check PHP-FPM engine..
#-----------------------------
#---------------------------------------
if $check_php_fpm ; then
for _version in $php_versions ; do
echo "" > $LOCK_DIR/extra_msg.txt
if $LOGGING ; then
echo -e "\nChecking PHP-FPM v$_version (FastCGI Process Manager) on \"`hostname -f`\".."
fi
PID=$(ps aux | grep "php-fpm: " | grep "master" | grep -E "[-|\(]${php_fpm_pid_search_path[$_version]}" | grep -v grep | awk '{print$2}')
if [ "X${PID}X" = "XX" ];then
echo -e "\nPHP-FPM v$_version is not running! - Try to start the service.." > $LOCK_DIR/extra_msg.txt
echo -e "====================================================" >> $LOCK_DIR/extra_msg.txt
error "PHP-FPM v$_version is not running! - Try to start the service.."
restart_php_fpm $_version
elif [[ -n ${php_fpm_ping_path[$_version]} ]] ; then
if ! $(curl -Is -m 10 http://${curl_check_host}/${php_fpm_ping_path[$_version]} | head -n 1 | grep "200 OK" > /dev/null) ; then
echo -e "\nPHP-FPM v$_version seems to be running, but is NOT responding! - Try to restart service.." >> $LOCK_DIR/extra_msg.txt
echo -e "====================================================================================" >> $LOCK_DIR/extra_msg.txt
error "PHP-FPM v$_version seems to be running, but is NOT responding! - Try to restart service.."
restart_php_fpm $_version
else
if $LOGGING ; then
ok "PHP-FPM v$_version engine on \"`hostname -f`\" seems to be running.."
fi
fi
fi
done
fi
#---------------------------------------
#-----------------------------
# Check website..
#-----------------------------
#---------------------------------------
if $check_website ; then
send_msg=false
if $LOGGING ; then
echo -e "\nDo some extra checks.."
fi
declare -i rval=$(curl -s $is_working_url 2> /dev/null | grep "$check_string" | wc -l)
if [[ $rval -lt 1 ]] ; then
echo -e "The website does not respond as expected!" > $LOCK_DIR/extra_msg.txt
echo -e "=========================================" >> $LOCK_DIR/extra_msg.txt
error "The website $is_working_url does not respond as expected."
# - Stop services..
# -
stop_apache false
if [[ "$php_version_of_working_url" != "None" ]]; then
if [[ -n "$php_version_of_working_url" ]];then
stop_php_fpm $php_version_of_working_url false
else
for _version in $php_versions ; do
stop_php_fpm $_version false
done
fi
fi
# - Clean up cache/session files..
# -
if $include_cleanup_function ; then
info "Execute configured cleanup tasks.."
echo -e "\n\t[ Info ]: Execute configured cleanup tasks .." >> $LOCK_DIR/extra_msg.txt
source $cleanup_functions_file
fi
# - Start services
# -
if [[ "$php_version_of_working_url" != "None" ]]; then
if [[ -n "$php_version_of_working_url" ]];then
start_php_fpm $php_version_of_working_url false
else
for _version in $php_versions ; do
start_php_fpm $_version false
done
fi
fi
start_apache false
info "Sleeping for 30 seconds.."
sleep 30
declare -i rval=$(curl -s $is_working_url 2> /dev/null | grep "$check_string" | wc -l)
if [[ $rval -lt 1 ]] ; then
echo -e "\n\t[ Error ]: URL $is_working_url is still NOT responding as expected !!!" >> $LOCK_DIR/extra_msg.txt
error "URL $is_working_url is still NOT responding as expected !!!"
# - Only send extra message, if cleaning up and restarting wasn't successfully
# -
send_msg=true
else
ok "Cleaning up and restarting services was successfully."
fi
else
if $LOGGING ; then
ok "URL $is_working_url is responding as expected."
fi
fi
if $send_msg ; then
subject="[ Error ]: Shop System at `hostname -f` failed -- $datum"
msg="\n\n`cat $LOCK_DIR/extra_msg.txt`\n"
if [[ -n "$extra_alert_address" ]]; then
to_addresses="$to_addresses $extra_alert_address"
fi
for _to_address in $to_addresses ; do
echo -e "To:${_to_address}\n${content_type}\nSubject:$subject\n\n${msg}\n" \
| /usr/sbin/sendmail -F "Webservice Monitor" -f $from_address $_to_address
done
fi
rm -rf $cleanup_functions_file
fi
exit 0