From 903d594079d9b858a0f86bf93f0f664df35e979a Mon Sep 17 00:00:00 2001 From: Christoph Date: Sun, 12 Feb 2017 13:46:44 +0100 Subject: [PATCH] Initial import --- .gitignore | 2 + check_webservice_load.conf.sample | 145 ++++ check_webservice_load.sh | 1291 +++++++++++++++++++++++++++++ 3 files changed, 1438 insertions(+) create mode 100644 .gitignore create mode 100644 check_webservice_load.conf.sample create mode 100755 check_webservice_load.sh diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..7c9b3a5 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +check_webservice_load.conf +BAK/* diff --git a/check_webservice_load.conf.sample b/check_webservice_load.conf.sample new file mode 100644 index 0000000..5b29fed --- /dev/null +++ b/check_webservice_load.conf.sample @@ -0,0 +1,145 @@ +#--------------------------------------- +#----------------------------- +# Settings +#----------------------------- +#--------------------------------------- + +#LOGGING=true +LOGGING=false + +# - What to check +# - +check_load=true +check_mysql=true +check_apache=true +check_php_fpm=true +check_website=false + + +# - Additional Settings for check_mysql +# - +# - MySQL / MariaDB credentials +# - +# - Giving password on command line is insecure an sind mysql 5.5 +# - you will get a warning doing so. +# - +# - Reading username/password fro file ist also possible, using MySQL/MariaDB +# - commandline parameter '--defaults-file'. +# - +# - Since Mysql Version 5.6, you can read username/password from +# - encrypted file. +# - +# - Create (encrypted) option file: +# - $ mysql_config_editor set --login-path=local --socket=/tmp/mysql.sock --user=root --password +# - $ Password: +# - +# - Use of option file: +# - $ mysql --login-path=local ... +# - +# - Example +# - mysql_credential_args="--login-path=local" +# - mysql_credential_args="--defaults-file=/etc/mysql/debian.cnf" (Debian default) +# - mysql_credential_args="--defaults-file=/usr/local/mysql/sys-maint.cnf" +# - +mysql_credential_args="" + + +# - Additional Settings for check_php_fpm +# - +# - On Linux Vserver System set +# - curl_check_host=localhost +# - +# - On LX-Container set +# - curl_check_host=127.0.0.1 +# - +curl_check_host=127.0.0.1 + +# - Which PHP versions should be supported by this script. If more than one, +# - give a blank separated list +# - +# - Example: +# - php_versions="5.4 5.6 7.0 7.1" +# - +php_versions="" + +# - If PHP-FPM's ping.path setting does not match ping-$php_major_version, +# - set the value given in your ping.path setting here. Give ping_path also +# - the concerning php_version in form +# - : +# - +# - Multiple settings are possible, give a blank separated list. +# - +# - Example: +# - +# - ping_path="5.4:ping-site36_net 5.6:ping-oopen_de" +# - +ping_path="" + + +# - Additional Settings for check_website - checking (expected) website response +# - +# - example: +# - is_working_url="https://www.outoflineshop.de/" +# - check_string='ool-account-links' +# - include_cleanup_function=true +# - extra_alert_address="ilker@so36.net" +# - cleanup_function=' +# - rm -rf /var/www/www.outoflineshop.de/htdocs/var/cache/* +# - rm -rf /var/www/www.outoflineshop.de/htdocs/var/session/* +# - /usr/local/bin/redis-cli flushall > /dev/null 2>&1 +# - if [[ "$?" = "0" ]]; then +# - ok "I have cleaned up directory \"/var/www/www.outoflineshop.de/htdocs/var/cache/\"" +# - else +# - error "Cleaning up directory \"/var/www/www.outoflineshop.de/htdocs/var/cache/\" failed!" +# - fi +# - /etc/init.d/redis_6379 restart +# - if [[ "$?" = "0" ]]; then +# - ok "I restarted the redis service" +# - echo -e "\t[ Ok ]: I restarted the redis service" >> $LOCK_DIR/extra_msg.txt +# - else +# - error "Restarting the redis server failed!" +# - echo -e "\t[ Error ]: Restarting the redis server failed!" >> $LOCK_DIR/extra_msg.txt +# - fi +# - ' +# - +is_working_url='' + +check_string='' + +include_cleanup_function=true + +# - An extra e-mail address, which will be informed, if the given check URL +# - does not response as expected (check_string) AFTER script checking, restarting +# - servervices (webserver, php-fpm) and cleaning up (cleanup_function) was done. +# - +extra_alert_address='' + +# - php_version_of_working_url +# - +# - If given website (is_working_url) does not response as expected, this PHP FPM +# - engines will be restarted. +# - +# - Type "None" if site does not support php +# - +# - If php_version_of_working_url is not set, PHP FPM processes of ALL versions (php_versions) +# - will be restarted +# - +php_version_of_working_url='' + +# - Notice: +# - If single qoutes "'" not needed inside cleanup function, then use single quotes +# - to enclose variable "cleanup_function". Then you don't have do masquerade any +# - sign inside. +# - +# - Otherwise use double quotes and masq any sign to prevent bash from interpreting. +# - +cleanup_function=' +' + + +# - E-Mail settings for sending script messages +# - +from_address="root@`hostname -f`" +content_type='Content-Type: text/plain;\n charset="utf-8"' +to_addresses="root" + diff --git a/check_webservice_load.sh b/check_webservice_load.sh new file mode 100755 index 0000000..9e811a9 --- /dev/null +++ b/check_webservice_load.sh @@ -0,0 +1,1291 @@ +#!/usr/bin/env bash + +working_dir="$(dirname $(realpath $0))" +conf_file="${working_dir}/check_webservice_load.conf" + + +#--------------------------------------- +#----------------------------- +# Read Configurations from $conf_file +#----------------------------- +#--------------------------------------- + +if [[ ! -f "$conf_file" ]]; then + echo "" + echo -e " [ Fatal ] Configuration file '$(basename ${conf_file})' not found!" + echo "" + echo -e "\tScript terminated.." + echo "" + exit 1 +else + source "$conf_file" +fi + + + + + +# - Lock directory exists, until the script ends. So +# - we can check, if a previos instanze is already running. +# - +#LOCK_DIR=/tmp/check_webservice.lock +LOCK_DIR=`mktemp -d` + + +#--------------------------------------- +#----------------------------- +# Base Function(s) +#----------------------------- +#--------------------------------------- + +fatal(){ + echo "" + echo -e " [ Fatal ] $*" + echo "" + echo -e "\tScript terminated.." + echo "" + exit 1 +} + +error (){ + echo "" + echo -e " [ Error ] $*" + echo "" +} + +warn (){ + echo "" + echo -e " [ Warn ] $*" + echo "" +} + +info (){ + echo "" + echo -e " [ Info ] $*" + echo "" +} + +ok (){ + echo "" + echo -e " [ Ok ] $*" + echo "" +} + +trim() { + local var="$*" + var="${var#"${var%%[![:space:]]*}"}" # remove leading whitespace characters + var="${var%"${var##*[![:space:]]}"}" # remove trailing whitespace characters + echo -n "$var" +} + + +#--------------------------------------- +#----------------------------- +# Check some prerequisites +#----------------------------- +#--------------------------------------- + +if [ -z "`which basename`" ]; then + fatal 'It seems "basename" is not installed, but needed!' +fi + +if [ -z "`which sendmail`" ]; then + fatal 'It seems "sendmail" is not installed, but needed!' +fi + +if [ -z "`which realpath`" ]; then + fatal 'It seems "realpath" is not installed, but needed!' +fi + +if $check_load ; then + if [ -z "`which bc`" ]; then + fatal 'It seems "bc" is not installed, but needed!' + fi +fi + +# - Systemd supported ? +## - +systemd=$(which systemd) +systemctl=$(which systemctl) + +systemd_supported=false +if [[ -n "$systemd" ]] && [[ -n "$systemd" ]] ; then + systemd_supported=true +fi + + +if $check_mysql ; then + + MYSQLD=`realpath $(which mysqld) 2>/dev/null` + if [ -z "$MYSQLD" ]; then + if [ -x "/usr/local/mysql/bin/mysqld" ]; then + MYSQLD=`realpath "/usr/local/mysql/bin/mysqld"` + else + fatal 'Command \"mysqld\" not found!' + fi + fi + + MYSQLD_SAFE=`realpath $(which mysqld_safe) 2>/dev/null` + if [ -z "$MYSQLD_SAFE" ]; then + if [ -x "/usr/local/mysql/bin/mysqld_safe" ]; then + MYSQLD_SAFE=`realpath "/usr/local/mysql/bin/mysqld_safe"` + else + fatal 'Command \"mysqld_safe\" not found!' + fi + fi + + MYADMIN=`realpath $(which mysqladmin) 2>/dev/null` + if [ -z "$MYADMIN" ]; then + if [ -x "/usr/local/mysql/bin/mysqladmin" ]; then + MYADMIN=`realpath "/usr/local/mysql/bin/mysqladmin"` + else + fatal 'Command \"mysqladmin\" not found!' + fi + fi + + MYSQL_INIT_SCRIPT="" + if [ -x "/etc/init.d/mysql.server" ]; then + MYSQL_INIT_SCRIPT="/etc/init.d/mysql.server" + elif [ -x "/etc/init.d/mysql" ]; then + MYSQL_INIT_SCRIPT="/etc/init.d/mysql" + else + fatal 'Cannot find init-script for MySQL server!' + fi + +fi + +if $check_apache ; then + + HTTPD=`realpath $(which httpd) 2>/dev/null` + if [ -z "$HTTPD" ]; then + if [ -x "/usr/local/apache2/bin/httpd" ]; then + HTTPD=`realpath "/usr/local/apache2/bin/httpd"` + else + HTTPD=`realpath $(which apache2) 2>/dev/null` + if [ -z "$HTTPD" ]; then + fatal 'Command \"httpd\"/\"apache2\" not found!' + fi + fi + fi + + APACHE_INIT_SCRIPT="" + if [ -x "/etc/init.d/apache2" ]; then + APACHE_INIT_SCRIPT="/etc/init.d/apache2" + elif [ -x "/etc/init.d/apachectl" ]; then + APACHE_INIT_SCRIPT="/etc/init.d/apachectl" + else + fatal 'Cannot find init-script for Apache web server!' + fi +fi + +declare -A start_stop_method_php_fpm +declare -A start_stop_file_php_fpm +declare -A php_fpm_ping_path +declare -A php_fpm_pid_search_path +if $check_php_fpm ; then + + if [ -z "`which curl`" ]; then + fatal 'It seems "curl" is not installed, but needed!' + fi + + if [[ "$(trim $php_versions)" = "" ]]; then + fatal 'No PHP version is given. Setting of \"php_version\" is needed!' + fi + + + if [[ -n "$ping_path" ]];then + for _path in $ping_path ;do + + IFS=':' read -a _path_arr <<< "${_path}" + + # - some checks.. + # - + if [[ -z "${_path_arr[0]}" ]] || [[ -z "${_path_arr[0]}" ]] ; then + fatal "Wrong settings in variable \"ping_path\"!" + fi + if [[ ! "$php_versions" =~ "${_path_arr[0]}" ]] ; then + fatal "Unknown PHP version \"${_path_arr[0]}\" (variable ping_path)!\n\n PHP version must match given versions of var \"php_versions\"" + fi + + php_fpm_ping_path["${_path_arr[0]}"]="${_path_arr[1]}" + + done + fi + + for _version in $php_versions ; do + + PHP_FPM_SERVICE_FILE="" + if $systemd_supported ; then + # - Is Service exclusive controlled by systemd + # - + if systemctl -t service list-unit-files \ + | grep -e "^php-$_version" \ + | grep -q enabled 2> /devnull ; then + + PHP_FPM_SERVICE_FILE=$(systemctl -t service list-unit-files \ + | grep -e "^php-$_version" \ + | awk '{print$1}') + fi + fi + + if [[ -z "${php_fpm_ping_path[$_version]}" ]] ; then + php_fpm_ping_path["$_version"]="ping-$_version" + fi + + if [[ -n "$PHP_FPM_SERVICE_FILE" ]]; then + start_stop_method_php_fpm["$_version"]="systemd_service" + start_stop_file_php_fpm["$_version"]="$PHP_FPM_SERVICE_FILE" + #php_fpm_ping_path["$_version"]="ping-$_version" + php_fpm_pid_search_path["$_version"]="$_version" + elif [[ -x "/etc/init.d/php-${_version}-fpm" ]]; then + start_stop_method_php_fpm["$_version"]="init_script" + start_stop_file_php_fpm["$_version"]="/etc/init.d/php-${_version}-fpm" + #php_fpm_ping_path["$_version"]="ping-$_version" + php_fpm_pid_search_path["$_version"]="$_version" + elif [[ -x "/etc/init.d/php5-fpm" ]]; then + start_stop_method_php_fpm["$_version"]="init_script" + start_stop_file_php_fpm["$_version"]="/etc/init.d/php5-fpm" + #php_fpm_ping_path["$_version"]="$_ping_path" + php_fpm_pid_search_path["$_version"]="/etc/php5" + else + fatal "Cannot find systemd service or init-script for PHP FPM (v$_version) !" + fi + done +fi + +if $check_website ; then + + if [[ -n "$php_version_of_working_url" ]] && [[ "$php_version_of_working_url" != "None" ]] ; then + + if [[ ! "$php_versions" =~ $php_version_of_working_url ]]; then + fatal "Unknown PHP version \"$php_version_of_working_url\" (variable php_version_of_working_url)!\n\n Var \"php_version_of_working_url\" must match given versions of var \"php_versions\"" + fi + + PHP_FPM_SERVICE_FILE="" + if $systemd_supported ; then + # - Is Service exclusive controlled by systemd + # - + if systemctl -t service list-unit-files \ + | grep -e "^php-$php_version_of_working_url-" \ + | grep -q enabled 2> /devnull ; then + + PHP_FPM_SERVICE_FILE=$(systemctl -t service list-unit-files \ + | grep -e "^php-$php_version_of_working_url-" \ + | awk '{print$1}') + fi + fi + + if [[ -z "$PHP_FPM_SERVICE_FILE" ]]; then + if [[ ! -x "/etc/init.d/php-${php_version_of_working_url}-fpm" ]] ; then + fatal "Cannot find systemd service or init-script for PHP FPM (v$php_version_of_working_url)!\n Check Variable \"php_version_of_working_url\"." + fi + fi + fi + + if [[ -z "$(trim $is_working_url)" ]];then + fatal "Check respond of a certain website is requested, but no url (is_working_url) is given!" + fi + if [[ -z "$(trim $check_string)" ]];then + fatal "Check respond of a certain website is requested, but no check string is given!" + fi + if $include_cleanup_function && [[ -z $(trim $cleanup_function) ]]; then + fatal "include_cleanup_function is enabled, but no cleanup functions are given!\n Check Variable \"cleanup_function\"." + else + cleanup_functions_file=$(mktemp) + printf '%s' "$cleanup_function" > $cleanup_functions_file + fi +fi + +## - #--------------------------------------- +## - #----------------------------- +## - # Job is already running? +## - #----------------------------- +## - #--------------------------------------- +## - +## - ## - If job already runs, stop execution.. +## - ## - +## - if mkdir "$LOCK_DIR" 2> /dev/null ; then +## - +## - ## - Remove lockdir when the script finishes, or when it receives a signal +## - trap 'rm -rf "$LOCK_DIR"' 0 2 15 +## - +## - else +## - +## - datum=`date +"%d.%m.%Y"` +## - +## - msg="[ Error ]: A previos instance of \"`basename $0`\" seems already be running.\n\tExiting now.." +## - +## - echo "" +## - echo "[ Error ]: A previos instance of that script \"`basename $0`\" seems already be running." +## - echo "" +## - echo -e "\tExiting now.." +## - echo "" +## - +## - for _to_address in $to_addresses ; do +## - echo -e "To:${_to_address}\n${content_type}\nSubject:Error cronjob `basename $0` -- $datum\n${msg}\n" \ +## - | sendmail -F "Error `hostname -f`" -f $from_address $_to_address +## - done +## - +## - exit 1 +## - +## - fi + +## - Remove lockdir when the script finishes, or when it receives a signal +trap 'rm -rf "$LOCK_DIR"' 0 2 15 + + +#--------------------------------------- +#----------------------------- +# Start/Stop functions +#----------------------------- +#--------------------------------------- + +stop_apache() { + + send_msg=$1 + send_msg=${send_msg:=true} + + ## - Stop Apache Webservice + ## - + $APACHE_INIT_SCRIPT stop > /dev/null 2>&1 + + sleep 12 + + declare -i i=0 + PIDS=$(ps aux | grep "$HTTPD" | grep -v grep | awk '{print$2}') + + while [ "X$PIDS" != "X" ]; do + + if [ $i -eq 0 ]; then + echo "" >> $LOCK_DIR/extra_msg.txt + echo -e "\t[ Error ]: Stopping Apache Webservice failed !!!" >> $LOCK_DIR/extra_msg.txt + echo "" >> $LOCK_DIR/extra_msg.txt + echo -e "\t[ Info ]; Going to kill remaining httpd-processes.." >> $LOCK_DIR/extra_msg.txt + echo "" >> $LOCK_DIR/extra_msg.txt + + error "Stopping Apache Webservice failed !!!" + info "Going to kill remaining httpd-processes.." + fi + + if [ $i -gt 10 ]; then + + echo "" >> $LOCK_DIR/extra_msg.txt + echo -e "\t[ Error ]: Killing remaining httpd-processes failed !!!" >> $LOCK_DIR/extra_msg.txt + echo "" >> $LOCK_DIR/extra_msg.txt + + error "Killing remaining httpd-processes failed !!!" + + if $send_msg ; then + + subject="[ Error ]: Stopping Apache Webserver on `hostname -f` failed -- $datum" + msg="\n\n`cat $LOCK_DIR/extra_msg.txt`\n" + + for _to_address in $to_addresses ; do + echo -e "To:${_to_address}\n${content_type}\nSubject:$subject\n\n${msg}\n" \ + | /usr/sbin/sendmail -F "Webservice Monitor" -f $from_address $_to_address + done + + fi + + break + fi + + for _PID in $PIDS ; do + kill -9 $_PID > /dev/null 2>&1 + done + sleep 2 + PIDS=$(ps aux | grep "$HTTPD" | grep -v grep | awk '{print$2}') + i=i+1 + done + + +} + +stop_mysql() { + + send_msg=$1 + send_msg=${send_msg:=true} + + ## - Stop MySQL Service + ## - + $MYSQL_INIT_SCRIPT stop > /dev/null 2>&1 + + _msg=false + + sleep 2 + + declare -i i=0 + PIDS=$(ps aux | grep "$MYSQLD_SAFE" | grep -v grep | awk '{print$2}') + while [ "X$PIDS" != "X" ]; do + + if [ $i -eq 0 ]; then + echo "" >> $LOCK_DIR/extra_msg.txt + echo -e "\t[ Error ]: Stopping MySQL Service failed !!!" >> $LOCK_DIR/extra_msg.txt + echo "" >> $LOCK_DIR/extra_msg.txt + echo -e "\t[ Info ]; Going to kill MySQL Processes (mysqld_safe).." >> $LOCK_DIR/extra_msg.txt + echo "" >> $LOCK_DIR/extra_msg.txt + + error "Stopping MySQL Service failed !!!" + info "Going to kill MySQL Processes (mysqld_safe).." + fi + + if [ $i -gt 20 ]; then + + echo "" >> $LOCK_DIR/extra_msg.txt + echo -e "\t[ Error ]: Killing MySQL Processes (mysqld_safe) failed !!!" >> $LOCK_DIR/extra_msg.txt + echo "" >> $LOCK_DIR/extra_msg.txt + + error "Killing MySQL Processes (mysqld_safe) failed !!!" + + _msg=true + + break + fi + + for _PID in $PIDS ; do + kill -9 $_PID > /dev/null 2>&1 + done + + sleep 2 + + PIDS=$(ps aux | grep "$MYSQLD_SAFE" | grep -v grep | awk '{print$2}') + i=i+1 + done + + PIDS=$(ps aux | grep "$MYSQLD" | grep -v grep | awk '{print$2}') + declare -i i=0 + while [ "X$PIDS" != "X" ]; do + + if [ $i -eq 0 ]; then + echo "" >> $LOCK_DIR/extra_msg.txt + echo -e "\t[ Error ]: MySQL Processes (mysqld) still exists !!!" >> $LOCK_DIR/extra_msg.txt + echo "" >> $LOCK_DIR/extra_msg.txt + echo -e "\t[ Info ]; Going to kill remaining MySQL Processes (mysqld).." >> $LOCK_DIR/extra_msg.txt + echo "" >> $LOCK_DIR/extra_msg.txt + + error "MySQL Processes (mysqld) still exists !!!" + info "Going to kill remaining MySQL Processes (mysqld).." + fi + + if [ $i -gt 20 ]; then + + echo "" >> $LOCK_DIR/extra_msg.txt + echo -e "\t[ Error ]: Killing remaining MySQL Processes (mysqld) failed !!!" >> $LOCK_DIR/extra_msg.txt + echo "" >> $LOCK_DIR/extra_msg.txt + + error "Killing remaining MySQL Processes (mysqld) failed !!!" + + _msg=true + + break + fi + + for _PID in $PIDS ; do + kill -9 $_PID > /dev/null 2>&1 + done + sleep 2 + PIDS=i$(ps aux | grep "$MYSQLD" | grep -v grep | awk '{print$2}') + i=i+1 + + done + + if $send_msg && $_msg ; then + subject="[ Error ]: Stopping MySQL Engine on `hostname -f` failed -- $datum" + msg="\n\n`cat $LOCK_DIR/extra_msg.txt`\n" + + for _to_address in $to_addresses ; do + echo -e "To:${_to_address}\n${content_type}\nSubject:$subject\n\n${msg}\n" \ + | /usr/sbin/sendmail -F "Webservice Monitor" -f $from_address $_to_address + done + fi +} + +stop_php_fpm() { + + __version=$1 + + send_msg=$2 + send_msg=${send_msg:=true} + + _msg=false + + ## - Stop PHP-FPM Service + ## - + if [[ "${start_stop_method_php_fpm["$__version"]}" = "systemd_service" ]]; then + systemctl stop ${start_stop_file_php_fpm[$__version]} > /dev/null 2>&1 + elif [[ "${start_stop_method_php_fpm[$__version]}" = "init_script" ]]; then + ${start_stop_file_php_fpm[$__version]} stop > /dev/null 2>&1 + fi + + sleep 2 + + declare -i i=0 + + PIDS=$(ps aux | grep "php-fpm: " | grep " master " | grep -E "[-|\(]${php_fpm_pid_search_path[$__version]}" | grep -v grep | awk '{print$2}') + + while [ "X$PIDS" != "X" ]; do + + if [ $i -eq 0 ]; then + echo "" >> $LOCK_DIR/extra_msg.txt + echo -e "\t[ Error ]: Stopping PHP FPM v$__version failed !!!" >> $LOCK_DIR/extra_msg.txt + echo "" >> $LOCK_DIR/extra_msg.txt + echo -e "\t[ Info ]; Going to kill PHP FPM v$__version Processes (master).." >> $LOCK_DIR/extra_msg.txt + echo "" >> $LOCK_DIR/extra_msg.txt + + error "Stopping PHP FPM v$__version failed !!!" + info "Going to kill PHP FPM v$__version Processes (master).." + fi + + if [ $i -gt 20 ]; then + + echo "" >> $LOCK_DIR/extra_msg.txt + echo -e "\t[ Error ]: Killing PHP FPM Processes v$__version (master) failed !!!" >> $LOCK_DIR/extra_msg.txt + echo "" >> $LOCK_DIR/extra_msg.txt + + error "Killing PHP FPM Processes v$__version (master) failed !!!" + + _msg=true + + break + fi + + for _PID in $PIDS ; do + kill -9 $_PID > /dev/null 2>&1 + done + + # - Remove all sockets php-fpm sockets from thid version + # - + rm -f /tmp/php-${__version}-fpm*.sock + + sleep 2 + + PIDS=$(ps aux | grep "php-fpm: " | grep " master " | grep -E "[-|\(]${php_fpm_pid_search_path[$__version]}" | grep -v grep | awk '{print$2}') + + i=i+1 + done + + declare -i i=0 + + PIDS=$(ps aux | grep "php-fpm: " | grep -E "(pool|master)" | grep -E "\-${php_fpm_pid_search_path[$__version]}" | grep -v grep | awk '{print$2}') + + while [ "X$PIDS" != "X" ]; do + + if [ $i -eq 0 ]; then + echo "" >> $LOCK_DIR/extra_msg.txt + echo -e "\t[ Error ]: PHP FPM Processes v$__version (pool) still exists !!!" >> $LOCK_DIR/extra_msg.txt + echo "" >> $LOCK_DIR/extra_msg.txt + echo -e "\t[ Info ]; Going to kill remaining PHP FPM v$__version Processes (pool).." >> $LOCK_DIR/extra_msg.txt + echo "" >> $LOCK_DIR/extra_msg.txt + + error "PHP FPM Processes v$__version (pool) still exists !!!" + info "Going to kill remaining PHP FPM Processes v$__version (pool).." + fi + + if [ $i -gt 20 ]; then + + echo "" >> $LOCK_DIR/extra_msg.txt + echo -e "\t[ Error ]: Killing remaining PHP FPM Processes v$__version (pool) failed !!!" >> $LOCK_DIR/extra_msg.txt + echo "" >> $LOCK_DIR/extra_msg.txt + + error "Killing remaining PHP FPM Processes v$__version (pool) failed !!!" + + _msg=true + + break + fi + + for _PID in $PIDS ; do + kill -9 $_PID > /dev/null 2>&1 + done + sleep 2 + + PIDS=$(ps aux | grep "php-fpm: " | grep -E "(pool|master)" | grep -E "\-${php_fpm_pid_search_path[$__version]}" | grep -v grep | awk '{print$2}') + + i=i+1 + + done + + if $send_msg && $_msg ; then + subject="[ Error ]: Stopping PHP FPM v$__version on `hostname -f` failed -- $datum" + msg="\n\n`cat $LOCK_DIR/extra_msg.txt`\n" + + for _to_address in $to_addresses ; do + echo -e "To:${_to_address}\n${content_type}\nSubject:$subject\n\n${msg}\n" \ + | /usr/sbin/sendmail -F "Webservice Monitor" -f $from_address $_to_address + done + fi + +} + +start_mysql() { + + send_msg=$1 + send_msg=${send_msg:=true} + + $MYSQL_INIT_SCRIPT start > /dev/null 2>&1 + sleep 2 + + NEWPID=$(ps aux | grep "$MYSQLD_SAFE" | grep -v grep | grep root | awk '{print$2}') + + if [ "X${NEWPID}X" = "XX" ]; then + + echo "" >> $LOCK_DIR/extra_msg.txt + echo -e "\t[ Error ]: I have started the MySQL server, but the services is not running !!!" >> $LOCK_DIR/extra_msg.txt + echo "" >> $LOCK_DIR/extra_msg.txt + + error "I have started the MySQL server, but the service is not running !!!" + + datum=`date +"%d.%m.%Y %H:%Mh"` + + subject="[ Error ]: MySQL Service on `hostname -f` NOT RUNNING -- $datum" + msg="\n\n`cat $LOCK_DIR/extra_msg.txt`\n" + + for _to_address in $to_addresses ; do + echo -e "To:${_to_address}\n${content_type}\nSubject:$subject\n\n${msg}\n" \ + | /usr/sbin/sendmail -F "Webservice Monitor" -f $from_address $_to_address + done + + else + + echo "" >> $LOCK_DIR/extra_msg.txt + echo -e "\tI have started MySQL service. The new PID is $NEWPID" >> $LOCK_DIR/extra_msg.txt + echo "" >> $LOCK_DIR/extra_msg.txt + + ok "I have started MySQL service. The new PID is $NEWPID" + fi + + if $send_msg ; then + datum=`date +"%d.%m.%Y %H:%Mh"` + subject="Restarting MySQL on `hostname -f` invoked -- $datum" + msg=`cat $LOCK_DIR/extra_msg.txt` + + for _to_address in $to_addresses ; do + echo -e "To:${_to_address}\n${content_type}\nSubject:$subject\n\n${msg}\n" \ + | /usr/sbin/sendmail -F "Webservice Monitor" -f $from_address $_to_address + done + fi +} + +start_apache() { + + send_msg=$1 + send_msg=${send_msg:=true} + + $APACHE_INIT_SCRIPT start > /dev/null 2>&1 + sleep 2 + + NEWPID=$(ps aux | grep "$HTTPD" | grep -v grep | grep root | awk '{print$2}') + + if [ "X${NEWPID}X" = "XX" ]; then + + echo "" >> $LOCK_DIR/extra_msg.txt + echo -e "\t[ Error ]: I have tarted the webserver, but service is not running !!!" >> $LOCK_DIR/extra_msg.txt + echo "" >> $LOCK_DIR/extra_msg.txt + + error "I have started the webserver, but service is not running !!!" + + datum=`date +"%d.%m.%Y %H:%Mh"` + + subject="[ Error ]: Webservice on `hostname -f` NOT RUNNING -- $datum" + msg="\n\n`cat $LOCK_DIR/extra_msg.txt`\n" + + for _to_address in $to_addresses ; do + echo -e "To:${_to_address}\n${content_type}\nSubject:$subject\n\n${msg}\n" \ + | /usr/sbin/sendmail -F "Webservice Monitor" -f $from_address $_to_address + done + + else + + echo "" >> $LOCK_DIR/extra_msg.txt + echo -e "\tI have started the webserver. the new PID is $NEWPID" >> $LOCK_DIR/extra_msg.txt + echo "" >> $LOCK_DIR/extra_msg.txt + + ok "I have started the webserver. the new PID is $NEWPID" + + fi + + if $send_msg ; then + datum=`date +"%d.%m.%Y %H:%Mh"` + subject="Restarting Apache on `hostname -f` invoked -- $datum" + msg=`cat $LOCK_DIR/extra_msg.txt` + + for _to_address in $to_addresses ; do + echo -e "To:${_to_address}\n${content_type}\nSubject:$subject\n\n${msg}\n" \ + | /usr/sbin/sendmail -F "Webservice Monitor" -f $from_address $_to_address + done + fi +} + +start_php_fpm() { + + __version=$1 + + send_msg=$2 + send_msg=${send_msg:=true} + + start_php_fpm_failed=false + + if [[ "${start_stop_method_php_fpm[$__version]}" = "systemd_service" ]]; then + systemctl restart ${start_stop_file_php_fpm[$__version]} > /dev/null 2>&1 + elif [[ "${start_stop_method_php_fpm["$__version"]}" = "init_script" ]]; then + ${start_stop_file_php_fpm["$__version"]} start > /dev/null 2>&1 + else + error "Neither start/stop script nor systemd service file for PHP FPM v$__version found!" + + echo "" >> $LOCK_DIR/extra_msg.txt + echo -e "\t[ Error ]: Neither start/stop script nor systemd service file for PHP FPM v$__version found !!!" >> $LOCK_DIR/extra_msg.txt + echo "" >> $LOCK_DIR/extra_msg.txt + + start_php_fpm_failed=true + fi + sleep 2 + + NEWPID=`ps aux | grep "php-fpm: " | grep " master " | grep -E "[-|\(]${php_fpm_pid_search_path[$__version]}" | grep -v grep | awk '{print$2}'` + + + if [ "X${NEWPID}X" = "XX" ]; then + + if ! $start_php_fpm_failed ; then + echo "" >> $LOCK_DIR/extra_msg.txt + echo -e "\t[ Error ]: I have started PHP FPM v$__version, but service is not running !!!" >> $LOCK_DIR/extra_msg.txt + echo "" >> $LOCK_DIR/extra_msg.txt + + error "I have started PHP FPM v$__version, but service is not running !!!" + fi + + datum=`date +"%d.%m.%Y %H:%Mh"` + + subject="[ Error ]: PHP-FPM v$__version engine on `hostname -f` NOT RUNNING -- $datum" + msg="\n\n`cat $LOCK_DIR/extra_msg.txt`\n" + + for _to_address in $to_addresses ; do + echo -e "To:${_to_address}\n${content_type}\nSubject:$subject\n\n${msg}\n" \ + | /usr/sbin/sendmail -F "Webservice Monitor" -f $from_address $_to_address + done + + else + + echo "" >> $LOCK_DIR/extra_msg.txt + echo -e "\tI have started the PHP-FPM v$__version engine. the new PID is $NEWPID" >> $LOCK_DIR/extra_msg.txt + echo "" >> $LOCK_DIR/extra_msg.txt + + ok "I have started the PHP-FPM v$__version engine. the new PID is $NEWPID" + fi + + if $send_msg ; then + datum=`date +"%d.%m.%Y %H:%Mh"` + subject="Restarting PHP FPM v$__version engine on `hostname -f` invoked -- $datum" + msg=`cat $LOCK_DIR/extra_msg.txt` + + for _to_address in $to_addresses ; do + echo -e "To:${_to_address}\n${content_type}\nSubject:$subject\n\n${msg}\n" \ + | /usr/sbin/sendmail -F "Webservice Monitor" -f $from_address $_to_address + done + fi +} + +restart_apache() { + + ## - Stop Apache Webservice + ## - + stop_apache + + sleep 2 + + ## - Start Apache Webservice + ## - + start_apache +} + +restart_mysql(){ + + ## - Stop MySQL Servive + ## - + stop_mysql + + sleep 2 + + ## - Start MySQL Service + ## - + start_mysql +} + +restart_php_fpm(){ + + ## - Stop MySQL Servive + ## - + stop_php_fpm $1 + + sleep 2 + + ## - Start MySQL Service + ## - + start_php_fpm $1 +} + +graceful_restart_php_fpm() { + + __version=$1 + + send_msg=$2 + send_msg=${send_msg:=true} + + start_php_fpm_failed=false + + if [[ "$start_stop_method_php_fpm" = "systemd_service" ]]; then + systemctl restart $start_stop_method_php_fpm["$__version"] > /dev/null 2>&1 + elif [[ "$start_stop_method_php_fpm" = "init_script" ]]; then + $start_stop_method_php_fpm["$__version"] restart > /dev/null 2>&1 + else + error "Neither start/stop script nor systemd service file for PHP FPM v$__version found!" + + echo "" >> $LOCK_DIR/extra_msg.txt + echo -e "\t[ Error ]: Neither start/stop script nor systemd service file for PHP FPM v$__version found !!!" >> $LOCK_DIR/extra_msg.txt + echo "" >> $LOCK_DIR/extra_msg.txt + + start_php_fpm_failed=true + fi + sleep 2 + + NEWPID=`ps aux | grep "php-fpm: " | grep " master " | grep "[-|\(]${php_fpm_pid_search_path[\"$__version\"]}" | grep -v grep | awk '{print$2}'` + + if [ "X${NEWPID}X" = "XX" ]; then + + if ! $start_php_fpm_failed ; then + echo "" >> $LOCK_DIR/extra_msg.txt + echo -e "\t[ Error ]: I have restarted PHP-FPM, but service is not running !!!" >> $LOCK_DIR/extra_msg.txt + echo "" >> $LOCK_DIR/extra_msg.txt + + error "I have restarted PHP-FPM, but service is not running !!!" + fi + + datum=`date +"%d.%m.%Y %H:%Mh"` + + subject="[ Error ]: PHP-FPM engine on `hostname -f` NOT RUNNING -- $datum" + msg="\n\n`cat $LOCK_DIR/extra_msg.txt`\n" + + for _to_address in $to_addresses ; do + echo -e "To:${_to_address}\n${content_type}\nSubject:$subject\n\n${msg}\n" \ + | /usr/sbin/sendmail -F "Webservice Monitor" -f $from_address $_to_address + done + + else + + echo "" >> $LOCK_DIR/extra_msg.txt + echo -e "\tI have restarted the PHP-FPM engine. the new PID is $NEWPID" >> $LOCK_DIR/extra_msg.txt + echo "" >> $LOCK_DIR/extra_msg.txt + + ok "I have restarted the PHP-FPM engine. the new PID is $NEWPID" + fi + + if $send_msg ; then + datum=`date +"%d.%m.%Y %H:%Mh"` + subject="Restarting PHP-FPM engine on `hostname -f` invoked -- $datum" + msg=`cat $LOCK_DIR/extra_msg.txt` + + for _to_address in $to_addresses ; do + echo -e "To:${_to_address}\n${content_type}\nSubject:$subject\n\n${msg}\n" \ + | /usr/sbin/sendmail -F "Webservice Monitor" -f $from_address $_to_address + done + fi +} + + +#--------------------------------------- +#----------------------------- +# Checking Load.. +#----------------------------- +#--------------------------------------- + +if $check_load ; then + + #echo "" > $LOCK_DIR/extra_msg.txt + + if $LOGGING ; then + echo -e "\nChecking Load on \"`hostname -f`\".." + fi + + warn_msg=${LOCK_DIR}/warn.msg + echo "" > $warn_msg + + ## - Number of processors + ## - + declare -i number_processors=`grep -E "processor\s+:\s+" /proc/cpuinfo -c` + + if [ -z "$number_processors" -o $number_processors -eq 0 ]; then + fatal "Detecting number of prozessors ($number_processors) failed!" + fi + + ## - Thresholds + ## - + if [ $number_processors -gt 8 ]; then + + #threshod_1_min=`echo "scale=2; $number_processors / 1.95" | bc` + #threshod_5_min=`echo "scale=2; $number_processors / 2.95" | bc` + #threshod_15_min=`echo "scale=2; $number_processors / 3.45" | bc` + threshod_1_min=`echo "scale=2; $number_processors / 1.45" | bc` + threshod_5_min=`echo "scale=2; $number_processors / 1.95" | bc` + threshod_15_min=`echo "scale=2; $number_processors / 2.35" | bc` + else + threshod_1_min=`echo "scale=2; $number_processors / 1.00" | bc` + threshod_5_min=`echo "scale=2; $number_processors / 1.45" | bc` + threshod_15_min=`echo "scale=2; $number_processors / 1.95" | bc` + fi + + ## - Determin load avarage values + ## - + + ## - Last minute + load_avarage_1m=`cat /proc/loadavg | cut -d ' ' -f1` + ## - Last 5 minutes + load_avarage_5m=`cat /proc/loadavg | cut -d ' ' -f2` + ## -Last 15 minutes + load_avarage_15m=`cat /proc/loadavg | cut -d ' ' -f3` + + ## - Evaluate + ## - + result_1m=$(echo "scale=2; $load_avarage_1m > $threshod_1_min" | bc) + result_5m=$(echo "scale=2; $load_avarage_5m > $threshod_5_min" | bc) + result_15m=$(echo "scale=2; $load_avarage_15m > $threshod_15_min" | bc) + + #if [ "$result_1m" = "1" -o "$result_5m" = "1" -o "$result_15m" = "1" ] ; then + if [ "$result_1m" = "1" ] ; then + subject="[ Warning ]: High load on `hostname -f`" + + cat < $warn_msg + + High load on *`hostname -f`* + + + Server Name...................: `hostname -f` + + Date..........................: `date +%d.%m.%Y` + Time..........................: `date +%H:%M` h + + Load avarage last minute......: $load_avarage_1m + Load avarage last 5 minutes...: $load_avarage_5m + Load avarage last 15 minutes..: $load_avarage_15m + +EOF + + + for _to_address in $to_addresses ; do + echo -e "To: ${_to_address}\nSubject: $subject\n\n`cat $warn_msg`" \ + | sendmail -F "Load Monitor" -f $from_address $_to_address + done + + echo "" > $warn_msg + + fi + + _load_avarage_1m=$(echo "scale=0; $load_avarage_1m/1" | bc -l) + _load_avarage_5m=$(echo "scale=0; $load_avarage_5m/1" | bc -l) + if [ $_load_avarage_1m -gt 200 -a $_load_avarage_5m -gt 200 ]; then + cat<> $warn_msg + Server Name...................: `hostname -f` + + Date..........................: `date +%d.%m.%Y` + Time..........................: `date +%H:%M` h + + Load avarage last minute......: $load_avarage_1m + Load avarage last 5 minutes...: $load_avarage_5m + Load avarage last 15 minutes..: $load_avarage_15m + +EOF + + stop_apache + stop_mysql + + sleep 60 + + start_mysql + start_apache + + else + if $LOGGING ; then + ok "Load on \"`hostname -f`\" seems to be fine.." + cat< $LOCK_DIR/extra_msg.txt + + if $LOGGING ; then + echo -e "\nChecking Mysql databse service on \"`hostname -f`\".." + fi + + PID=`ps aux | grep "$MYSQLD_SAFE" | grep -v grep | awk '{print$2}'` + + if [ "X${PID}X" = "XX" ];then + + echo -e "\nMySQL is not running! - Try to start it.." >> $LOCK_DIR/extra_msg.txt + echo -e "=========================================" >> $LOCK_DIR/extra_msg.txt + + error "MySQL is not running! - Try to start it.." + + restart_mysql + + else + if [ -z "`$MYADMIN $mysql_credential_args ping 2>/dev/null`" ]; then + + echo -e "\nMySQL seems to be running, but NOT responding! - Try to restart service.." >> $LOCK_DIR/extra_msg.txt + echo -e "=======================================================================" >> $LOCK_DIR/extra_msg.txt + + error "MySQL seems to be running, but NOT responding! - Try to restart service.." + + restart_mysql + + else + + if $LOGGING ; then + ok "MySQL database service on \"`hostname -f`\" seems to be running.." + fi + fi + fi +fi + + +#--------------------------------------- +#----------------------------- +# Check Apache webservice.. +#----------------------------- +#--------------------------------------- + +if $check_apache ; then + + echo "" > $LOCK_DIR/extra_msg.txt + + if $LOGGING ; then + echo -e "\nChecking Apache webservice on \"`hostname -f`\".." + fi + + #_PID_FILE=`$HTTPD -t -D DUMP_RUN_CFG | grep -i -e "^PidFile: " | awk '{print$2}' \ + # | sed -e "s/^[\"']//" | sed -e "s/[\"']$//"` + #PID=`cat $_PID_FILE` + #if [ "X`ps ax | grep \"$HTTPD \" | grep -e \"^$PID \"`" = "X" ]; then + + PID=`ps aux | grep "$HTTPD " | grep -e "^root" | grep -v grep | awk '{print$2}'` + if [ "X${PID}X" = "XX" ];then + + + echo -e "\nApache Webservice is not running! - Try to start it.." > $LOCK_DIR/extra_msg.txt + echo -e "=====================================================" >> $LOCK_DIR/extra_msg.txt + + error "Apache webservice is not running! - Try to start it.." + + restart_apache + + else + + if ! $(curl -Is -m 10 http://$curl_check_host | head -n 1 | grep "200 OK" > /dev/null) ; then + echo -e "\nApache Webservice seems to be running, but is NOT responding! - Try to restart service.." >> $LOCK_DIR/extra_msg.txt + echo -e "========================================================================================" >> $LOCK_DIR/extra_msg.txt + + error "Apache Webservice seems to be running, but is NOT responding! - Try to restart service.." + + restart_apache + + else + + if $LOGGING ; then + ok "Apache Webserver on \"`hostname -f`\" seems to be running.." + fi + fi + fi +fi + + +#--------------------------------------- +#----------------------------- +# Check PHP-FPM engine.. +#----------------------------- +#--------------------------------------- + +if $check_php_fpm ; then + + for _version in $php_versions ; do + + echo "" > $LOCK_DIR/extra_msg.txt + + if $LOGGING ; then + echo -e "\nChecking PHP-FPM v$_version (FastCGI Process Manager) on \"`hostname -f`\".." + fi + PID=$(ps aux | grep "php-fpm: " | grep "master" | grep -E "[-|\(]${php_fpm_pid_search_path[$_version]}" | grep -v grep | awk '{print$2}') + + if [ "X${PID}X" = "XX" ];then + + echo -e "\nPHP-FPM v$_version is not running! - Try to start the service.." > $LOCK_DIR/extra_msg.txt + echo -e "====================================================" >> $LOCK_DIR/extra_msg.txt + + error "PHP-FPM v$_version is not running! - Try to start the service.." + + restart_php_fpm $_version + + elif [[ -n ${php_fpm_ping_path[$_version]} ]] ; then + + if ! $(curl -Is -m 10 http://${curl_check_host}/${php_fpm_ping_path[$_version]} | head -n 1 | grep "200 OK" > /dev/null) ; then + + echo -e "\nPHP-FPM v$_version seems to be running, but is NOT responding! - Try to restart service.." >> $LOCK_DIR/extra_msg.txt + echo -e "====================================================================================" >> $LOCK_DIR/extra_msg.txt + + error "PHP-FPM v$_version seems to be running, but is NOT responding! - Try to restart service.." + + restart_php_fpm $_version + + else + if $LOGGING ; then + ok "PHP-FPM v$_version engine on \"`hostname -f`\" seems to be running.." + fi + fi + fi + done +fi + + +#--------------------------------------- +#----------------------------- +# Check website.. +#----------------------------- +#--------------------------------------- + +if $check_website ; then + + send_msg=false + + if $LOGGING ; then + echo -e "\nDo some extra checks.." + fi + + declare -i rval=$(curl -s $is_working_url 2> /dev/null | grep "$check_string" | wc -l) + + if [[ $rval -lt 1 ]] ; then + + echo -e "The website does not respond as expected!" > $LOCK_DIR/extra_msg.txt + echo -e "=========================================" >> $LOCK_DIR/extra_msg.txt + + error "The website $is_working_url does not respond as expected." + + # - Stop services.. + # - + stop_apache false + + if [[ "$php_version_of_working_url" != "None" ]]; then + if [[ -n "$php_version_of_working_url" ]];then + stop_php_fpm $php_version_of_working_url false + else + for _version in $php_versions ; do + + stop_php_fpm $_version false + + done + fi + fi + + # - Clean up cache/session files.. + # - + if $include_cleanup_function ; then + + info "Execute configured cleanup tasks.." + echo -e "\n\t[ Info ]: Execute configured cleanup tasks .." >> $LOCK_DIR/extra_msg.txt + + source $cleanup_functions_file + + fi + + # - Start services + # - + if [[ "$php_version_of_working_url" != "None" ]]; then + if [[ -n "$php_version_of_working_url" ]];then + start_php_fpm $php_version_of_working_url false + else + for _version in $php_versions ; do + + start_php_fpm $_version false + + done + fi + fi + + start_apache false + + info "Sleeping for 30 seconds.." + sleep 30 + + declare -i rval=$(curl -s $is_working_url 2> /dev/null | grep "$check_string" | wc -l) + if [[ $rval -lt 1 ]] ; then + + echo -e "\n\t[ Error ]: URL $is_working_url is still NOT responding as expected !!!" >> $LOCK_DIR/extra_msg.txt + + error "URL $is_working_url is still NOT responding as expected !!!" + + # - Only send extra message, if cleaning up and restarting wasn't successfully + # - + send_msg=true + + else + + ok "Cleaning up and restarting services was successfully." + + fi + + else + + if $LOGGING ; then + ok "URL $is_working_url is responding as expected." + fi + + fi + + if $send_msg ; then + + subject="[ Error ]: Shop System at `hostname -f` failed -- $datum" + msg="\n\n`cat $LOCK_DIR/extra_msg.txt`\n" + + if [[ -n "$extra_alert_address" ]]; then + to_addresses="$to_addresses $extra_alert_address" + fi + + for _to_address in $to_addresses ; do + echo -e "To:${_to_address}\n${content_type}\nSubject:$subject\n\n${msg}\n" \ + | /usr/sbin/sendmail -F "Webservice Monitor" -f $from_address $_to_address + done + + fi + + rm -rf $cleanup_functions_file + +fi + +exit 0