monitoring/check_local_webservice.sh

#!/usr/bin/env bash


script_name="$(basename $(realpath $0))"
working_dir="$(dirname $(realpath $0))"
conf_file="${working_dir}/conf/${script_name%%.*}.conf"

LOCK_DIR="/tmp/check_local_webservice.LOCK"

RESTART_CHECK_FILE="/tmp/${script_name%%.*}.NEED-RESTART"

declare -a alert_email_arr

# -------------
# --- Read Configurations from $conf_file
# -------------

if [[ ! -f "$conf_file" ]]; then
   echo ""
   echo -e "   [ Fatal ] Configuration file '$(basename ${conf_file})' not found!"
   echo ""
   echo -e "\tScript terminated.."
   echo ""
   exit 1
else
   source "$conf_file"
fi

for _email in $alert_email_addresses ; do
   alert_email_arr+=("$_email")
done

[[ -n "$sender_address" ]] || sender_address="check_local_webservice@$(hostname -f)"
[[ -n "$content_type" ]] || content_type='Content-Type: text/plain;\n charset="utf-8"'

[[ -n "$TIME_OUT" ]] || TIME_OUT=30
TIME_OUT_MAX="$(expr ${TIME_OUT} + 5)"


# -------------
# - Job is already running?
# -------------

# - If job already runs, stop execution..
# -
if mkdir "$LOCK_DIR" 2> /dev/null ; then

   ## - Remove lockdir when the script finishes, or when it receives a signal
   trap clean_up SIGHUP SIGINT SIGTERM

else

   datum="$(date +"%d.%m.%Y %H:%M")"

   msg="[ Error ]: A previos instance of \"`basename $0`\" seems already be running.\n\n           Exiting now.."

   echo ""
   echo "[ Error ]: A previos instance of that script \"`basename $0`\" seems already be running."
   echo ""
   echo -e "           Exiting now.."
   echo ""

   for _email in ${alert_email_arr[@]} ; do
      echo -e "To:${_email}\n${content_type}\nSubject:Error cronjob `basename $0` -- $datum\n${msg}\n" \
         | sendmail -F "Error `hostname -f`" -f $sender_address $_email
   done

   exit 1

fi


# -------------
# --- Some functions
# -------------

clean_up() {

   # Perform program exit housekeeping
   rm -rf "$LOCK_DIR"
   if $LOGGING ; then
      echo ""
   fi
   exit $1
}

echononl(){
   if $terminal && $LOGGING ; then
      echo X\\c > /tmp/shprompt$$
      if [ `wc -c /tmp/shprompt$$ | awk '{print $1}'` -eq 1 ]; then
         echo -e -n "$*\\c" 1>&2
      else
         echo -e -n "$*" 1>&2
      fi
      rm /tmp/shprompt$$
   fi
}

fatal(){
   echo ""
   if $terminal ; then
      echo -e "[ \033[31m\033[1mError\033[m ]: $*"
      echo ""
      echo -e "\t\033[31m\033[1mScript was interupted\033[m!"
   else
      echo "   [ Fatal ]: $*"
      echo ""
      echo "              Script was terminated...."
   fi
   echo ""
   clean_up 1
}

error (){
   echo ""
   if $terminal ; then
      echo -e "\t[ \033[31m\033[1mError\033[m ]: $*"
   else
      echo "[ Error ]: $*"
   fi
   echo ""
}

warn (){
   echo ""
   if $terminal ; then
      echo -e "\t[ \033[33m\033[1mWarning\033[m ]: $*"
   else
      echo "[ Warning ]: $*"
   fi
   echo ""
}

echo_done() {
   if $terminal && $LOGGING ; then
      echo -e "\033[75G[ \033[32mdone\033[m ]"
   else
      if $LOGGING ; then
         echo " [ done ]"
      fi
   fi
}
echo_ok() {
   if $terminal && $LOGGING ; then
      echo -e "\033[75G[ \033[32mok\033[m ]"
   else
      if $LOGGING ; then
         echo " [ ok ]"
      fi
   fi
}
echo_failed(){
   if $terminal && $LOGGING ; then
      echo -e "\033[75G[ \033[1;31mfailed\033[m ]"
   else
      if $LOGGING ; then
         echo " [ failed ]"
      fi
   fi
}
echo_skipped() {
   if $terminal && $LOGGING ; then
      echo -e "\033[75G[ \033[33m\033[1mskipped\033[m ]"
   else
      if $LOGGING ; then
         echo " [ skipped ]"
      fi
   fi
}

reboot_system() {

   content_type='Content-Type: text/plain;\n charset="utf-8"'
   datum="$(date +"%d.%m.%Y")"
   from_address="root@$(hostname --long)"
 
 
   for _email in ${alert_email_arr[@]} ; do

      echo -e "To:${_email}\n${content_type}\nSubject:[Fatal: Local Webservice NOT rsponding] - Reboot System\n$*" \
         | sendmail -F "Error `hostname -f`" -f $sender_address $_email
   done

   sleep 10
   /sbin/reboot -f

}





# -------------
# --- Check some prerequisites
# -------------

# - Running in a terminal?
# -
if [[ -t 1 ]] ; then
   terminal=true
   LOGGING=true
else
   terminal=false
   LOGGING=false
fi


# - Systemd supported ?
# -
systemd=$(which systemd)
systemctl=$(which systemctl)

systemd_supported=false
if [[ -n "$systemd" ]] && [[ -n "$systemctl" ]] ; then
   systemd_supported=true
fi

 
if $LOGGING ; then
   echo ""
fi

if [[ -n "$LOCAL_WEBSITES_TO_CHECK" ]] ; then
   for _site in $LOCAL_WEBSITES_TO_CHECK ; do
      echononl "   Check site \033[1m$_site\033[m .."

      declare -i i=0

      while [[ $i -lt 3 ]] ; do 
         response="$(curl --max-time $TIME_OUT_MAX --connect-timeout $TIME_OUT \
                     -I -k -L --write-out %{http_code} --silent --output /dev/null $_site 2> ${LOCK_DIR}/err.msg)"

         # 200 - OK
         # 401 - Unauthorized (response of roundcube webmailer (since version 1.4)
         if [[ "$response" -eq 200 ]] || [[ "$response" -eq 401 ]]; then
            echo_ok
            if [[ -f "$RESTART_CHECK_FILE" ]] ; then
                rm -f "$RESTART_CHECK_FILE"
            fi
            clean_up 0
         fi
         sleep 2
         ((i++))
      done
      echo_failed

   done
else
   warn "No Website to check given (empty var 'LOCAL_WEBSITES_TO_CHECK')"
   clean_up 10
fi


if [[ ! -f "$RESTART_CHECK_FILE" ]]; then

   touch "$RESTART_CHECK_FILE"

   error "The local webservice seems to be down. Some Websites do not respond as expected:"
   err_msg="\n[ Error ]: The local webservice seems to be down. Some Websites do not respond as expected::\n\n"

   for _site in $LOCAL_WEBSITES_TO_CHECK ; do
      err_msg+="   $_site\n"
      if $LOGGING ; then
         echo -e "\t           \033[1m$_site\033[m"
      else
         echo "   $_site"
      fi

   done

   datum="$(date +"%d.%m.%Y %H:%M")"
   for _email in ${alert_email_arr[@]} ; do

      echo -e "To:${_email}\n${content_type}\nSubject:[Warn: Local Webservice NOT rsponding] - Do nothing for now\n$err_msg" \
         | sendmail -F "Error `hostname -f`" -f $sender_address $_email
  done

   clean_up 20
else

   rm -f "$RESTART_CHECK_FILE"
   reboot_system "The local web service is still down - a system restart is triggered."

fi

 
if $LOGGING ; then
   echo ""
fi
clean_up 0