605 lines
12 KiB
Bash
Executable File
605 lines
12 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
|
|
script_name="$(basename $(realpath $0))"
|
|
working_dir="$(dirname $(realpath $0))"
|
|
|
|
conf_file="${working_dir}/conf/${script_name%%.*}.conf"
|
|
|
|
LOCK_DIR="/tmp/$(basename $0).$$.LOCK"
|
|
log_file="${LOCK_DIR}/${script_name%%.*}.log"
|
|
|
|
PROBLEM_FOUND=false
|
|
|
|
LINK_UP=false
|
|
RESTART_CHECK_FILE="/tmp/${script_name%%.*}.NEED-RESTART"
|
|
|
|
|
|
# ----------
|
|
# Base Function(s)
|
|
# ----------
|
|
|
|
clean_up() {
|
|
|
|
# Perform program exit housekeeping
|
|
rm -rf "$LOCK_DIR"
|
|
blank_line
|
|
exit $1
|
|
}
|
|
|
|
usage() {
|
|
|
|
|
|
[[ -n "$1" ]] && error "$1"
|
|
|
|
|
|
[[ $terminal ]] && echo -e "
|
|
\033[1mUsage:\033[m
|
|
|
|
$(basename $0)
|
|
|
|
\033[1mDescription\033[m
|
|
|
|
Script checks presence of some basic network settings and do some basic connectivity
|
|
tests. If fundamental problems found, then restart machine if this i not the firts time.
|
|
If this i the first time, file \033[1m$RESTART_CHECK_FILE\033[m is created.
|
|
|
|
\033[1mOptions\033[m
|
|
|
|
No Options present
|
|
|
|
\033[1mFiles\033[m
|
|
|
|
$conf_file: Configuration file
|
|
|
|
\033[1mExample:\033[m
|
|
|
|
Chech connectivity of this machine
|
|
|
|
$(basename $0)
|
|
|
|
"
|
|
|
|
clean_up 1
|
|
|
|
}
|
|
|
|
|
|
echononl(){
|
|
if $terminal ; then
|
|
echo X\\c > /tmp/shprompt$$
|
|
if [ `wc -c /tmp/shprompt$$ | awk '{print $1}'` -eq 1 ]; then
|
|
echo -e -n " $*\\c" 1>&2
|
|
else
|
|
echo -e -n " $*" 1>&2
|
|
fi
|
|
rm /tmp/shprompt$$
|
|
fi
|
|
}
|
|
|
|
error (){
|
|
echo ""
|
|
if $terminal ; then
|
|
echo -e " [ \033[31m\033[1mError\033[m ] $*"
|
|
else
|
|
echo " [ Error ] $*"
|
|
fi
|
|
echo ""
|
|
}
|
|
|
|
warn (){
|
|
echo ""
|
|
if $terminal ; then
|
|
echo -e " [ \033[33m\033[1mWarn\033[m ] $*"
|
|
else
|
|
echo " [ Warn ] $*"
|
|
fi
|
|
echo ""
|
|
}
|
|
|
|
info (){
|
|
if $terminal ; then
|
|
echo ""
|
|
echo -e " [ \033[32m\033[1mInfo\033[m ] $*"
|
|
echo ""
|
|
fi
|
|
}
|
|
|
|
echo_ok() {
|
|
if $terminal ; then
|
|
echo -e "\033[75G[ \033[32mok\033[m ]"
|
|
fi
|
|
}
|
|
echo_failed(){
|
|
if $terminal ; then
|
|
echo -e "\033[75G[ \033[1;31mfailed\033[m ]"
|
|
fi
|
|
}
|
|
echo_warn() {
|
|
if $terminal ; then
|
|
echo -e "\033[75G[ \033[33m\033[1mwarn\033[m ]"
|
|
fi
|
|
}
|
|
echo_skipped() {
|
|
if $terminal ; then
|
|
echo -e "\033[75G[ \033[37m\033[1mskipped\033[m ]"
|
|
fi
|
|
}
|
|
|
|
trim() {
|
|
local var="$*"
|
|
var="${var#"${var%%[![:space:]]*}"}" # remove leading whitespace characters
|
|
var="${var%"${var##*[![:space:]]}"}" # remove trailing whitespace characters
|
|
echo -n "$var"
|
|
}
|
|
|
|
blank_line() {
|
|
if $terminal ; then
|
|
echo ""
|
|
fi
|
|
}
|
|
|
|
|
|
# Check if at least one interface has physical link is up
|
|
#
|
|
online() {
|
|
|
|
# Call this functionin a subshell to avoid exiting main programm
|
|
#
|
|
# Example:
|
|
# $(online) && echo -e '\033[32m\033[1mOnline\033[m' || echo -e '\033[31m\033[1mOFFLINE\033[m'
|
|
#
|
|
find /sys/class/net/ -maxdepth 1 -mindepth 1 \
|
|
! -name "*lo*" \
|
|
! -name "*tun*" \
|
|
! -name "*veth*" \
|
|
-exec sh -c 'cat "$0"/carrier 2>&1' {} \; | grep -q -E '^1$' && exit 0
|
|
|
|
# using ethtool also works:
|
|
#
|
|
#-exec sh -c 'ethtool $(basename "$0") 2>&1' {} \; | grep 'ink detected' | grep -i -q 'yes' && exit 0
|
|
|
|
}
|
|
|
|
reboot_system() {
|
|
|
|
content_type='Content-Type: text/plain;\n charset="utf-8"'
|
|
datum="$(date +"%d.%m.%Y")"
|
|
from_address="root@$(hostname --long)"
|
|
|
|
echo -e "To:${ADMIN_EMAIL}\n${content_type}\nSubject:[Connectivity Error] $datum - Reboot System\n$*" | /usr/sbin/sendmail -F "Connectivity Check" -f $from_address $ADMIN_EMAIL
|
|
|
|
sleep 10
|
|
/sbin/reboot -f
|
|
|
|
}
|
|
|
|
# ----------
|
|
# - Jobhandling
|
|
# ----------
|
|
|
|
# - Run 'clean_up' for signals SIGHUP SIGINT SIGTERM
|
|
# -
|
|
trap clean_up SIGHUP SIGINT SIGTERM
|
|
|
|
# - Create lock directory '$LOCK_DIR"
|
|
#
|
|
mkdir "$LOCK_DIR"
|
|
|
|
|
|
# ----------
|
|
# - Some checks ..
|
|
# ----------
|
|
|
|
# - Running in a terminal?
|
|
# -
|
|
if [[ -t 1 ]] ; then
|
|
terminal=true
|
|
else
|
|
terminal=false
|
|
fi
|
|
|
|
# - Print help?
|
|
# -
|
|
if [[ "$(trim $*)" = "-h" ]] || [[ "$(trim $*)" = "--help" ]] ; then
|
|
usage
|
|
fi
|
|
|
|
|
|
# ==========
|
|
# - Begin Main Script
|
|
# ==========
|
|
|
|
# ----------
|
|
# - Headline
|
|
# ----------
|
|
|
|
if $terminal ; then
|
|
echo ""
|
|
echo -e "\033[1m----------\033[m"
|
|
echo -e "\033[32m\033[1mRunning script \033[m\033[1m$script_name\033[32m .. \033[m"
|
|
echo -e "\033[1m----------\033[m"
|
|
echo ""
|
|
fi
|
|
|
|
# ----------
|
|
# Read Configurations from $conf_file
|
|
# ----------
|
|
|
|
|
|
# - Give your default values here
|
|
# -
|
|
DEFAULT_IPV6_PRESENT=true
|
|
DEFAULT_CHECK_HOSTS="www.oopen.de www.google.de www.heise.de www.debian.org"
|
|
DEFAULT_ADMIN_EMAIL="argus@oopen.de"
|
|
|
|
echononl "Load Configurations from file '$(basename "$conf_file")'."
|
|
if [[ -f "$conf_file" ]]; then
|
|
source "$conf_file"
|
|
echo_ok
|
|
else
|
|
echo_skipped
|
|
msg="No configuration file found. Script default values where loaded.."
|
|
if $terminal ; then
|
|
echo ""
|
|
echo -e " [ \033[33m\033[1mWarn\033[m ] $msg"
|
|
echo ""
|
|
fi
|
|
#if [[ -z "$msg_warn" ]]; then
|
|
# msg_warn="$msg"
|
|
#else
|
|
# msg_warn="$msg_warn
|
|
# $msg"
|
|
#fi
|
|
fi
|
|
|
|
[[ -z "$IPV6_PRESENT" ]] && IPV6_PRESENT="$DEFAULT_IPV6_PRESENT"
|
|
[[ -z "$DNS_CHECK_HOSTS" ]] && DNS_CHECK_HOSTS="$DEFAULT_CHECK_HOSTS"
|
|
[[ -z "$HTTPS_CHECK_HOSTS" ]] && HTTPS_CHECK_HOSTS="$DEFAULT_CHECK_HOSTS"
|
|
[[ -z "$ADMIN_EMAIL" ]] && ADMIN_EMAIL="$DEFAULT_ADMIN_EMAIL"
|
|
|
|
|
|
blank_line
|
|
|
|
|
|
echononl "Check if at least one interface has physical link is up"
|
|
if ! $(online) ; then
|
|
echo_failed
|
|
PROBLEM_FOUND=true
|
|
else
|
|
echo_ok
|
|
LINK_UP=true
|
|
fi
|
|
|
|
if $LINK_UP ; then
|
|
msg="Machine has at least one interface with physical link is up."
|
|
if [[ -z "$msg_info" ]]; then
|
|
msg_info="$msg"
|
|
else
|
|
msg_info="$msg_info
|
|
$msg"
|
|
fi
|
|
else
|
|
error 'Machine is OFFLINE'
|
|
if [[ -f "$RESTART_CHECK_FILE" ]] ; then
|
|
|
|
_file_mod_time=$(date --utc --reference=$RESTART_CHECK_FILE +%s)
|
|
_now=$(date +%s)
|
|
_delta=$(($_now-$_file_mod_time))
|
|
echo "_delta=: $_delta"
|
|
|
|
if [[ $_delta -gt 3900 ]] ; then
|
|
touch "$RESTART_CHECK_FILE"
|
|
else
|
|
reboot_system "No interface with physical link is up - Machine is OFFLINE."
|
|
fi
|
|
else
|
|
touch "$RESTART_CHECK_FILE"
|
|
fi
|
|
clean_up 1
|
|
fi
|
|
|
|
|
|
# ---
|
|
# Routing ok?
|
|
# ---
|
|
|
|
# Determin IPv4 default gateway(s)
|
|
#
|
|
DEFAULT_GW_4="$(/sbin/ip -4 route | awk '/default/ { print $3 }')"
|
|
if [[ -z "$DEFAULT_GW_4" ]] ; then
|
|
DEFAULT_GW_4="$(/sbin/ip -4 route | awk '/nexthop/ { print $3 }')"
|
|
fi
|
|
|
|
declare -a DEFAULT_GW_4_ARR=()
|
|
for _ip in $DEFAULT_GW_4 ; do
|
|
DEFAULT_GW_4_ARR+=("$_ip")
|
|
done
|
|
|
|
echononl "Check if IPv4 default route is set.."
|
|
|
|
if [[ ${#DEFAULT_GW_4_ARR[@]} -gt 0 ]] ; then
|
|
echo_ok
|
|
if [[ ${#DEFAULT_GW_4_ARR[@]} -eq 1 ]] ; then
|
|
msg="Default IPv4 Gateway(s): ${DEFAULT_GW_4_ARR[@]}"
|
|
if [[ -z "$msg_info" ]]; then
|
|
msg_info="$msg"
|
|
else
|
|
msg_info="$msg_info
|
|
|
|
$msg"
|
|
fi
|
|
else
|
|
msg="Default IPv4 Gateway(s): ${DEFAULT_GW_4_ARR[0]}"
|
|
for index in "${!DEFAULT_GW_4_ARR[@]}"; do
|
|
[[ $index -eq 0 ]] && continue
|
|
msg="$msg
|
|
${DEFAULT_GW_4_ARR[$index]}"
|
|
done
|
|
if [[ -z "$msg_info" ]]; then
|
|
msg_info="$msg"
|
|
else
|
|
msg_info="$msg_info
|
|
|
|
$msg"
|
|
fi
|
|
fi
|
|
else
|
|
echo_failed
|
|
PROBLEM_FOUND=true
|
|
|
|
error "No default route IPV4 is set."
|
|
|
|
if [[ -f "$RESTART_CHECK_FILE" ]] ; then
|
|
|
|
_file_mod_time=$(date --utc --reference=$RESTART_CHECK_FILE +%s)
|
|
_now=$(date +%s)
|
|
_delta=$(($_now-$_file_mod_time))
|
|
echo "_delta=: $_delta"
|
|
|
|
if [[ $_delta -gt 3900 ]] ; then
|
|
touch "$RESTART_CHECK_FILE"
|
|
else
|
|
reboot_system "No default route IPv4 is set."
|
|
fi
|
|
else
|
|
touch "$RESTART_CHECK_FILE"
|
|
fi
|
|
clean_up 1
|
|
fi
|
|
|
|
|
|
# Determin IPv6 default gateway(s)
|
|
#
|
|
if $IPV6_PRESENT ; then
|
|
DEFAULT_GW_6="$(/sbin/ip -6 route | awk '/default/ { print $3 }')"
|
|
|
|
declare -a DEFAULT_GW_6_ARR=()
|
|
for _ip in $DEFAULT_GW_6 ; do
|
|
DEFAULT_GW_6_ARR+=("$_ip")
|
|
done
|
|
|
|
echononl "Check if IPv6 default route is set.."
|
|
if [[ ${#DEFAULT_GW_6_ARR[@]} -gt 0 ]] ; then
|
|
echo_ok
|
|
if [[ ${#DEFAULT_GW_6_ARR[@]} -eq 1 ]] ; then
|
|
msg="Default IPv6 Gateway(s): ${DEFAULT_GW_6_ARR[@]}"
|
|
if [[ -z "$msg_info" ]]; then
|
|
msg_info="$msg"
|
|
else
|
|
msg_info="$msg_info
|
|
|
|
$msg"
|
|
fi
|
|
else
|
|
msg="Default IPv6 Gateway(s): ${DEFAULT_GW_6_ARR[0]}"
|
|
for index in "${!DEFAULT_GW_6_ARR[@]}"; do
|
|
[[ $index -eq 0 ]] && continue
|
|
msg="$msg
|
|
${DEFAULT_GW_6_ARR[$index]}"
|
|
done
|
|
|
|
if [[ -z "$msg_info" ]]; then
|
|
msg_info="$msg"
|
|
else
|
|
msg_info="$msg_info
|
|
|
|
$msg"
|
|
fi
|
|
|
|
fi
|
|
else
|
|
echo_warn
|
|
|
|
msg="No default route IPv6 is set."
|
|
if [[ -z "$msg_warn" ]]; then
|
|
msg_warn="$msg"
|
|
else
|
|
msg_warn="$msg_warn
|
|
|
|
$msg"
|
|
fi
|
|
fi
|
|
|
|
fi
|
|
|
|
|
|
# ---
|
|
# Nameservice ok?
|
|
# ---
|
|
|
|
FOUND_NS_PROBLEM=true
|
|
|
|
# Check if nameserver entry at /etc/resolv.conf ist present
|
|
#
|
|
echononl "Check if nameserver entry at /etc/resolv.conf ist present.."
|
|
FIRST_NS="$(cat /etc/resolv.conf | awk '/^nameserver/ {print $2}' | awk 'NR == 1 {print; exit}')"
|
|
if [[ -n "$FIRST_NS" ]] ; then
|
|
echo_ok
|
|
FOUND_NS_PROBLEM=false
|
|
|
|
msg="Found nameserver \033[1m$FIRST_NS\033[m at '/etc/resolv.conf'."
|
|
if [[ -z "$msg_info" ]]; then
|
|
msg_info="$msg"
|
|
else
|
|
msg_info="$msg_info
|
|
|
|
$msg"
|
|
fi
|
|
|
|
else
|
|
echo_failed
|
|
error "No nameserverentry at '/etc/resolv.conf' found."
|
|
fi
|
|
|
|
|
|
# Check if nameservice works..
|
|
#
|
|
echononl "Check if first nameserver works.."
|
|
if ! $FOUND_NS_PROBLEM ; then
|
|
|
|
FOUND_NS_PROBLEM=true
|
|
|
|
for _hostname in $DNS_CHECK_HOSTS ; do
|
|
if [[ -n "$(dig +short @$FIRST_NS $_hostname)" ]] ; then
|
|
FOUND_NS_PROBLEM=false
|
|
break
|
|
fi
|
|
done
|
|
|
|
if $FOUND_NS_PROBLEM ; then
|
|
echo_failed
|
|
PROBLEM_FOUND=true
|
|
|
|
error "First nameserver located at /etc/resolv.conf does not work properly."
|
|
|
|
if [[ -f "$RESTART_CHECK_FILE" ]] ; then
|
|
|
|
_file_mod_time=$(date --utc --reference=$RESTART_CHECK_FILE +%s)
|
|
_now=$(date +%s)
|
|
_delta=$(($_now-$_file_mod_time))
|
|
echo "_delta=: $_delta"
|
|
|
|
if [[ $_delta -gt 3900 ]] ; then
|
|
touch "$RESTART_CHECK_FILE"
|
|
else
|
|
reboot_system "First nameserver located at /etc/resolv.conf does not work properly."
|
|
fi
|
|
else
|
|
touch "$RESTART_CHECK_FILE"
|
|
fi
|
|
clean_up 1
|
|
else
|
|
echo_ok
|
|
|
|
msg="Nameserver works as expected."
|
|
if [[ -z "$msg_info" ]]; then
|
|
msg_info="$msg"
|
|
else
|
|
msg_info="$msg_info
|
|
|
|
$msg"
|
|
fi
|
|
|
|
fi
|
|
else
|
|
echo_skipped
|
|
msg="Resolver test was not triggert because of nameserver problems.."
|
|
if [[ -z "$msg_warn" ]]; then
|
|
msg_warn="$msg"
|
|
else
|
|
msg_warn="$msg_warn
|
|
|
|
$msg"
|
|
fi
|
|
fi
|
|
|
|
|
|
# ---
|
|
# Website reachable?
|
|
# ---
|
|
|
|
if ! $FOUND_NS_PROBLEM ; then
|
|
FOUND_CHECK_HOST_PROBLEM=true
|
|
else
|
|
FOUND_CHECK_HOST_PROBLEM=false
|
|
fi
|
|
|
|
# Check if some websites reachable..
|
|
#
|
|
for _host in $HTTPS_CHECK_HOSTS ; do
|
|
echononl "Check if website \033[1m$_host\033[m is reachable.."
|
|
if ! $FOUND_NS_PROBLEM ; then
|
|
|
|
case "$(curl -s --max-time 2 -I $_host | sed 's/^[^ ]* *\([0-9]\).*/\1/; 1q')" in
|
|
[23]) echo_ok
|
|
FOUND_CHECK_HOST_PROBLEM=false
|
|
break
|
|
;;
|
|
5) echo_failed
|
|
;;
|
|
|
|
*) echo_failed
|
|
;;
|
|
esac
|
|
else
|
|
echo_skipped
|
|
msg="Website accessibility test was not triggert because of nameserver problems.."
|
|
if [[ -z "$msg_warn" ]]; then
|
|
msg_warn="$msg"
|
|
else
|
|
msg_warn="$msg_warn
|
|
|
|
$msg"
|
|
fi
|
|
break
|
|
fi
|
|
|
|
done
|
|
|
|
if $FOUND_CHECK_HOST_PROBLEM ; then
|
|
|
|
PROBLEM_FOUND=true
|
|
|
|
error "No (configured) website could be reached."
|
|
|
|
if [[ -f "$RESTART_CHECK_FILE" ]] ; then
|
|
|
|
_file_mod_time=$(date --utc --reference=$RESTART_CHECK_FILE +%s)
|
|
_now=$(date +%s)
|
|
_delta=$(($_now-$_file_mod_time))
|
|
echo "_delta=: $_delta"
|
|
|
|
if [[ $_delta -gt 3900 ]] ; then
|
|
touch "$RESTART_CHECK_FILE"
|
|
else
|
|
reboot_system "No website could be reached."
|
|
fi
|
|
else
|
|
touch "$RESTART_CHECK_FILE"
|
|
fi
|
|
clean_up 1
|
|
else
|
|
if ! $FOUND_NS_PROBLEM ; then
|
|
msg="Extern Websites are reachable."
|
|
if [[ -z "$msg_info" ]]; then
|
|
msg_info="$msg"
|
|
else
|
|
msg_info="$msg_info
|
|
|
|
$msg"
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
if [[ -n "$msg_info" ]]; then
|
|
info "$msg_info"
|
|
fi
|
|
|
|
if [[ -n "$msg_warn" ]]; then
|
|
warn "$msg_warn"
|
|
fi
|
|
|
|
|
|
! $PROBLEM_FOUND && rm -f "$RESTART_CHECK_FILE"
|
|
clean_up 0
|