script/webalizer
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Initial commit

Browse Source
This commit is contained in:
Christoph
2017-11-17 21:20:54 +01:00
commit 8031aee65d
4 changed files with 700 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored Normal file
View File

@@ -0,0 +1,2 @@
/conf/*.conf
*.swp

34
conf/webalizer.conf.sample Normal file
View File

@@ -0,0 +1,34 @@
# ---
# - Configuration file for scripts 'generate_webstats.sh' und 'mk_webstats.sh'
# ---
# - WEBALSIZER_BINARY
# -
# - Where to find the webalizer program
# -
# - Defaults to: WEBALSIZER_BINARY="/usr/local/webalizer/bin/webalizer"
# -
#WEBALSIZER_BINARY="/usr/local/webalizer/bin/webalizer"
# - WEBALSIZER_CONF_DIR
# -
# - The directory where the configuration files are saved
# -
# - Defaults to: WEBALSIZER_CONF_DIR="/usr/local/webalizer/etc"
# -
#WEBALSIZER_CONF_DIR="/usr/local/webalizer/etc"
# - WEB_BASE_DIR
# -
# - Defaults to: WEB_BASE_DIR="/var/www"
# -
#WEB_BASE_DIR="/var/www"
# - STAT_DIR
# -
# - Path to the diredctory where the webstatistics should be generated. The Path
# - is relative to the sites documentroot directory.
# -
# - Defaults to: STAT_DIR="webstats"
# -
#STAT_DIR="webstats"

486
config_new_site.sh Executable file
View File

@@ -0,0 +1,486 @@
#!/usr/bin/env bash
_src_base_dir="$(dirname $(realpath $0))"
conf_file="${_src_base_dir}/conf/webalizer.conf"
tmp_dir="$(mktemp -d)"
# -------------
# - Functions
# -------------
clean_up() {
# Perform program exit housekeeping
rm -rf "$tmp_dir"
if $terminal ; then
echo ""
fi
exit $1
}
echononl(){
echo X\\c > /tmp/shprompt$$
if [ `wc -c /tmp/shprompt$$ | awk '{print $1}'` -eq 1 ]; then
echo "$*\\c" 1>&2
else
echo -n "$*" 1>&2
fi
rm /tmp/shprompt$$
}
fatal(){
echo ""
echo Fehler: $*
echo -e "\n\t\033[31m\033[1mSkript wird abgebrochen\033[m\033[m\n"
clean_up 1
}
warn (){
echo ""
echo -e "\t[ \033[33m\033[1mWarnung\033[m ]: $*"
echo ""
}
success(){
echo ""
echo -e "\n\t\033[32m\033[1mKonfiguration zur Erstellung von Webstatistiken\033[m\033[m"
echo -e "\t\033[32m\033[1mfuer \"${SITE}\" wurde erstellt.\033[m\033[m\n"
clean_up 1
}
# -------------
# --- Check some prerequisites
# -------------
# - Running in a terminal?
# -
if [[ -t 1 ]] ; then
terminal=true
else
terminal=false
fi
# -------------
# --- Default values
# -------------
DEFAULT_WEBALSIZER_BINARY="/usr/local/webalizer/bin/webalizer"
DEFAULT_WEBALSIZER_CONF_DIR="/usr/local/webalizer/etc"
DEFAULT_WEB_BASE_DIR="/var/www"
DEFAULT_STAT_DIR="webstats"
# -------------
# --- Read Configurations from $conf_file
# -------------
if [[ -f "$conf_file" ]]; then
source "$conf_file"
fi
[[ -n "$WEBALSIZER_BINARY" ]] || WEBALSIZER_BINARY="$DEFAULT_WEBALSIZER_BINARY"
[[ -n "$WEBALSIZER_CONF_DIR" ]] || WEBALSIZER_CONF_DIR="$DEFAULT_WEBALSIZER_CONF_DIR"
[[ -n "$WEB_BASE_DIR" ]] || WEB_BASE_DIR="$DEFAULT_WEB_BASE_DIR"
[[ -n "$STAT_DIR" ]] || STAT_DIR="$DEFAULT_STAT_DIR"
curdir=`pwd`
rc_done="\033[71G\033[32mdone\033[m"
rc_failed="\033[71G\033[31m\033[1mfailed\033[m"
Site=$1
LOG_DIR=""
CACHE_FILE="${tmp_dir}/dns_cache.db"
clear
clear
echo -e "\033[11G\033[32mErzeugung von automatisch generierten Webstatistiken \033[m"
echo
echo ""
echo "--"
echo ""
echo "Fuer welche site ( www.<domain>.<tld> ) sollen Statistiken erstellt werde ?"
echo ""
echononl "Site [ $Site ]: "
read SITE
while [ "X$SITE" == "X" ];do
if [ "X$Site" != "X" ]; then
SITE=$Site
else
echo
echo "Geben sie die Site an fuer die die Webstatistiken eingerichtet"
echononl "werden sollen oder brechen sie mit <CTRL>+C ab: "
read logfile
fi
done
_HOST=`echo $SITE | cut -s -d"." -f 1`
_DOMAIN=`echo $SITE | cut -s -d"." -f 2`
_TDL=`echo $SITE | cut -s -d"." -f 3`
if [ "X$_DOMAIN" == "X" ]; then
OK=FALSE
else
OK=TRUE
fi
while [ "X$SITE" == "X" -o $OK == "FALSE" ]
do
echo -e "\n\tdie Eingabe \"$SITE\" ist fehlerhaft."
echo -e "\tGeben sie den Namen fuer die Site erneut ein\n"
echononl "Site : "
read SITE
_HOST=`echo $SITE | cut -s -d"." -f 1`
_DOMAIN=`echo $SITE | cut -s -d"." -f 2`
_TDL=`echo $SITE | cut -s -d"." -f 3`
if [ "X$_DOMAIN" == "X" ]; then
OK=FALSE
else
OK=TRUE
fi
done
if [ "X$_TDL" == "X" ]; then
_DOMAIN=$_HOST
_FQN_DOMAIN=$SITE
else
_FQN_DOMAIN=`echo $SITE | cut -s -d"." -f 2,3`
fi
CONF_FILE=${_DOMAIN}.conf
echo "HOST: $_HOST"
echo "DOMAIN: $_DOMAIN"
echo "FQN_DOMAIN: $_FQN_DOMAIN"
echo "WEB_BASE_DIR: $WEB_BASE_DIR"
echo "\${WEB_BASE_DIR}/\${_FQN_DOMAIN}: ${WEB_BASE_DIR}/${_FQN_DOMAIN}"
if [[ -d "${WEB_BASE_DIR}/${_DOMAIN}" ]]; then
if [[ -d "${WEB_BASE_DIR}/${_DOMAIN}/logs" ]]; then
LOG_DIR="${WEB_BASE_DIR}/${_DOMAIN}/logs"
elif [[ -d "${WEB_BASE_DIR}/${_DOMAIN}/log" ]]; then
LOG_DIR="${WEB_BASE_DIR}/${_DOMAIN}/log"
fi
if [[ -d "${WEB_BASE_DIR}/${_DOMAIN}/htdocs" ]]; then
OUT_DIR="${WEB_BASE_DIR}/${_DOMAIN}/htdocs/${STAT_DIR}"
elif [[ -d "${WEB_BASE_DIR}/${_DOMAIN}/public" ]]; then
OUT_DIR="${WEB_BASE_DIR}/${_DOMAIN}/public/${STAT_DIR}"
fi
elif [[ -d "${WEB_BASE_DIR}/${_HOST}" ]]; then
if [[ -d "${WEB_BASE_DIR}/${_HOST}/logs" ]]; then
LOG_DIR="${WEB_BASE_DIR}/${_HOST}/logs"
elif [[ -d "${WEB_BASE_DIR}/${_HOST}/log" ]]; then
LOG_DIR="${WEB_BASE_DIR}/${_HOST}/log"
fi
if [[ -d "${WEB_BASE_DIR}/${_HOST}/htdocs" ]]; then
OUT_DIR="${WEB_BASE_DIR}/${_HOST}/htdocs/${STAT_DIR}"
elif [[ -d "${WEB_BASE_DIR}/${_HOST}/public" ]]; then
OUT_DIR="${WEB_BASE_DIR}/${_HOST}/public/${STAT_DIR}"
fi
elif [[ -d "${WEB_BASE_DIR}/${_FQN_DOMAIN}" ]]; then
if [[ -d "${WEB_BASE_DIR}/${_FQN_DOMAIN}/logs" ]]; then
LOG_DIR="${WEB_BASE_DIR}/${_FQN_DOMAIN}/logs"
elif [[ -d "${WEB_BASE_DIR}/${_FQN_DOMAIN}/log" ]]; then
LOG_DIR="${WEB_BASE_DIR}/${_FQN_DOMAIN}/log"
fi
if [[ -d "${WEB_BASE_DIR}/${_FQN_DOMAIN}/htdocs" ]]; then
OUT_DIR="${WEB_BASE_DIR}/${_FQN_DOMAIN}/htdocs/${STAT_DIR}"
elif [[ -d "${WEB_BASE_DIR}/${_FQN_DOMAIN}/public" ]]; then
OUT_DIR="${WEB_BASE_DIR}/${_FQN_DOMAIN}/public/${STAT_DIR}"
fi
fi
if [[ -z "$LOG_DIR" ]] ; then
if [[ -d "/var/log/apache2" ]]; then
LOG_DIR="/var/log/apache2"
fi
fi
echo "LOG_DIR: $LOG_DIR/"
if [[ -d "$LOG_DIR" ]]; then
if [[ -f "$LOG_DIR/${_DOMAIN}-access.log" ]]; then
LOGFILE="$LOG_DIR/${_DOMAIN}-access.log"
elif [[ -f "$LOG_DIR/${_DOMAIN}.log" ]]; then
LOGFILE="$LOG_DIR/${_DOMAIN}.log"
elif [[ -f "$LOG_DIR/${SITE}-access.log" ]] ; then
LOGFILE="$LOG_DIR/${SITE}-access.log"
elif [[ -f "$LOG_DIR/${SITE}.log" ]] ; then
LOGFILE="$LOG_DIR/${SITE}.log"
elif [[ -f "$LOG_DIR/${_FQN_DOMAIN}-access.log" ]] ; then
LOGFILE="$LOG_DIR/${_FQN_DOMAIN}-access.log"
elif [[ -f "$LOG_DIR/${_FQN_DOMAIN}.log" ]] ; then
LOGFILE="$LOG_DIR/${_FQN_DOMAIN}.log"
fi
fi
echo
echo "Geben sie den Ort der zu untersuchende Logdatei ein "
echo "oder druecken Sie die <Return-Taste> um die Vorgabe"
echononl "zu akzeptieren [ $LOGFILE ]: "
read logfile
while [ "X$logfile" == "X" ];do
if [ "X$LOGFILE" != "X" ];then
logfile=$LOGFILE
else
echo
echo "Geben sie den Ort der zu untersuchende Logdatei ein "
echononl "oder brechen sie mit <CTRL>+C ab: "
read logfile
fi
done
LOGFILE=$logfile
if [ ! -f $LOGFILE ] ; then
fatal kann Logdatei nicht finden
fi
echo
echo "Geben sie den Ort des Verzeichnisses ein, in welches"
echo "die Statistikdateien gespeichert werden solle, oder"
echo "druecken Sie die <Retutn-Taste> um die Vorgabe"
echononl "zu akzeptieren [ $OUT_DIR ]: "
read out_dir
while [ "X$out_dir" == "X" ];do
if [ "X$OUT_DIR" != "X" ]; then
out_dir=$OUT_DIR
else
echo
echo "Geben Sie ein Verzeichnis an, in welches die"
echo "Statistikdateien gespeichert werden sollen oder"
echononl "brechen sie mit <CTRL>+C ab: "
read out_dir
fi
done
OUT_DIR=$out_dir
if [ ! -d $OUT_DIR ] ; then
echo
echo "Das gewuenschte Verzeichnis existietrt nicht."
echononl "Soll es angelegt werden? [ja/nein]: "
read OK
while [ "X$OK" != "Xyes" -a "X$OK" != "XYes" -a "X$OK" != "Xja" -a "X$OK" != "XJa" \
-a "X$OK" != "XNo" -a "X$OK" != "Xno" -a "X$OK" != "Xn" -a "X$OK" != "Xnein" -a "X$OK" != "XNein" ]
do
echononl "falsche Angabe! [ja/nein] :"
read OK
done
[ $OK = "Yes" -o $OK = "yes" -o "$OK" = "ja" -o "$OK" = "Ja" ] || fatal wiederhole Installation zur Eingabe anderer Parameter
mkdir -p $OUT_DIR
if test "$?" != "0"
then
echo
echo -e "\tFehler beim Erstellen des Verzeichnisses \"$OUT_DIR\" !!"
echo -e "\tSkript wird abgebrochen\n"
cd $curdir
clean_up 1
fi
fi
echo -e "\n\tSite : $SITE"
echo -e "\tDomain : $_DOMAIN"
echo -e "\tfullqualified Domainname : $_FQN_DOMAIN"
echo -e "\tLogfile : $LOGFILE"
echo -e "\tOutput Dir : ${OUT_DIR}\n"
echo -e "\tConfiguration File: ${WEBALSIZER_CONF_DIR}/${CONF_FILE}\n"
if [[ "${LOGFILE##*.}" = "log" ]]; then
warn "Falls die Erstellung der Westatistiken im Abschnitt 'postrotate' des Logrotation Scripts
\t fuer den Webserver definiert ist (empfohlen), sollte als Parameter \"LogFile\" die soeben
\t Datei verwendet werden, je nach configuration:
\t ${LOGFILE}.0
\t oder
\t ${LOGFILE}.1"
fi
echononl "Sind die Angaben richtig [ja/nein] :"
read OK
while [ "X$OK" != "Xyes" -a "X$OK" != "XYes" -a "X$OK" != "Xja" -a "X$OK" != "XJa" \
-a "X$OK" != "XNo" -a "X$OK" != "Xno" -a "X$OK" != "Xn" -a "X$OK" != "Xnein" -a "X$OK" != "XNein" ]
do
echononl "falsche Angabe! [ja/nein] :"
read OK
done
[ $OK = "Yes" -o $OK = "yes" -o "$OK" = "ja" -o "$OK" = "Ja" ] || fatal wiederhole Installation zur Eingabe anderer Parameter
if [ -f ${WEBALSIZER_CONF_DIR}/$CONF_FILE ];then
_moved=false
for i in 00 01 02 03 04 05 06 07 08 09 10 ; do
if [ -f ${WEBALSIZER_CONF_DIR}/$CONF_FILE.$i ]; then
continue
else
echo -e -n "\n\tSichere alte Konfiguration in Datei:\n\t ${WEBALSIZER_CONF_DIR}/${CONF_FILE}.$i ..."
mv ${WEBALSIZER_CONF_DIR}/$CONF_FILE ${WEBALSIZER_CONF_DIR}/${CONF_FILE}.$i
_moved=true
break
fi
done
if ! $_moved ; then
echo -e -n "\n\tSichere alte Konfiguration in Datei:\n\t ${WEBALSIZER_CONF_DIR}/${CONF_FILE}.`date +%s` ..."
mv ${WEBALSIZER_CONF_DIR}/$CONF_FILE ${WEBALSIZER_CONF_DIR}/${CONF_FILE}.`date +%s`
fi
echo -e "$rc_done"
fi
echo -e -n "\n\tEingaben wurden akzeptiert. Schreibe Konfigurationsdatei ..."
date=`date`
cat<<EOF >${WEBALSIZER_CONF_DIR}/$CONF_FILE
#############################################################
## ##
## this file was autogenerated by mk_webstats.sh, a little ##
## script to generate a config file for use with webalizer ##
## ##
## Date = $date ##
## ##
#############################################################
LogFile $LOGFILE
LogType clf
OutputDir $OUT_DIR
Incremental yes
HostName $SITE
PageType htm*
PageType cgi
PageType phtml
PageType shtml
PageType php*
PageType pl
#DNSCache /tmp/dns_cache.db
#DNSChildren 10
AllSites yes
AllURLs yes
AllReferrers yes
AllAgents yes
AllSearchStr yes
AllUsers yes
#HideURL *.gif
#HideURL *.GIF
#HideURL *.jpg
#HideURL *.JPG
#HideURL *.png
#HideURL *.PNG
# .ra - real audio sound file (Realplayer
HideURL *.ra
# ignore embedded files
IgnoreURL *.gif
IgnoreURL *.GIF
IgnoreURL *.jpg
IgnoreURL *.JPG
IgnoreURL *.png
IgnoreURL *.PNG
IgnoreURL *.js
IgnoreURL *.JS
IgnoreURL *.css
IgnoreURL *.CSS
IgnoreURL robots.txt
IgnoreURL favicon.ico
# ignore bots
IgnoreAgent bot
IgnoreAgent Bot
IgnoreAgent crawl
IgnoreAgent Crawl
IgnoreAgent search
IgnoreAgent get
IgnoreAgent spider
IgnoreAgent find
IgnoreAgent java
IgnoreAgent Java
IgnoreAgent Google
IgnoreAgent yahoo
IgnoreAgent ask.com
IgnoreAgent contaxe.com
IgnoreAgent Yandex
IgnoreAgent libwww-perl
IgnoreSite crawl.yahoo.net
IgnoreSite googlebot.com
IgnoreSite search.live.com
IgnoreSite yandex.ru
IgnoreSite ask.com
IgnoreSite bigfinder.de
GroupURL /cgi-bin/* CGI Scripts
GroupURL /images/* Images
SearchEngine yahoo.com p=
SearchEngine altavista.com q=
SearchEngine google.com q=
SearchEngine eureka.com q=
SearchEngine lycos.com query=
SearchEngine hotbot.com MT=
SearchEngine msn.com MT=
SearchEngine infoseek.com qt=
SearchEngine webcrawler searchText=
SearchEngine excite search=
SearchEngine netscape.com search=
SearchEngine mamma.com query=
SearchEngine alltheweb.com query=
SearchEngine northernlight.com qr=
EOF
echo ""
echo ""
echononl "Sollen Webstatistiken jetzt generiert werden [ja/nein] :"
read OK
while [ "X$OK" != "Xyes" -a "X$OK" != "XYes" -a "X$OK" != "Xja" -a "X$OK" != "XJa" \
-a "X$OK" != "XNo" -a "X$OK" != "Xno" -a "X$OK" != "Xn" -a "X$OK" != "Xnein" -a "X$OK" != "XNein" ]
do
echononl "falsche Angabe! [ja/nein] :"
read OK
done
[ $OK = "Yes" -o $OK = "yes" -o "$OK" = "ja" -o "$OK" = "Ja" ] || success
echo -e "$rc_done"
echo -e -n "\n\tErstelle Statistik ..."
$WEBALSIZER_BINARY -Q -p -M4 -N20 -D $CACHE_FILE -c ${WEBALSIZER_CONF_DIR}/${CONF_FILE} > /dev/null 2>&1
if test $? != "0"
then
echo -e "$rc_failed"
echo
echo -e "\t\033[31mEs konnte keine Zugriffsstatistik erstellt werden !!\033[m"
echo
cd $curdir
clean_up 1
fi
echo -e "$rc_done"
success
clean_up 0

178
generate_webstats.sh Executable file
View File

@@ -0,0 +1,178 @@
#!/usr/bin/env bash
_src_base_dir="$(dirname $(realpath $0))"
conf_file="${_src_base_dir}/conf/webalizer.conf"
tmp_dir="$(mktemp -d)"
log_file="$(mktemp)"
# -------------
# - Functions
# -------------
clean_up() {
# Perform program exit housekeeping
rm -f "$log_file"
rm -rf "$tmp_dir"
if $terminal ; then
echo ""
fi
exit $1
}
echononl(){
if $terminal ; then
echo X\\c > /tmp/shprompt$$
if [ `wc -c /tmp/shprompt$$ | awk '{print $1}'` -eq 1 ]; then
echo -e -n "$*\\c" 1>&2
else
echo -e -n "$*" 1>&2
fi
rm /tmp/shprompt$$
fi
}
info (){
if $terminal ; then
echo ""
echo -e "\t[ \033[32m\033[1mInfo\033[m ]: $*"
echo ""
fi
}
warn (){
if $terminal ; then
echo ""
echo -e "\t[ \033[33m\033[1mWarning\033[m ]: $*"
echo ""
else
echo "Warning: $*"
fi
}
error(){
echo ""
if $terminal ; then
echo -e "\t[ \033[31m\033[1mFehler\033[m ]: $*"
else
echo "Error: $*"
fi
echo ""
}
echo_done() {
if $terminal ; then
echo -e "\033[75G[ \033[32mdone\033[m ]"
fi
}
echo_failed(){
if $terminal ; then
echo -e "\033[75G[ \033[1;31mfailed\033[m ]"
fi
}
echo_skipped() {
if $terminal ; then
echo -e "\033[75G[ \033[37mskipped\033[m ]"
fi
}
# -------------
# --- Check some prerequisites
# -------------
# - Running in a terminal?
# -
if [[ -t 1 ]] ; then
terminal=true
else
terminal=false
fi
# -------------
# --- Default values
# -------------
DEFAULT_WEBALSIZER_BINARY="/usr/local/webalizer/bin/webalizer"
DEFAULT_WEBALSIZER_CONF_DIR="/usr/local/webalizer/etc"
# -------------
# --- Read Configurations from $conf_file
# -------------
if [[ -f "$conf_file" ]]; then
source "$conf_file"
fi
[[ -n "$WEBALSIZER_BINARY" ]] || WEBALSIZER_BINARY="$DEFAULT_WEBALSIZER_BINARY"
[[ -n "$WEBALSIZER_CONF_DIR" ]] || WEBALSIZER_CONF_DIR="$DEFAULT_WEBALSIZER_CONF_DIR"
_CACHE_FILE="${tmp_dir}/dns_cache.db"
#/bin/rm -f ${TMP_DIR}/$CACHE_FILE
#
#for i in /var/log/apache/*.log; do
#
# /usr/local/webalizer/bin/webazolver -Q -o ${TMP_DIR} -N20 -D $CACHE_FILE $i
#
#done
# - Check if script was invoked from logrotation script
# -
# - Note: Its the parent of the parent
# -
_stat=($(</proc/$PPID/stat))
_PPPID="${_stat[3]}"
_COMMAND="$(ps -o cmd= $_PPPID)"
if [[ "$_COMMAND" =~ /usr/sbin/logrotate ]]; then
invoked_from_logrotate=true
else
invoked_from_logrotate=false
fi
info "Generate web-statistics at host '$(hostname -f)'"
while IFS='' read -r -d '' _conf_file ; do
_websites_log_file="$(grep -E "^\s*LogFile" $_conf_file | awk '{print$2}')"
_site="$(grep -e "^\s*HostName" $_conf_file | awk '{print$2}')"
# - Warn if script was called from logrotation script
# -
if $invoked_from_logrotate ; then
if [[ "${_websites_log_file##*.}" = "log" ]]; then
warn "${_site}: Maybe you are going to inspect an empty - because just even rotated - logfile."
fi
fi
echononl " Generate statistics for site '${_site}'.."
rm -f "$_CACHE_FILE"
# - Take care log file is not empty
# -
if [[ ! -s "${_websites_log_file}" ]] ; then
echo_skipped
if ! $terminal ; then
echo "[ Warning ]: No web-statistics for site '${_site}' generated!"
echo " LogFile "${_websites_log_file}" not found or empty."
fi
continue
fi
$WEBALSIZER_BINARY -Q -p -M4 -N20 -D "$_CACHE_FILE" -c "${_conf_file}" > $log_file 2>&1
if [[ $? -ne 0 ]]; then
if ! $terminal ; then
echo "[ Error ]: Generatin webstatistics for site '${_site}' failed!"
fi
echo_failed
error "$(cat "$log_file")"
else
echo_done
fi
done < <(find "${WEBALSIZER_CONF_DIR}" -maxdepth 1 -type f -name "*.conf" -print0)
clean_up 0