commit 8031aee65d847a093a165ea37d17f71c7de24e00 Author: Christoph Date: Fri Nov 17 21:20:54 2017 +0100 Initial commit diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..28ba115 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +/conf/*.conf +*.swp diff --git a/conf/webalizer.conf.sample b/conf/webalizer.conf.sample new file mode 100644 index 0000000..d080ac9 --- /dev/null +++ b/conf/webalizer.conf.sample @@ -0,0 +1,34 @@ +# --- +# - Configuration file for scripts 'generate_webstats.sh' und 'mk_webstats.sh' +# --- + +# - WEBALSIZER_BINARY +# - +# - Where to find the webalizer program +# - +# - Defaults to: WEBALSIZER_BINARY="/usr/local/webalizer/bin/webalizer" +# - +#WEBALSIZER_BINARY="/usr/local/webalizer/bin/webalizer" + +# - WEBALSIZER_CONF_DIR +# - +# - The directory where the configuration files are saved +# - +# - Defaults to: WEBALSIZER_CONF_DIR="/usr/local/webalizer/etc" +# - +#WEBALSIZER_CONF_DIR="/usr/local/webalizer/etc" + +# - WEB_BASE_DIR +# - +# - Defaults to: WEB_BASE_DIR="/var/www" +# - +#WEB_BASE_DIR="/var/www" + +# - STAT_DIR +# - +# - Path to the diredctory where the webstatistics should be generated. The Path +# - is relative to the sites documentroot directory. +# - +# - Defaults to: STAT_DIR="webstats" +# - +#STAT_DIR="webstats" diff --git a/config_new_site.sh b/config_new_site.sh new file mode 100755 index 0000000..84be58b --- /dev/null +++ b/config_new_site.sh @@ -0,0 +1,486 @@ +#!/usr/bin/env bash + +_src_base_dir="$(dirname $(realpath $0))" +conf_file="${_src_base_dir}/conf/webalizer.conf" + +tmp_dir="$(mktemp -d)" + + +# ------------- +# - Functions +# ------------- + +clean_up() { + + # Perform program exit housekeeping + rm -rf "$tmp_dir" + if $terminal ; then + echo "" + fi + exit $1 +} + +echononl(){ + echo X\\c > /tmp/shprompt$$ + if [ `wc -c /tmp/shprompt$$ | awk '{print $1}'` -eq 1 ]; then + echo "$*\\c" 1>&2 + else + echo -n "$*" 1>&2 + fi + rm /tmp/shprompt$$ +} + +fatal(){ + echo "" + echo Fehler: $* + echo -e "\n\t\033[31m\033[1mSkript wird abgebrochen\033[m\033[m\n" + clean_up 1 +} +warn (){ + echo "" + echo -e "\t[ \033[33m\033[1mWarnung\033[m ]: $*" + echo "" +} + + +success(){ + echo "" + echo -e "\n\t\033[32m\033[1mKonfiguration zur Erstellung von Webstatistiken\033[m\033[m" + echo -e "\t\033[32m\033[1mfuer \"${SITE}\" wurde erstellt.\033[m\033[m\n" + clean_up 1 +} + +# ------------- +# --- Check some prerequisites +# ------------- + +# - Running in a terminal? +# - +if [[ -t 1 ]] ; then + terminal=true +else + terminal=false +fi + +# ------------- +# --- Default values +# ------------- + +DEFAULT_WEBALSIZER_BINARY="/usr/local/webalizer/bin/webalizer" +DEFAULT_WEBALSIZER_CONF_DIR="/usr/local/webalizer/etc" +DEFAULT_WEB_BASE_DIR="/var/www" +DEFAULT_STAT_DIR="webstats" + + +# ------------- +# --- Read Configurations from $conf_file +# ------------- + +if [[ -f "$conf_file" ]]; then + source "$conf_file" +fi + +[[ -n "$WEBALSIZER_BINARY" ]] || WEBALSIZER_BINARY="$DEFAULT_WEBALSIZER_BINARY" +[[ -n "$WEBALSIZER_CONF_DIR" ]] || WEBALSIZER_CONF_DIR="$DEFAULT_WEBALSIZER_CONF_DIR" +[[ -n "$WEB_BASE_DIR" ]] || WEB_BASE_DIR="$DEFAULT_WEB_BASE_DIR" +[[ -n "$STAT_DIR" ]] || STAT_DIR="$DEFAULT_STAT_DIR" + + + + +curdir=`pwd` +rc_done="\033[71G\033[32mdone\033[m" +rc_failed="\033[71G\033[31m\033[1mfailed\033[m" + + +Site=$1 + +LOG_DIR="" + + + +CACHE_FILE="${tmp_dir}/dns_cache.db" + +clear +clear +echo -e "\033[11G\033[32mErzeugung von automatisch generierten Webstatistiken \033[m" +echo +echo "" +echo "--" +echo "" +echo "Fuer welche site ( www.. ) sollen Statistiken erstellt werde ?" +echo "" +echononl "Site [ $Site ]: " +read SITE +while [ "X$SITE" == "X" ];do + if [ "X$Site" != "X" ]; then + SITE=$Site + else + echo + echo "Geben sie die Site an fuer die die Webstatistiken eingerichtet" + echononl "werden sollen oder brechen sie mit +C ab: " + read logfile + fi +done + + +_HOST=`echo $SITE | cut -s -d"." -f 1` +_DOMAIN=`echo $SITE | cut -s -d"." -f 2` +_TDL=`echo $SITE | cut -s -d"." -f 3` + +if [ "X$_DOMAIN" == "X" ]; then + OK=FALSE +else + OK=TRUE +fi + +while [ "X$SITE" == "X" -o $OK == "FALSE" ] +do + echo -e "\n\tdie Eingabe \"$SITE\" ist fehlerhaft." + echo -e "\tGeben sie den Namen fuer die Site erneut ein\n" + echononl "Site : " + read SITE + _HOST=`echo $SITE | cut -s -d"." -f 1` + _DOMAIN=`echo $SITE | cut -s -d"." -f 2` + _TDL=`echo $SITE | cut -s -d"." -f 3` + if [ "X$_DOMAIN" == "X" ]; then + OK=FALSE + else + OK=TRUE + fi +done + +if [ "X$_TDL" == "X" ]; then + _DOMAIN=$_HOST + _FQN_DOMAIN=$SITE +else + _FQN_DOMAIN=`echo $SITE | cut -s -d"." -f 2,3` +fi + +CONF_FILE=${_DOMAIN}.conf + +echo "HOST: $_HOST" +echo "DOMAIN: $_DOMAIN" +echo "FQN_DOMAIN: $_FQN_DOMAIN" +echo "WEB_BASE_DIR: $WEB_BASE_DIR" +echo "\${WEB_BASE_DIR}/\${_FQN_DOMAIN}: ${WEB_BASE_DIR}/${_FQN_DOMAIN}" + +if [[ -d "${WEB_BASE_DIR}/${_DOMAIN}" ]]; then + if [[ -d "${WEB_BASE_DIR}/${_DOMAIN}/logs" ]]; then + LOG_DIR="${WEB_BASE_DIR}/${_DOMAIN}/logs" + elif [[ -d "${WEB_BASE_DIR}/${_DOMAIN}/log" ]]; then + LOG_DIR="${WEB_BASE_DIR}/${_DOMAIN}/log" + fi + if [[ -d "${WEB_BASE_DIR}/${_DOMAIN}/htdocs" ]]; then + OUT_DIR="${WEB_BASE_DIR}/${_DOMAIN}/htdocs/${STAT_DIR}" + elif [[ -d "${WEB_BASE_DIR}/${_DOMAIN}/public" ]]; then + OUT_DIR="${WEB_BASE_DIR}/${_DOMAIN}/public/${STAT_DIR}" + fi +elif [[ -d "${WEB_BASE_DIR}/${_HOST}" ]]; then + if [[ -d "${WEB_BASE_DIR}/${_HOST}/logs" ]]; then + LOG_DIR="${WEB_BASE_DIR}/${_HOST}/logs" + elif [[ -d "${WEB_BASE_DIR}/${_HOST}/log" ]]; then + LOG_DIR="${WEB_BASE_DIR}/${_HOST}/log" + fi + if [[ -d "${WEB_BASE_DIR}/${_HOST}/htdocs" ]]; then + OUT_DIR="${WEB_BASE_DIR}/${_HOST}/htdocs/${STAT_DIR}" + elif [[ -d "${WEB_BASE_DIR}/${_HOST}/public" ]]; then + OUT_DIR="${WEB_BASE_DIR}/${_HOST}/public/${STAT_DIR}" + fi +elif [[ -d "${WEB_BASE_DIR}/${_FQN_DOMAIN}" ]]; then + if [[ -d "${WEB_BASE_DIR}/${_FQN_DOMAIN}/logs" ]]; then + LOG_DIR="${WEB_BASE_DIR}/${_FQN_DOMAIN}/logs" + elif [[ -d "${WEB_BASE_DIR}/${_FQN_DOMAIN}/log" ]]; then + LOG_DIR="${WEB_BASE_DIR}/${_FQN_DOMAIN}/log" + fi + if [[ -d "${WEB_BASE_DIR}/${_FQN_DOMAIN}/htdocs" ]]; then + OUT_DIR="${WEB_BASE_DIR}/${_FQN_DOMAIN}/htdocs/${STAT_DIR}" + elif [[ -d "${WEB_BASE_DIR}/${_FQN_DOMAIN}/public" ]]; then + OUT_DIR="${WEB_BASE_DIR}/${_FQN_DOMAIN}/public/${STAT_DIR}" + fi +fi + +if [[ -z "$LOG_DIR" ]] ; then + if [[ -d "/var/log/apache2" ]]; then + LOG_DIR="/var/log/apache2" + fi +fi + +echo "LOG_DIR: $LOG_DIR/" + +if [[ -d "$LOG_DIR" ]]; then + if [[ -f "$LOG_DIR/${_DOMAIN}-access.log" ]]; then + LOGFILE="$LOG_DIR/${_DOMAIN}-access.log" + elif [[ -f "$LOG_DIR/${_DOMAIN}.log" ]]; then + LOGFILE="$LOG_DIR/${_DOMAIN}.log" + elif [[ -f "$LOG_DIR/${SITE}-access.log" ]] ; then + LOGFILE="$LOG_DIR/${SITE}-access.log" + elif [[ -f "$LOG_DIR/${SITE}.log" ]] ; then + LOGFILE="$LOG_DIR/${SITE}.log" + elif [[ -f "$LOG_DIR/${_FQN_DOMAIN}-access.log" ]] ; then + LOGFILE="$LOG_DIR/${_FQN_DOMAIN}-access.log" + elif [[ -f "$LOG_DIR/${_FQN_DOMAIN}.log" ]] ; then + LOGFILE="$LOG_DIR/${_FQN_DOMAIN}.log" + fi +fi + + +echo +echo "Geben sie den Ort der zu untersuchende Logdatei ein " +echo "oder druecken Sie die um die Vorgabe" +echononl "zu akzeptieren [ $LOGFILE ]: " +read logfile +while [ "X$logfile" == "X" ];do + if [ "X$LOGFILE" != "X" ];then + logfile=$LOGFILE + else + echo + echo "Geben sie den Ort der zu untersuchende Logdatei ein " + echononl "oder brechen sie mit +C ab: " + read logfile + fi +done +LOGFILE=$logfile + +if [ ! -f $LOGFILE ] ; then + fatal kann Logdatei nicht finden +fi + + +echo +echo "Geben sie den Ort des Verzeichnisses ein, in welches" +echo "die Statistikdateien gespeichert werden solle, oder" +echo "druecken Sie die um die Vorgabe" +echononl "zu akzeptieren [ $OUT_DIR ]: " +read out_dir + +while [ "X$out_dir" == "X" ];do + + if [ "X$OUT_DIR" != "X" ]; then + out_dir=$OUT_DIR + else + echo + echo "Geben Sie ein Verzeichnis an, in welches die" + echo "Statistikdateien gespeichert werden sollen oder" + echononl "brechen sie mit +C ab: " + read out_dir + fi +done +OUT_DIR=$out_dir + +if [ ! -d $OUT_DIR ] ; then + echo + echo "Das gewuenschte Verzeichnis existietrt nicht." + echononl "Soll es angelegt werden? [ja/nein]: " + read OK + + while [ "X$OK" != "Xyes" -a "X$OK" != "XYes" -a "X$OK" != "Xja" -a "X$OK" != "XJa" \ + -a "X$OK" != "XNo" -a "X$OK" != "Xno" -a "X$OK" != "Xn" -a "X$OK" != "Xnein" -a "X$OK" != "XNein" ] + do + echononl "falsche Angabe! [ja/nein] :" + read OK + done + + [ $OK = "Yes" -o $OK = "yes" -o "$OK" = "ja" -o "$OK" = "Ja" ] || fatal wiederhole Installation zur Eingabe anderer Parameter + mkdir -p $OUT_DIR + + if test "$?" != "0" + then + echo + echo -e "\tFehler beim Erstellen des Verzeichnisses \"$OUT_DIR\" !!" + echo -e "\tSkript wird abgebrochen\n" + cd $curdir + clean_up 1 + fi + +fi + + +echo -e "\n\tSite : $SITE" +echo -e "\tDomain : $_DOMAIN" +echo -e "\tfullqualified Domainname : $_FQN_DOMAIN" +echo -e "\tLogfile : $LOGFILE" +echo -e "\tOutput Dir : ${OUT_DIR}\n" +echo -e "\tConfiguration File: ${WEBALSIZER_CONF_DIR}/${CONF_FILE}\n" + +if [[ "${LOGFILE##*.}" = "log" ]]; then + warn "Falls die Erstellung der Westatistiken im Abschnitt 'postrotate' des Logrotation Scripts +\t fuer den Webserver definiert ist (empfohlen), sollte als Parameter \"LogFile\" die soeben +\t Datei verwendet werden, je nach configuration: +\t ${LOGFILE}.0 +\t oder +\t ${LOGFILE}.1" +fi + +echononl "Sind die Angaben richtig [ja/nein] :" +read OK +while [ "X$OK" != "Xyes" -a "X$OK" != "XYes" -a "X$OK" != "Xja" -a "X$OK" != "XJa" \ + -a "X$OK" != "XNo" -a "X$OK" != "Xno" -a "X$OK" != "Xn" -a "X$OK" != "Xnein" -a "X$OK" != "XNein" ] +do + echononl "falsche Angabe! [ja/nein] :" + read OK +done +[ $OK = "Yes" -o $OK = "yes" -o "$OK" = "ja" -o "$OK" = "Ja" ] || fatal wiederhole Installation zur Eingabe anderer Parameter + +if [ -f ${WEBALSIZER_CONF_DIR}/$CONF_FILE ];then + _moved=false + for i in 00 01 02 03 04 05 06 07 08 09 10 ; do + + if [ -f ${WEBALSIZER_CONF_DIR}/$CONF_FILE.$i ]; then + continue + else + echo -e -n "\n\tSichere alte Konfiguration in Datei:\n\t ${WEBALSIZER_CONF_DIR}/${CONF_FILE}.$i ..." + mv ${WEBALSIZER_CONF_DIR}/$CONF_FILE ${WEBALSIZER_CONF_DIR}/${CONF_FILE}.$i + _moved=true + break + fi + + done + + if ! $_moved ; then + echo -e -n "\n\tSichere alte Konfiguration in Datei:\n\t ${WEBALSIZER_CONF_DIR}/${CONF_FILE}.`date +%s` ..." + mv ${WEBALSIZER_CONF_DIR}/$CONF_FILE ${WEBALSIZER_CONF_DIR}/${CONF_FILE}.`date +%s` + fi + + echo -e "$rc_done" + +fi + +echo -e -n "\n\tEingaben wurden akzeptiert. Schreibe Konfigurationsdatei ..." + + +date=`date` + +cat<${WEBALSIZER_CONF_DIR}/$CONF_FILE +############################################################# +## ## +## this file was autogenerated by mk_webstats.sh, a little ## +## script to generate a config file for use with webalizer ## +## ## +## Date = $date ## +## ## +############################################################# + +LogFile $LOGFILE +LogType clf + +OutputDir $OUT_DIR + +Incremental yes + +HostName $SITE + +PageType htm* +PageType cgi +PageType phtml +PageType shtml +PageType php* +PageType pl + +#DNSCache /tmp/dns_cache.db +#DNSChildren 10 + +AllSites yes +AllURLs yes +AllReferrers yes +AllAgents yes +AllSearchStr yes +AllUsers yes + +#HideURL *.gif +#HideURL *.GIF +#HideURL *.jpg +#HideURL *.JPG +#HideURL *.png +#HideURL *.PNG +# .ra - real audio sound file (Realplayer +HideURL *.ra + +# ignore embedded files +IgnoreURL *.gif +IgnoreURL *.GIF +IgnoreURL *.jpg +IgnoreURL *.JPG +IgnoreURL *.png +IgnoreURL *.PNG +IgnoreURL *.js +IgnoreURL *.JS +IgnoreURL *.css +IgnoreURL *.CSS +IgnoreURL robots.txt +IgnoreURL favicon.ico + +# ignore bots +IgnoreAgent bot +IgnoreAgent Bot +IgnoreAgent crawl +IgnoreAgent Crawl +IgnoreAgent search +IgnoreAgent get +IgnoreAgent spider +IgnoreAgent find +IgnoreAgent java +IgnoreAgent Java +IgnoreAgent Google +IgnoreAgent yahoo +IgnoreAgent ask.com +IgnoreAgent contaxe.com +IgnoreAgent Yandex +IgnoreAgent libwww-perl +IgnoreSite crawl.yahoo.net +IgnoreSite googlebot.com +IgnoreSite search.live.com +IgnoreSite yandex.ru +IgnoreSite ask.com +IgnoreSite bigfinder.de + +GroupURL /cgi-bin/* CGI Scripts +GroupURL /images/* Images + +SearchEngine yahoo.com p= +SearchEngine altavista.com q= +SearchEngine google.com q= +SearchEngine eureka.com q= +SearchEngine lycos.com query= +SearchEngine hotbot.com MT= +SearchEngine msn.com MT= +SearchEngine infoseek.com qt= +SearchEngine webcrawler searchText= +SearchEngine excite search= +SearchEngine netscape.com search= +SearchEngine mamma.com query= +SearchEngine alltheweb.com query= +SearchEngine northernlight.com qr= +EOF +echo "" +echo "" +echononl "Sollen Webstatistiken jetzt generiert werden [ja/nein] :" +read OK +while [ "X$OK" != "Xyes" -a "X$OK" != "XYes" -a "X$OK" != "Xja" -a "X$OK" != "XJa" \ + -a "X$OK" != "XNo" -a "X$OK" != "Xno" -a "X$OK" != "Xn" -a "X$OK" != "Xnein" -a "X$OK" != "XNein" ] +do + echononl "falsche Angabe! [ja/nein] :" + read OK +done +[ $OK = "Yes" -o $OK = "yes" -o "$OK" = "ja" -o "$OK" = "Ja" ] || success + +echo -e "$rc_done" +echo -e -n "\n\tErstelle Statistik ..." + +$WEBALSIZER_BINARY -Q -p -M4 -N20 -D $CACHE_FILE -c ${WEBALSIZER_CONF_DIR}/${CONF_FILE} > /dev/null 2>&1 + +if test $? != "0" +then + echo -e "$rc_failed" + echo + echo -e "\t\033[31mEs konnte keine Zugriffsstatistik erstellt werden !!\033[m" + echo + cd $curdir + clean_up 1 +fi + +echo -e "$rc_done" +success + +clean_up 0 + diff --git a/generate_webstats.sh b/generate_webstats.sh new file mode 100755 index 0000000..d0882c6 --- /dev/null +++ b/generate_webstats.sh @@ -0,0 +1,178 @@ +#!/usr/bin/env bash + +_src_base_dir="$(dirname $(realpath $0))" +conf_file="${_src_base_dir}/conf/webalizer.conf" + +tmp_dir="$(mktemp -d)" +log_file="$(mktemp)" + + + +# ------------- +# - Functions +# ------------- + +clean_up() { + + # Perform program exit housekeeping + rm -f "$log_file" + rm -rf "$tmp_dir" + if $terminal ; then + echo "" + fi + exit $1 +} + +echononl(){ + if $terminal ; then + echo X\\c > /tmp/shprompt$$ + if [ `wc -c /tmp/shprompt$$ | awk '{print $1}'` -eq 1 ]; then + echo -e -n "$*\\c" 1>&2 + else + echo -e -n "$*" 1>&2 + fi + rm /tmp/shprompt$$ + fi +} +info (){ + if $terminal ; then + echo "" + echo -e "\t[ \033[32m\033[1mInfo\033[m ]: $*" + echo "" + fi +} +warn (){ + if $terminal ; then + echo "" + echo -e "\t[ \033[33m\033[1mWarning\033[m ]: $*" + echo "" + else + echo "Warning: $*" + fi +} +error(){ + echo "" + if $terminal ; then + echo -e "\t[ \033[31m\033[1mFehler\033[m ]: $*" + else + echo "Error: $*" + fi + echo "" +} + + +echo_done() { + if $terminal ; then + echo -e "\033[75G[ \033[32mdone\033[m ]" + fi +} + +echo_failed(){ + if $terminal ; then + echo -e "\033[75G[ \033[1;31mfailed\033[m ]" + fi +} +echo_skipped() { + if $terminal ; then + echo -e "\033[75G[ \033[37mskipped\033[m ]" + fi +} + + + +# ------------- +# --- Check some prerequisites +# ------------- + +# - Running in a terminal? +# - +if [[ -t 1 ]] ; then + terminal=true +else + terminal=false +fi + +# ------------- +# --- Default values +# ------------- + +DEFAULT_WEBALSIZER_BINARY="/usr/local/webalizer/bin/webalizer" +DEFAULT_WEBALSIZER_CONF_DIR="/usr/local/webalizer/etc" + +# ------------- +# --- Read Configurations from $conf_file +# ------------- + +if [[ -f "$conf_file" ]]; then + source "$conf_file" +fi + +[[ -n "$WEBALSIZER_BINARY" ]] || WEBALSIZER_BINARY="$DEFAULT_WEBALSIZER_BINARY" +[[ -n "$WEBALSIZER_CONF_DIR" ]] || WEBALSIZER_CONF_DIR="$DEFAULT_WEBALSIZER_CONF_DIR" + +_CACHE_FILE="${tmp_dir}/dns_cache.db" + +#/bin/rm -f ${TMP_DIR}/$CACHE_FILE +# +#for i in /var/log/apache/*.log; do +# +# /usr/local/webalizer/bin/webazolver -Q -o ${TMP_DIR} -N20 -D $CACHE_FILE $i +# +#done + +# - Check if script was invoked from logrotation script +# - +# - Note: Its the parent of the parent +# - +_stat=($( $log_file 2>&1 + if [[ $? -ne 0 ]]; then + if ! $terminal ; then + echo "[ Error ]: Generatin webstatistics for site '${_site}' failed!" + fi + echo_failed + error "$(cat "$log_file")" + else + echo_done + fi + +done < <(find "${WEBALSIZER_CONF_DIR}" -maxdepth 1 -type f -name "*.conf" -print0) + +clean_up 0