From be12e2d74a2a7bf4494590e707586fce11293f10 Mon Sep 17 00:00:00 2001 From: Christoph Date: Thu, 10 Oct 2019 17:53:15 +0200 Subject: [PATCH] Redesign of script. Add sample configuration file. --- .gitignore | 1 + conf/trigger_piwik_archives.conf.sample | 71 +++++ trigger_piwik_archives.sh | 371 +++++++++++++++++------- 3 files changed, 334 insertions(+), 109 deletions(-) create mode 100644 conf/trigger_piwik_archives.conf.sample diff --git a/.gitignore b/.gitignore index c64e6ce..6ac35e8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ /BAK/* +/conf/*.conf *.swp diff --git a/conf/trigger_piwik_archives.conf.sample b/conf/trigger_piwik_archives.conf.sample new file mode 100644 index 0000000..969fe57 --- /dev/null +++ b/conf/trigger_piwik_archives.conf.sample @@ -0,0 +1,71 @@ +# ========== +# Settings for script trigger_piwik_archives.sh +# ========== + +# - MYSQL_CREDENTIAL_ARGS +# - +# - MySQL / MariaDB credentials +# - +# - Giving password on command line is insecure an sind mysql 5.5 +# - you will get a warning doing so. +# - +# - Reading username/password fro file ist also possible, using MySQL/MariaDB +# - commandline parameter '--defaults-file'. +# - +# - Since Mysql Version 5.6, you can read username/password from +# - encrypted file. +# - +# - Create (encrypted) option file: +# - $ mysql_config_editor set --login-path=piwik_admin --socket=/tmp/mysql.sock --user=piwik_admin --password +# - $ Password: +# - +# - Use of option file: +# - $ mysql --login-path=local ... +# - +# - Example +# - mysql_credential_args="--login-path=piwik_admin" +# - mysql_credential_args="--defaults-file=/etc/mysql/debian.cnf" (Debian default) +# - mysql_credential_args="--defaults-file=/usr/local/mysql/sys-maint.cnf" +# - +# - Defaults to: +# - MYSQL_CREDENTIAL_ARGS="--defaults-file=/usr/local/mysql/sys-maint.cnf" +# - +#MYSQL_CREDENTIAL_ARGS="--defaults-file=/usr/local/mysql/sys-maint.cnf" + + +# - STATS_HOSTNAME +# - +# - Hostname which provides matomo statistics +# - +# - Example: +# - STATS_HOSTNAME="stats.warenform.de" +# - +# - Defaults to stats. +# - +#STATS_HOSTNAME="" + + +# - STATS_BASE_DIR +# - +# - Base directory where all matomo (piwik) instances are located. +# - +# - Example: +# - STATS_BASE_DIR="/var/www/stats.warenform.de" +# - +# Defaults to: +# - STATS_BASE_DIR="/var/www/${STATS_HOSTNAME}"" +# - +#STATS_BASE_DIR="" + + +# - MATOMO_PREFIX +# - +# - Databases and Matomo instances named as ${MATOMO_PREFIX}_. +# - +# - Example: +# - MATOMO_PREFIX="piwik" +# - +# - Defaults to: +# - MATOMO_PREFIX="matomo" +# - +#MATOMO_PREFIX="matomo" diff --git a/trigger_piwik_archives.sh b/trigger_piwik_archives.sh index 8586958..5e9ab19 100755 --- a/trigger_piwik_archives.sh +++ b/trigger_piwik_archives.sh @@ -1,97 +1,134 @@ #!/usr/bin/env bash -PATH=/usr/local/apache2/bin:/usr/local/php/bin:/usr/local/mysql/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +script_name="$(basename $(realpath $0))" +working_dir="$(dirname $(realpath $0))" -# - Is this script running on terminal ? -# - -if [[ -t 1 ]] ; then - terminal=true - LOGGING=true -else - terminal=false - LOGGING=false -fi +conf_file="${working_dir}/conf/${script_name%%.*}.conf" + +LOCK_DIR="/tmp/${script_name%%.*}.LOCK" + +error_log="${LOCK_DIR}/error.log" -stats_base_dir="/var/www/stats.warenform.de" +# ------------- +# - Default values +# ------------- php_bin=/usr/local/php/bin/php -http_user=www-data -# - MySQL / MariaDB credentials -# - -# - Giving password on command line is insecure an sind mysql 5.5 -# - you will get a warning doing so. -# - -# - Reading username/password fro file ist also possible, using MySQL/MariaDB -# - commandline parameter '--defaults-file'. -# - -# - Since Mysql Version 5.6, you can read username/password from -# - encrypted file. -# - -# - Create (encrypted) option file: -# - $ mysql_config_editor set --login-path=piwik_admin --socket=/tmp/mysql.sock --user=piwik_admin --password -# - $ Password: -# - -# - Use of option file: -# - $ mysql --login-path=local ... -# - -# - Example -# - mysql_credential_args="--login-pat=piwik_admin" -# - mysql_credential_args="--defaults-file=/etc/mysql/debian.cnf" (Debian default) -# - mysql_credential_args="--defaults-file=/usr/local/mysql/sys-maint.cnf" -# - +DEFAULT_MATOMO_PREFIX="matomo" -mysql_credential_args="--login-path=piwik_admin" +DEFAULT_MYSQL_CREDENTIAL_ARGS="--defaults-file=/usr/local/mysql/sys-maint.cnf" +DEFAULT_HTTP_USER=www-data -error_log="/tmp/$$.err" +VERBOSE=false + + + +# ------------- +# --- Some functions +# ------------- + +clean_up() { + + # Perform program exit housekeeping + rm -rf "$LOCK_DIR" + blank_line + exit $1 +} -# --------------------------------------------------- # -# ------------------- Funktionen -------------------- # -# usage() { - echo - [ -n "$1" ] && echo -e "Error: $1\n" -cat<] + + +\033[1mDescription\033[m Script archives piwik reports. - Options: - -h Prints this help. +\033[1mOptions\033[m - -v Verbose Mode + -h + Prints this help. - Applies only, if script is running in a console, or in otherwords: - parameter has no effect, if script is running as a cronjob. + -s + The directory contains the matomo instances on this server. + + -v + Verbose Mode. Applies only, if script is running in a console, or in otherwords: + parameter has no effect, if script is running as a cronjob. + +\033[1mFiles\033[m + + $conf_file: Configuration file +" - -EOF -exit +clean_up 1 } echononl(){ - echo X\\c > /tmp/shprompt$$ - if [ `wc -c /tmp/shprompt$$ | awk '{print $1}'` -eq 1 ]; then - echo "$*\\c" 1>&2 - else - echo -e -n "$*" 1>&2 + if $terminal ; then + echo X\\c > /tmp/shprompt$$ + if [ `wc -c /tmp/shprompt$$ | awk '{print $1}'` -eq 1 ]; then + echo "$*\\c" 1>&2 + else + echo -e -n "$*" 1>&2 + fi + rm /tmp/shprompt$$ fi - rm /tmp/shprompt$$ } info (){ - echo "" - echo -e "\t[ \033[32m\033[1mInfo\033[m ]: $*" - echo "" + if $terminal ; then + echo "" + echo -e " [ \033[32m\033[1mInfo\033[m ]: $*" + echo "" + fi } +warn (){ + if $terminal ; then + echo "" + echo -e " [ \033[33m\033[1mWarn\033[m ] $*" + echo "" + fi +} + +fatal(){ + echo "" + if $terminal ; then + echo -e " [ \033[31m\033[1mFatal\033[m ] $*" + else + echo -e " [ Fatal ] $*" + fi + echo "" + if $terminal ; then + echo -e " \033[1mScript terminated\033[m.." + else + echo -e " Script terminated.." + fi + echo "" + rm -rf $LOCK_DIR + exit 1 +} + + error (){ echo "" - echo -e "\t[ \033[31m\033[1mError\033[m ]: $*" + if $terminal ; then + echo -e " [ \033[31m\033[1mError\033[m ]: $*" + else + echo " [ Error ] $*" + fi echo "" } + error_cron(){ echo echo "----" @@ -101,26 +138,118 @@ error_cron(){ echo } echo_ok() { - echo -e "\033[85G[ \033[32mok\033[m ]" - ## echo -e " [ ok ]" + if $terminal ; then + echo -e "\033[85G[ \033[32mok\033[m ]" + fi } echo_failed(){ - echo -e "\033[85G[ \033[1;31mfailed\033[m ]" - ## echo -e " [ failed ]" + if $terminal ; then + echo -e "\033[85G[ \033[1;31mfailed\033[m ]" + fi } -# -# ------------------ Ende Funktionen ---------------- # -# --------------------------------------------------- # -VERBOSE=false +blank_line() { + if $terminal ; then + echo "" + fi +} -# --- +get_sld_domain() { + + # Set IFS to dot, so that we can split $@ on dots instead of spaces. + local IFS='.' + + # Break up arguments passed to shorthost so that each domain zone is + # a new index in an array. + zones=($@) + + index_tld="$((${#zones[@]}-1))" + index_sld="$((index_tld-1))" + + echo ${zones[$index_sld]}.${zones[$index_tld]} + +} + + + +# ------------- +# - Job is already running? +# ------------- + +# - If job already runs, stop execution.. +# - +if mkdir "$LOCK_DIR" 2> /dev/null ; then + + ## - Remove lockdir when the script finishes, or when it receives a signal + trap clean_up SIGHUP SIGINT SIGTERM + +else + + datum="$(date +"%d.%m.%Y %H:%M")" + + msg="[ Error ]: A previos instance of \"`basename $0`\" seems already be running.\n\n Exiting now.." + + echo "" + echo "[ Error ]: A previos instance of that script \"`basename $0`\" seems already be running." + echo "" + echo -e " Exiting now.." + echo "" + + exit 1 + +fi + + +# ------------- +# - Check some prerequisites +# ------------- + +# - Is this script running on terminal ? +# - +if [[ -t 1 ]] ; then + terminal=true +else + terminal=false +fi + + +# ========== +# - Begin Main Script +# ========== + +# ---------- +# - Headline +# ---------- + +if $terminal ; then + echo "" + echo -e "\033[1m----------\033[m" + echo -e "\033[32m\033[1mRunning script \033[m\033[1m$script_name\033[32m .. \033[m" + echo -e "\033[1m----------\033[m" +fi + + +# ---------- +# - Read Configurations from $conf_file +# ---------- + +if [[ -f "$conf_file" ]]; then + source "$conf_file" +else + warn "No configuration file '$conf_file' present." +fi + + +# ---------- # - Read in Commandline arguments -# --- -while getopts hv opt ; do +# ---------- + +while getopts hs:v opt ; do case $opt in v) VERBOSE=true ;; + s) STATS_BASE_DIR=$OPTARG + ;; h) usage ;; esac @@ -128,25 +257,70 @@ done cd /tmp -databases=`mysql $mysql_credential_args -N -s -e "show databases" | grep piwik_` +[[ -n "$MYSQL_CREDENTIAL_ARGS" ]] || MYSQL_CREDENTIAL_ARGS="$DEFAULT_MYSQL_CREDENTIAL_ARGS" +[[ -n "$HTTP_USER" ]] || HTTP_USER="$DEFAULT_HTTP_USER" -if $LOGGING ; then - echo -e "\n\t\033[1;32mStarting Script for Archiving Reports of Piwik Databases\033[1;37m\033[m\n\n" +if [[ -z "$STATS_HOSTNAME" ]]; then + STATS_HOSTNAME="stats.$(get_sld_domain "$(hostname -f)")" +fi + +[[ -n "$STATS_BASE_DIR" ]] || STATS_BASE_DIR="/var/www/${STATS_HOSTNAME}" + +if [[ ! -d "$STATS_BASE_DIR" ]]; then + fatal "Directory \033[1m$STATS_BASE_DIR\033[m do not exits!." +fi + +if [[ -z "$MATOMO_PREFIX" ]]; then + if $(ls ${STATS_BASE_DIR}/matomo* > /dev/null 2>&1) ; then + MATOMO_PREFIX="matomo" + elif $(ls ${STATS_BASE_DIR}/piwik* > /dev/null 2>&1) ; then + MATOMO_PREFIX="piwik" + else + error "No matomo instances found on this server." + fi +else + if ! $(ll ${STATS_BASE_DIR}/${MATOMO_PREFIX}* > /dev/null 2>&1) ; then + error "No matomo instances found on this server (${STATS_BASE_DIR}/${MATOMO_PREFIX}*)." + fi +fi + +if $terminal ; then + echo "" + echo " HTTP_USER.................: $HTTP_USER" + echo " MYSQL_CREDENTIAL_ARGS.....: $MYSQL_CREDENTIAL_ARGS" + echo " STATS_HOSTNAME............: $STATS_HOSTNAME" + echo " STATS_BASE_DIR............: $STATS_BASE_DIR" + echo " MATOMO_PREFIX.............: $MATOMO_PREFIX" + echo "" +fi + + + +echononl " Get matomo databases (names)" +databases="$(mysql $MYSQL_CREDENTIAL_ARGS -N -s -e "show databases" 2> $error_log | grep ${MATOMO_PREFIX}_ 2>> $error_log)" +if [[ $? -gt 0 ]] ;then + echo_failed + [[ -s "$error_log" ]] && error "$(cat $error_log)" +else + echo_ok +fi + +if [[ -z "$databases" ]] ; then + error "No matomo databses found." + clean_up 1 fi declare -i sum_errors=0 for db_name in $databases ; do - if [ -f "$stats_base_dir/$db_name/console" ];then + if [ -f "$STATS_BASE_DIR/$db_name/console" ];then - if $LOGGING ; then - - echononl "\tArchiving Reports of Piwik Database \033[1m$db_name\033[m" + echononl " Archiving Reports of Piwik Database \033[1m$db_name\033[m" # - Archive Reports # - - su $http_user -s /bin/bash \ - -c "$php_bin $stats_base_dir/$db_name/console core:archive --url=https://stats.warenform.de/$db_name/" \ + su $HTTP_USER -s /bin/bash \ + -c "$php_bin $STATS_BASE_DIR/$db_name/console core:archive --url=https://${STATS_HOSTNAME}/$db_name/" \ > $error_log 2>&1 if [[ $? -ne 0 ]]; then let "sum_errors += 1" @@ -162,39 +336,18 @@ for db_name in $databases ; do fi fi - else - - # - Archive Reports - # - - su $http_user -s /bin/bash \ - -c "$php_bin $stats_base_dir/$db_name/console core:archive --url=https://stats.warenform.de/$db_name/" \ - > $error_log 2>&1 - if [[ $? -ne 0 ]]; then - let "sum_errors += 1" - error_cron "Error while executing archive script for database \"$db_name\"" "$(cat $error_log)" - fi - - fi - else - if $LOGGING ; then - error "Archive script \"$stats_base_dir/$db_name/console\" for database \"$db_name\" not found!" - else - error_cron "Error: archive script for database \"$db_name\" not found" "$stats_base_dir/$db_name/console" - fi + error "Archive script \"$STATS_BASE_DIR/$db_name/console\" for database \"$db_name\" not found!" continue fi done -if $LOGGING ; then - if [[ $sum_errors -eq 0 ]]; then - info "Script ended successfully." - else - error "Script ended with \033[1m${sum_errors}\033[m error(s)!" - fi +if [[ $sum_errors -eq 0 ]]; then + info "Script ended successfully." +else + error "Script ended with \033[1m${sum_errors}\033[m error(s)!" fi -rm -f $error_log -exit 0 +clean_up 0