Optimize/Repair table scripts: ommit tables with table engine InnoDB.

conf/optimize_mysql_tables.conf.sample

@@ -0,0 +1,62 @@
+# ----------------------------------------------------
+# ---
+# - Parameter Settings for Script 'optimize_mysql_tables.sh'
+# ---
+# ----------------------------------------------------
+
+
+# - MySQL / MariaDB / Percona credentials
+
+# - mysql_credential_args
+# -
+# - Giving password on command line is insecure an sind mysql 5.5
+# - you will get a warning doing so.
+# - 
+# - Reading username/password fro file ist also possible, using MySQL/MariaDB
+# - commandline parameter '--defaults-file'.
+# - 
+# - Since Version 5.6, that method is considered as insecure. 
+# - To avoid giving the password on command line, we use an 
+# - encrypted option file
+# -
+# -    Create (encrypted) option file:
+# -    $ mysql_config_editor set --login-path=local --socket=/var/run/mysqld/mysqld.sock  --user=backup --password
+# -    $ Password:
+# -
+# -    Use of option file:
+# -    $ mysql --login-path=local ...
+# -
+# - Example
+# -    mysql_credential_args="--login-path=local"
+# -    mysql_credential_args="--defaults-file=/etc/mysql/debian.cnf" (Debian default)
+# -    mysql_credential_args="--defaults-file=/usr/local/mysql/sys-maint.cnf"
+# -
+# - Defaults to mysql_credential_args="--login-path=local"
+# -
+#mysql_credential_args="--login-path=local"
+
+
+# - mysql_credential_args_arr
+# -
+# - If multiple MySQL / MariaDB / Percona Installations are present, for each installation, 
+# - you can give its own credentials.
+# - 
+# - Note:
+# -   If this parameter ist present, the parameter 'mysql_credential_args' will be ignored.
+# -
+# - Example
+# -   mysql_credential_args_arr="10.2.13:--defaults-file=/etc/mysql/debian.cnf 5.2.7:--login-path=local"
+# -   msql_credential_args_arr="5.6:--login-path=local-5.6 5.7:--login-path=local"
+# -
+# -
+# - Note:
+# -    the forst value (before the colon ':') is only user for logging output and has no
+# -    further relevance .
+# -
+#msql_credential_args_arr=""
+
+# - log_file
+# -
+# - Defaults to "/var/log/optimize_mysql_tables.log"
+# -
+#log_file="/var/log/optimize_mysql_tables.log"

optimize_mysql_tables-ND.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 
 working_dir="$(dirname $(realpath $0))"
-conf_file="${working_dir}/conf/mysql_credetials.conf"
+conf_file="${working_dir}/conf/optimize_mysql_tables.conf"
 
 tmp_log_file="$(mktemp)"
 
@@ -10,12 +10,54 @@ tmp_log_file="$(mktemp)"
 # - Variable settings
 # -------------
 DEFAULT_MYSQL_CREDENTIAL_ARGS="--login-path=local"
+DEFAULT_LOG_FILE="/var/log/optimize_mysql_tables.log"
+VERBOSE=false
 
 
 # -------------
 # --- Some functions
 # -------------
 
+usage() {
+
+   [[ -n  "$1" ]] && error "$1"
+
+   [[ $terminal ]] && echo -e "
+\033[1mUsage:\033[m
+
+   $(basename $0) [-h|-v] [DB-Name]
+
+\033[1mDescription\033[m
+
+   Script checks (and reorganizes) all tables of all databases by executing MySQL
+   command 'OPTIMIZE TABLE'.
+
+   If a database is given at the command line, only tables of that database
+   will be checked.
+
+   If a check on a table fails, MySQL command 'REPAIR TABLE' will be executed on
+   that table.
+
+\033[1mOptions\033[m
+
+   -h
+      Prints this help
+
+   -v
+      Verbose output if running on a terminal.
+
+\033[1mFiles\033[m
+
+   $conf_file: Configuration file
+
+
+"
+
+   clean_up 1
+
+
+}
+
 
 clean_up() {
 
@@ -108,14 +150,18 @@ echo_skipped() {
    fi
 }
 
+trim() {
+    local var="$*"
+    var="${var#"${var%%[![:space:]]*}"}"   # remove leading whitespace characters
+    var="${var%"${var##*[![:space:]]}"}"   # remove trailing whitespace characters
+    echo -n "$var"
+}
+
+
 
 trap clean_up SIGHUP SIGINT SIGTERM
 
 
-
-
-
-
 # - Is this script running on terminal ?
 # -
 if [[ -t 1 ]] ; then
@@ -129,16 +175,47 @@ mysql=`which mysql`
 if [ -z "$mysql" ]; then
    if [ -x "/usr/local/mysql/bin/mysql" ]; then
       mysql=/usr/local/mysql/bin/mysql
-   fi
+   else
       fatal "No binary 'mysql' found!"
    fi
+fi
 
 
+# -------------
+# - Read Commandline Parameters
+# -------------
+
+while getopts hv opt ; do
+   case $opt in
+      v) VERBOSE=true
+         ;;
+      h) usage
+          ;;
+      *) usage
+   esac
+done
+
+shift $(expr $OPTIND - 1)
+[[ "$#" -gt 1 ]] && usage "Wrong number or order of arguments given!"
+
+
+# - Print help?
+# -
+if [[ "$(trim $*)" =~ " -h" ]] || [[ "$(trim $*)" =~ " --help" ]] ; then
+   usage
+fi
+
 
 # -------------
 # - Load Settings from configuration file
 # -------------
 
+if [[ -n "$1 " ]] ; then
+   DATABASES="$1"
+else
+   DATABASES=""
+fi
+
 if $terminal ; then
    echo ""
 fi
@@ -160,6 +237,7 @@ fi
 
 
 [[ -z "$mysql_credential_args" ]] && mysql_credential_args="$DEFAULT_MYSQL_CREDENTIAL_ARGS"
+[[ -z "$log_file" ]] && log_file="$DEFAULT_LOG_FILE"
 
 if [[ ${#mysql_credential_args_arr[@]} -eq 0 ]]; then
    [[ -z "$mysql_credential_args" ]] && mysql_credential_args="$DEFAULT_MYSQL_CREDENTIAL_ARGS"
@@ -181,8 +259,11 @@ for _val in ${mysql_credential_args_arr[@]} ; do
       echo ""
       echo -e "[ \033[37m\033[1mMySQL $mysql_version\033[m ]: optimize (and repair) tables of databases at host '$(hostname -f)'."
    fi
+   echo -e "[ MySQL $mysql_version ]: optimize (and repair) tables of databases at host '$(hostname -f)'." > $log_file
 
+   if [[ -z "$DATABASES" ]] ; then
       DATABASES=`$mysql $mysql_credential_args -N -s -e "show databases"`
+   fi
 
    length_table_name=0
    _service_extension=PRODUCTION.$$
@@ -196,6 +277,8 @@ for _val in ${mysql_credential_args_arr[@]} ; do
          echo ""
          echo -e "   [$(date)] Optimize tables in database '${db}'.."
       fi
+      echo "" >> $log_file
+      echo -e "   [$(date)] Optimize tables in database '${db}'.." >> $log_file
 
       _htdocs_nd_moved=false
       _htdocs_nd_symlinked=false
@@ -273,6 +356,17 @@ for _val in ${mysql_credential_args_arr[@]} ; do
 
       for table in $TABLES ; do
 
+         # - Ommit InnoDB tables
+         # -
+         _engine="$($mysql $mysql_credential_args -N -s -e "SELECT ENGINE FROM information_schema.TABLES WHERE TABLE_SCHEMA = '$db' AND TABLE_NAME = '$table'")"
+        if [[ "${_engine,,}" = 'innodb' ]] ; then
+            if $VERBOSE ; then
+               echo -e "   [$(date)] Ommit table '$table' - storage engine is InnoDB"
+            fi
+            echo -e "   [$(date)] Ommit table '$table' - storage engine is InnoDB" >> $log_file
+            continue
+         fi 
+
          if $terminal ; then
             blank_signs=""
             if [[ $length_table_name -gt ${#table} ]]; then
@@ -285,6 +379,7 @@ for _val in ${mysql_credential_args_arr[@]} ; do
                echo -en "\033[1G"
             fi
             echo -en "\033[1G   \033[32mOptimize table \033[1m$table\033[m$blank_signs\033[1G"
+            echo -e "   [$(date)] Optimize table '$table'" >> $log_file
          fi
          length_table_name=${#table}
 
@@ -294,6 +389,10 @@ for _val in ${mysql_credential_args_arr[@]} ; do
          if [[ $? -ne 0 ]]; then
 
             warn "Optimizing table \"${table}\" of database \"$db\" failed. Trying to repair.."
+            echo "" >> $log_file
+            echo "  [$(date)] warning: Optimizing table \"${table}\" of database \"$db\" failed." >> $log_file
+            echo "" >> $log_file
+            echo "  [$(date)] Repair  table '$table'" >> $log_file
 
             $mysql $mysql_credential_args $db -N -s -e "REPAIR TABLE \`$table\`" > $tmp_log_file 2>&1
 
@@ -302,14 +401,21 @@ for _val in ${mysql_credential_args_arr[@]} ; do
                _all_success=false
                error "Repairing table '$table' failed.\n$(cat "$tmp_log_file")"
                error_messages_arr+=("MySQL $mysql_version: Error while repairing table '${table}' of database '$db'.")
+               echo "" >> $log_file
+               echo -e "  [$(date)] error: Repairing table '$table' failed of database \"$db\" failed..\n$(cat "$tmp_log_file")" >> $log_file
+               echo "" >> $log_file
 
             else
 
+               echo -e "   [$(date)] Optimize table '$table'" >> $log_file
                $mysql $mysql_credential_args  $db -N -s -e "OPTIMIZE TABLE \`$table\`" > $tmp_log_file 2>&1
 
                if [[ $? -ne 0 ]]; then
                   error "Reoptimizing table \"${table}\" of database \"$db\" failed.\n$(cat "$tmp_log_file")"
                   error_messages_arr+=("MySQL $mysql_version: Error while (re-)optimizing table '${table}' of database '$db'.")
+                  echo "" >> $log_file
+                  echo -e "  [$(date)] error: Reoptimizing table \"${table}\" of database \"$db\" failed.\n$(cat "$tmp_log_file")" >> $log_file
+                  echo "" >> $log_file
                else
                   info "Reoptimizing table \"${table}\" of database \"$db\" was successfully."
                fi
@@ -371,11 +477,12 @@ for _val in ${mysql_credential_args_arr[@]} ; do
          echo -en "\033[1G   [$(date)] End optimize tables in database '${db}'.."
          echo
       fi
+      echo -e "   [$(date)] End optimize tables in database '${db}'" >> $log_file
 
    done
 
    if $_all_success ; then
-      info_messages_arr+=("MySQL $mysql_version: The optimization of the MySQL tables of all databases was successful.")
+      info_messages_arr+=("MySQL $mysql_version: The optimization of the MySQL tables of all databases were successful.")
    fi
 
    if $terminal ; then
@@ -383,6 +490,9 @@ for _val in ${mysql_credential_args_arr[@]} ; do
       echo -e "[ \033[37m\033[1mMySQL $mysql_version\033[m ]: Finished optimizing MySQL databases at host $(hostname -f)."
       echo ""
    fi
+   echo "" >> $log_file
+   echo "[ MySQL $mysql_version ]: Finished optimizing MySQL databases at host $(hostname -f)." >> $log_file
+   echo "" >> $log_file
 done
 
 if [[ ${#info_messages_arr[@]} -gt 0 ]]; then
@@ -390,6 +500,7 @@ if [[ ${#info_messages_arr[@]} -gt 0 ]]; then
       if $terminal ; then
          info "$msg"
       fi
+      echo "[ Info ]: $msg" >> $log_file
    done
 fi
 
@@ -398,7 +509,12 @@ if [[ ${#error_messages_arr[@]} -gt 0 ]]; then
       if $terminal ; then
          error "$msg"
       fi
+      echo "[ Error ]: $msg" >> $log_file
    done
 fi
 
+if $terminal ; then
+   info "See logfile '$log_file' for more details."
+fi
+
 clean_up 0

optimize_mysql_tables.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 
 working_dir="$(dirname $(realpath $0))"
-conf_file="${working_dir}/conf/mysql_credetials.conf"
+conf_file="${working_dir}/conf/optimize_mysql_tables.conf"
 
 tmp_log_file="$(mktemp)"
 
@@ -10,6 +10,8 @@ tmp_log_file="$(mktemp)"
 # - Variable settings
 # -------------
 DEFAULT_MYSQL_CREDENTIAL_ARGS="--login-path=local"
+DEFAULT_LOG_FILE="/var/log/optimize_mysql_tables.log"
+VERBOSE=false
 
 
 # -------------
@@ -23,7 +25,7 @@ usage() {
    [[ $terminal ]] && echo -e "
 \033[1mUsage:\033[m
 
-   $(basename $0) [DB-Name]
+   $(basename $0) [-h|-v] [DB-Name]
 
 \033[1mDescription\033[m
 
@@ -38,7 +40,11 @@ usage() {
 
 \033[1mOptions\033[m
 
-   No Options available
+   -h
+      Prints this help
+
+   -v
+      Verbose output if running on a terminal.
 
 \033[1mFiles\033[m
 
@@ -174,10 +180,27 @@ if [ -z "$mysql" ]; then
    fi
 fi
 
+# -------------
+# - Read Commandline Parameters
+# -------------
+
+while getopts hv opt ; do
+   case $opt in
+      v) VERBOSE=true
+         ;;
+      h) usage
+          ;;
+      *) usage
+   esac
+done
+
+shift $(expr $OPTIND - 1)
+[[ "$#" -gt 1 ]] && usage "Wrong number or order of arguments given!"
+
 
 # - Print help?
 # -
-if [[ "$(trim $*)" = "-h" ]] || [[ "$(trim $*)" = "--help" ]] ; then
+if [[ "$(trim $*)" =~ " -h" ]] || [[ "$(trim $*)" =~ " --help" ]] ; then
    usage
 fi
 
@@ -213,6 +236,7 @@ fi
 
 
 [[ -z "$mysql_credential_args" ]] && mysql_credential_args="$DEFAULT_MYSQL_CREDENTIAL_ARGS"
+[[ -z "$log_file" ]] && log_file="$DEFAULT_LOG_FILE"
 
 if [[ ${#mysql_credential_args_arr[@]} -eq 0 ]]; then
    [[ -z "$mysql_credential_args" ]] && mysql_credential_args="$DEFAULT_MYSQL_CREDENTIAL_ARGS"
@@ -234,6 +258,7 @@ for _val in ${mysql_credential_args_arr[@]} ; do
       echo ""
       echo -e "[ \033[37m\033[1mMySQL $mysql_version\033[m ]: optimize (and repair) tables of databases at host '$(hostname -f)'."
    fi
+    echo -e "[ MySQL $mysql_version ]: optimize (and repair) tables of databases at host '$(hostname -f)'." > $log_file
 
 	if [[ -z "$DATABASES" ]] ; then
    	DATABASES="$($mysql $mysql_credential_args -N -s -e "show databases")"
@@ -250,11 +275,24 @@ for _val in ${mysql_credential_args_arr[@]} ; do
          echo ""
          echo -e "   [$(date)] Optimize tables in database '${db}'.."
       fi
+      echo "" >> $log_file
+      echo -e "   [$(date)] Optimize tables in database '${db}'.." >> $log_file
 
       TABLES=`$mysql $mysql_credential_args $db -N -s -e "show tables"`
 
       for table in $TABLES ; do
 
+         # - Ommit InnoDB tables
+         # -
+         _engine="$($mysql $mysql_credential_args -N -s -e "SELECT ENGINE FROM information_schema.TABLES WHERE TABLE_SCHEMA = '$db' AND TABLE_NAME = '$table'")"
+         if [[ "${_engine,,}" = 'innodb' ]] ; then 
+            if $VERBOSE ; then
+               echo -e "   [$(date)] Ommit table '$table' - storage engine is InnoDB"
+            fi
+            echo -e "   [$(date)] Ommit table '$table' - storage engine is InnoDB" >> $log_file
+            continue
+         fi
+
          if $terminal ; then
             blank_signs=""
             if [[ $length_table_name -gt ${#table} ]]; then
@@ -267,6 +305,7 @@ for _val in ${mysql_credential_args_arr[@]} ; do
                echo -en "\033[1G"
             fi
             echo -en "\033[1G   \033[32mOptimize table \033[1m$table\033[m$blank_signs\033[1G"
+            echo -e "   [$(date)] Optimize table '$table'" >> $log_file
          fi
          length_table_name=${#table}
 
@@ -276,6 +315,10 @@ for _val in ${mysql_credential_args_arr[@]} ; do
          if [[ $? -ne 0 ]]; then
 
             warn "Optimizing table \"${table}\" of database \"$db\" failed. Trying to repair.."
+            echo "" >> $log_file
+            echo "  [$(date)] warning: Optimizing table \"${table}\" of database \"$db\" failed." >> $log_file 
+            echo "" >> $log_file
+            echo "  [$(date)] Repair  table '$table'" >> $log_file
 
             $mysql $mysql_credential_args $db -N -s -e "REPAIR TABLE \`$table\`" > $tmp_log_file 2>&1
 
@@ -284,14 +327,22 @@ for _val in ${mysql_credential_args_arr[@]} ; do
                _all_success=false
                error "Repairing table '$table' failed.\n$(cat "$tmp_log_file")"
                error_messages_arr+=("MySQL $mysql_version: Error while repairing table '${table}' of database '$db'.")
+               echo "" >> $log_file
+               echo -e "  [$(date)] error: Repairing table '$table' failed of database \"$db\" failed..\n$(cat "$tmp_log_file")" >> $log_file
+               echo "" >> $log_file
+
 
             else
 
+               echo -e "   [$(date)] Optimize table '$table'" >> $log_file
                $mysql $mysql_credential_args  $db -N -s -e "OPTIMIZE TABLE \`$table\`" > $tmp_log_file 2>&1
 
                if [[ $? -ne 0 ]]; then
                   error "Reoptimizing table \"${table}\" of database \"$db\" failed.\n$(cat "$tmp_log_file")"
                   error_messages_arr+=("MySQL $mysql_version: Error while (re-)optimizing table '${table}' of database '$db'.")
+                  echo "" >> $log_file
+                  echo -e "  [$(date)] error: Reoptimizing table \"${table}\" of database \"$db\" failed.\n$(cat "$tmp_log_file")" >> $log_file
+                  echo "" >> $log_file
                else
                   info "Reoptimizing table \"${table}\" of database \"$db\" was successfully."
                fi
@@ -308,11 +359,12 @@ for _val in ${mysql_credential_args_arr[@]} ; do
          echo -en "\033[1G   [$(date)] End optimize tables in database '${db}'.."
          echo
       fi
+      echo -e "   [$(date)] End optimize tables in database '${db}'" >> $log_file
 
    done
 
    if $_all_success ; then
-      info_messages_arr+=("MySQL $mysql_version: The optimization of the MySQL tables of all databases was successful.")
+      info_messages_arr+=("MySQL $mysql_version: The optimization of the MySQL tables of all databases were successful.")
    fi
 
    if $terminal ; then
@@ -320,6 +372,9 @@ for _val in ${mysql_credential_args_arr[@]} ; do
       echo -e "[ \033[37m\033[1mMySQL $mysql_version\033[m ]: Finished optimizing MySQL databases at host $(hostname -f)."
       echo ""
    fi
+   echo "" >> $log_file
+   echo "[ MySQL $mysql_version ]: Finished optimizing MySQL databases at host $(hostname -f)." >> $log_file
+   echo "" >> $log_file
 done
 
 if [[ ${#info_messages_arr[@]} -gt 0 ]]; then
@@ -327,6 +382,7 @@ if [[ ${#info_messages_arr[@]} -gt 0 ]]; then
       if $terminal ; then
          info "$msg"
       fi
+      echo "[ Info ]: $msg" >> $log_file
    done
 fi
 
@@ -335,7 +391,12 @@ if [[ ${#error_messages_arr[@]} -gt 0 ]]; then
       if $terminal ; then
          error "$msg"
       fi
+      echo "[ Error ]: $msg" >> $log_file
    done
 fi
 
+if $terminal ; then
+   info "See logfile '$log_file' for more details."
+fi
+
 clean_up 0

repair_mysql_tables.sh

@@ -286,6 +286,14 @@ for _val in ${mysql_credential_args_arr[@]} ; do
 
       for table in $TABLES ; do
 
+         # - Ommit InnoDB tables
+         # -
+         _engine="$($mysql $mysql_credential_args -N -s -e "SELECT ENGINE FROM information_schema.TABLES WHERE TABLE_SCHEMA = '$db' AND TABLE_NAME = '$table'")"
+         if [[ "${_engine,,}" = 'innodb' ]] ; then
+            echo -e "   [$(date)] Ommit table '$table' - The storage engine (InnoDB) doesn't support repair"
+            continue
+         fi
+
          if $terminal ; then
             blank_signs=""
             if [[ $length_table_name -gt ${#table} ]]; then