diff --git a/conf/optimize_mysql_tables.conf.sample b/conf/optimize_mysql_tables.conf.sample new file mode 100644 index 0000000..85d417e --- /dev/null +++ b/conf/optimize_mysql_tables.conf.sample @@ -0,0 +1,62 @@ +# ---------------------------------------------------- +# --- +# - Parameter Settings for Script 'optimize_mysql_tables.sh' +# --- +# ---------------------------------------------------- + + +# - MySQL / MariaDB / Percona credentials + +# - mysql_credential_args +# - +# - Giving password on command line is insecure an sind mysql 5.5 +# - you will get a warning doing so. +# - +# - Reading username/password fro file ist also possible, using MySQL/MariaDB +# - commandline parameter '--defaults-file'. +# - +# - Since Version 5.6, that method is considered as insecure. +# - To avoid giving the password on command line, we use an +# - encrypted option file +# - +# - Create (encrypted) option file: +# - $ mysql_config_editor set --login-path=local --socket=/var/run/mysqld/mysqld.sock --user=backup --password +# - $ Password: +# - +# - Use of option file: +# - $ mysql --login-path=local ... +# - +# - Example +# - mysql_credential_args="--login-path=local" +# - mysql_credential_args="--defaults-file=/etc/mysql/debian.cnf" (Debian default) +# - mysql_credential_args="--defaults-file=/usr/local/mysql/sys-maint.cnf" +# - +# - Defaults to mysql_credential_args="--login-path=local" +# - +#mysql_credential_args="--login-path=local" + + +# - mysql_credential_args_arr +# - +# - If multiple MySQL / MariaDB / Percona Installations are present, for each installation, +# - you can give its own credentials. +# - +# - Note: +# - If this parameter ist present, the parameter 'mysql_credential_args' will be ignored. +# - +# - Example +# - mysql_credential_args_arr="10.2.13:--defaults-file=/etc/mysql/debian.cnf 5.2.7:--login-path=local" +# - msql_credential_args_arr="5.6:--login-path=local-5.6 5.7:--login-path=local" +# - +# - +# - Note: +# - the forst value (before the colon ':') is only user for logging output and has no +# - further relevance . +# - +#msql_credential_args_arr="" + +# - log_file +# - +# - Defaults to "/var/log/optimize_mysql_tables.log" +# - +#log_file="/var/log/optimize_mysql_tables.log" diff --git a/optimize_mysql_tables-ND.sh b/optimize_mysql_tables-ND.sh index bf8a4ba..7632cad 100755 --- a/optimize_mysql_tables-ND.sh +++ b/optimize_mysql_tables-ND.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash working_dir="$(dirname $(realpath $0))" -conf_file="${working_dir}/conf/mysql_credetials.conf" +conf_file="${working_dir}/conf/optimize_mysql_tables.conf" tmp_log_file="$(mktemp)" @@ -10,12 +10,54 @@ tmp_log_file="$(mktemp)" # - Variable settings # ------------- DEFAULT_MYSQL_CREDENTIAL_ARGS="--login-path=local" +DEFAULT_LOG_FILE="/var/log/optimize_mysql_tables.log" +VERBOSE=false # ------------- # --- Some functions # ------------- +usage() { + + [[ -n "$1" ]] && error "$1" + + [[ $terminal ]] && echo -e " +\033[1mUsage:\033[m + + $(basename $0) [-h|-v] [DB-Name] + +\033[1mDescription\033[m + + Script checks (and reorganizes) all tables of all databases by executing MySQL + command 'OPTIMIZE TABLE'. + + If a database is given at the command line, only tables of that database + will be checked. + + If a check on a table fails, MySQL command 'REPAIR TABLE' will be executed on + that table. + +\033[1mOptions\033[m + + -h + Prints this help + + -v + Verbose output if running on a terminal. + +\033[1mFiles\033[m + + $conf_file: Configuration file + + +" + + clean_up 1 + + +} + clean_up() { @@ -108,14 +150,18 @@ echo_skipped() { fi } +trim() { + local var="$*" + var="${var#"${var%%[![:space:]]*}"}" # remove leading whitespace characters + var="${var%"${var##*[![:space:]]}"}" # remove trailing whitespace characters + echo -n "$var" +} + + trap clean_up SIGHUP SIGINT SIGTERM - - - - # - Is this script running on terminal ? # - if [[ -t 1 ]] ; then @@ -129,16 +175,47 @@ mysql=`which mysql` if [ -z "$mysql" ]; then if [ -x "/usr/local/mysql/bin/mysql" ]; then mysql=/usr/local/mysql/bin/mysql + else + fatal "No binary 'mysql' found!" fi - fatal "No binary 'mysql' found!" fi +# ------------- +# - Read Commandline Parameters +# ------------- + +while getopts hv opt ; do + case $opt in + v) VERBOSE=true + ;; + h) usage + ;; + *) usage + esac +done + +shift $(expr $OPTIND - 1) +[[ "$#" -gt 1 ]] && usage "Wrong number or order of arguments given!" + + +# - Print help? +# - +if [[ "$(trim $*)" =~ " -h" ]] || [[ "$(trim $*)" =~ " --help" ]] ; then + usage +fi + # ------------- # - Load Settings from configuration file # ------------- +if [[ -n "$1 " ]] ; then + DATABASES="$1" +else + DATABASES="" +fi + if $terminal ; then echo "" fi @@ -160,6 +237,7 @@ fi [[ -z "$mysql_credential_args" ]] && mysql_credential_args="$DEFAULT_MYSQL_CREDENTIAL_ARGS" +[[ -z "$log_file" ]] && log_file="$DEFAULT_LOG_FILE" if [[ ${#mysql_credential_args_arr[@]} -eq 0 ]]; then [[ -z "$mysql_credential_args" ]] && mysql_credential_args="$DEFAULT_MYSQL_CREDENTIAL_ARGS" @@ -181,8 +259,11 @@ for _val in ${mysql_credential_args_arr[@]} ; do echo "" echo -e "[ \033[37m\033[1mMySQL $mysql_version\033[m ]: optimize (and repair) tables of databases at host '$(hostname -f)'." fi + echo -e "[ MySQL $mysql_version ]: optimize (and repair) tables of databases at host '$(hostname -f)'." > $log_file - DATABASES=`$mysql $mysql_credential_args -N -s -e "show databases"` + if [[ -z "$DATABASES" ]] ; then + DATABASES=`$mysql $mysql_credential_args -N -s -e "show databases"` + fi length_table_name=0 _service_extension=PRODUCTION.$$ @@ -196,6 +277,8 @@ for _val in ${mysql_credential_args_arr[@]} ; do echo "" echo -e " [$(date)] Optimize tables in database '${db}'.." fi + echo "" >> $log_file + echo -e " [$(date)] Optimize tables in database '${db}'.." >> $log_file _htdocs_nd_moved=false _htdocs_nd_symlinked=false @@ -273,6 +356,17 @@ for _val in ${mysql_credential_args_arr[@]} ; do for table in $TABLES ; do + # - Ommit InnoDB tables + # - + _engine="$($mysql $mysql_credential_args -N -s -e "SELECT ENGINE FROM information_schema.TABLES WHERE TABLE_SCHEMA = '$db' AND TABLE_NAME = '$table'")" + if [[ "${_engine,,}" = 'innodb' ]] ; then + if $VERBOSE ; then + echo -e " [$(date)] Ommit table '$table' - storage engine is InnoDB" + fi + echo -e " [$(date)] Ommit table '$table' - storage engine is InnoDB" >> $log_file + continue + fi + if $terminal ; then blank_signs="" if [[ $length_table_name -gt ${#table} ]]; then @@ -285,6 +379,7 @@ for _val in ${mysql_credential_args_arr[@]} ; do echo -en "\033[1G" fi echo -en "\033[1G \033[32mOptimize table \033[1m$table\033[m$blank_signs\033[1G" + echo -e " [$(date)] Optimize table '$table'" >> $log_file fi length_table_name=${#table} @@ -294,6 +389,10 @@ for _val in ${mysql_credential_args_arr[@]} ; do if [[ $? -ne 0 ]]; then warn "Optimizing table \"${table}\" of database \"$db\" failed. Trying to repair.." + echo "" >> $log_file + echo " [$(date)] warning: Optimizing table \"${table}\" of database \"$db\" failed." >> $log_file + echo "" >> $log_file + echo " [$(date)] Repair table '$table'" >> $log_file $mysql $mysql_credential_args $db -N -s -e "REPAIR TABLE \`$table\`" > $tmp_log_file 2>&1 @@ -302,14 +401,21 @@ for _val in ${mysql_credential_args_arr[@]} ; do _all_success=false error "Repairing table '$table' failed.\n$(cat "$tmp_log_file")" error_messages_arr+=("MySQL $mysql_version: Error while repairing table '${table}' of database '$db'.") + echo "" >> $log_file + echo -e " [$(date)] error: Repairing table '$table' failed of database \"$db\" failed..\n$(cat "$tmp_log_file")" >> $log_file + echo "" >> $log_file else + echo -e " [$(date)] Optimize table '$table'" >> $log_file $mysql $mysql_credential_args $db -N -s -e "OPTIMIZE TABLE \`$table\`" > $tmp_log_file 2>&1 if [[ $? -ne 0 ]]; then error "Reoptimizing table \"${table}\" of database \"$db\" failed.\n$(cat "$tmp_log_file")" error_messages_arr+=("MySQL $mysql_version: Error while (re-)optimizing table '${table}' of database '$db'.") + echo "" >> $log_file + echo -e " [$(date)] error: Reoptimizing table \"${table}\" of database \"$db\" failed.\n$(cat "$tmp_log_file")" >> $log_file + echo "" >> $log_file else info "Reoptimizing table \"${table}\" of database \"$db\" was successfully." fi @@ -371,11 +477,12 @@ for _val in ${mysql_credential_args_arr[@]} ; do echo -en "\033[1G [$(date)] End optimize tables in database '${db}'.." echo fi + echo -e " [$(date)] End optimize tables in database '${db}'" >> $log_file done if $_all_success ; then - info_messages_arr+=("MySQL $mysql_version: The optimization of the MySQL tables of all databases was successful.") + info_messages_arr+=("MySQL $mysql_version: The optimization of the MySQL tables of all databases were successful.") fi if $terminal ; then @@ -383,6 +490,9 @@ for _val in ${mysql_credential_args_arr[@]} ; do echo -e "[ \033[37m\033[1mMySQL $mysql_version\033[m ]: Finished optimizing MySQL databases at host $(hostname -f)." echo "" fi + echo "" >> $log_file + echo "[ MySQL $mysql_version ]: Finished optimizing MySQL databases at host $(hostname -f)." >> $log_file + echo "" >> $log_file done if [[ ${#info_messages_arr[@]} -gt 0 ]]; then @@ -390,6 +500,7 @@ if [[ ${#info_messages_arr[@]} -gt 0 ]]; then if $terminal ; then info "$msg" fi + echo "[ Info ]: $msg" >> $log_file done fi @@ -398,7 +509,12 @@ if [[ ${#error_messages_arr[@]} -gt 0 ]]; then if $terminal ; then error "$msg" fi + echo "[ Error ]: $msg" >> $log_file done fi +if $terminal ; then + info "See logfile '$log_file' for more details." +fi + clean_up 0 diff --git a/optimize_mysql_tables.sh b/optimize_mysql_tables.sh index bf366ca..06d9033 100755 --- a/optimize_mysql_tables.sh +++ b/optimize_mysql_tables.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash working_dir="$(dirname $(realpath $0))" -conf_file="${working_dir}/conf/mysql_credetials.conf" +conf_file="${working_dir}/conf/optimize_mysql_tables.conf" tmp_log_file="$(mktemp)" @@ -10,6 +10,8 @@ tmp_log_file="$(mktemp)" # - Variable settings # ------------- DEFAULT_MYSQL_CREDENTIAL_ARGS="--login-path=local" +DEFAULT_LOG_FILE="/var/log/optimize_mysql_tables.log" +VERBOSE=false # ------------- @@ -23,7 +25,7 @@ usage() { [[ $terminal ]] && echo -e " \033[1mUsage:\033[m - $(basename $0) [DB-Name] + $(basename $0) [-h|-v] [DB-Name] \033[1mDescription\033[m @@ -38,7 +40,11 @@ usage() { \033[1mOptions\033[m - No Options available + -h + Prints this help + + -v + Verbose output if running on a terminal. \033[1mFiles\033[m @@ -174,10 +180,27 @@ if [ -z "$mysql" ]; then fi fi +# ------------- +# - Read Commandline Parameters +# ------------- + +while getopts hv opt ; do + case $opt in + v) VERBOSE=true + ;; + h) usage + ;; + *) usage + esac +done + +shift $(expr $OPTIND - 1) +[[ "$#" -gt 1 ]] && usage "Wrong number or order of arguments given!" + # - Print help? # - -if [[ "$(trim $*)" = "-h" ]] || [[ "$(trim $*)" = "--help" ]] ; then +if [[ "$(trim $*)" =~ " -h" ]] || [[ "$(trim $*)" =~ " --help" ]] ; then usage fi @@ -213,6 +236,7 @@ fi [[ -z "$mysql_credential_args" ]] && mysql_credential_args="$DEFAULT_MYSQL_CREDENTIAL_ARGS" +[[ -z "$log_file" ]] && log_file="$DEFAULT_LOG_FILE" if [[ ${#mysql_credential_args_arr[@]} -eq 0 ]]; then [[ -z "$mysql_credential_args" ]] && mysql_credential_args="$DEFAULT_MYSQL_CREDENTIAL_ARGS" @@ -234,6 +258,7 @@ for _val in ${mysql_credential_args_arr[@]} ; do echo "" echo -e "[ \033[37m\033[1mMySQL $mysql_version\033[m ]: optimize (and repair) tables of databases at host '$(hostname -f)'." fi + echo -e "[ MySQL $mysql_version ]: optimize (and repair) tables of databases at host '$(hostname -f)'." > $log_file if [[ -z "$DATABASES" ]] ; then DATABASES="$($mysql $mysql_credential_args -N -s -e "show databases")" @@ -250,11 +275,24 @@ for _val in ${mysql_credential_args_arr[@]} ; do echo "" echo -e " [$(date)] Optimize tables in database '${db}'.." fi + echo "" >> $log_file + echo -e " [$(date)] Optimize tables in database '${db}'.." >> $log_file TABLES=`$mysql $mysql_credential_args $db -N -s -e "show tables"` for table in $TABLES ; do + # - Ommit InnoDB tables + # - + _engine="$($mysql $mysql_credential_args -N -s -e "SELECT ENGINE FROM information_schema.TABLES WHERE TABLE_SCHEMA = '$db' AND TABLE_NAME = '$table'")" + if [[ "${_engine,,}" = 'innodb' ]] ; then + if $VERBOSE ; then + echo -e " [$(date)] Ommit table '$table' - storage engine is InnoDB" + fi + echo -e " [$(date)] Ommit table '$table' - storage engine is InnoDB" >> $log_file + continue + fi + if $terminal ; then blank_signs="" if [[ $length_table_name -gt ${#table} ]]; then @@ -267,6 +305,7 @@ for _val in ${mysql_credential_args_arr[@]} ; do echo -en "\033[1G" fi echo -en "\033[1G \033[32mOptimize table \033[1m$table\033[m$blank_signs\033[1G" + echo -e " [$(date)] Optimize table '$table'" >> $log_file fi length_table_name=${#table} @@ -276,6 +315,10 @@ for _val in ${mysql_credential_args_arr[@]} ; do if [[ $? -ne 0 ]]; then warn "Optimizing table \"${table}\" of database \"$db\" failed. Trying to repair.." + echo "" >> $log_file + echo " [$(date)] warning: Optimizing table \"${table}\" of database \"$db\" failed." >> $log_file + echo "" >> $log_file + echo " [$(date)] Repair table '$table'" >> $log_file $mysql $mysql_credential_args $db -N -s -e "REPAIR TABLE \`$table\`" > $tmp_log_file 2>&1 @@ -284,14 +327,22 @@ for _val in ${mysql_credential_args_arr[@]} ; do _all_success=false error "Repairing table '$table' failed.\n$(cat "$tmp_log_file")" error_messages_arr+=("MySQL $mysql_version: Error while repairing table '${table}' of database '$db'.") + echo "" >> $log_file + echo -e " [$(date)] error: Repairing table '$table' failed of database \"$db\" failed..\n$(cat "$tmp_log_file")" >> $log_file + echo "" >> $log_file + else + echo -e " [$(date)] Optimize table '$table'" >> $log_file $mysql $mysql_credential_args $db -N -s -e "OPTIMIZE TABLE \`$table\`" > $tmp_log_file 2>&1 if [[ $? -ne 0 ]]; then error "Reoptimizing table \"${table}\" of database \"$db\" failed.\n$(cat "$tmp_log_file")" error_messages_arr+=("MySQL $mysql_version: Error while (re-)optimizing table '${table}' of database '$db'.") + echo "" >> $log_file + echo -e " [$(date)] error: Reoptimizing table \"${table}\" of database \"$db\" failed.\n$(cat "$tmp_log_file")" >> $log_file + echo "" >> $log_file else info "Reoptimizing table \"${table}\" of database \"$db\" was successfully." fi @@ -308,11 +359,12 @@ for _val in ${mysql_credential_args_arr[@]} ; do echo -en "\033[1G [$(date)] End optimize tables in database '${db}'.." echo fi + echo -e " [$(date)] End optimize tables in database '${db}'" >> $log_file done if $_all_success ; then - info_messages_arr+=("MySQL $mysql_version: The optimization of the MySQL tables of all databases was successful.") + info_messages_arr+=("MySQL $mysql_version: The optimization of the MySQL tables of all databases were successful.") fi if $terminal ; then @@ -320,6 +372,9 @@ for _val in ${mysql_credential_args_arr[@]} ; do echo -e "[ \033[37m\033[1mMySQL $mysql_version\033[m ]: Finished optimizing MySQL databases at host $(hostname -f)." echo "" fi + echo "" >> $log_file + echo "[ MySQL $mysql_version ]: Finished optimizing MySQL databases at host $(hostname -f)." >> $log_file + echo "" >> $log_file done if [[ ${#info_messages_arr[@]} -gt 0 ]]; then @@ -327,6 +382,7 @@ if [[ ${#info_messages_arr[@]} -gt 0 ]]; then if $terminal ; then info "$msg" fi + echo "[ Info ]: $msg" >> $log_file done fi @@ -335,7 +391,12 @@ if [[ ${#error_messages_arr[@]} -gt 0 ]]; then if $terminal ; then error "$msg" fi + echo "[ Error ]: $msg" >> $log_file done fi +if $terminal ; then + info "See logfile '$log_file' for more details." +fi + clean_up 0 diff --git a/repair_mysql_tables.sh b/repair_mysql_tables.sh index a7f8a3d..144c542 100755 --- a/repair_mysql_tables.sh +++ b/repair_mysql_tables.sh @@ -286,6 +286,14 @@ for _val in ${mysql_credential_args_arr[@]} ; do for table in $TABLES ; do + # - Ommit InnoDB tables + # - + _engine="$($mysql $mysql_credential_args -N -s -e "SELECT ENGINE FROM information_schema.TABLES WHERE TABLE_SCHEMA = '$db' AND TABLE_NAME = '$table'")" + if [[ "${_engine,,}" = 'innodb' ]] ; then + echo -e " [$(date)] Ommit table '$table' - The storage engine (InnoDB) doesn't support repair" + continue + fi + if $terminal ; then blank_signs="" if [[ $length_table_name -gt ${#table} ]]; then