From 321156508c49bedecdf255aee3fcb406c71cf96e Mon Sep 17 00:00:00 2001 From: Christoph Date: Mon, 8 May 2023 02:15:40 +0200 Subject: [PATCH] sync_from_old_server.sh: redesign mysql transfer.. --- sync_from_old_server.sh | 339 +++++++++++++++++++++++++++++++++------- 1 file changed, 282 insertions(+), 57 deletions(-) diff --git a/sync_from_old_server.sh b/sync_from_old_server.sh index 17b347e..3523385 100755 --- a/sync_from_old_server.sh +++ b/sync_from_old_server.sh @@ -1,20 +1,20 @@ #!/usr/bin/env bash -ipv4_old="" +ipv4_old="83.223.86.121" old_server_ip=$ipv4_old tmp_err_msg=$(mktemp) log_dir=/root/sync_from_old_server_logs +cur_date="$(date +%Y-%m-%d-%H%M)" + # - Sync Home Directoties # - -# - Example: -# - sync_home_dirs=" -# - /data/home/ilker -# - /data/home/jan-kout -# - /data/home/max -# - " -# - +#sync_home_dirs=" +# /data/home/ilker +# /data/home/jan-kout +# /data/home/max +#" sync_home_dirs="" @@ -49,21 +49,15 @@ sync_web_dirs="" # - sync_vhost_dirs="" - # - Only needed to replace ip-addresse in virtual host configuration files # - -ipv4_new="" -ipv6_old="" -ipv6_new="" +ipv4_new="83.223.90.20" +ipv6_old="2a01:30:0:13:2e0:7bff:fe9a:b18a" +ipv6_new="2a01:31:0:2::f851:1" # - Sync dehydrated cert directory # - -# - Example: -# - sync_dehydrated_dirs=" -# - /var/lib/dehydrated/certs -# - " -# - sync_dehydrated_dirs="" @@ -83,20 +77,11 @@ sync_dehydrated_dirs="" # - the basename of the destination directory must be the same as # - the basename of the source directory. # - -# - Example: -# - sync_other_dirs=" -# - /root/bin/mysql/databases -# - " sync_other_dirs="" # - Sync files # - -# - sync_files=" -# - /var/lib/dehydrated/domains.txt -# - /etc/pure-ftpd/pureftpd.passwd -# - " -# - sync_files="" @@ -126,28 +111,18 @@ sync_pgsql_databases="" # - sync_mysql_databases="" - # - mysql_remote_credential_args # - mysql_local_credential_args # - -# - -# - Values (Examples) for credential args: -# - -# - -u root -S /run/mysqld/mysqld.sock -# - --defaults-file=/usr/local/mysql/sys-maint.cnf -# - --defaults-file=/etc/mysql/debian.cnf -# - --login-path=local -# - -u -p'' -# - # - Example -# - -# - mysql_remote_credential_args="--login-path=local" -# - mysql_local_credential_args="-u root -S /run/mysqld/mysqld.sock" +# - mysql_credential_args="-u root -S /run/mysqld/mysqld.sock" +# - mysql_credential_args="--defaults-file=/usr/local/mysql/sys-maint.cnf" +# - mysql_credential_args="--login-path=local" +# - mysql_credential_args="-u -p''" # - mysql_remote_credential_args="" mysql_local_credential_args="" - # ------------- # --- Some functions # ------------- @@ -207,6 +182,61 @@ echo_skipped() { echo -e "\033[80G[ \033[33m\033[1mskipped\033[m ]" } +detect_mysql_version () { + + _MYSQLD_VERSION="$(mysqld -V 2>/dev/null)" + + if [[ -z "$_MYSQLD_VERSION" ]]; then + fatal "No installed MySQL server or distribution found!" + elif [[ -d "/usr/local/mysql" ]] && [[ "$(basename "$(realpath "/usr/local/mysql")")" =~ percona- ]]; then + MYSQL_CUR_DISTRIBUTION="Percona" + elif [[ "$_MYSQLD_VERSION" =~ MariaDB ]]; then + MYSQL_CUR_DISTRIBUTION="MariaDB" + elif [[ "$_MYSQLD_VERSION" =~ MySQL ]]; then + MYSQL_CUR_DISTRIBUTION="MySQL" + elif [[ -d "/usr/local/mysql" ]] && [[ "$(basename "$(realpath "/usr/local/mysql")")" =~ mysql- ]]; then + MYSQL_CUR_DISTRIBUTION="MySQL" + elif [[ -d "/usr/local/mysql" ]] && [[ "$(basename "$(realpath "/usr/local/mysql")")" =~ mariadb- ]]; then + MYSQL_CUR_DISTRIBUTION="MariaDB" + else + error "MySQL Instalation found, but cannot determin the distribution!" + + MYSQL_CUR_DISTRIBUTION= + echo "" + echo " Select the MySQL distribution to install." + echo "" + echo " [1] MySQL (the original community edition)" + echo " [2] Percona Server for MySQL" + echo " [3] MariaDB" + echo "" + echononl " Eingabe [1/2/3]: " + + while [ "$MYSQL_CUR_DISTRIBUTION" != "MySQL" -a "$MYSQL_CUR_DISTRIBUTION" != "MariaDB" -a "$MYSQL_CUR_DISTRIBUTION" != "Percona" ];do + read OPTION + case $OPTION in + 1) MYSQL_CUR_DISTRIBUTION="MySQL" + ;; + 2) MYSQL_CUR_DISTRIBUTION="Percona" + ;; + 3) MYSQL_CUR_DISTRIBUTION="MariaDB" + ;; + *) echo "" + echo -e "\tFalsche Eingabe ! [ 1 = MySQL ; 2 = Percona ; 3 = MariaDB ]" + echo "" + echononl " Eingabe:" + ;; + esac + done + fi + + MYSQL_VERSION="$(echo $_MYSQLD_VERSION | grep -o -E "[0-9]+\.[0-9]+\.[0-9]+(-[0-9]+)?" | head -n 1)" + MYSQL_MAJOR_VERSION="$(echo $MYSQL_VERSION | cut -d '.' -f1)" + MYSQL_MINOR_VERSION="$(echo $MYSQL_VERSION | cut -d '.' -f2)" + MYSQL_PATCH_LEVEL="$(echo $MYSQL_VERSION | cut -d '.' -f3)" + MYSQL_MAIN_VERSION="$(echo $MYSQL_VERSION | cut -d '.' -f1,2)" + +} + service_exists() { local n=$1 if [[ $(systemctl list-units --all -t service --full --no-legend "$n.service" | cut -f1 -d' ') == $n.service ]]; then @@ -340,15 +370,52 @@ fi # - echo -e "\n\n \033[37m\033[1mSyncing MySQL Databases..\033[m\n" -_got_mysql_databases=false -sync_mysql_databases="${sync_mysql_databases##*( )}" -sync_mysql_databases="${sync_mysql_databases%%*( )}" - if [[ -z "$sync_mysql_databases" ]]; then warn "No MySQL databases for syncing configured." -elif [[ "$sync_mysql_databases" = "ALL" ]]; then +else + detect_mysql_version + + MAJOR_VERSION="$MYSQL_MAJOR_VERSION" + MINOR_VERSION="$MYSQL_MINOR_VERSION" + PATCH_LEVEL="$MYSQL_PATCH_LEVEL" + + # - get remote mysqldump command + # - + if $(ssh $old_server_ip [[ -x "/usr/local/mysql/bin/mysqldump" ]]) ; then + r_mysqldump="/usr/local/mysql/bin/mysqldump" + else + r_mysqldump="$(ssh $old_server_ip which mysqldump)" + fi + + # - get remote mysql command + # - + if $(ssh $old_server_ip [[ -x "/usr/local/mysql/bin/mysql" ]]) ; then + r_mysql="/usr/local/mysql/bin/mysql" + else + r_mysql="$(ssh $old_server_ip which mysql)" + fi + + ssh_options="-o BatchMode=yes -o ConnectTimeout=360" + + # - get local mysql command + # - + if [[ -x "/usr/local/mysql/bin/mysql" ]] ; then + l_mysql="/usr/local/mysql/bin/mysql" + else + l_mysql="$(which mysql)" + fi + +fi + + +if [[ "$sync_mysql_databases" = "ALL" ]]; then + + _got_mysql_databases=false + sync_mysql_databases="${sync_mysql_databases##*( )}" + sync_mysql_databases="${sync_mysql_databases%%*( )}" + echononl " Get MySQL databases from $old_server_ip.." - _mysql_databases_remote=$(ssh $old_server_ip "/usr/local/mysql/bin/mysql $mysql_remote_credential_args -N -s -e \"show databases\"") + _mysql_databases_remote=$(ssh $old_server_ip "${r_mysql} $mysql_remote_credential_args -N -s -e \"show databases\"") if [[ $? -eq 0 ]];then echo_ok _got_mysql_databases=true @@ -363,13 +430,13 @@ fi if $_got_mysql_databases ; then log_file=${log_dir}/sync_mysql.log > $log_file - mysqldump_flags="--protocol=SOCKET --max_allowed_packet=128M --skip-definer --skip-opt --add-drop-table --add-locks --create-options --quick --set-charset --disable-keys --lock-tables --routines" + mysqldump_flags="--protocol=SOCKET --max_allowed_packet=512M --skip-opt --add-drop-table --add-locks --create-options --quick --compress --set-charset --disable-keys --lock-tables --routines" # - GET current (global) Autocommit value # - - echononl " GET current (global) Autocommit value" - CUR_AUTOCOMMIT="$(mysql $mysql_local_credential_args -N -s -e "SHOW GLOBAL VARIABLES LIKE 'autocommit'" | awk '{print$2}')" >> $log_file 2> $tmp_err_msg + echononl " GET current (global) 'autocommit' value" + CUR_AUTOCOMMIT="$(${l_mysql} $mysql_local_credential_args -N -s -e "SHOW GLOBAL VARIABLES LIKE 'autocommit'" | awk '{print$2}')" >> $log_file 2> $tmp_err_msg if [[ $? -eq 0 ]];then echo_ok else @@ -378,10 +445,34 @@ if $_got_mysql_databases ; then fi - # - Set Autocommit OFF - # - - echononl " Set Autocommit to OFF" - mysql $mysql_local_credential_args -N -s -e "SET GLOBAL AUTOCOMMIT='OFF'" >> $log_file 2> $tmp_err_msg + # - GET current (global) value for 'unique_checks' + # - + echononl " GET current (global) 'unique_checks' value" + CUR_UNIQUE_CHECKS="$(${l_mysql} $mysql_local_credential_args -N -s -e "SHOW GLOBAL VARIABLES LIKE 'unique_checks'" | awk '{print$2}')" >> $log_file 2> $tmp_err_msg + if [[ $? -eq 0 ]];then + echo_ok + else + echo_failed + error "$(cat $tmp_err_msg)" + fi + + + # - GET current (global) value for 'foreign_key_checks' + # - + echononl " GET current (global) 'foreign_key_checks' value" + CUR_FOREIGN_KEY_CHECKS="$(${l_mysql} $mysql_local_credential_args -N -s -e "SHOW GLOBAL VARIABLES LIKE 'foreign_key_checks'" | awk '{print$2}')" >> $log_file 2> $tmp_err_msg + if [[ $? -eq 0 ]];then + echo_ok + else + echo_failed + error "$(cat $tmp_err_msg)" + fi + + + # - GET current (global) value for 'innodb_flush_log_at_trx_commit' + # - + echononl " GET current (global) 'innodb_flush_log_at_trx_commit' value" + CUR_INNODB_FLUSH_LOG_AT_TRX_COMMIT="$(${l_mysql} $mysql_local_credential_args -N -s -e "SHOW GLOBAL VARIABLES LIKE 'innodb_flush_log_at_trx_commit'" | awk '{print$2}')" >> $log_file 2> $tmp_err_msg if [[ $? -eq 0 ]];then echo_ok else @@ -390,15 +481,62 @@ if $_got_mysql_databases ; then fi echo "" + + # - Set Autocommit OFF + # - + echononl " Set 'autocommit' to OFF" + ${l_mysql} $mysql_local_credential_args -N -s -e "SET GLOBAL autocommit='OFF'" >> $log_file 2> $tmp_err_msg + if [[ $? -eq 0 ]];then + echo_ok + else + echo_failed + error "$(cat $tmp_err_msg)" + fi + + + # - Set UNIQUE_CHECKS OFF + # - + echononl " Set 'unique_checks' to OFF" + ${l_mysql} $mysql_local_credential_args -N -s -e "SET GLOBAL unique_checks='OFF'" >> $log_file 2> $tmp_err_msg + if [[ $? -eq 0 ]];then + echo_ok + else + echo_failed + error "$(cat $tmp_err_msg)" + fi + + + # - Set FOREIGN_KEY_CHECKS OFF + # - + echononl " Set 'foreign_key_checks' to OFF" + ${l_mysql} $mysql_local_credential_args -N -s -e "SET GLOBAL foreign_key_checks='OFF'" >> $log_file 2> $tmp_err_msg + if [[ $? -eq 0 ]];then + echo_ok + else + echo_failed + error "$(cat $tmp_err_msg)" + fi + + + # - Set innodb_flush_log_at_trx_commit OFF + # - + echononl " Set 'innodb_flush_log_at_trx_commit' to 2" + ${l_mysql} $mysql_local_credential_args -N -s -e "SET GLOBAL innodb_flush_log_at_trx_commit=2" >> $log_file 2> $tmp_err_msg + if [[ $? -eq 0 ]];then + echo_ok + else + echo_failed + error "$(cat $tmp_err_msg)" + fi + + for _val in $_mysql_databases_remote ; do IFS=':' read -a _val_arr <<< "${_val}" _src_db="${_val_arr[0]}" if [[ -n "${_val_arr[1]}" ]] ; then _dst_db="${_val_arr[1]}" - echononl " Sync Database '$_src_db' --> '$_dst_db'.. " else - echononl " Sync Database '$_src_db'.. " _dst_db="$_src_db" fi @@ -422,21 +560,108 @@ if $_got_mysql_databases ; then echo_skipped continue fi - ssh $old_server_ip "/usr/local/mysql/bin/mysqldump $mysql_remote_credential_args $_src_db" | mysql $mysql_local_credential_args $_dst_db >> $log_file 2> $tmp_err_msg + + echo "" + if [[ -n "${_val_arr[1]}" ]] ; then + echononl " Sync Database '$_src_db' --> '$_dst_db'.. " + else + echononl " Sync Database '$_src_db'.. " + fi + ssh $old_server_ip "${r_mysqldump} $mysql_remote_credential_args $mysqldump_flags $_src_db" \ + | ${l_mysql} $mysql_local_credential_args $_dst_db >> $log_file 2> $tmp_err_msg if [[ $? -eq 0 ]];then echo_ok else echo_failed error "$(cat $tmp_err_msg)" + + echo -e "\n\n \033[33mSomething went wromg! I'm trying an alternative way ..\033[m\n" + + + echononl " Read remote DB '${_src_db}' into local file '/tmp/${_dst_db}-${cur_date}.sql'.." + ssh $old_server_ip "${r_mysqldump} $mysql_remote_credential_args $mysqldump_flags $_src_db" \ + > "/tmp/${_dst_db}-${cur_date}.sql" 2> $tmp_err_msg + if [[ $? -eq 0 ]];then + echo_ok + else + echo_failed + error "$(cat $tmp_err_msg)" + fi + + echononl " Get rid of DEFINER statements from Mysql dump .." + sed -i 's/DEFINER=[^*]*\*/\*/g' "/tmp/${_dst_db}-${cur_date}.sql" 2> $tmp_err_msg + if [[ $? -eq 0 ]];then + echo_ok + else + echo_failed + error "$(cat $tmp_err_msg)" + fi + + echononl " Read in MySQL database '${_dst_db}'.." + ${l_mysql} $mysql_local_credential_args $_dst_db < "/tmp/${_dst_db}-${cur_date}.sql" 2> $tmp_err_msg + if [[ $? -eq 0 ]];then + echo_ok + else + echo_failed + error "$(cat $tmp_err_msg)" + fi + + echononl " Remove dump file '/tmp/${_dst_db}-${cur_date}.sql'.." + rm -rf "/tmp/${_dst_db}-${cur_date}.sql" 2> $tmp_err_msg + if [[ $? -eq 0 ]];then + echo_ok + else + echo_failed + error "$(cat $tmp_err_msg)" + fi + fi + + done # - Reset (global) Autocommit value # - echo "" echononl " Reset (global) Autocommit value to '$CUR_AUTOCOMMIT'" - mysql $mysql_local_credential_args -N -s -e "SET GLOBAL AUTOCOMMIT='$CUR_AUTOCOMMIT'" >> $log_file 2> $tmp_err_msg + ${l_mysql} $mysql_local_credential_args -N -s -e "SET GLOBAL AUTOCOMMIT='$CUR_AUTOCOMMIT'" >> $log_file 2> $tmp_err_msg + if [[ $? -eq 0 ]];then + echo_ok + else + echo_failed + error "$(cat $tmp_err_msg)" + fi + + # - Reset (global) value for 'unique_checks' + # - + echo "" + echononl " Reset (global) 'unique_checks' value to '$CUR_UNIQUE_CHECKS'" + ${l_mysql} $mysql_local_credential_args -N -s -e "SET GLOBAL unique_checks='$CUR_UNIQUE_CHECKS'" >> $log_file 2> $tmp_err_msg + if [[ $? -eq 0 ]];then + echo_ok + else + echo_failed + error "$(cat $tmp_err_msg)" + fi + + # - Reset (global) value for 'foreign_key_checks' + # - + echo "" + echononl " Reset (global) 'foreign_key_checks' value to '$CUR_FOREIGN_KEY_CHECKS'" + ${l_mysql} $mysql_local_credential_args -N -s -e "SET GLOBAL foreign_key_checks='$CUR_FOREIGN_KEY_CHECKS'" >> $log_file 2> $tmp_err_msg + if [[ $? -eq 0 ]];then + echo_ok + else + echo_failed + error "$(cat $tmp_err_msg)" + fi + + # - Reset (global) value for 'innodb_flush_log_at_trx_commit' + # - + echo "" + echononl " Reset (global) 'innodb_flush_log_at_trx_commit' value to '$CUR_INNODB_FLUSH_LOG_AT_TRX_COMMIT'" + ${l_mysql} $mysql_local_credential_args -N -s -e "SET GLOBAL innodb_flush_log_at_trx_commit=$CUR_INNODB_FLUSH_LOG_AT_TRX_COMMIT" >> $log_file 2> $tmp_err_msg if [[ $? -eq 0 ]];then echo_ok else