diff --git a/README.migrate-MySQL-to-PostgreSQL b/README.migrate-MySQL-to-PostgreSQL index 5c91c53..92f1be1 100644 --- a/README.migrate-MySQL-to-PostgreSQL +++ b/README.migrate-MySQL-to-PostgreSQL @@ -1,7 +1,405 @@ # see: -# +# # https://docs.mattermost.com/deploy/postgres-migration-assist-tool.html # https://docs.mattermost.com/deploy/manual-postgres-migration.html # # https://docs.mattermost.com/deploy/postgres-migration.html + +MATTERMOST_VERSION="9.11.6" + +# Postgresql +psql_db_host="10.0.3.42" +psql_db_port="5432" +psql_db_name="mattermost" +psql_db_user="mattermost" +psql_db_password='aiqu7oghae1eZai3' + +MYSQL_DSN="${mysql_db_user}:${mysql_db_password}@tcp(${mysql_db_host}:${mysql_db_port})/${mysql_db_name}" + +# Mysql +mysql_db_host="10.0.3.42" +mysql_db_port="3306" +mysql_db_name="mattermost2" +mysql_db_user="mattermost2" +mysql_db_password='aiqu7oghae1eZai3' + +POSTGRES_DSN="postgres://${psql_db_user}:${psql_db_password}@${psql_db_host}:${psql_db_port}/${psql_db_name}" + +pgloader_bin='/usr/local/src/pgloader/pgloader/build/bin/pgloader' + + +# --- +# Compile/Install pgloader from source +# --- + +apt purge pgloader +apt --purge autoremove +mkdir -p "/usr/local/src/pgloader/" +cd "/usr/local/src/pgloader/" +git clone https://github.com/dimitri/pgloader.git +apt-get install sbcl unzip libsqlite3-dev make curl gawk freetds-dev libzip-dev +cd pgloader +/build/bin/pgloader --help + + +# --- +# Completely deinstall postgresql +# +# --- + +DEBIAN_FRONTEND=noninteractive apt purge -y postgresql postgresql-* +rm -rf /var/lib/postgresql +rm -rf /etc/postgresql +rm -rf /var/log/postgresql + +DEBIAN_FRONTEND=noninteractive apt install -y --allow-unauthenticated postgresql + +# Adjust files +# +# /etc/postgresql/15/main/pg_hba.conf +# /etc/postgresql/15/main/postgresql.conf +# +# +# /etc/postgresql/15/main/postgresql.conf +# +# Find the following line: #listen_addresses = 'localhost' +# Uncomment the line and change localhost to *: listen_addresses = '*' +# Restart PostgreSQL for the change to take effect by running: +# +# +# /etc/postgresql/{version}/main/pg_hba.conf +# +# Find the following lines: +# +# local all all peer +# +# host all all ::1/128 ident +# +# Change peer and ident to trust: +# +# local all all trust +# +# host all all ::1/128 trust +# +# +# Remote Database (separate server) +# +# If the Mattermost server and the database are on different machines: +# +# /etc/postgresql/{version}/main/pg_hba.conf in a text editor as root user. +# +# Add the following line to the end of the file, where {mattermost-server-IP} is the +# IP address of the Mattermost server: +# +# host all all {mattermost-server-IP}/32 md5. +# +cp -a /etc/postgresql/15/main/pg_hba.conf /etc/postgresql/15/main/pg_hba.conf.ORIG +cp /root/postgresql_15_main/pg_hba.conf /etc/postgresql/15/main/pg_hba.conf + +cp -a /etc/postgresql/15/main/postgresql.conf /etc/postgresql/15/main/postgresql.conf.ORIG +cp /root/postgresql_15_main/postgresql.conf /etc/postgresql/15/main/postgresql.conf + +systemctl restart postgresql + + +# Create PostgreSQL Database om mattermost server.. +# +# seee: https://docs.mattermost.com/install/prepare-mattermost-database.html +# +# sudo -u postgres psql +# +# CREATE DATABASE mm; +# +# \connect mm +# +# CREATE USER mmuser WITH PASSWORD 'mmuser-password'; +# +# +# If you're configuring PostgreSQL v15.x or later: +# +# GRANT ALL PRIVILEGES ON DATABASE mattermost to mmuser; +# +# ALTER DATABASE mattermost OWNER TO mmuser; +# +# GRANT USAGE, CREATE ON SCHEMA PUBLIC TO mmuser; +# +# +# Note: +# Error Occurred: could not check schema owner: the user "mmuser" is not owner of the "public" schema +# +# Fix: on postreSQL monitor insert: +# +# ALTER SCHEMA public OWNER TO mmuser; +# GRANT ALL ON SCHEMA public to mmuser; +# +cd /tmp +sudo -u postgres psql -c "CREATE DATABASE ${psql_db_name};" + +sudo -u postgres psql ${psql_db_name} -c "CREATE USER ${psql_db_user} WITH PASSWORD '${psql_db_password}';" +sudo -u postgres psql ${psql_db_name} -c "GRANT ALL PRIVILEGES ON DATABASE ${psql_db_name} to ${psql_db_user};" +sudo -u postgres psql ${psql_db_name} -c "ALTER DATABASE ${psql_db_name} OWNER TO ${psql_db_user};" +sudo -u postgres psql ${psql_db_name} -c "GRANT USAGE, CREATE ON SCHEMA PUBLIC TO ${psql_db_user};" +sudo -u postgres psql ${psql_db_name} -c "ALTER SCHEMA public OWNER TO ${psql_db_user};" +sudo -u postgres psql ${psql_db_name} -c "GRANT ALL ON SCHEMA public to ${psql_db_user};" + +systemctl stop postgresql +systemctl start postgresql + + +# ============================== +# Automated PostgreSQL migration +# ============================== + +# Step 1 - Check the MySQL database schema +# ======================================== +# +# run the following command: +# +# migration-assist mysql "" --fix-artifacts --fix-unicode --fix-varchar +# +# +# Example MySQ_ DSN: "user:password@tcp(address:3306)/db_name" +# +migration-assist mysql ${MYSQL_DSN} \ + --fix-artifacts \ + --fix-unicode \ + --fix-varchar + +# Output was: +# +# 2024-12-25 12:59:01 pinging mysql... +# 2024-12-25 12:59:01 connected to mysql successfully... +# 2024-12-25 12:59:01 running checks for artifacts... +# 2024-12-25 12:59:01 a fix is required for: schema_migrations +# 2024-12-25 12:59:01 the fix query has been executed successfully. +# 2024-12-25 12:59:01 4 checks been made, all good for artifacts +# 2024-12-25 12:59:01 running checks for unicode... +# 2024-12-25 12:59:02 11 checks been made, all good for unicode +# 2024-12-25 12:59:02 running checks for varchar... +# 2024-12-25 12:59:02 8 checks been made, all good for varchar +# 2024-12-25 12:59:02 running checks for varchar-extended... +# 2024-12-25 12:59:02 a fix is required for: linkmetadata.url +# 2024-12-25 12:59:02 the fix query has been executed successfully. +# 2024-12-25 12:59:02 12 checks been made, all good for varchar-extended +# + + + +# Step 2 - Create the PostgreSQL database schema +# ============================================== +# +# run the following command: +# +# migration-assist postgres "" --run-migrations --mattermost-version="" +# +# +# Example POSTGRES_DSN: "postgres://user:password@address:5432/db_name" +# +# Example MATTERMOST_VERSION: "v9.4.0" +# +#migration-assist postgres ${POSTGRES_DSN} --run-migrations --mattermost-version=${MATTERMOST_VERSION} + +migration-assist postgres ${POSTGRES_DSN} --run-migrations --mattermost-version ${MATTERMOST_VERSION} + +# Output was: +# +# 2024-12-25 13:14:41 pinging postgres... +# 2024-12-25 13:14:41 connected to postgres successfully. +# 2024-12-25 13:14:41 schema owner check passed. +# 2024-12-25 13:14:41 checking if tables are empty... +# 2024-12-25 13:14:41 cloning repository@9.11.1 +# 2024-12-25 13:14:41 git version: git version 2.39.5 +# 2024-12-25 13:14:42 checking out... +# 2024-12-25 13:14:42 moving migration files into a better place.. +# 2024-12-25 13:14:42 running migrations.. +# 2024-12-25 13:14:43 migrations applied. +# + + +# Step 3 - Generate a pgloader configuration +# ========================================== +# +# run the following command: +# +# migration-assist pgloader --mysql="" --postgres="" > migration.load +# +migration-assist pgloader --mysql $MYSQL_DSN --postgres $POSTGRES_DSN | tee migration.load + +# Output was: +# +# LOAD DATABASE +# FROM mysql://mmuser:aiqu7oghae1eZai3@10.0.3.42:3306/mm +# INTO pgsql://mmuser:aiqu7oghae1eZai3@10.0.3.42:5432/mm +# +# WITH data only, +# workers = 8, concurrency = 1, +# multiple readers per thread, rows per range = 10000, +# prefetch rows = 10000, batch rows = 2500, +# create no tables, create no indexes, +# preserve index names +# +# SET PostgreSQL PARAMETERS +# maintenance_work_mem to '128MB', +# work_mem to '12MB' +# +# SET MySQL PARAMETERS +# net_read_timeout = '120', +# net_write_timeout = '120' +# +# CAST column Channels.Type to "channel_type" drop typemod, +# column Teams.Type to "team_type" drop typemod, +# column UploadSessions.Type to "upload_session_type" drop typemod, +# column ChannelBookmarks.Type to "channel_bookmark_type" drop typemod, +# column Drafts.Priority to text, +# type int when (= precision 11) to integer drop typemod, +# type bigint when (= precision 20) to bigint drop typemod, +# type text to varchar drop typemod using remove-null-characters, +# type tinyint when (<= precision 4) to boolean using tinyint-to-boolean, +# type json to jsonb drop typemod using remove-null-characters +# +# EXCLUDING TABLE NAMES MATCHING ~, ~, ~, 'schema_migrations', 'db_migrations', 'db_lock', +# 'configurations', 'configurationfiles', 'db_config_migrations' +# +# BEFORE LOAD DO +# $$ ALTER SCHEMA public RENAME TO mm; $$, +# $$ TRUNCATE TABLE mm.systems; $$, +# $$ DROP INDEX IF EXISTS mm.idx_posts_message_txt; $$, +# $$ DROP INDEX IF EXISTS mm.idx_fileinfo_content_txt; $$ +# +# AFTER LOAD DO +# $$ UPDATE mm.db_migrations set name='add_createat_to_teamembers' where version=92; $$, +# $$ ALTER SCHEMA mm RENAME TO public; $$, +# $$ SELECT pg_catalog.set_config('search_path', '"$user", "$user", public', false); $$, +# $$ ALTER USER mmuser SET SEARCH_PATH TO '"$user", public'; $$; + + +# Step 4 - Run pgloader +# ===================== +# +# Run pgloader with the generated configuration file: +# +# pgloader migration.load > migration.log +# +${pgloader_bin} migration.load | tee migration.log + + +# Step 5 - Restore full-text indexes +# ================================== +# +# Run the following command to create the full-text indexes for the Posts and FileInfo tables: +# +# migration-assist postgres post-migrate "" +# +# This command creates the full-text indexes for the Posts and FileInfo tables. See the +# Restore full-text indexes documentation for more information. +# +migration-assist postgres post-migrate $POSTGRES_DSN | tee postgres-post-migrate.log + + +# Step 6 - Complete plugin migrations +# =================================== +# +# Generate migration configuration for collaborative playbooks, boards and calls: +# +# migration-assist pgloader boards --mysql="" --postgres="" > boards.load +# migration-assist pgloader playbooks --mysql="" --postgres="" > playbooks.load +# migration-assist pgloader calls --mysql="" --postgres="" > calls.load +# +# Then run pgloader with the generated configuration files: +# +# pgloader boards.load > boards_migration.log +# pgloader playbooks.load > playbooks_migration.log +# pgloader calls.load > calls.log +# +# Note: +# Carefully read the log file to analyze whether there were any errors during the migration process. +# See the Plugin migrations documentation +# https://docs.mattermost.com/deploy/manual-postgres-migration.html#plugin-migrations) +# for information on migrating Playbooks, Boards and Calls. +# +migration-assist pgloader boards --mysql="${MYSQL_DSN}" --postgres="${POSTGRES_DSN}" | tee boards.load +migration-assist pgloader playbooks --mysql="${MYSQL_DSN}" --postgres="${POSTGRES_DSN}" | tee playbooks.load +migration-assist pgloader calls --mysql=""${MYSQL_DSN} --postgres=""${POSTGRES_DSN} | tee calls.load + + +# Prepare boards.load +# +# Delte entries: +# +# column focalboard_teams.settings to "json" drop typemod using remove-null-characters, +# ... +# $$ UPDATE mattermost2.focalboard_teams SET "settings" = '{}'::json WHERE "settings"::text = ''; $$, +# +sed -i '/focalboard_teams.settings/d' boards.load +sed -i '/mattermost.focalboard_teams/d' boards.load + +${pgloader_bin} boards.load | tee boards_migration.log + + +# playbooks.load +# +# Add missing entries into database: +CREATE TABLE public.ir_category ( + id character varying(26) NOT NULL, + name character varying(512) NOT NULL, + teamid character varying(26) NOT NULL, + userid character varying(26) NOT NULL, + collapsed boolean DEFAULT false, + createat bigint NOT NULL, + updateat bigint DEFAULT 0 NOT NULL, + deleteat bigint DEFAULT 0 NOT NULL +); +ALTER TABLE public.ir_category OWNER TO mattermost; + +CREATE TABLE public.ir_category_item ( + type character varying(1) NOT NULL, + categoryid character varying(26) NOT NULL, + itemid character varying(26) NOT NULL +); +ALTER TABLE public.ir_category_item OWNER TO mattermost; + + +ALTER TABLE ONLY public.ir_category_item + ADD CONSTRAINT ir_category_item_pkey PRIMARY KEY (categoryid, itemid, type); + +ALTER TABLE ONLY public.ir_category + ADD CONSTRAINT ir_category_pkey PRIMARY KEY (id); + +CREATE INDEX ir_category_item_categoryid ON public.ir_category_item USING btree (categoryid); +CREATE INDEX ir_category_teamid_userid ON public.ir_category USING btree (teamid, userid); + +ALTER TABLE ONLY public.ir_category_item + ADD CONSTRAINT ir_category_item_categoryid_fkey FOREIGN KEY (categoryid) REFERENCES public.ir_category(id); +# + +${pgloader_bin} playbooks.load | tee playbooks_migration.log + +${pgloader_bin} calls.load | tee calls.log + + + +# Important on postgres monitor console (psql mattermost), the requests +# +# SHOW SEARCH_PATH; +# snd +# SELECT CURRENT_SCHEMA(); +# +# must response as shown: +# + +mattermost=# SHOW SEARCH_PATH; + search_path +----------------- + "$user", public +(1 row) + +mattermost=# SELECT CURRENT_SCHEMA(); + current_schema +---------------- + public +(1 row) + +mattermost=# + +