install/mattermost
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
543ee65fe0
mattermost/README.migrate-MySQL-to-PostgreSQL

406 lines
13 KiB
Plaintext
Raw Blame History

# see:
#
# https://docs.mattermost.com/deploy/postgres-migration-assist-tool.html
# https://docs.mattermost.com/deploy/manual-postgres-migration.html
#
# https://docs.mattermost.com/deploy/postgres-migration.html
MATTERMOST_VERSION="9.11.6"
# Postgresql
psql_db_host="10.0.3.42"
psql_db_port="5432"
psql_db_name="mattermost"
psql_db_user="mattermost"
psql_db_password='aiqu7oghae1eZai3'
MYSQL_DSN="${mysql_db_user}:${mysql_db_password}@tcp(${mysql_db_host}:${mysql_db_port})/${mysql_db_name}"
# Mysql
mysql_db_host="10.0.3.42"
mysql_db_port="3306"
mysql_db_name="mattermost2"
mysql_db_user="mattermost2"
mysql_db_password='aiqu7oghae1eZai3'
POSTGRES_DSN="postgres://${psql_db_user}:${psql_db_password}@${psql_db_host}:${psql_db_port}/${psql_db_name}"
pgloader_bin='/usr/local/src/pgloader/pgloader/build/bin/pgloader'
# ---
# Compile/Install pgloader from source
# ---
apt purge pgloader
apt --purge autoremove
mkdir -p "/usr/local/src/pgloader/"
cd "/usr/local/src/pgloader/"
git clone https://github.com/dimitri/pgloader.git
apt-get install sbcl unzip libsqlite3-dev make curl gawk freetds-dev libzip-dev
cd pgloader
/build/bin/pgloader --help
# ---
# Completely deinstall postgresql
#
# ---
DEBIAN_FRONTEND=noninteractive apt purge -y postgresql postgresql-*
rm -rf /var/lib/postgresql
rm -rf /etc/postgresql
rm -rf /var/log/postgresql
DEBIAN_FRONTEND=noninteractive apt install -y --allow-unauthenticated postgresql
# Adjust files
#
# /etc/postgresql/15/main/pg_hba.conf
# /etc/postgresql/15/main/postgresql.conf
#
#
# /etc/postgresql/15/main/postgresql.conf
#
# Find the following line: #listen_addresses = 'localhost'
# Uncomment the line and change localhost to *: listen_addresses = '*'
# Restart PostgreSQL for the change to take effect by running:
#
#
# /etc/postgresql/{version}/main/pg_hba.conf
#
# Find the following lines:
#
# local all all peer
#
# host all all ::1/128 ident
#
# Change peer and ident to trust:
#
# local all all trust
#
# host all all ::1/128 trust
#
#
# Remote Database (separate server)
#
# If the Mattermost server and the database are on different machines:
#
# /etc/postgresql/{version}/main/pg_hba.conf in a text editor as root user.
#
# Add the following line to the end of the file, where {mattermost-server-IP} is the
# IP address of the Mattermost server:
#
# host all all {mattermost-server-IP}/32 md5.
#
cp -a /etc/postgresql/15/main/pg_hba.conf /etc/postgresql/15/main/pg_hba.conf.ORIG
cp /root/postgresql_15_main/pg_hba.conf /etc/postgresql/15/main/pg_hba.conf
cp -a /etc/postgresql/15/main/postgresql.conf /etc/postgresql/15/main/postgresql.conf.ORIG
cp /root/postgresql_15_main/postgresql.conf /etc/postgresql/15/main/postgresql.conf
systemctl restart postgresql
# Create PostgreSQL Database om mattermost server..
#
# seee: https://docs.mattermost.com/install/prepare-mattermost-database.html
#
# sudo -u postgres psql
#
# CREATE DATABASE mm;
#
# \connect mm
#
# CREATE USER mmuser WITH PASSWORD 'mmuser-password';
#
#
# If you're configuring PostgreSQL v15.x or later:
#
# GRANT ALL PRIVILEGES ON DATABASE mattermost to mmuser;
#
# ALTER DATABASE mattermost OWNER TO mmuser;
#
# GRANT USAGE, CREATE ON SCHEMA PUBLIC TO mmuser;
#
#
# Note:
# Error Occurred: could not check schema owner: the user "mmuser" is not owner of the "public" schema
#
# Fix: on postreSQL monitor insert:
#
# ALTER SCHEMA public OWNER TO mmuser;
# GRANT ALL ON SCHEMA public to mmuser;
#
cd /tmp
sudo -u postgres psql -c "CREATE DATABASE ${psql_db_name};"
sudo -u postgres psql ${psql_db_name} -c "CREATE USER ${psql_db_user} WITH PASSWORD '${psql_db_password}';"
sudo -u postgres psql ${psql_db_name} -c "GRANT ALL PRIVILEGES ON DATABASE ${psql_db_name} to ${psql_db_user};"
sudo -u postgres psql ${psql_db_name} -c "ALTER DATABASE ${psql_db_name} OWNER TO ${psql_db_user};"
sudo -u postgres psql ${psql_db_name} -c "GRANT USAGE, CREATE ON SCHEMA PUBLIC TO ${psql_db_user};"
sudo -u postgres psql ${psql_db_name} -c "ALTER SCHEMA public OWNER TO ${psql_db_user};"
sudo -u postgres psql ${psql_db_name} -c "GRANT ALL ON SCHEMA public to ${psql_db_user};"
systemctl stop postgresql
systemctl start postgresql
# ==============================
# Automated PostgreSQL migration
# ==============================
# Step 1 - Check the MySQL database schema
# ========================================
#
# run the following command:
#
# migration-assist mysql "<MYSQL_DSN>" --fix-artifacts --fix-unicode --fix-varchar
#
#
# Example MySQ_ DSN: "user:password@tcp(address:3306)/db_name"
#
migration-assist mysql ${MYSQL_DSN} \
--fix-artifacts \
--fix-unicode \
--fix-varchar
# Output was:
#
# 2024-12-25 12:59:01 pinging mysql...
# 2024-12-25 12:59:01 connected to mysql successfully...
# 2024-12-25 12:59:01 running checks for artifacts...
# 2024-12-25 12:59:01 a fix is required for: schema_migrations
# 2024-12-25 12:59:01 the fix query has been executed successfully.
# 2024-12-25 12:59:01 4 checks been made, all good for artifacts
# 2024-12-25 12:59:01 running checks for unicode...
# 2024-12-25 12:59:02 11 checks been made, all good for unicode
# 2024-12-25 12:59:02 running checks for varchar...
# 2024-12-25 12:59:02 8 checks been made, all good for varchar
# 2024-12-25 12:59:02 running checks for varchar-extended...
# 2024-12-25 12:59:02 a fix is required for: linkmetadata.url
# 2024-12-25 12:59:02 the fix query has been executed successfully.
# 2024-12-25 12:59:02 12 checks been made, all good for varchar-extended
#
# Step 2 - Create the PostgreSQL database schema
# ==============================================
#
# run the following command:
#
# migration-assist postgres "<POSTGRES_DSN>" --run-migrations --mattermost-version="<MATTERMOST_VERSION>"
#
#
# Example POSTGRES_DSN: "postgres://user:password@address:5432/db_name"
#
# Example MATTERMOST_VERSION: "v9.4.0"
#
#migration-assist postgres ${POSTGRES_DSN} --run-migrations --mattermost-version=${MATTERMOST_VERSION}
migration-assist postgres ${POSTGRES_DSN} --run-migrations --mattermost-version ${MATTERMOST_VERSION}
# Output was:
#
# 2024-12-25 13:14:41 pinging postgres...
# 2024-12-25 13:14:41 connected to postgres successfully.
# 2024-12-25 13:14:41 schema owner check passed.
# 2024-12-25 13:14:41 checking if tables are empty...
# 2024-12-25 13:14:41 cloning repository@9.11.1
# 2024-12-25 13:14:41 git version: git version 2.39.5
# 2024-12-25 13:14:42 checking out...
# 2024-12-25 13:14:42 moving migration files into a better place..
# 2024-12-25 13:14:42 running migrations..
# 2024-12-25 13:14:43 migrations applied.
#
# Step 3 - Generate a pgloader configuration
# ==========================================
#
# run the following command:
#
# migration-assist pgloader --mysql="<MYSQL_DSN>" --postgres="<POSTGRES_DSN>" > migration.load
#
migration-assist pgloader --mysql $MYSQL_DSN --postgres $POSTGRES_DSN | tee migration.load
# Output was:
#
# LOAD DATABASE
# FROM mysql://mmuser:aiqu7oghae1eZai3@10.0.3.42:3306/mm
# INTO pgsql://mmuser:aiqu7oghae1eZai3@10.0.3.42:5432/mm
#
# WITH data only,
# workers = 8, concurrency = 1,
# multiple readers per thread, rows per range = 10000,
# prefetch rows = 10000, batch rows = 2500,
# create no tables, create no indexes,
# preserve index names
#
# SET PostgreSQL PARAMETERS
# maintenance_work_mem to '128MB',
# work_mem to '12MB'
#
# SET MySQL PARAMETERS
# net_read_timeout = '120',
# net_write_timeout = '120'
#
# CAST column Channels.Type to "channel_type" drop typemod,
# column Teams.Type to "team_type" drop typemod,
# column UploadSessions.Type to "upload_session_type" drop typemod,
# column ChannelBookmarks.Type to "channel_bookmark_type" drop typemod,
# column Drafts.Priority to text,
# type int when (= precision 11) to integer drop typemod,
# type bigint when (= precision 20) to bigint drop typemod,
# type text to varchar drop typemod using remove-null-characters,
# type tinyint when (<= precision 4) to boolean using tinyint-to-boolean,
# type json to jsonb drop typemod using remove-null-characters
#
# EXCLUDING TABLE NAMES MATCHING ~<IR_>, ~<focalboard>, ~<calls>, 'schema_migrations', 'db_migrations', 'db_lock',
# 'configurations', 'configurationfiles', 'db_config_migrations'
#
# BEFORE LOAD DO
# $$ ALTER SCHEMA public RENAME TO mm; $$,
# $$ TRUNCATE TABLE mm.systems; $$,
# $$ DROP INDEX IF EXISTS mm.idx_posts_message_txt; $$,
# $$ DROP INDEX IF EXISTS mm.idx_fileinfo_content_txt; $$
#
# AFTER LOAD DO
# $$ UPDATE mm.db_migrations set name='add_createat_to_teamembers' where version=92; $$,
# $$ ALTER SCHEMA mm RENAME TO public; $$,
# $$ SELECT pg_catalog.set_config('search_path', '"$user", "$user", public', false); $$,
# $$ ALTER USER mmuser SET SEARCH_PATH TO '"$user", public'; $$;
# Step 4 - Run pgloader
# =====================
#
# Run pgloader with the generated configuration file:
#
# pgloader migration.load > migration.log
#
${pgloader_bin} migration.load | tee migration.log
# Step 5 - Restore full-text indexes
# ==================================
#
# Run the following command to create the full-text indexes for the Posts and FileInfo tables:
#
# migration-assist postgres post-migrate "<POSTGRES_DSN>"
#
# This command creates the full-text indexes for the Posts and FileInfo tables. See the
# Restore full-text indexes documentation for more information.
#
migration-assist postgres post-migrate $POSTGRES_DSN | tee postgres-post-migrate.log
# Step 6 - Complete plugin migrations
# ===================================
#
# Generate migration configuration for collaborative playbooks, boards and calls:
#
# migration-assist pgloader boards --mysql="<MYSQL_DSN>" --postgres="<POSTGRES_DSN>" > boards.load
# migration-assist pgloader playbooks --mysql="<MYSQL_DSN>" --postgres="<POSTGRES_DSN>" > playbooks.load
# migration-assist pgloader calls --mysql="<MYSQL_DSN>" --postgres="<POSTGRES_DSN>" > calls.load
#
# Then run pgloader with the generated configuration files:
#
# pgloader boards.load > boards_migration.log
# pgloader playbooks.load > playbooks_migration.log
# pgloader calls.load > calls.log
#
# Note:
# Carefully read the log file to analyze whether there were any errors during the migration process.
# See the Plugin migrations documentation
# https://docs.mattermost.com/deploy/manual-postgres-migration.html#plugin-migrations)
# for information on migrating Playbooks, Boards and Calls.
#
migration-assist pgloader boards --mysql="${MYSQL_DSN}" --postgres="${POSTGRES_DSN}" | tee boards.load
migration-assist pgloader playbooks --mysql="${MYSQL_DSN}" --postgres="${POSTGRES_DSN}" | tee playbooks.load
migration-assist pgloader calls --mysql=""${MYSQL_DSN} --postgres=""${POSTGRES_DSN} | tee calls.load
# Prepare boards.load
#
# Delte entries:
#
# column focalboard_teams.settings to "json" drop typemod using remove-null-characters,
# ...
# $$ UPDATE mattermost2.focalboard_teams SET "settings" = '{}'::json WHERE "settings"::text = ''; $$,
#
sed -i '/focalboard_teams.settings/d' boards.load
sed -i '/mattermost.focalboard_teams/d' boards.load
${pgloader_bin} boards.load | tee boards_migration.log
# playbooks.load
#
# Add missing entries into database:
CREATE TABLE public.ir_category (
id character varying(26) NOT NULL,
name character varying(512) NOT NULL,
teamid character varying(26) NOT NULL,
userid character varying(26) NOT NULL,
collapsed boolean DEFAULT false,
createat bigint NOT NULL,
updateat bigint DEFAULT 0 NOT NULL,
deleteat bigint DEFAULT 0 NOT NULL
);
ALTER TABLE public.ir_category OWNER TO mattermost;
CREATE TABLE public.ir_category_item (
type character varying(1) NOT NULL,
categoryid character varying(26) NOT NULL,
itemid character varying(26) NOT NULL
);
ALTER TABLE public.ir_category_item OWNER TO mattermost;
ALTER TABLE ONLY public.ir_category_item
ADD CONSTRAINT ir_category_item_pkey PRIMARY KEY (categoryid, itemid, type);
ALTER TABLE ONLY public.ir_category
ADD CONSTRAINT ir_category_pkey PRIMARY KEY (id);
CREATE INDEX ir_category_item_categoryid ON public.ir_category_item USING btree (categoryid);
CREATE INDEX ir_category_teamid_userid ON public.ir_category USING btree (teamid, userid);
ALTER TABLE ONLY public.ir_category_item
ADD CONSTRAINT ir_category_item_categoryid_fkey FOREIGN KEY (categoryid) REFERENCES public.ir_category(id);
#
${pgloader_bin} playbooks.load | tee playbooks_migration.log
${pgloader_bin} calls.load | tee calls.log
# Important on postgres monitor console (psql mattermost), the requests
#
# SHOW SEARCH_PATH;
# snd
# SELECT CURRENT_SCHEMA();
#
# must response as shown:
#
mattermost=# SHOW SEARCH_PATH;
search_path
-----------------
"$user", public
(1 row)
mattermost=# SELECT CURRENT_SCHEMA();
current_schema
----------------
public
(1 row)
mattermost=#
Reference in New Issue View Git Blame Copy Permalink