README.migrate-MySQL-to-PostgreSQL: documentation migration of 'mm-irights.oopen.de'.
This commit is contained in:
parent
66d3210184
commit
543ee65fe0
@ -1,7 +1,405 @@
|
||||
|
||||
# see:
|
||||
#
|
||||
#
|
||||
# https://docs.mattermost.com/deploy/postgres-migration-assist-tool.html
|
||||
# https://docs.mattermost.com/deploy/manual-postgres-migration.html
|
||||
#
|
||||
# https://docs.mattermost.com/deploy/postgres-migration.html
|
||||
|
||||
MATTERMOST_VERSION="9.11.6"
|
||||
|
||||
# Postgresql
|
||||
psql_db_host="10.0.3.42"
|
||||
psql_db_port="5432"
|
||||
psql_db_name="mattermost"
|
||||
psql_db_user="mattermost"
|
||||
psql_db_password='aiqu7oghae1eZai3'
|
||||
|
||||
MYSQL_DSN="${mysql_db_user}:${mysql_db_password}@tcp(${mysql_db_host}:${mysql_db_port})/${mysql_db_name}"
|
||||
|
||||
# Mysql
|
||||
mysql_db_host="10.0.3.42"
|
||||
mysql_db_port="3306"
|
||||
mysql_db_name="mattermost2"
|
||||
mysql_db_user="mattermost2"
|
||||
mysql_db_password='aiqu7oghae1eZai3'
|
||||
|
||||
POSTGRES_DSN="postgres://${psql_db_user}:${psql_db_password}@${psql_db_host}:${psql_db_port}/${psql_db_name}"
|
||||
|
||||
pgloader_bin='/usr/local/src/pgloader/pgloader/build/bin/pgloader'
|
||||
|
||||
|
||||
# ---
|
||||
# Compile/Install pgloader from source
|
||||
# ---
|
||||
|
||||
apt purge pgloader
|
||||
apt --purge autoremove
|
||||
mkdir -p "/usr/local/src/pgloader/"
|
||||
cd "/usr/local/src/pgloader/"
|
||||
git clone https://github.com/dimitri/pgloader.git
|
||||
apt-get install sbcl unzip libsqlite3-dev make curl gawk freetds-dev libzip-dev
|
||||
cd pgloader
|
||||
/build/bin/pgloader --help
|
||||
|
||||
|
||||
# ---
|
||||
# Completely deinstall postgresql
|
||||
#
|
||||
# ---
|
||||
|
||||
DEBIAN_FRONTEND=noninteractive apt purge -y postgresql postgresql-*
|
||||
rm -rf /var/lib/postgresql
|
||||
rm -rf /etc/postgresql
|
||||
rm -rf /var/log/postgresql
|
||||
|
||||
DEBIAN_FRONTEND=noninteractive apt install -y --allow-unauthenticated postgresql
|
||||
|
||||
# Adjust files
|
||||
#
|
||||
# /etc/postgresql/15/main/pg_hba.conf
|
||||
# /etc/postgresql/15/main/postgresql.conf
|
||||
#
|
||||
#
|
||||
# /etc/postgresql/15/main/postgresql.conf
|
||||
#
|
||||
# Find the following line: #listen_addresses = 'localhost'
|
||||
# Uncomment the line and change localhost to *: listen_addresses = '*'
|
||||
# Restart PostgreSQL for the change to take effect by running:
|
||||
#
|
||||
#
|
||||
# /etc/postgresql/{version}/main/pg_hba.conf
|
||||
#
|
||||
# Find the following lines:
|
||||
#
|
||||
# local all all peer
|
||||
#
|
||||
# host all all ::1/128 ident
|
||||
#
|
||||
# Change peer and ident to trust:
|
||||
#
|
||||
# local all all trust
|
||||
#
|
||||
# host all all ::1/128 trust
|
||||
#
|
||||
#
|
||||
# Remote Database (separate server)
|
||||
#
|
||||
# If the Mattermost server and the database are on different machines:
|
||||
#
|
||||
# /etc/postgresql/{version}/main/pg_hba.conf in a text editor as root user.
|
||||
#
|
||||
# Add the following line to the end of the file, where {mattermost-server-IP} is the
|
||||
# IP address of the Mattermost server:
|
||||
#
|
||||
# host all all {mattermost-server-IP}/32 md5.
|
||||
#
|
||||
cp -a /etc/postgresql/15/main/pg_hba.conf /etc/postgresql/15/main/pg_hba.conf.ORIG
|
||||
cp /root/postgresql_15_main/pg_hba.conf /etc/postgresql/15/main/pg_hba.conf
|
||||
|
||||
cp -a /etc/postgresql/15/main/postgresql.conf /etc/postgresql/15/main/postgresql.conf.ORIG
|
||||
cp /root/postgresql_15_main/postgresql.conf /etc/postgresql/15/main/postgresql.conf
|
||||
|
||||
systemctl restart postgresql
|
||||
|
||||
|
||||
# Create PostgreSQL Database om mattermost server..
|
||||
#
|
||||
# seee: https://docs.mattermost.com/install/prepare-mattermost-database.html
|
||||
#
|
||||
# sudo -u postgres psql
|
||||
#
|
||||
# CREATE DATABASE mm;
|
||||
#
|
||||
# \connect mm
|
||||
#
|
||||
# CREATE USER mmuser WITH PASSWORD 'mmuser-password';
|
||||
#
|
||||
#
|
||||
# If you're configuring PostgreSQL v15.x or later:
|
||||
#
|
||||
# GRANT ALL PRIVILEGES ON DATABASE mattermost to mmuser;
|
||||
#
|
||||
# ALTER DATABASE mattermost OWNER TO mmuser;
|
||||
#
|
||||
# GRANT USAGE, CREATE ON SCHEMA PUBLIC TO mmuser;
|
||||
#
|
||||
#
|
||||
# Note:
|
||||
# Error Occurred: could not check schema owner: the user "mmuser" is not owner of the "public" schema
|
||||
#
|
||||
# Fix: on postreSQL monitor insert:
|
||||
#
|
||||
# ALTER SCHEMA public OWNER TO mmuser;
|
||||
# GRANT ALL ON SCHEMA public to mmuser;
|
||||
#
|
||||
cd /tmp
|
||||
sudo -u postgres psql -c "CREATE DATABASE ${psql_db_name};"
|
||||
|
||||
sudo -u postgres psql ${psql_db_name} -c "CREATE USER ${psql_db_user} WITH PASSWORD '${psql_db_password}';"
|
||||
sudo -u postgres psql ${psql_db_name} -c "GRANT ALL PRIVILEGES ON DATABASE ${psql_db_name} to ${psql_db_user};"
|
||||
sudo -u postgres psql ${psql_db_name} -c "ALTER DATABASE ${psql_db_name} OWNER TO ${psql_db_user};"
|
||||
sudo -u postgres psql ${psql_db_name} -c "GRANT USAGE, CREATE ON SCHEMA PUBLIC TO ${psql_db_user};"
|
||||
sudo -u postgres psql ${psql_db_name} -c "ALTER SCHEMA public OWNER TO ${psql_db_user};"
|
||||
sudo -u postgres psql ${psql_db_name} -c "GRANT ALL ON SCHEMA public to ${psql_db_user};"
|
||||
|
||||
systemctl stop postgresql
|
||||
systemctl start postgresql
|
||||
|
||||
|
||||
# ==============================
|
||||
# Automated PostgreSQL migration
|
||||
# ==============================
|
||||
|
||||
# Step 1 - Check the MySQL database schema
|
||||
# ========================================
|
||||
#
|
||||
# run the following command:
|
||||
#
|
||||
# migration-assist mysql "<MYSQL_DSN>" --fix-artifacts --fix-unicode --fix-varchar
|
||||
#
|
||||
#
|
||||
# Example MySQ_ DSN: "user:password@tcp(address:3306)/db_name"
|
||||
#
|
||||
migration-assist mysql ${MYSQL_DSN} \
|
||||
--fix-artifacts \
|
||||
--fix-unicode \
|
||||
--fix-varchar
|
||||
|
||||
# Output was:
|
||||
#
|
||||
# 2024-12-25 12:59:01 pinging mysql...
|
||||
# 2024-12-25 12:59:01 connected to mysql successfully...
|
||||
# 2024-12-25 12:59:01 running checks for artifacts...
|
||||
# 2024-12-25 12:59:01 a fix is required for: schema_migrations
|
||||
# 2024-12-25 12:59:01 the fix query has been executed successfully.
|
||||
# 2024-12-25 12:59:01 4 checks been made, all good for artifacts
|
||||
# 2024-12-25 12:59:01 running checks for unicode...
|
||||
# 2024-12-25 12:59:02 11 checks been made, all good for unicode
|
||||
# 2024-12-25 12:59:02 running checks for varchar...
|
||||
# 2024-12-25 12:59:02 8 checks been made, all good for varchar
|
||||
# 2024-12-25 12:59:02 running checks for varchar-extended...
|
||||
# 2024-12-25 12:59:02 a fix is required for: linkmetadata.url
|
||||
# 2024-12-25 12:59:02 the fix query has been executed successfully.
|
||||
# 2024-12-25 12:59:02 12 checks been made, all good for varchar-extended
|
||||
#
|
||||
|
||||
|
||||
|
||||
# Step 2 - Create the PostgreSQL database schema
|
||||
# ==============================================
|
||||
#
|
||||
# run the following command:
|
||||
#
|
||||
# migration-assist postgres "<POSTGRES_DSN>" --run-migrations --mattermost-version="<MATTERMOST_VERSION>"
|
||||
#
|
||||
#
|
||||
# Example POSTGRES_DSN: "postgres://user:password@address:5432/db_name"
|
||||
#
|
||||
# Example MATTERMOST_VERSION: "v9.4.0"
|
||||
#
|
||||
#migration-assist postgres ${POSTGRES_DSN} --run-migrations --mattermost-version=${MATTERMOST_VERSION}
|
||||
|
||||
migration-assist postgres ${POSTGRES_DSN} --run-migrations --mattermost-version ${MATTERMOST_VERSION}
|
||||
|
||||
# Output was:
|
||||
#
|
||||
# 2024-12-25 13:14:41 pinging postgres...
|
||||
# 2024-12-25 13:14:41 connected to postgres successfully.
|
||||
# 2024-12-25 13:14:41 schema owner check passed.
|
||||
# 2024-12-25 13:14:41 checking if tables are empty...
|
||||
# 2024-12-25 13:14:41 cloning repository@9.11.1
|
||||
# 2024-12-25 13:14:41 git version: git version 2.39.5
|
||||
# 2024-12-25 13:14:42 checking out...
|
||||
# 2024-12-25 13:14:42 moving migration files into a better place..
|
||||
# 2024-12-25 13:14:42 running migrations..
|
||||
# 2024-12-25 13:14:43 migrations applied.
|
||||
#
|
||||
|
||||
|
||||
# Step 3 - Generate a pgloader configuration
|
||||
# ==========================================
|
||||
#
|
||||
# run the following command:
|
||||
#
|
||||
# migration-assist pgloader --mysql="<MYSQL_DSN>" --postgres="<POSTGRES_DSN>" > migration.load
|
||||
#
|
||||
migration-assist pgloader --mysql $MYSQL_DSN --postgres $POSTGRES_DSN | tee migration.load
|
||||
|
||||
# Output was:
|
||||
#
|
||||
# LOAD DATABASE
|
||||
# FROM mysql://mmuser:aiqu7oghae1eZai3@10.0.3.42:3306/mm
|
||||
# INTO pgsql://mmuser:aiqu7oghae1eZai3@10.0.3.42:5432/mm
|
||||
#
|
||||
# WITH data only,
|
||||
# workers = 8, concurrency = 1,
|
||||
# multiple readers per thread, rows per range = 10000,
|
||||
# prefetch rows = 10000, batch rows = 2500,
|
||||
# create no tables, create no indexes,
|
||||
# preserve index names
|
||||
#
|
||||
# SET PostgreSQL PARAMETERS
|
||||
# maintenance_work_mem to '128MB',
|
||||
# work_mem to '12MB'
|
||||
#
|
||||
# SET MySQL PARAMETERS
|
||||
# net_read_timeout = '120',
|
||||
# net_write_timeout = '120'
|
||||
#
|
||||
# CAST column Channels.Type to "channel_type" drop typemod,
|
||||
# column Teams.Type to "team_type" drop typemod,
|
||||
# column UploadSessions.Type to "upload_session_type" drop typemod,
|
||||
# column ChannelBookmarks.Type to "channel_bookmark_type" drop typemod,
|
||||
# column Drafts.Priority to text,
|
||||
# type int when (= precision 11) to integer drop typemod,
|
||||
# type bigint when (= precision 20) to bigint drop typemod,
|
||||
# type text to varchar drop typemod using remove-null-characters,
|
||||
# type tinyint when (<= precision 4) to boolean using tinyint-to-boolean,
|
||||
# type json to jsonb drop typemod using remove-null-characters
|
||||
#
|
||||
# EXCLUDING TABLE NAMES MATCHING ~<IR_>, ~<focalboard>, ~<calls>, 'schema_migrations', 'db_migrations', 'db_lock',
|
||||
# 'configurations', 'configurationfiles', 'db_config_migrations'
|
||||
#
|
||||
# BEFORE LOAD DO
|
||||
# $$ ALTER SCHEMA public RENAME TO mm; $$,
|
||||
# $$ TRUNCATE TABLE mm.systems; $$,
|
||||
# $$ DROP INDEX IF EXISTS mm.idx_posts_message_txt; $$,
|
||||
# $$ DROP INDEX IF EXISTS mm.idx_fileinfo_content_txt; $$
|
||||
#
|
||||
# AFTER LOAD DO
|
||||
# $$ UPDATE mm.db_migrations set name='add_createat_to_teamembers' where version=92; $$,
|
||||
# $$ ALTER SCHEMA mm RENAME TO public; $$,
|
||||
# $$ SELECT pg_catalog.set_config('search_path', '"$user", "$user", public', false); $$,
|
||||
# $$ ALTER USER mmuser SET SEARCH_PATH TO '"$user", public'; $$;
|
||||
|
||||
|
||||
# Step 4 - Run pgloader
|
||||
# =====================
|
||||
#
|
||||
# Run pgloader with the generated configuration file:
|
||||
#
|
||||
# pgloader migration.load > migration.log
|
||||
#
|
||||
${pgloader_bin} migration.load | tee migration.log
|
||||
|
||||
|
||||
# Step 5 - Restore full-text indexes
|
||||
# ==================================
|
||||
#
|
||||
# Run the following command to create the full-text indexes for the Posts and FileInfo tables:
|
||||
#
|
||||
# migration-assist postgres post-migrate "<POSTGRES_DSN>"
|
||||
#
|
||||
# This command creates the full-text indexes for the Posts and FileInfo tables. See the
|
||||
# Restore full-text indexes documentation for more information.
|
||||
#
|
||||
migration-assist postgres post-migrate $POSTGRES_DSN | tee postgres-post-migrate.log
|
||||
|
||||
|
||||
# Step 6 - Complete plugin migrations
|
||||
# ===================================
|
||||
#
|
||||
# Generate migration configuration for collaborative playbooks, boards and calls:
|
||||
#
|
||||
# migration-assist pgloader boards --mysql="<MYSQL_DSN>" --postgres="<POSTGRES_DSN>" > boards.load
|
||||
# migration-assist pgloader playbooks --mysql="<MYSQL_DSN>" --postgres="<POSTGRES_DSN>" > playbooks.load
|
||||
# migration-assist pgloader calls --mysql="<MYSQL_DSN>" --postgres="<POSTGRES_DSN>" > calls.load
|
||||
#
|
||||
# Then run pgloader with the generated configuration files:
|
||||
#
|
||||
# pgloader boards.load > boards_migration.log
|
||||
# pgloader playbooks.load > playbooks_migration.log
|
||||
# pgloader calls.load > calls.log
|
||||
#
|
||||
# Note:
|
||||
# Carefully read the log file to analyze whether there were any errors during the migration process.
|
||||
# See the Plugin migrations documentation
|
||||
# https://docs.mattermost.com/deploy/manual-postgres-migration.html#plugin-migrations)
|
||||
# for information on migrating Playbooks, Boards and Calls.
|
||||
#
|
||||
migration-assist pgloader boards --mysql="${MYSQL_DSN}" --postgres="${POSTGRES_DSN}" | tee boards.load
|
||||
migration-assist pgloader playbooks --mysql="${MYSQL_DSN}" --postgres="${POSTGRES_DSN}" | tee playbooks.load
|
||||
migration-assist pgloader calls --mysql=""${MYSQL_DSN} --postgres=""${POSTGRES_DSN} | tee calls.load
|
||||
|
||||
|
||||
# Prepare boards.load
|
||||
#
|
||||
# Delte entries:
|
||||
#
|
||||
# column focalboard_teams.settings to "json" drop typemod using remove-null-characters,
|
||||
# ...
|
||||
# $$ UPDATE mattermost2.focalboard_teams SET "settings" = '{}'::json WHERE "settings"::text = ''; $$,
|
||||
#
|
||||
sed -i '/focalboard_teams.settings/d' boards.load
|
||||
sed -i '/mattermost.focalboard_teams/d' boards.load
|
||||
|
||||
${pgloader_bin} boards.load | tee boards_migration.log
|
||||
|
||||
|
||||
# playbooks.load
|
||||
#
|
||||
# Add missing entries into database:
|
||||
CREATE TABLE public.ir_category (
|
||||
id character varying(26) NOT NULL,
|
||||
name character varying(512) NOT NULL,
|
||||
teamid character varying(26) NOT NULL,
|
||||
userid character varying(26) NOT NULL,
|
||||
collapsed boolean DEFAULT false,
|
||||
createat bigint NOT NULL,
|
||||
updateat bigint DEFAULT 0 NOT NULL,
|
||||
deleteat bigint DEFAULT 0 NOT NULL
|
||||
);
|
||||
ALTER TABLE public.ir_category OWNER TO mattermost;
|
||||
|
||||
CREATE TABLE public.ir_category_item (
|
||||
type character varying(1) NOT NULL,
|
||||
categoryid character varying(26) NOT NULL,
|
||||
itemid character varying(26) NOT NULL
|
||||
);
|
||||
ALTER TABLE public.ir_category_item OWNER TO mattermost;
|
||||
|
||||
|
||||
ALTER TABLE ONLY public.ir_category_item
|
||||
ADD CONSTRAINT ir_category_item_pkey PRIMARY KEY (categoryid, itemid, type);
|
||||
|
||||
ALTER TABLE ONLY public.ir_category
|
||||
ADD CONSTRAINT ir_category_pkey PRIMARY KEY (id);
|
||||
|
||||
CREATE INDEX ir_category_item_categoryid ON public.ir_category_item USING btree (categoryid);
|
||||
CREATE INDEX ir_category_teamid_userid ON public.ir_category USING btree (teamid, userid);
|
||||
|
||||
ALTER TABLE ONLY public.ir_category_item
|
||||
ADD CONSTRAINT ir_category_item_categoryid_fkey FOREIGN KEY (categoryid) REFERENCES public.ir_category(id);
|
||||
#
|
||||
|
||||
${pgloader_bin} playbooks.load | tee playbooks_migration.log
|
||||
|
||||
${pgloader_bin} calls.load | tee calls.log
|
||||
|
||||
|
||||
|
||||
# Important on postgres monitor console (psql mattermost), the requests
|
||||
#
|
||||
# SHOW SEARCH_PATH;
|
||||
# snd
|
||||
# SELECT CURRENT_SCHEMA();
|
||||
#
|
||||
# must response as shown:
|
||||
#
|
||||
|
||||
mattermost=# SHOW SEARCH_PATH;
|
||||
search_path
|
||||
-----------------
|
||||
"$user", public
|
||||
(1 row)
|
||||
|
||||
mattermost=# SELECT CURRENT_SCHEMA();
|
||||
current_schema
|
||||
----------------
|
||||
public
|
||||
(1 row)
|
||||
|
||||
mattermost=#
|
||||
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user