From ed024de082a1048a94261dbbfa1f4a56180faf25 Mon Sep 17 00:00:00 2001 From: Bruno Sakoto Date: Wed, 5 Jan 2022 17:44:11 -0500 Subject: [PATCH] Upgrade timescaledb Timescaledb is upgraded to 2.5.1-pg14 to have postgres 14.1 Issue-ID: CPS-842 Change-Id: I65f2f090c0ff7346c28bf89f6bac3d5697ec1768 Signed-off-by: Bruno Sakoto --- README.md | 2 +- docker-compose.yml | 2 +- docs/content/deployment.rst | 130 ++++++++++++++++++++- .../repository/containers/TimescaleContainer.java | 2 +- 4 files changed, 129 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index bf2210f..b9aded5 100755 --- a/README.md +++ b/README.md @@ -93,7 +93,7 @@ docker run --name postgres-cps-temporal -p 5433:5432 -d \ -e POSTGRES_DB=cpstemporaldb \ -e POSTGRES_USER=cpstemporal \ -e POSTGRES_PASSWORD=cpstemporal \ - timescale/timescaledb:2.1.1-pg13 + timescale/timescaledb:2.5.1-pg14 ``` [Liquibase](https://www.liquibase.org/) is used to manage database schema changes and versions. diff --git a/docker-compose.yml b/docker-compose.yml index afe9019..5ab43dd 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -41,7 +41,7 @@ services: timescaledb: container_name: timescaledb - image: timescale/timescaledb:2.1.1-pg13 + image: timescale/timescaledb:2.5.1-pg14 ports: - '5433:5432' environment: diff --git a/docs/content/deployment.rst b/docs/content/deployment.rst index 0013466..726d25c 100644 --- a/docs/content/deployment.rst +++ b/docs/content/deployment.rst @@ -12,6 +12,7 @@ CPS Temporal Deployment * Deployment_ * Configuration_ * `Running With Docker`_ +* `Upgrade from Istanbul to Jakarta`_ Deployment ========== @@ -128,13 +129,134 @@ Docker. Refer to `README.md `_ files for more details. -Upgrade -======= +Upgrade from Istanbul to Jakarta +================================ -**Cps data-updated-event schema v2** +Cps data-updated-event schema v2 +-------------------------------- Cps data-updated-event schema v2 used in Jakarta is backward compatible with data-updated-event schema v1 used in Istanbul. It means that consumers using the schema v2 can process events generated by producers using the schema v2 or v1. -This implies that cps temporal (consumer) must be upgraded from Istanbul to Jakarta at the same time or before cps core (producer) is upgraded from Istanbul to Jakarta. \ No newline at end of file +This implies that cps temporal (consumer) must be upgraded from Istanbul to Jakarta at the same time or before cps core (producer) is upgraded from Istanbul to Jakarta. + +Database +-------- + +In Jakarta, CPS Temporal database is upgraded from TimescaleDB ``2.1.1`` running PostgresSQL ``13.2`` to +TimescaleDB ``2.5.1`` running PosgresSQL ``14.1``. This is a major PostgresSQL upgrade subject to change data storage +format. Then, any existing CPS Temporal data from Istanbul needs to be migrated before it can be used in Jakarta. + +The migration needs to be done in 2 main sequential steps for both TimescaleDB and PostgresSQL: + +#. Upgrade TimescaleDB from 2.1.1 to 2.5.1 +#. Upgrade PostgresSQL form 13.2 to 14.1 + +Bellow are the detailed steps to be completed for data migration (manual docker steps, to be reviewed for +production upgrade). + +* Stop CPS Temporal Istanbul service. + +* Backup 2.1.1-pg13 data. This backup is to be used to restore data if needed. + +.. code:: text + + # Start timescale 2.1.1-pg13 db instance + docker run --name postgres-cps-temporal-2.1.1-13 -d \ + -e POSTGRES_DB=cpstemporaldb \ + -e POSTGRES_USER=cpstemporal \ + -e POSTGRES_PASSWORD=cpstemporal \ + -v cps-temporal_data:/var/lib/postgresql/data \ + -v cps-temporal_backup-2.1.1-13:/var/lib/postgresql/backup \ + timescale/timescaledb:2.1.1-pg13 + + docker exec -it postgres-cps-temporal-2.1.1-13 psql -d cpstemporaldb -U cpstemporal -c "select version();" + docker exec -it postgres-cps-temporal-2.1.1-13 psql -d cpstemporaldb -U cpstemporal -c "\dx timescaledb" + + # Backup 2.1.1-pg13 data + docker exec -it postgres-cps-temporal-2.1.1-13 pg_basebackup -U cpstemporal -D /var/lib/postgresql/backup/ + docker exec -it postgres-cps-temporal-2.1.1-13 ls -l /var/lib/postgresql/backup/ + + # Stop db instance + docker container stop postgres-cps-temporal-2.1.1-13 + docker container rm postgres-cps-temporal-2.1.1-13 + +* Upgrade data from 2.1.1-pg13 to 2.5.1-pg-13 + +.. code:: text + + # Start timescale 2.5.1.-pg13 + docker run --name postgres-cps-temporal-2.5.1-13 -d \ + -e POSTGRES_DB=cpstemporaldb \ + -e POSTGRES_USER=cpstemporal \ + -e POSTGRES_PASSWORD=cpstemporal \ + -v cps-temporal_data:/var/lib/postgresql/data \ + -v cps-temporal_dump-2.5.1-13:/var/lib/postgresql/dump \ + timescale/timescaledb:2.5.1-pg13 + + # Upgrade data to 2.5.1-pg13 + docker exec -it postgres-cps-temporal-2.5.1-13 psql -d cpstemporaldb -U cpstemporal -c "\dx timescaledb" + docker exec -it postgres-cps-temporal-2.5.1-13 psql -X -d cpstemporaldb -U cpstemporal -c "ALTER EXTENSION timescaledb UPDATE;" + docker exec -it postgres-cps-temporal-2.5.1-13 psql -d cpstemporaldb -U cpstemporal -c "\dx timescaledb" + +* Create a dump of 2.5.1-pg-13 data + +.. code:: text + + # Dump 2.5.1-pg13 data + docker exec -it postgres-cps-temporal-2.5.1-13 pg_dump -d cpstemporaldb -U cpstemporal -Fc -f /var/lib/postgresql/dump/dumpfile + docker exec -it postgres-cps-temporal-2.5.1-13 ls -l /var/lib/postgresql/dump/dumpfile + + # Stop db instance + docker container stop postgres-cps-temporal-2.5.1-13 + docker container rm postgres-cps-temporal-2.5.1-13 + +* Upgrade data from 2.5.1-pg13 to 2.5.1-pg-14, by importing the dump in PostgresSQL 14 instance + +.. code:: text + + # Start timescale 2.5.1-pg14 + docker run --name postgres-cps-temporal-2.5.1-14 -p 5432:5432 -d \ + -e POSTGRES_DB=cpstemporaldb \ + -e POSTGRES_USER=cpstemporal \ + -e POSTGRES_PASSWORD=cpstemporal \ + -v cps-temporal_data-2.5.1-14:/var/lib/postgresql/data \ + -v cps-temporal_dump-2.5.1-13:/var/lib/postgresql/dump \ + timescale/timescaledb:2.5.1-pg14 + + docker exec -it postgres-cps-temporal-2.5.1-14 psql -d cpstemporaldb -U cpstemporal -c "select version();" + docker exec -it postgres-cps-temporal-2.5.1-14 psql -d cpstemporaldb -U cpstemporal -c "\dx timescaledb" + + # Upgrade data to 2.5.1-pg14 by restoring the dump + docker exec -it postgres-cps-temporal-2.5.1-14 pg_restore -d cpstemporaldb -U cpstemporal -Fc /var/lib/postgresql/dump/dumpfile + docker exec -it postgres-cps-temporal-2.5.1-14 psql -d cpstemporaldb -U cpstemporal -c "select count(*) from network_data;" + + # Stop db instances + docker container stop postgres-cps-temporal-2.5.1-14 + docker container rm postgres-cps-temporal-2.5.1-14 + +* Copy 2.5.1-pg14 data to data volume + +.. code:: text + + # Start busybox + docker run -it --rm \ + -v cps-temporal_data:/data \ + -v cps-temporal_data-2.5.1-14:/data-2.5.1-14 \ + busybox:1.34.1 + + # Run commands in busybox + rm -rf /data/* + cp -rp /data-2.5.1-14/* /data + diff -r /data /data-2.5.1-14 + exit + +* Start Cps Temporal Jakarta service + +* Cleanup volumes that are not needed anymore + +For more details about TimescaleDB and PostgresSQL upgrades, refer to: + +* `Updating TimescaleDB versions `_ +* `Upgrading a PostgreSQL Cluster `_ diff --git a/src/test/java/org/onap/cps/temporal/repository/containers/TimescaleContainer.java b/src/test/java/org/onap/cps/temporal/repository/containers/TimescaleContainer.java index a6ad5db..9ca7212 100644 --- a/src/test/java/org/onap/cps/temporal/repository/containers/TimescaleContainer.java +++ b/src/test/java/org/onap/cps/temporal/repository/containers/TimescaleContainer.java @@ -26,7 +26,7 @@ import org.testcontainers.utility.DockerImageName; */ public class TimescaleContainer extends PostgreSQLContainer { - private static final String IMAGE_NAME = "timescale/timescaledb:2.1.1-pg13"; + private static final String IMAGE_NAME = "timescale/timescaledb:2.5.1-pg14"; private static final DockerImageName DOCKER_IMAGE_NAME = DockerImageName.parse(IMAGE_NAME).asCompatibleSubstituteFor("postgres"); -- 2.16.6