3 # ============LICENSE_START=======================================================
5 # ================================================================================
6 # Copyright (C) 2022 Nordix Foundation.
7 # Modification Copyright 2024 Nordix Foundation.
8 # ================================================================================
9 # Licensed under the Apache License, Version 2.0 (the "License");
10 # you may not use this file except in compliance with the License.
11 # You may obtain a copy of the License at
13 # http://www.apache.org/licenses/LICENSE-2.0
15 # Unless required by applicable law or agreed to in writing, software
16 # distributed under the License is distributed on an "AS IS" BASIS,
17 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18 # See the License for the specific language governing permissions and
19 # limitations under the License.
20 # ============LICENSE_END=========================================================
22 # #####################################################################
24 # Upgrade/Downgrade SQL File Name Format:
26 # <VERSION>-<pdp|feature-name>[-description](.upgrade|.downgrade).sql
28 # This tool operates on a migration working directory at
30 # $POLICY_HOME/etc/db/migration
32 # Upgrade/Downgrade files for each schema (aka database) names to be maintained
33 # by this tool are located at
35 # $POLICY_HOME/etc/db/migration/<schema-name>/sql
37 # The nature of the migration directories is dynamic.
38 # A number of environment variables needs to be initialized
39 # prior to running db-migrator.
40 # These variables specify the connection details for the database
41 # to be upgraded/downgraded.
43 # The repository of upgrade/downgrade scripts is located in the
44 # /home/${SQL_DB}/sql directory.
45 # Two additional scripts have been provided to prepare the directories/files
46 # needed to perform the upgrade/downgrade.
48 # At any given time the following invariant must be preserved in any given
49 # $POLICY_HOME/etc/db/migration/<schema-name>/sql directory
51 # There is only upgrade scripts, or only downgrade scripts, or none.
53 # #####################################################################
55 source ${POLICY_HOME}/etc/profile.d/env.sh
58 METADATA_TABLE=schema_versions
59 MIGRATION_DIR=${POLICY_HOME}/etc/db/migration
62 UPGRADE_SQL_SUFFIX=".upgrade.sql"
63 DOWNGRADE_SQL_SUFFIX=".downgrade.sql"
65 SQL_QUOTES="SET SESSION SQL_MODE=ANSI_QUOTES;"
67 #####################################################
69 #####################################################
73 echo -e "syntax: $(basename "$0") "
74 echo -e "\t -s <schema-name> "
75 echo -e "\t [-b <migration-dir>] "
76 echo -e "\t [-f <from-version>]"
77 echo -e "\t [-t <target-version>]"
78 echo -e "\t -o <operations> "
80 echo -e "\t where <operations>=upgrade|downgrade|auto|version|erase|report"
83 echo -e "Configuration Options:"
84 echo -e "\t -s|--schema|--database: schema to operate on ('ALL' to apply on all)"
85 echo -e "\t -b|--basedir: overrides base DB migration directory"
86 echo -e "\t -f|--from: overrides current release version for operations"
87 echo -e "\t -t|--target: overrides target release to upgrade/downgrade"
90 echo -e "\t upgrade: upgrade operation"
91 echo -e "\t downgrade: performs a downgrade operation"
92 echo -e "\t auto: autonomous operation, determines upgrade or downgrade"
93 echo -e "\t version: returns current version, and in conjunction if '-f' sets the current version"
94 echo -e "\t erase: erase all data related <schema> (use with care)"
95 echo -e "\t report: migration detailed report on an schema"
96 echo -e "\t ok: is the migration status valid"
101 #####################################################
103 #####################################################
105 function current_schema() {
106 if [ "${DEBUG}" = "y" ]; then
107 echo "-- current_schema --"
112 local query="SELECT count(table_name) from information_schema.tables where table_schema='${SCHEMA}'"
114 PSQL="psql -h ${SQL_HOST} -U ${SQL_USER} -d ${SQL_DB}"
115 TABLES_IN_SCHEMA=$(${PSQL} -X -A --tuples-only --quiet --command "${query}")
116 if [ ${TABLES_IN_SCHEMA} -gt 0 ] && [ "${BASE_VERSION}" \> "${CURRENT_RELEASE}" ]; then
117 set_current_release "${BASE_VERSION}"
124 #####################################################
125 # ensure global metadata
126 #####################################################
128 function ensure_metadata() {
129 if [ "${DEBUG}" = "y" ]; then
130 echo "-- ensure_metadata --"
136 PSQL="psql -h ${SQL_HOST} -U ${SQL_USER} -d ${METADATA_DB}"
137 sql="SELECT 'CREATE DATABASE migration' WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = 'migration');"
138 ${PSQL} --command "${sql}"
140 if [ ${rc} -ne 0 ]; then
144 sql="CREATE TABLE IF NOT EXISTS ${METADATA_TABLE} "
145 sql=${sql}"(name VARCHAR(60) NOT NULL, version VARCHAR(20), "
146 sql=${sql}"PRIMARY KEY(name));"
147 ${PSQL} --command "${sql}"
151 #####################################################
152 # ensure metadata on a per schema basis
153 #####################################################
155 function ensure_metadata_schema() {
156 if [ "${DEBUG}" = "y" ]; then
157 echo "-- ensure_metadata_schema --"
163 PSQL="psql -h ${SQL_HOST} -U ${SQL_USER} -d ${METADATA_DB}"
164 sql="CREATE TABLE IF NOT EXISTS ${METADATA_HISTORY} "
165 sql=${sql}"(ID bigserial, script VARCHAR(80) NOT NULL, "
166 sql=${sql}"operation VARCHAR(10), from_version VARCHAR(20), to_version VARCHAR(20), tag VARCHAR(20), "
167 sql=${sql}"success VARCHAR(1), atTime TIMESTAMP DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY(ID));"
168 ${PSQL} --command "${sql}"
170 if [ ${rc} -ne 0 ]; then
174 sql="SELECT 'CREATE DATABASE migration' WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = 'policyadmin');"
175 ${PSQL} --command "${sql}"
179 #####################################################
181 #####################################################
183 function target_release() {
184 if [ "${DEBUG}" = "y" ]; then
185 echo "-- target_release --"
189 local sql sqlName upgradeSqls downgradeSqls
191 TARGET_UPGRADE_RELEASE=${ZERO_VERSION}
192 TARGET_DOWNGRADE_RELEASE=${ZERO_VERSION}
194 upgradeSqls=$(ls -v -r "${UPGRADE_DIR}"/*/upgrade/*.sql 2>/dev/null)
195 for sql in ${upgradeSqls}; do
196 TARGET_UPGRADE_RELEASE="$(basename $(dirname $(dirname $sql)))"
200 # default unless overriden
201 TARGET_DOWNGRADE_RELEASE="${ZERO_VERSION}"
204 #####################################################
206 #####################################################
208 function is_upgrade() {
209 if [ "${DEBUG}" = "y" ]; then
210 echo "-- is_upgrade --"
216 upgradeSqls=$(ls -v "${UPGRADE_DIR}"/*/upgrade/*.sql 2>/dev/null)
217 if [ -z "${upgradeSqls}" ]; then
224 #####################################################
226 #####################################################
228 function is_downgrade() {
229 if [ "${DEBUG}" = "y" ]; then
230 echo "-- is_downgrade --"
236 downgradeSqls=$(ls -v -r "${DOWNGRADE_DIR}"/*/downgrade/*.sql 2>/dev/null)
237 if [ -z "${downgradeSqls}" ]; then
244 #####################################################
245 # set_current_release
246 #####################################################
248 function set_current_release() {
249 if [ "${DEBUG}" = "y" ]; then
250 echo "-- set_current_release --"
254 CURRENT_RELEASE="${1}"
257 sql="INSERT INTO ${METADATA_TABLE} (name, version) "
258 sql=${sql}"VALUES('${SCHEMA}', '${CURRENT_RELEASE}') "
259 sql=${sql}"ON CONFLICT (name) DO UPDATE SET version='${CURRENT_RELEASE}';"
261 PSQL="psql -h ${SQL_HOST} -U ${SQL_USER} -d ${METADATA_DB}"
262 ${PSQL} --command "${sql}"
266 #####################################################
268 #####################################################
270 function current_release() {
271 if [ "${DEBUG}" = "y" ]; then
272 echo "-- current_release --"
277 local query="SELECT version FROM schema_versions WHERE name='${SQL_DB}'"
279 PSQL="psql -h ${SQL_HOST} -U ${SQL_USER} -d ${METADATA_DB}"
280 CURRENT_RELEASE=$(${PSQL} -X -A -t --command "${query}")
282 if [ -z "${CURRENT_RELEASE}" ]; then
283 set_current_release "${ZERO_VERSION}"
290 #####################################################
292 #####################################################
294 function previous_release() {
295 if [ "${DEBUG}" = "y" ]; then
296 echo "-- previous_release --"
299 local current_release="${1}"
300 local current_release_int_val previous_release_int_val previous_release
302 if [ $current_release == $ZERO_VERSION ] || [ $current_release == $BASE_VERSION ]; then
303 PREVIOUS_RELEASE=$ZERO_VERSION
305 current_release_int_val=$(echo $current_release | awk '{$0=int($0)}1')
306 let previous_release_int_val=$current_release_int_val-100
307 if [ $previous_release_int_val -lt 1000 ]; then
308 previous_release="0"$previous_release_int_val
310 previous_release=$previous_release_int_val
312 PREVIOUS_RELEASE=$previous_release
318 #####################################################
319 # last_operation_status
320 #####################################################
322 function last_operation_status() {
323 if [ "${DEBUG}" = "y" ]; then
324 echo "-- last_operation_status --"
330 query="SELECT script, operation, from_version, to_version, success FROM "
331 query=${query}"${METADATA_HISTORY} WHERE id="
332 query=${query}"(SELECT MAX(id) from ${METADATA_HISTORY}) "
333 query=${query}"AND success::integer=0"
335 PSQL="psql -h ${SQL_HOST} -U ${SQL_USER} -d ${METADATA_DB}"
336 row=$(${PSQL} --tuples-only --quiet --command "${query}")
338 if [ "${row}" != "" ]; then
339 LAST_SCRIPT=$(echo $row | awk '{print $1}')
340 LAST_OPERATION=$(echo $row | awk '{print $2}')
341 LAST_FROM_VERSION=$(echo $row | awk '{print $3}')
342 LAST_TO_VERSION=$(echo $row | awk '{print $4}')
343 LAST_STATUS=$(echo $row | awk '{print $5}')
352 #####################################################
353 # execute sql script history
354 #####################################################
356 function track_script() {
357 if [ "${DEBUG}" = "y" ]; then
358 echo "-- track_script $* --"
362 local script="${1}" operation="${2}" success="${3}" from_version="${4}" to_version="${5}" tag="${6}"
363 if [ $operation == "downgrade" ]; then
364 to_version=${TARGET_DOWNGRADE_RELEASE}
367 local sql="INSERT INTO ${METADATA_HISTORY}(script,operation,from_version,to_version,tag,success,atTime) "
368 sql=${sql}"VALUES ('${script}','${operation}','${from_version}','${to_version}','${tag}','${success}',now()) "
370 PSQL="psql -h ${SQL_HOST} -U ${SQL_USER} -d ${METADATA_DB}"
371 ${PSQL} --command "${sql}"
375 #####################################################
377 #####################################################
379 function run_script() {
380 if [ "${DEBUG}" == "y" ]; then
381 echo "-- run_script $* --"
385 local operation="${1}" script="${2}" scriptPath="${3}" fromVersion="${4}" toVersion="${5}" tag="${6}"
388 echo "> ${operation} ${script}"
390 PSQL="psql -h ${SQL_HOST} -U ${SQL_USER} -d ${SQL_DB}"
391 ${PSQL} <"${scriptPath}"
393 if [ ${rc} -ne 0 ]; then
399 track_script "${script}" "${operation}" "${success}" "${fromVersion}" "${toVersion}" "${tag}"
404 #####################################################
406 #####################################################
409 if [ "${DEBUG}" = "y" ]; then
414 local sqlName sqlFile schemaVersion upgradeSqls rc tagDate tag
415 tagDate=$(date +%d%m%y%H%M%S)
417 echo "upgrade: ${CURRENT_RELEASE} -> ${TARGET_UPGRADE_RELEASE}"
419 if [ ${CURRENT_RELEASE} \< ${TARGET_UPGRADE_RELEASE} ]; then
420 upgradeSqls=$(ls -v "${UPGRADE_DIR}"/*/upgrade/*.sql 2>/dev/null)
421 for sqlFile in ${upgradeSqls}; do
422 sqlName=$(basename "${sqlFile}")
423 schemaVersion="$(basename $(dirname $(dirname $sqlFile)))"
424 previous_release $schemaVersion
425 if [ "${schemaVersion}" -gt "${CURRENT_RELEASE}" ] &&
426 [ "${schemaVersion}" -le "${TARGET_UPGRADE_RELEASE}" ]; then
427 tag=${tagDate}"${schemaVersion}u"
428 if [ $RETRY -eq 1 ] &&
429 # Compare the numeric portion of the filename because shell doesn't support string comparison
430 [ $(echo ${sqlName} | awk -F- '{print $1}') -${COMPARE} $(echo ${LAST_SCRIPT} | awk -F- '{print $1}') ]; then
437 if [ ${rc} -eq 0 ]; then
438 run_script "upgrade" "${sqlName}" "${sqlFile}" "${PREVIOUS_RELEASE}" "${schemaVersion}" "${tag}"
440 if [ ${rc} -ne 0 ]; then
441 echo "${SCHEMA}: upgrade aborted at ${schemaVersion} by script ${sqlName}"
442 set_current_release "${schemaVersion}"
449 set_current_release "${TARGET_UPGRADE_RELEASE}"
455 #####################################################
457 #####################################################
459 function downgrade() {
460 if [ "${DEBUG}" = "y" ]; then
461 echo "-- downgrade --"
465 local sqlName sqlFile schemaVersion downgradeSqls rc tagDate tag
466 tagDate=$(date +%d%m%y%H%M%S)
468 echo "downgrade: ${CURRENT_RELEASE} -> ${TARGET_DOWNGRADE_RELEASE}"
470 if [ ${CURRENT_RELEASE} \> ${TARGET_DOWNGRADE_RELEASE} ]; then
471 downgradeSqls=$(ls -v -r "${DOWNGRADE_DIR}"/*/downgrade/*.sql 2>/dev/null)
472 for sqlFile in ${downgradeSqls}; do
473 sqlName=$(basename "${sqlFile}")
474 schemaVersion="$(basename $(dirname $(dirname $sqlFile)))"
475 if [ "${schemaVersion}" -le "${CURRENT_RELEASE}" ] &&
476 [ "${schemaVersion}" -gt "${TARGET_DOWNGRADE_RELEASE}" ]; then
477 tag=${tagDate}"${schemaVersion}d"
478 if [ $RETRY -eq 1 ] &&
479 # Compare the numeric portion of the filename because shell doesn't support string comparison
480 [ $(echo ${sqlName} | awk -F- '{print $1}') -${COMPARE} $(echo ${LAST_SCRIPT} | awk -F- '{print $1}') ]; then
485 if [ ${rc} -eq 0 ]; then
486 run_script "downgrade" "${sqlName}" "${sqlFile}" "${schemaVersion}" "${PREVIOUS_RELEASE}" ${tag}
488 if [ ${rc} -ne 0 ]; then
489 echo "${SCHEMA}: downgrade aborted at ${schemaVersion} by script ${sqlName}"
490 set_current_release "${schemaVersion}"
497 set_current_release "${TARGET_DOWNGRADE_RELEASE}"
503 #####################################################
505 #####################################################
508 if [ "${DEBUG}" = "y" ]; then
513 PSQL="psql -h ${SQL_HOST} -U ${SQL_USER} -d ${METADATA_DB}"
514 local updateMetadata="UPDATE ${METADATA_TABLE} SET version='${ZERO_VERSION}';"
515 ${PSQL} --command "${updateMetadata}"
517 local deleteHistory="DELETE FROM ${METADATA_HISTORY};"
518 ${PSQL} --command "${deleteHistory}"
520 local dropDB="DROP DATABASE IF EXISTS ${SCHEMA_DB}"
521 ${PSQL} --command "${dropDB}"
524 #####################################################
526 #####################################################
529 if [ "${DEBUG}" = "y" ]; then
534 PSQL="psql -h ${SQL_HOST} -U ${SQL_USER} -d ${METADATA_DB}"
535 local versionSql="SELECT * FROM ${METADATA_TABLE} WHERE name='${SCHEMA}';"
536 ${PSQL} --command "${versionSql}"
538 local historySql="SELECT * FROM ${METADATA_HISTORY} ORDER BY id, atTime ASC;"
539 ${PSQL} --command "${historySql}"
545 if [ "${DEBUG}" = "y" ]; then
552 if [ "${CURRENT_RELEASE}" = "${TARGET_UPGRADE_RELEASE}" ]; then
553 echo "${SCHEMA}: OK @ ${CURRENT_RELEASE}"
555 echo "${SCHEMA}: upgrade available: ${CURRENT_RELEASE} -> ${TARGET_UPGRADE_RELEASE}"
559 if [ "${CURRENT_RELEASE}" = "${TARGET_DOWNGRADE_RELEASE}" ]; then
560 echo "${SCHEMA}: OK @ ${CURRENT_RELEASE}"
562 echo "${SCHEMA}: downgrade available: ${CURRENT_RELEASE} -> ${TARGET_DOWNGRADE_RELEASE}"
570 #####################################################
572 #####################################################
574 if [ "${DEBUG}" = "y" ]; then
578 until [ -z "$1" ]; do
580 -s | --schema | --database)
590 INPUT_TARGET_RELEASE=$1
594 INPUT_FROM_RELEASE=$1
617 echo "error: invalid operation provided"
623 if [ -z "${SCHEMA}" ]; then
624 echo "error: a database name must be provided"
629 source "${POLICY_HOME}"/etc/profile.d/env.sh
631 if [ -z "${SQL_HOST}" ] || [ -z "${SQL_USER}" ] || [ -z "${SQL_PASSWORD}" ]; then
632 echo "error: no database has been set up"
636 PSQL="psql -h ${SQL_HOST} -U ${SQL_USER} -d ${SQL_DB}"
637 if ! ${PSQL} --command '\l'; then
638 echo "error: No DB connectivity to ${SQL_HOST} for ${SQL_USER}"
642 if [ "${SCHEMA}" = "ALL" ]; then
646 SCHEMA_S=$(ls -d "${MIGRATION_DIR}"/${SCHEMA}/ 2>/dev/null)
647 if [ -z "${SCHEMA_S}" ]; then
648 echo "error: no databases available"
652 if ! ensure_metadata; then
653 echo "error: migration metadata not accessible"
658 for dbPath in ${SCHEMA_S}; do
659 SCHEMA=$(basename "${dbPath}")
660 SCHEMA_DB="'${SCHEMA}'"
661 UPGRADE_DIR="${MIGRATION_DIR}"/"${SCHEMA}"/postgres
662 DOWNGRADE_DIR=${UPGRADE_DIR}
663 METADATA_HISTORY="${SCHEMA}_schema_changelog"
664 TARGET_RELEASE=${INPUT_TARGET_RELEASE}
666 if is_upgrade && is_downgrade; then
667 echo "${SCHEMA}: failure: invalid configuration: ${UPGRADE_SQL_SUFFIX} and " \
668 "${DOWNGRADE_SQL_SUFFIX} exist under ${DOWNGRADE_DIR}"
673 if [ "${operation}" = "auto" ]; then
681 if ! ensure_metadata_schema; then
682 echo "${SCHEMA}: failure: metadata not accessible for this schema"
686 if [ -z "${TARGET_RELEASE}" ]; then
689 # user asked to override
690 TARGET_UPGRADE_RELEASE="${TARGET_RELEASE}"
691 TARGET_DOWNGRADE_RELEASE="${TARGET_RELEASE}"
694 if [ -z "${CURRENT_RELEASE}" ]; then
695 if ! current_release; then
696 echo "${SCHEMA}: failure: cannot obtain current release"
700 if ! set_current_release "${CURRENT_RELEASE}"; then
701 echo "${SCHEMA}: failure: cannot set current release"
706 PSQL="psql -h ${SQL_HOST} -U ${SQL_USER} -d ${SQL_DB};"
707 # Check if the schema has already been installed
711 last_operation_status
713 if [ $last_status -eq 0 ]; then
714 echo "Partial $LAST_OPERATION detected"
716 if [ "${OPERATION}" == 'downgrade' ] && [ "${LAST_OPERATION}" == 'downgrade' ]; then
717 CURRENT_RELEASE="${LAST_FROM_VERSION}"
720 echo "Downgrade will attempt to resume at $LAST_SCRIPT"
721 elif [ "${OPERATION}" == 'downgrade' ] && [ "${LAST_OPERATION}" == 'upgrade' ]; then
722 CURRENT_RELEASE="${LAST_TO_VERSION}"
723 # greater than or equal to
725 echo "Downgrade will attempt to resume before $LAST_SCRIPT"
726 elif [ "${OPERATION}" == 'upgrade' ] && [ "${LAST_OPERATION}" == 'upgrade' ]; then
727 CURRENT_RELEASE="${LAST_FROM_VERSION}"
730 echo "Upgrade will attempt to resume at $LAST_SCRIPT"
732 CURRENT_RELEASE="${LAST_TO_VERSION}"
735 echo "Upgrade will attempt to resume after $LAST_SCRIPT"
739 if [ -n "${INPUT_FROM_RELEASE}" ]; then
740 if [ "${CURRENT_RELEASE}" \> "${INPUT_FROM_RELEASE}" ] || [ "${CURRENT_RELEASE}" \< "${INPUT_FROM_RELEASE}" ]; then
741 echo "${SCHEMA}: On version ${CURRENT_RELEASE} cannot ${OPERATION} from ${INPUT_FROM_RELEASE}"
744 CURRENT_RELEASE=${INPUT_FROM_RELEASE}
751 echo "${SCHEMA}: OK: upgrade (${CURRENT_RELEASE})"
754 echo "${SCHEMA}: failure: upgrade to release ${TARGET_UPGRADE_RELEASE} (${CURRENT_RELEASE})"
759 echo "${SCHEMA}: OK: downgrade (${CURRENT_RELEASE})"
762 echo "${SCHEMA}: failure: downgrade to release ${TARGET_DOWNGRADE_RELEASE} (${CURRENT_RELEASE})"
766 echo "${SCHEMA}: ${CURRENT_RELEASE}"