3 # ============LICENSE_START=======================================================
5 # ================================================================================
6 # Copyright (C) 2022 Nordix Foundation.
7 # ================================================================================
8 # Licensed under the Apache License, Version 2.0 (the "License");
9 # you may not use this file except in compliance with the License.
10 # You may obtain a copy of the License at
12 # http://www.apache.org/licenses/LICENSE-2.0
14 # Unless required by applicable law or agreed to in writing, software
15 # distributed under the License is distributed on an "AS IS" BASIS,
16 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17 # See the License for the specific language governing permissions and
18 # limitations under the License.
19 # ============LICENSE_END=========================================================
21 # #####################################################################
23 # Upgrade/Downgrade SQL File Name Format:
25 # <VERSION>-<pdp|feature-name>[-description](.upgrade|.downgrade).sql
27 # This tool operates on a migration working directory at
29 # $POLICY_HOME/etc/db/migration
31 # Upgrade/Downgrade files for each schema (aka database) names to be maintained
32 # by this tool are located at
34 # $POLICY_HOME/etc/db/migration/<schema-name>/sql
36 # The nature of the migration directories is dynamic.
37 # A number of environment variables needs to be initialized
38 # prior to running db-migrator.
39 # These vaiables specifiy the connection details for the database
40 # to be upgraded/downgraded.
42 # The repository of upgrade/downgrade scripts is located in the
43 # /home/policy/sql directory.
44 # Two additional scripts have been provided to prepare the directories/files
45 # needed to perform the upgrade/downgrade.
47 # At any given time the following invariant must be preserved in any given
48 # $POLICY_HOME/etc/db/migration/<schema-name>/sql directory
50 # There is only upgrade scripts, or only downgrade scripts, or none.
52 # #####################################################################
54 source ${POLICY_HOME}/etc/profile.d/env.sh
57 METADATA_TABLE=schema_versions
58 MIGRATION_DIR=${POLICY_HOME}/etc/db/migration
61 UPGRADE_SQL_SUFFIX=".upgrade.sql"
62 DOWNGRADE_SQL_SUFFIX=".downgrade.sql"
64 SQL_QUOTES="SET SESSION SQL_MODE=ANSI_QUOTES;"
66 #####################################################
68 #####################################################
72 echo -e "syntax: $(basename "$0") "
73 echo -e "\t -s <schema-name> "
74 echo -e "\t [-b <migration-dir>] "
75 echo -e "\t [-f <from-version>]"
76 echo -e "\t [-t <target-version>]"
77 echo -e "\t -o <operations> "
79 echo -e "\t where <operations>=upgrade|downgrade|auto|version|erase|report"
82 echo -e "Configuration Options:"
83 echo -e "\t -s|--schema|--database: schema to operate on ('ALL' to apply on all)"
84 echo -e "\t -b|--basedir: overrides base DB migration directory"
85 echo -e "\t -f|--from: overrides current release version for operations"
86 echo -e "\t -t|--target: overrides target release to upgrade/downgrade"
89 echo -e "\t upgrade: upgrade operation"
90 echo -e "\t downgrade: performs a downgrade operation"
91 echo -e "\t auto: autonomous operation, determines upgrade or downgrade"
92 echo -e "\t version: returns current version, and in conjunction if '-f' sets the current version"
93 echo -e "\t erase: erase all data related <schema> (use with care)"
94 echo -e "\t report: migration detailed report on an schema"
95 echo -e "\t ok: is the migration status valid"
100 #####################################################
102 #####################################################
104 function current_schema() {
105 if [ "${DEBUG}" = "y" ]; then
106 echo "-- current_schema --"
111 local query="SELECT count(table_name) from information_schema.tables where table_schema='${SCHEMA}'"
113 PSQL="psql -h ${SQL_HOST} -U ${SQL_USER} -d ${SQL_DB}"
114 TABLES_IN_SCHEMA=$(${PSQL} -X -A --tuples-only --quiet --command "${query}")
115 if [ ${TABLES_IN_SCHEMA} -gt 0 ] && [ "${BASE_VERSION}" \> "${CURRENT_RELEASE}" ]; then
116 set_current_release "${BASE_VERSION}"
123 #####################################################
124 # ensure global metadata
125 #####################################################
127 function ensure_metadata() {
128 if [ "${DEBUG}" = "y" ]; then
129 echo "-- ensure_metadata --"
135 PSQL="psql -h ${SQL_HOST} -U ${SQL_USER} -d ${METADATA_DB}"
136 sql="SELECT 'CREATE DATABASE migration' WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = 'migration');"
137 ${PSQL} --command "${sql}"
139 if [ ${rc} -ne 0 ]; then
143 sql="CREATE TABLE IF NOT EXISTS ${METADATA_TABLE} "
144 sql=${sql}"(name VARCHAR(60) NOT NULL, version VARCHAR(20), "
145 sql=${sql}"PRIMARY KEY(name));"
146 ${PSQL} --command "${sql}"
150 #####################################################
151 # ensure metadata on a per schema basis
152 #####################################################
154 function ensure_metadata_schema() {
155 if [ "${DEBUG}" = "y" ]; then
156 echo "-- ensure_metadata_schema --"
162 PSQL="psql -h ${SQL_HOST} -U ${SQL_USER} -d ${METADATA_DB}"
163 sql="CREATE TABLE IF NOT EXISTS ${METADATA_HISTORY} "
164 sql=${sql}"(ID bigserial, script VARCHAR(80) NOT NULL, "
165 sql=${sql}"operation VARCHAR(10), from_version VARCHAR(20), to_version VARCHAR(20), tag VARCHAR(20), "
166 sql=${sql}"success VARCHAR(1), atTime TIMESTAMP DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY(ID));"
167 ${PSQL} --command "${sql}"
169 if [ ${rc} -ne 0 ]; then
173 sql="SELECT 'CREATE DATABASE migration' WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = 'policyadmin');"
174 ${PSQL} --command "${sql}"
178 #####################################################
180 #####################################################
182 function target_release() {
183 if [ "${DEBUG}" = "y" ]; then
184 echo "-- target_release --"
188 local sql sqlName upgradeSqls downgradeSqls
190 TARGET_UPGRADE_RELEASE=${ZERO_VERSION}
191 TARGET_DOWNGRADE_RELEASE=${ZERO_VERSION}
193 upgradeSqls=$(ls -v -r "${UPGRADE_DIR}"/*/upgrade/*.sql 2>/dev/null)
194 for sql in ${upgradeSqls}; do
195 TARGET_UPGRADE_RELEASE="$(basename $(dirname $(dirname $sql)))"
199 # default unless overriden
200 TARGET_DOWNGRADE_RELEASE="${ZERO_VERSION}"
203 #####################################################
205 #####################################################
207 function is_upgrade() {
208 if [ "${DEBUG}" = "y" ]; then
209 echo "-- is_upgrade --"
215 upgradeSqls=$(ls -v "${UPGRADE_DIR}"/*/upgrade/*.sql 2>/dev/null)
216 if [ -z "${upgradeSqls}" ]; then
223 #####################################################
225 #####################################################
227 function is_downgrade() {
228 if [ "${DEBUG}" = "y" ]; then
229 echo "-- is_downgrade --"
235 downgradeSqls=$(ls -v -r "${DOWNGRADE_DIR}"/*/downgrade/*.sql 2>/dev/null)
236 if [ -z "${downgradeSqls}" ]; then
243 #####################################################
244 # set_current_release
245 #####################################################
247 function set_current_release() {
248 if [ "${DEBUG}" = "y" ]; then
249 echo "-- set_current_release --"
253 CURRENT_RELEASE="${1}"
256 sql="INSERT INTO ${METADATA_TABLE} (name, version) "
257 sql=${sql}"VALUES('${SCHEMA}', '${CURRENT_RELEASE}') "
258 sql=${sql}"ON CONFLICT (name) DO UPDATE SET version='${CURRENT_RELEASE}';"
260 PSQL="psql -h ${SQL_HOST} -U ${SQL_USER} -d ${METADATA_DB}"
261 ${PSQL} --command "${sql}"
265 #####################################################
267 #####################################################
269 function current_release() {
270 if [ "${DEBUG}" = "y" ]; then
271 echo "-- current_release --"
276 local query="SELECT version FROM schema_versions WHERE name='${SQL_DB}'"
278 PSQL="psql -h ${SQL_HOST} -U ${SQL_USER} -d ${METADATA_DB}"
279 CURRENT_RELEASE=$(${PSQL} -X -A -t --command "${query}")
281 if [ -z "${CURRENT_RELEASE}" ]; then
282 set_current_release "${ZERO_VERSION}"
289 #####################################################
291 #####################################################
293 function previous_release() {
294 if [ "${DEBUG}" = "y" ]; then
295 echo "-- previous_release --"
298 local current_release="${1}"
299 local current_release_int_val previous_release_int_val previous_release
301 if [ $current_release == $ZERO_VERSION ] || [ $current_release == $BASE_VERSION ]; then
302 PREVIOUS_RELEASE=$ZERO_VERSION
304 current_release_int_val=$(echo $current_release | awk '{$0=int($0)}1')
305 let previous_release_int_val=$current_release_int_val-100
306 if [ $previous_release_int_val -lt 1000 ]; then
307 previous_release="0"$previous_release_int_val
309 previous_release=$previous_release_int_val
311 PREVIOUS_RELEASE=$previous_release
317 #####################################################
318 # last_operation_status
319 #####################################################
321 function last_operation_status() {
322 if [ "${DEBUG}" = "y" ]; then
323 echo "-- last_operation_status --"
329 query="SELECT script, operation, from_version, to_version, success FROM "
330 query=${query}"${METADATA_HISTORY} WHERE id="
331 query=${query}"(SELECT MAX(id) from ${METADATA_HISTORY}) "
332 query=${query}"AND success::integer=0"
334 PSQL="psql -h ${SQL_HOST} -U ${SQL_USER} -d ${METADATA_DB}"
335 row=$(${PSQL} --tuples-only --quiet --command "${query}")
337 if [ "${row}" != "" ]; then
338 LAST_SCRIPT=$(echo $row | awk '{print $1}')
339 LAST_OPERATION=$(echo $row | awk '{print $2}')
340 LAST_FROM_VERSION=$(echo $row | awk '{print $3}')
341 LAST_TO_VERSION=$(echo $row | awk '{print $4}')
342 LAST_STATUS=$(echo $row | awk '{print $5}')
351 #####################################################
352 # execute sql script history
353 #####################################################
355 function track_script() {
356 if [ "${DEBUG}" = "y" ]; then
357 echo "-- track_script $* --"
361 local script="${1}" operation="${2}" success="${3}" from_version="${4}" to_version="${5}" tag="${6}"
362 if [ $operation == "downgrade" ]; then
363 to_version=${TARGET_DOWNGRADE_RELEASE}
366 local sql="INSERT INTO ${METADATA_HISTORY}(script,operation,from_version,to_version,tag,success,atTime) "
367 sql=${sql}"VALUES ('${script}','${operation}','${from_version}','${to_version}','${tag}','${success}',now()) "
369 PSQL="psql -h ${SQL_HOST} -U ${SQL_USER} -d ${METADATA_DB}"
370 ${PSQL} --command "${sql}"
374 #####################################################
376 #####################################################
378 function run_script() {
379 if [ "${DEBUG}" == "y" ]; then
380 echo "-- run_script $* --"
384 local operation="${1}" script="${2}" scriptPath="${3}" fromVersion="${4}" toVersion="${5}" tag="${6}"
387 echo "> ${operation} ${script}"
389 PSQL="psql -h ${SQL_HOST} -U ${SQL_USER} -d ${SQL_DB}"
390 ${PSQL} <"${scriptPath}"
392 if [ ${rc} -ne 0 ]; then
398 track_script "${script}" "${operation}" "${success}" "${fromVersion}" "${toVersion}" "${tag}"
403 #####################################################
405 #####################################################
408 if [ "${DEBUG}" = "y" ]; then
413 local sqlName sqlFile schemaVersion upgradeSqls rc tagDate tag
414 tagDate=$(date +%d%m%y%H%M%S)
416 echo "upgrade: ${CURRENT_RELEASE} -> ${TARGET_UPGRADE_RELEASE}"
418 if [ ${CURRENT_RELEASE} \< ${TARGET_UPGRADE_RELEASE} ]; then
419 upgradeSqls=$(ls -v "${UPGRADE_DIR}"/*/upgrade/*.sql 2>/dev/null)
420 for sqlFile in ${upgradeSqls}; do
421 sqlName=$(basename "${sqlFile}")
422 schemaVersion="$(basename $(dirname $(dirname $sqlFile)))"
423 previous_release $schemaVersion
424 if [ "${schemaVersion}" -gt "${CURRENT_RELEASE}" ] &&
425 [ "${schemaVersion}" -le "${TARGET_UPGRADE_RELEASE}" ]; then
426 tag=${tagDate}"${schemaVersion}u"
427 if [ $RETRY -eq 1 ] &&
428 # Compare the numeric portion of the filename because shell doesn't support string comparison
429 [ $(echo ${sqlName} | awk -F- '{print $1}') -${COMPARE} $(echo ${LAST_SCRIPT} | awk -F- '{print $1}') ]; then
436 if [ ${rc} -eq 0 ]; then
437 run_script "upgrade" "${sqlName}" "${sqlFile}" "${PREVIOUS_RELEASE}" "${schemaVersion}" "${tag}"
439 if [ ${rc} -ne 0 ]; then
440 echo "${SCHEMA}: upgrade aborted at ${schemaVersion} by script ${sqlName}"
441 set_current_release "${schemaVersion}"
448 set_current_release "${TARGET_UPGRADE_RELEASE}"
454 #####################################################
456 #####################################################
458 function downgrade() {
459 if [ "${DEBUG}" = "y" ]; then
460 echo "-- downgrade --"
464 local sqlName sqlFile schemaVersion downgradeSqls rc tagDate tag
465 tagDate=$(date +%d%m%y%H%M%S)
467 echo "downgrade: ${CURRENT_RELEASE} -> ${TARGET_DOWNGRADE_RELEASE}"
469 if [ ${CURRENT_RELEASE} \> ${TARGET_DOWNGRADE_RELEASE} ]; then
470 downgradeSqls=$(ls -v -r "${DOWNGRADE_DIR}"/*/downgrade/*.sql 2>/dev/null)
471 for sqlFile in ${downgradeSqls}; do
472 sqlName=$(basename "${sqlFile}")
473 schemaVersion="$(basename $(dirname $(dirname $sqlFile)))"
474 if [ "${schemaVersion}" -le "${CURRENT_RELEASE}" ] &&
475 [ "${schemaVersion}" -gt "${TARGET_DOWNGRADE_RELEASE}" ]; then
476 tag=${tagDate}"${schemaVersion}d"
477 if [ $RETRY -eq 1 ] &&
478 # Compare the numeric portion of the filename because shell doesn't support string comparison
479 [ $(echo ${sqlName} | awk -F- '{print $1}') -${COMPARE} $(echo ${LAST_SCRIPT} | awk -F- '{print $1}') ]; then
484 if [ ${rc} -eq 0 ]; then
485 run_script "downgrade" "${sqlName}" "${sqlFile}" "${schemaVersion}" "${PREVIOUS_RELEASE}" ${tag}
487 if [ ${rc} -ne 0 ]; then
488 echo "${SCHEMA}: downgrade aborted at ${schemaVersion} by script ${sqlName}"
489 set_current_release "${schemaVersion}"
496 set_current_release "${TARGET_DOWNGRADE_RELEASE}"
502 #####################################################
504 #####################################################
507 if [ "${DEBUG}" = "y" ]; then
512 PSQL="psql -h ${SQL_HOST} -U ${SQL_USER} -d ${METADATA_DB}"
513 local updateMetadata="UPDATE ${METADATA_TABLE} SET version='${ZERO_VERSION}';"
514 ${PSQL} --command "${updateMetadata}"
516 local deleteHistory="DELETE FROM ${METADATA_HISTORY};"
517 ${PSQL} --command "${deleteHistory}"
519 local dropDB="DROP DATABASE IF EXISTS ${SCHEMA_DB}"
520 ${PSQL} --command "${dropDB}"
523 #####################################################
525 #####################################################
528 if [ "${DEBUG}" = "y" ]; then
533 PSQL="psql -h ${SQL_HOST} -U ${SQL_USER} -d ${METADATA_DB}"
534 local versionSql="SELECT * FROM ${METADATA_TABLE} WHERE name='${SCHEMA}';"
535 ${PSQL} --command "${versionSql}"
537 local historySql="SELECT * FROM ${METADATA_HISTORY} ORDER BY id, atTime ASC;"
538 ${PSQL} --command "${historySql}"
544 if [ "${DEBUG}" = "y" ]; then
551 if [ "${CURRENT_RELEASE}" = "${TARGET_UPGRADE_RELEASE}" ]; then
552 echo "${SCHEMA}: OK @ ${CURRENT_RELEASE}"
554 echo "${SCHEMA}: upgrade available: ${CURRENT_RELEASE} -> ${TARGET_UPGRADE_RELEASE}"
558 if [ "${CURRENT_RELEASE}" = "${TARGET_DOWNGRADE_RELEASE}" ]; then
559 echo "${SCHEMA}: OK @ ${CURRENT_RELEASE}"
561 echo "${SCHEMA}: downgrade available: ${CURRENT_RELEASE} -> ${TARGET_DOWNGRADE_RELEASE}"
569 #####################################################
571 #####################################################
573 if [ "${DEBUG}" = "y" ]; then
577 until [ -z "$1" ]; do
579 -s | --schema | --database)
589 INPUT_TARGET_RELEASE=$1
593 INPUT_FROM_RELEASE=$1
616 echo "error: invalid operation provided"
622 if [ -z "${SCHEMA}" ]; then
623 echo "error: a database name must be provided"
628 source "${POLICY_HOME}"/etc/profile.d/env.sh
630 if [ -z "${SQL_HOST}" ] || [ -z "${SQL_USER}" ] || [ -z "${SQL_PASSWORD}" ]; then
631 echo "error: no database has been set up"
635 PSQL="psql -h ${SQL_HOST} -U ${SQL_USER} -d ${SQL_DB}"
636 if ! ${PSQL} --command '\l'; then
637 echo "error: No DB connectivity to ${SQL_HOST} for ${SQL_USER}"
641 if [ "${SCHEMA}" = "ALL" ]; then
645 SCHEMA_S=$(ls -d "${MIGRATION_DIR}"/${SCHEMA}/ 2>/dev/null)
646 if [ -z "${SCHEMA_S}" ]; then
647 echo "error: no databases available"
651 if ! ensure_metadata; then
652 echo "error: migration metadata not accessible"
657 for dbPath in ${SCHEMA_S}; do
658 SCHEMA=$(basename "${dbPath}")
659 SCHEMA_DB="'${SCHEMA}'"
660 UPGRADE_DIR="${MIGRATION_DIR}"/"${SCHEMA}"/postgres
661 DOWNGRADE_DIR=${UPGRADE_DIR}
662 METADATA_HISTORY="${SCHEMA}_schema_changelog"
663 TARGET_RELEASE=${INPUT_TARGET_RELEASE}
665 if is_upgrade && is_downgrade; then
666 echo "${SCHEMA}: failure: invalid configuration: ${UPGRADE_SQL_SUFFIX} and " \
667 "${DOWNGRADE_SQL_SUFFIX} exist under ${DOWNGRADE_DIR}"
672 if [ "${operation}" = "auto" ]; then
680 if ! ensure_metadata_schema; then
681 echo "${SCHEMA}: failure: metadata not accessible for this schema"
685 if [ -z "${TARGET_RELEASE}" ]; then
688 # user asked to override
689 TARGET_UPGRADE_RELEASE="${TARGET_RELEASE}"
690 TARGET_DOWNGRADE_RELEASE="${TARGET_RELEASE}"
693 if [ -z "${CURRENT_RELEASE}" ]; then
694 if ! current_release; then
695 echo "${SCHEMA}: failure: cannot obtain current release"
699 if ! set_current_release "${CURRENT_RELEASE}"; then
700 echo "${SCHEMA}: failure: cannot set current release"
705 PSQL="psql -h ${SQL_HOST} -U ${SQL_USER} -d ${SQL_DB};"
706 # Check if the schema has already been installed
710 last_operation_status
712 if [ $last_status -eq 0 ]; then
713 echo "Partial $LAST_OPERATION detected"
715 if [ "${OPERATION}" == 'downgrade' ] && [ "${LAST_OPERATION}" == 'downgrade' ]; then
716 CURRENT_RELEASE="${LAST_FROM_VERSION}"
719 echo "Downgrade will attempt to resume at $LAST_SCRIPT"
720 elif [ "${OPERATION}" == 'downgrade' ] && [ "${LAST_OPERATION}" == 'upgrade' ]; then
721 CURRENT_RELEASE="${LAST_TO_VERSION}"
722 # greater than or equal to
724 echo "Downgrade will attempt to resume before $LAST_SCRIPT"
725 elif [ "${OPERATION}" == 'upgrade' ] && [ "${LAST_OPERATION}" == 'upgrade' ]; then
726 CURRENT_RELEASE="${LAST_FROM_VERSION}"
729 echo "Upgrade will attempt to resume at $LAST_SCRIPT"
731 CURRENT_RELEASE="${LAST_TO_VERSION}"
734 echo "Upgrade will attempt to resume after $LAST_SCRIPT"
738 if [ -n "${INPUT_FROM_RELEASE}" ]; then
739 if [ "${CURRENT_RELEASE}" \> "${INPUT_FROM_RELEASE}" ] || [ "${CURRENT_RELEASE}" \< "${INPUT_FROM_RELEASE}" ]; then
740 echo "${SCHEMA}: On version ${CURRENT_RELEASE} cannot ${OPERATION} from ${INPUT_FROM_RELEASE}"
743 CURRENT_RELEASE=${INPUT_FROM_RELEASE}
750 echo "${SCHEMA}: OK: upgrade (${CURRENT_RELEASE})"
753 echo "${SCHEMA}: failure: upgrade to release ${TARGET_UPGRADE_RELEASE} (${CURRENT_RELEASE})"
758 echo "${SCHEMA}: OK: downgrade (${CURRENT_RELEASE})"
761 echo "${SCHEMA}: failure: downgrade to release ${TARGET_DOWNGRADE_RELEASE} (${CURRENT_RELEASE})"
765 echo "${SCHEMA}: ${CURRENT_RELEASE}"