3 # ============LICENSE_START=======================================================
5 # ================================================================================
6 # Copyright (C) 2017-2021 AT&T Intellectual Property. All rights reserved.
7 # Modifications Copyright (C) 2021-2022 Nordix Foundation.
8 # ================================================================================
9 # Licensed under the Apache License, Version 2.0 (the "License");
10 # you may not use this file except in compliance with the License.
11 # You may obtain a copy of the License at
13 # http://www.apache.org/licenses/LICENSE-2.0
15 # Unless required by applicable law or agreed to in writing, software
16 # distributed under the License is distributed on an "AS IS" BASIS,
17 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18 # See the License for the specific language governing permissions and
19 # limitations under the License.
20 # ============LICENSE_END=========================================================
22 # #####################################################################
24 # Upgrade/Downgrade SQL File Name Format:
26 # <VERSION>-<pdp|feature-name>[-description](.upgrade|.downgrade).sql
28 # This tool operates on a migration working directory at
30 # $POLICY_HOME/etc/db/migration
32 # Upgrade/Downgrade files for each schema (aka database) names to be maintained
33 # by this tool are located at
35 # $POLICY_HOME/etc/db/migration/<schema-name>/sql
37 # The nature of the migration directories is dynamic.
38 # A number of environment variables needs to be initialized
39 # prior to running db-migrator.
40 # These vaiables specifiy the connection details for the database
41 # to be upgraded/downgraded.
43 # The repository of upgrade/downgrade scripts is located in the
44 # /home/policy/sql directory.
45 # Two additional scripts have been provided to prepare the directories/files
46 # needed to perform the upgrade/downgrade.
48 # At any given time the following invariant must be preserved in any given
49 # $POLICY_HOME/etc/db/migration/<schema-name>/sql directory
51 # There is only upgrade scripts, or only downgrade scripts, or none.
53 # #####################################################################
55 source ${POLICY_HOME}/etc/profile.d/env.sh
58 METADATA_TABLE=${METADATA_DB}.schema_versions
59 MIGRATION_DIR=${POLICY_HOME}/etc/db/migration
62 UPGRADE_SQL_SUFFIX=".upgrade.sql"
63 DOWNGRADE_SQL_SUFFIX=".downgrade.sql"
65 SQL_QUOTES="SET SESSION SQL_MODE=ANSI_QUOTES;"
67 #####################################################
69 #####################################################
73 echo -e "syntax: $(basename "$0") "
74 echo -e "\t -s <schema-name> "
75 echo -e "\t [-b <migration-dir>] "
76 echo -e "\t [-f <from-version>]"
77 echo -e "\t [-t <target-version>]"
78 echo -e "\t -o <operations> "
80 echo -e "\t where <operations>=upgrade|downgrade|auto|version|erase|report"
83 echo -e "Configuration Options:"
84 echo -e "\t -s|--schema|--database: schema to operate on ('ALL' to apply on all)"
85 echo -e "\t -b|--basedir: overrides base DB migration directory"
86 echo -e "\t -f|--from: overrides current release version for operations"
87 echo -e "\t -t|--target: overrides target release to upgrade/downgrade"
90 echo -e "\t upgrade: upgrade operation"
91 echo -e "\t downgrade: performs a downgrade operation"
92 echo -e "\t auto: autonomous operation, determines upgrade or downgrade"
93 echo -e "\t version: returns current version, and in conjunction if '-f' sets the current version"
94 echo -e "\t erase: erase all data related <schema> (use with care)"
95 echo -e "\t report: migration detailed report on an schema"
96 echo -e "\t ok: is the migration status valid"
101 #####################################################
103 #####################################################
105 function current_schema() {
106 if [ "${DEBUG}" = "y" ]; then
107 echo "-- current_schema --"
112 local query="SELECT count(table_name) from information_schema.tables where table_schema='${SCHEMA}'"
114 TABLES_IN_SCHEMA=$(${MYSQL} --skip-column-names --silent --execute "${query}")
115 if [ ${TABLES_IN_SCHEMA} -gt 0 ] && [ "${BASE_VERSION}" \> "${CURRENT_RELEASE}" ]; then
116 set_current_release "${BASE_VERSION}"
123 #####################################################
124 # ensure global metadata
125 #####################################################
127 function ensure_metadata() {
128 if [ "${DEBUG}" = "y" ]; then
129 echo "-- ensure_metadata --"
135 sql="CREATE DATABASE IF NOT EXISTS ${METADATA_DB};"
136 ${MYSQL} --execute "${sql}"
138 if [ ${rc} -ne 0 ]; then
142 sql="CREATE TABLE IF NOT EXISTS ${METADATA_TABLE} "
143 sql=${sql}"(name VARCHAR(60) NOT NULL, version VARCHAR(20), "
144 sql=${sql}"PRIMARY KEY(name));"
145 ${MYSQL} --execute "${sql}"
149 #####################################################
150 # ensure metadata on a per schema basis
151 #####################################################
153 function ensure_metadata_schema() {
154 if [ "${DEBUG}" = "y" ]; then
155 echo "-- ensure_metadata_schema --"
161 sql="CREATE TABLE IF NOT EXISTS ${METADATA_HISTORY} "
162 sql=${sql}"(ID bigint(20) unsigned NOT NULL AUTO_INCREMENT, script VARCHAR(80) NOT NULL, "
163 sql=${sql}"operation VARCHAR(10), from_version VARCHAR(20), to_version VARCHAR(20), tag VARCHAR(20), "
164 sql=${sql}"success VARCHAR(1), atTime TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, "
165 sql=${sql}"PRIMARY KEY(ID));"
166 ${MYSQL} --execute "${sql}"
168 if [ ${rc} -ne 0 ]; then
172 sql="CREATE DATABASE IF NOT EXISTS ${SCHEMA_DB};"
173 ${MYSQL} --execute "${sql}"
177 #####################################################
179 #####################################################
181 function target_release() {
182 if [ "${DEBUG}" = "y" ]; then
183 echo "-- target_release --"
187 local sql sqlName upgradeSqls downgradeSqls
189 TARGET_UPGRADE_RELEASE=${ZERO_VERSION}
190 TARGET_DOWNGRADE_RELEASE=${ZERO_VERSION}
192 upgradeSqls=$(ls -v -r "${UPGRADE_DIR}"/*/upgrade/*.sql 2>/dev/null)
193 for sql in ${upgradeSqls}; do
194 TARGET_UPGRADE_RELEASE="$(basename $(dirname $(dirname $sql)))"
198 # default unless overriden
199 TARGET_DOWNGRADE_RELEASE="${ZERO_VERSION}"
202 #####################################################
204 #####################################################
206 function is_upgrade() {
207 if [ "${DEBUG}" = "y" ]; then
208 echo "-- is_upgrade --"
214 upgradeSqls=$(ls -v "${UPGRADE_DIR}"/*/upgrade/*.sql 2>/dev/null)
215 if [ -z "${upgradeSqls}" ]; then
222 #####################################################
224 #####################################################
226 function is_downgrade() {
227 if [ "${DEBUG}" = "y" ]; then
228 echo "-- is_downgrade --"
234 downgradeSqls=$(ls -v -r "${DOWNGRADE_DIR}"/*/downgrade/*.sql 2>/dev/null)
235 if [ -z "${downgradeSqls}" ]; then
242 #####################################################
243 # set_current_release
244 #####################################################
246 function set_current_release() {
247 if [ "${DEBUG}" = "y" ]; then
248 echo "-- set_current_release --"
252 CURRENT_RELEASE="${1}"
255 sql="INSERT INTO ${METADATA_TABLE} (name, version) "
256 sql=${sql}"VALUES('${SCHEMA}', '${CURRENT_RELEASE}') "
257 sql=${sql}"ON DUPLICATE KEY UPDATE version='${CURRENT_RELEASE}';"
259 ${MYSQL} --execute "${sql}"
263 #####################################################
265 #####################################################
267 function current_release() {
268 if [ "${DEBUG}" = "y" ]; then
269 echo "-- current_release --"
274 local query="SELECT version FROM ${METADATA_TABLE} WHERE name='${SCHEMA}'"
276 CURRENT_RELEASE=$(${MYSQL} --skip-column-names --silent --execute "${query}")
277 if [ -z "${CURRENT_RELEASE}" ]; then
278 set_current_release "${ZERO_VERSION}"
285 #####################################################
287 #####################################################
289 function previous_release() {
290 if [ "${DEBUG}" = "y" ]; then
291 echo "-- previous_release --"
294 local current_release="${1}"
295 local current_release_int_val previous_release_int_val previous_release
297 if [ $current_release == $ZERO_VERSION ] || [ $current_release == $BASE_VERSION ]; then
298 PREVIOUS_RELEASE=$ZERO_VERSION
300 current_release_int_val=$(echo $current_release | awk '{$0=int($0)}1')
301 let previous_release_int_val=$current_release_int_val-100
302 if [ $previous_release_int_val -lt 1000 ]; then
303 previous_release="0"$previous_release_int_val
305 previous_release=$previous_release_int_val
307 PREVIOUS_RELEASE=$previous_release
313 #####################################################
314 # last_operation_status
315 #####################################################
317 function last_operation_status() {
318 if [ "${DEBUG}" = "y" ]; then
319 echo "-- last_operation_status --"
325 query="SELECT script, operation, from_version, to_version, success FROM "
326 query=${query}"${METADATA_HISTORY} WHERE id="
327 query=${query}"(SELECT MAX(id) from ${METADATA_HISTORY}) "
328 query=${query}"AND success=0"
330 row=$(${MYSQL} --skip-column-names --silent --execute "${query}")
332 if [ "${row}" != "" ]; then
333 LAST_SCRIPT=$(echo $row | awk '{print $1}')
334 LAST_OPERATION=$(echo $row | awk '{print $2}')
335 LAST_FROM_VERSION=$(echo $row | awk '{print $3}')
336 LAST_TO_VERSION=$(echo $row | awk '{print $4}')
337 LAST_STATUS=$(echo $row | awk '{print $5}')
346 #####################################################
347 # execute sql script history
348 #####################################################
350 function track_script() {
351 if [ "${DEBUG}" = "y" ]; then
352 echo "-- track_script $* --"
356 local script="${1}" operation="${2}" success="${3}" from_version="${4}" to_version="${5}" tag="${6}"
357 if [ $operation == "downgrade" ]; then
358 to_version=${TARGET_DOWNGRADE_RELEASE}
361 local sql="INSERT INTO ${METADATA_HISTORY}(script,operation,from_version,to_version,tag,success,atTime) "
362 sql=${sql}"VALUES ('${script}','${operation}','${from_version}','${to_version}','${tag}','${success}',now()) "
364 ${MYSQL} --execute "${sql}"
368 #####################################################
370 #####################################################
372 function run_script() {
373 if [ "${DEBUG}" == "y" ]; then
374 echo "-- run_script $* --"
378 local operation="${1}" script="${2}" scriptPath="${3}" fromVersion="${4}" toVersion="${5}" tag="${6}"
381 echo "> ${operation} ${script}"
383 ${MYSQL} ${SCHEMA} --verbose <"${scriptPath}"
385 if [ ${rc} -ne 0 ]; then
391 track_script "${script}" "${operation}" "${success}" "${fromVersion}" "${toVersion}" "${tag}"
396 #####################################################
398 #####################################################
401 if [ "${DEBUG}" = "y" ]; then
406 local sqlName sqlFile schemaVersion upgradeSqls rc tagDate tag
407 tagDate=$(date +%d%m%y%H%M%S)
409 echo "upgrade: ${CURRENT_RELEASE} -> ${TARGET_UPGRADE_RELEASE}"
411 if [ ${CURRENT_RELEASE} \< ${TARGET_UPGRADE_RELEASE} ]; then
412 upgradeSqls=$(ls -v "${UPGRADE_DIR}"/*/upgrade/*.sql 2>/dev/null)
413 for sqlFile in ${upgradeSqls}; do
414 sqlName=$(basename "${sqlFile}")
415 schemaVersion="$(basename $(dirname $(dirname $sqlFile)))"
416 previous_release $schemaVersion
417 if [ "${schemaVersion}" -gt "${CURRENT_RELEASE}" ] &&
418 [ "${schemaVersion}" -le "${TARGET_UPGRADE_RELEASE}" ]; then
419 tag=${tagDate}"${schemaVersion}u"
420 if [ $RETRY -eq 1 ] &&
421 # Compare the numeric portion of the filename because shell doesn't support string comparison
422 [ $(echo ${sqlName} | awk -F- '{print $1}') -${COMPARE} $(echo ${LAST_SCRIPT} | awk -F- '{print $1}') ]; then
427 if [ ${rc} -eq 0 ]; then
428 run_script "upgrade" "${sqlName}" "${sqlFile}" "${PREVIOUS_RELEASE}" "${schemaVersion}" "${tag}"
430 if [ ${rc} -ne 0 ]; then
431 echo "${SCHEMA}: upgrade aborted at ${schemaVersion} by script ${sqlName}"
432 set_current_release "${schemaVersion}"
439 set_current_release "${TARGET_UPGRADE_RELEASE}"
445 #####################################################
447 #####################################################
449 function downgrade() {
450 if [ "${DEBUG}" = "y" ]; then
451 echo "-- downgrade --"
455 local sqlName sqlFile schemaVersion downgradeSqls rc tagDate tag
456 tagDate=$(date +%d%m%y%H%M%S)
458 echo "downgrade: ${CURRENT_RELEASE} -> ${TARGET_DOWNGRADE_RELEASE}"
460 if [ ${CURRENT_RELEASE} \> ${TARGET_DOWNGRADE_RELEASE} ]; then
461 downgradeSqls=$(ls -v -r "${DOWNGRADE_DIR}"/*/downgrade/*.sql 2>/dev/null)
462 for sqlFile in ${downgradeSqls}; do
463 sqlName=$(basename "${sqlFile}")
464 schemaVersion="$(basename $(dirname $(dirname $sqlFile)))"
465 if [ "${schemaVersion}" -le "${CURRENT_RELEASE}" ] &&
466 [ "${schemaVersion}" -gt "${TARGET_DOWNGRADE_RELEASE}" ]; then
467 tag=${tagDate}"${schemaVersion}d"
468 if [ $RETRY -eq 1 ] &&
469 # Compare the numeric portion of the filename because shell doesn't support string comparison
470 [ $(echo ${sqlName} | awk -F- '{print $1}') -${COMPARE} $(echo ${LAST_SCRIPT} | awk -F- '{print $1}') ]; then
475 if [ ${rc} -eq 0 ]; then
476 run_script "downgrade" "${sqlName}" "${sqlFile}" "${schemaVersion}" "${PREVIOUS_RELEASE}" ${tag}
478 if [ ${rc} -ne 0 ]; then
479 echo "${SCHEMA}: downgrade aborted at ${schemaVersion} by script ${sqlName}"
480 set_current_release "${schemaVersion}"
487 set_current_release "${TARGET_DOWNGRADE_RELEASE}"
493 #####################################################
495 #####################################################
498 if [ "${DEBUG}" = "y" ]; then
503 local updateMetadata="UPDATE ${METADATA_TABLE} SET version='${ZERO_VERSION}';"
504 ${MYSQL} --execute "${updateMetadata}"
506 local deleteHistory="DELETE FROM ${METADATA_HISTORY};"
507 ${MYSQL} --execute "${deleteHistory}"
509 local dropDB="DROP DATABASE IF EXISTS ${SCHEMA_DB}"
510 ${MYSQL} --execute "${dropDB}"
513 #####################################################
515 #####################################################
518 if [ "${DEBUG}" = "y" ]; then
523 local versionSql="SELECT * FROM ${METADATA_TABLE} WHERE name='${SCHEMA}';"
524 ${MYSQL} --execute "${versionSql}"
526 local historySql="SELECT * FROM ${METADATA_HISTORY} ORDER BY id, atTime ASC;"
527 ${MYSQL} --execute "${historySql}"
533 if [ "${DEBUG}" = "y" ]; then
540 if [ "${CURRENT_RELEASE}" = "${TARGET_UPGRADE_RELEASE}" ]; then
541 echo "${SCHEMA}: OK @ ${CURRENT_RELEASE}"
543 echo "${SCHEMA}: upgrade available: ${CURRENT_RELEASE} -> ${TARGET_UPGRADE_RELEASE}"
547 if [ "${CURRENT_RELEASE}" = "${TARGET_DOWNGRADE_RELEASE}" ]; then
548 echo "${SCHEMA}: OK @ ${CURRENT_RELEASE}"
550 echo "${SCHEMA}: downgrade available: ${CURRENT_RELEASE} -> ${TARGET_DOWNGRADE_RELEASE}"
558 #####################################################
560 #####################################################
562 if [ "${DEBUG}" = "y" ]; then
566 until [ -z "$1" ]; do
568 -s | --schema | --database)
578 INPUT_TARGET_RELEASE=$1
582 INPUT_FROM_RELEASE=$1
605 echo "error: invalid operation provided"
611 if [ -z "${SCHEMA}" ]; then
612 echo "error: a database name must be provided"
617 source "${POLICY_HOME}"/etc/profile.d/env.sh
619 if [ -z "${SQL_HOST}" ] || [ -z "${SQL_USER}" ] || [ -z "${SQL_PASSWORD}" ]; then
620 echo "error: no database has been set up"
624 if [ -z "$MYSQL_CMD" ]; then
628 MYSQL="${MYSQL_CMD} -u${SQL_USER} -p${SQL_PASSWORD} -h ${SQL_HOST}"
629 if ! ${MYSQL} -h"${SQL_HOST}" --execute "show databases;" >/dev/null 2>&1; then
630 echo "error: No DB connectivity to ${SQL_HOST} for ${SQL_USER}"
634 if [ "${SCHEMA}" = "ALL" ]; then
638 SCHEMA_S=$(ls -d "${MIGRATION_DIR}"/${SCHEMA}/ 2>/dev/null)
639 if [ -z "${SCHEMA_S}" ]; then
640 echo "error: no databases available"
644 if ! ensure_metadata; then
645 echo "error: migration metadata not accessible"
650 for dbPath in ${SCHEMA_S}; do
651 SCHEMA=$(basename "${dbPath}")
652 SCHEMA_DB="\`${SCHEMA}\`"
653 UPGRADE_DIR="${MIGRATION_DIR}"/"${SCHEMA}"/sql
654 DOWNGRADE_DIR=${UPGRADE_DIR}
655 METADATA_HISTORY="${METADATA_DB}.\`${SCHEMA}_schema_changelog\`"
656 TARGET_RELEASE=${INPUT_TARGET_RELEASE}
658 if is_upgrade && is_downgrade; then
659 echo "${SCHEMA}: failure: invalid configuration: ${UPGRADE_SQL_SUFFIX} and " \
660 "${DOWNGRADE_SQL_SUFFIX} exist under ${DOWNGRADE_DIR}"
665 if [ "${operation}" = "auto" ]; then
673 if ! ensure_metadata_schema; then
674 echo "${SCHEMA}: failure: metadata not accessible for this schema"
678 if [ -z "${TARGET_RELEASE}" ]; then
681 # user asked to override
682 TARGET_UPGRADE_RELEASE="${TARGET_RELEASE}"
683 TARGET_DOWNGRADE_RELEASE="${TARGET_RELEASE}"
686 if [ -z "${CURRENT_RELEASE}" ]; then
687 if ! current_release; then
688 echo "${SCHEMA}: failure: cannot obtain current release"
692 if ! set_current_release "${CURRENT_RELEASE}"; then
693 echo "${SCHEMA}: failure: cannot set current release"
698 # Check if the schema has already been installed
702 last_operation_status
704 if [ $last_status -eq 0 ]; then
705 echo "Partial $LAST_OPERATION detected"
707 if [ "${OPERATION}" == 'downgrade' ] && [ "${LAST_OPERATION}" == 'downgrade' ]; then
708 CURRENT_RELEASE="${LAST_FROM_VERSION}"
711 echo "Downgrade will attempt to resume at $LAST_SCRIPT"
712 elif [ "${OPERATION}" == 'downgrade' ] && [ "${LAST_OPERATION}" == 'upgrade' ]; then
713 CURRENT_RELEASE="${LAST_TO_VERSION}"
714 # greater than or equal to
716 echo "Downgrade will attempt to resume before $LAST_SCRIPT"
717 elif [ "${OPERATION}" == 'upgrade' ] && [ "${LAST_OPERATION}" == 'upgrade' ]; then
718 CURRENT_RELEASE="${LAST_FROM_VERSION}"
721 echo "Upgrade will attempt to resume at $LAST_SCRIPT"
723 CURRENT_RELEASE="${LAST_TO_VERSION}"
726 echo "Upgrade will attempt to resume after $LAST_SCRIPT"
730 if [ -n "${INPUT_FROM_RELEASE}" ]; then
731 if [ "${CURRENT_RELEASE}" \> "${INPUT_FROM_RELEASE}" ] || [ "${CURRENT_RELEASE}" \< "${INPUT_FROM_RELEASE}" ]; then
732 echo "${SCHEMA}: On version ${CURRENT_RELEASE} cannot ${OPERATION} from ${INPUT_FROM_RELEASE}"
735 CURRENT_RELEASE=${INPUT_FROM_RELEASE}
742 echo "${SCHEMA}: OK: upgrade (${CURRENT_RELEASE})"
745 echo "${SCHEMA}: failure: upgrade to release ${TARGET_UPGRADE_RELEASE} (${CURRENT_RELEASE})"
750 echo "${SCHEMA}: OK: downgrade (${CURRENT_RELEASE})"
753 echo "${SCHEMA}: failure: downgrade to release ${TARGET_DOWNGRADE_RELEASE} (${CURRENT_RELEASE})"
757 echo "${SCHEMA}: ${CURRENT_RELEASE}"