Support of DB-enabled features 77/7477/1
authorJorge Hernandez <jh1730@att.com>
Mon, 14 Aug 2017 18:34:53 +0000 (13:34 -0500)
committerJorge Hernandez <jh1730@att.com>
Mon, 14 Aug 2017 18:43:07 +0000 (13:43 -0500)
Isolation and data migration support overall and on a per
feature basis.

policy@drools:/opt/app/policy$ policy.sh status

[drools-pdp-controllers]
 L []: Policy Management (no pidfile) is NOT running
1 cron jobs installed.

[features]
name                 version         status
----                 -------         ------
eelf                 1.1.0           disabled
healthcheck          1.1.0           disabled
session-persistence  1.1.0           enabled

[migration]
session-persistence: upgrade out-of-sync: 0 -> 201702

policy@drools:/opt/app/policy$ features enable session-persistence
session-persistence: upgrade out-of-sync: 0 -> 201702

name                 version         status
----                 -------         ------
eelf                 1.1.0           disabled
healthcheck          1.1.0           disabled
session-persistence  1.1.0           enabled

policy@drools:/opt/app/policy$ db-migrator -s session-persistence -o upgrade
upgrade: 0 -> 201702

> upgrade 201701-blah.upgrade.sql
--------------
create table blah (a varchar(15), b varchar(20))
--------------

> upgrade 201702-blah2.upgrade.sql
--------------
create table blah2 (a varchar(15), b varchar(20))
--------------

session-persistence: OK: upgrade (201702)

policy@drools:/opt/app/policy$ db-migrator -s ALL -o report
+---------------------+---------+
| name                | version |
+---------------------+---------+
| session-persistence | 201702  |
+---------------------+---------+
+--------------------------+-----------+---------+---------------------+
| script                   | operation | success | atTime              |
+--------------------------+-----------+---------+---------------------+
| 201701-blah.upgrade.sql  | upgrade   | 1       | 2017-08-14 16:01:32 |
| 201702-blah2.upgrade.sql | upgrade   | 1       | 2017-08-14 16:01:32 |
+--------------------------+-----------+---------+---------------------+

policy@drools:/opt/app/policy$ features disable session-persistence

name                 version         status
----                 -------         ------
eelf                 1.1.0           disabled
healthcheck          1.1.0           disabled
session-persistence  1.1.0           disabled

policy@drools:/opt/app/policy$ db-migrator -s ALL -o downgrade
downgrade: 201702 -> 0

> downgrade 201702-blah2.downgrade.sql
--------------
drop table if exists blah2
--------------

> downgrade 201701-blah.downgrade.sql
--------------
drop table if exists blah
--------------

session-persistence: OK: downgrade (0)

policy@drools:/opt/app/policy$ db-migrator -s ALL -o report
+---------------------+---------+
| name                | version |
+---------------------+---------+
| session-persistence | 0       |
+---------------------+---------+
+----------------------------+-----------+---------+---------------------+
| script                     | operation | success | atTime              |
+----------------------------+-----------+---------+---------------------+
| 201701-blah.upgrade.sql    | upgrade   | 1       | 2017-08-14 16:01:32 |
| 201702-blah2.upgrade.sql   | upgrade   | 1       | 2017-08-14 16:01:32 |
| 201701-blah.downgrade.sql  | downgrade | 1       | 2017-08-14 16:13:49 |
| 201702-blah2.downgrade.sql | downgrade | 1       | 2017-08-14 16:13:49 |
+----------------------------+-----------+---------+---------------------+
session-persistence: OK @ 0

Change-Id: Ie185f5d7a8463cb349ac452d8c2b4b05928b3e56
Issue-ID: POLICY-96
Signed-off-by: Jorge Hernandez <jh1730@att.com>
packages/base/src/files/bin/policy.sh
policy-management/src/main/server-gen/bin/db-migrator [new file with mode: 0644]
policy-management/src/main/server-gen/bin/features

index 17ad281..d5fabde 100644 (file)
@@ -66,10 +66,21 @@ function policy_status() {
                set -x
        fi
        
+       echo
        policy_op "status"
 
        NUM_CRONS=$(crontab -l 2> /dev/null | wc -l)
        echo "  ${NUM_CRONS} cron jobs installed."
+
+       echo
+       echo "[features]"
+       features status
+       
+       local databases=$(ls -d "${POLICY_HOME}"/etc/db/migration/*/ 2> /dev/null)
+       if [[ -n ${databases} ]]; then
+               echo "[migration]"
+               db-migrator -s ALL -o ok
+       fi
        
 }
 
diff --git a/policy-management/src/main/server-gen/bin/db-migrator b/policy-management/src/main/server-gen/bin/db-migrator
new file mode 100644 (file)
index 0000000..328ad0b
--- /dev/null
@@ -0,0 +1,625 @@
+#!/bin/bash 
+#
+# ============LICENSE_START=======================================================
+# ONAP
+# ================================================================================
+# Copyright (C) 2017 AT&T Intellectual Property. All rights reserved.
+# ================================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============LICENSE_END=========================================================
+
+# #####################################################################
+#
+# Upgrade/Downgrade SQL File Name Format:  
+#
+#      <VERSION>-<pdp|feature-name>[-description](.upgrade|.downgrade).sql
+#
+# This tool operates on a migration working directory at
+# 
+#      $POLICY_HOME/etc/db/migration
+#
+# Upgrade/Downgrade files for each schema (aka database) names to be maintained 
+# by this tool are located at
+#
+#      $POLICY_HOME/etc/db/migration/<schema-name>/sql
+#
+# The nature of the migration directories is dynamic.
+# Other tooling aware of when migrations are needed are in charge to populate
+# the migrations directory accordingly.   
+#
+# One of these tools is the 'features' when a feature with DB requirements
+# is 'enabled', the upgrade scripts will be made present in the migration directory.
+# When a features is 'disabled' downgrade scripts will be made available in the
+# migration directory.
+#
+# The 'policy' tool via its operations 'status' or 'start' will signal the 
+# need to perform upgrade or downgrade for a given schema.
+#
+# At any given time the following invariant must be preserved in any given
+# $POLICY_HOME/etc/db/migration/<schema-name>/sql directory
+#
+#      There is only upgrade scripts, or only downgrade scripts, or none.
+#
+# #####################################################################
+
+METADATA_DB=migration
+METADATA_TABLE="${METADATA_DB}".metadata_versions
+MIGRATION_DIR="${POLICY_HOME}"/etc/db/migration
+
+ZERO_VERSION="0"
+UPGRADE_SQL_SUFFIX=".upgrade.sql"
+DOWNGRADE_SQL_SUFFIX=".downgrade.sql"
+
+SQL_QUOTES="SET SESSION SQL_MODE=ANSI_QUOTES;"
+
+#####################################################
+# usage
+#####################################################
+
+function usage() {
+       echo
+       echo -e "syntax: $(basename "$0") "
+       echo -e "\t -s <schema-name> "
+       echo -e "\t [-b <migration-dir>] "
+       echo -e "\t [-f <from-version>]"
+       echo -e "\t [-t <target-version>]"
+       echo -e "\t -o <operations> "
+       echo 
+       echo -e "\t where <operations>=upgrade|downgrade|auto|version|erase|report"
+       echo
+       echo
+       echo -e "Configuration Options:"
+       echo -e "\t -s|--schema|--database:  schema to operate on ('ALL' to apply on all)"              
+       echo -e "\t -b|--basedir: overrides base DB migration directory"
+       echo -e "\t -f|--from: overrides current release version for operations"
+       echo -e "\t -t|--target: overrides target release to upgrade/downgrade"
+       echo
+       echo -e "Operations:"
+       echo -e "\t upgrade: upgrade operation"
+       echo -e "\t downgrade: performs a downgrade operation"
+       echo -e "\t auto: autonomous operation, determines upgrade or downgrade"
+       echo -e "\t version: returns current version, and in conjunction if '-f' sets the current version"
+       echo -e "\t erase: erase all data related <schema> (use with care)"
+       echo -e "\t report: migration detailed report on an schema"
+       echo -e "\t ok: is the migration status valid"
+       echo
+       echo
+}
+
+#####################################################
+# ensure global metadata
+#####################################################
+
+function ensure_metadata
+{
+       if [[ ${DEBUG} == y ]]; then
+               echo "-- ${FUNCNAME[0]} --"
+               set -x
+       fi
+       
+       local sql rc
+       
+    sql="CREATE DATABASE IF NOT EXISTS ${METADATA_DB};"
+       ${MYSQL} --execute "${sql}"
+       rc=$?
+       if [[ ${rc} != 0 ]]; then
+               return ${rc}
+       fi
+    
+    sql="CREATE TABLE IF NOT EXISTS ${METADATA_TABLE} "
+    sql+="(name VARCHAR(60) NOT NULL, version VARCHAR(20), PRIMARY KEY(name));"
+       ${MYSQL} --execute "${sql}"
+       return $?
+}
+
+
+#####################################################
+# ensure metadata on a per schema basis
+#####################################################
+
+function ensure_metadata_schema
+{
+       if [[ ${DEBUG} == y ]]; then
+               echo "-- ${FUNCNAME[0]} --"
+               set -x
+       fi
+       
+       local sql rc
+       
+       sql="CREATE TABLE IF NOT EXISTS ${METADATA_HISTORY} "
+       sql+="(script VARCHAR(80) NOT NULL, operation VARCHAR(10), success VARCHAR(1), "
+       sql+="atTime TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, "
+       sql+="PRIMARY KEY(script));"
+       ${MYSQL} --execute "${sql}"
+       rc=$?
+       if [[ ${rc} != 0 ]]; then
+               return ${rc}
+       fi
+       
+       sql="CREATE DATABASE IF NOT EXISTS ${SCHEMA_DB};"
+       ${MYSQL} --execute "${sql}"
+       return $?
+}
+
+
+#####################################################
+# target_release
+#####################################################
+
+function target_release
+{
+       if [[ ${DEBUG} == y ]]; then
+               echo "-- ${FUNCNAME[0]} --"
+               set -x
+       fi
+       
+       local sql sqlName upgradeSqls downgradeSqls
+       
+       TARGET_UPGRADE_RELEASE=${ZERO_VERSION}
+       TARGET_DOWNGRADE_RELEASE=${ZERO_VERSION}
+       
+       upgradeSqls=$(ls -v -r "${UPGRADE_DIR}"/*"${UPGRADE_SQL_SUFFIX}" 2> /dev/null)
+       for sql in ${upgradeSqls}; do
+               sqlName=$(basename "${sql}")
+               TARGET_UPGRADE_RELEASE="${sqlName%-*}"
+               break
+       done
+       
+       # default unless overriden
+       TARGET_DOWNGRADE_RELEASE="${ZERO_VERSION}"
+}
+
+#####################################################
+# is_upgrade
+#####################################################
+
+function is_upgrade
+{
+       if [[ ${DEBUG} == y ]]; then
+               echo "-- ${FUNCNAME[0]} --"
+               set -x
+       fi
+       
+       local upgradeSqls
+       
+       upgradeSqls=$(ls "${UPGRADE_DIR}"/*"${UPGRADE_SQL_SUFFIX}" 2> /dev/null)
+       if [[ -z ${upgradeSqls} ]]; then
+               return 1
+       else
+               return 0
+       fi
+}
+
+
+#####################################################
+# is_downgrade
+#####################################################
+
+function is_downgrade
+{
+       if [[ ${DEBUG} == y ]]; then
+               echo "-- ${FUNCNAME[0]} --"
+               set -x
+       fi
+       
+       local downgradeSqls
+       
+       downgradeSqls=$(ls "${DOWNGRADE_DIR}"/*"${DOWNGRADE_SQL_SUFFIX}" 2> /dev/null)
+       if [[ -z ${downgradeSqls} ]]; then
+               return 1
+       else
+               return 0
+       fi
+}
+
+
+#####################################################
+# current_release
+#####################################################
+
+function current_release
+{
+       if [[ ${DEBUG} == y ]]; then
+               echo "-- ${FUNCNAME[0]} --"
+               set -x
+       fi
+       
+       local rc
+       local query="SELECT version FROM ${METADATA_TABLE} WHERE name='${SCHEMA}'"
+       
+       CURRENT_RELEASE=$(${MYSQL} --skip-column-names --silent --execute "${query}")   
+       if [[ -z ${CURRENT_RELEASE} ]]; then
+               set_current_release "${ZERO_VERSION}"
+               return $?
+       fi
+       
+       return 0
+}
+
+
+#####################################################
+# set_current_release
+#####################################################
+
+function set_current_release
+{
+       if [[ ${DEBUG} == y ]]; then
+               echo "-- ${FUNCNAME[0]} $* --"
+               set -x
+       fi
+       
+       CURRENT_RELEASE="${1}"
+       
+       local sql="INSERT INTO ${METADATA_TABLE} (name, version) "
+       sql+="VALUES('${SCHEMA}', '${CURRENT_RELEASE}') "
+       sql+="ON DUPLICATE KEY UPDATE version='${CURRENT_RELEASE}';"
+                                       
+       ${MYSQL} --execute "${sql}"
+       return $?
+}
+
+
+#####################################################
+# execute sql script history
+#####################################################
+
+function track_script
+{
+       if [[ ${DEBUG} == y ]]; then
+               echo "-- ${FUNCNAME[0]} $* --"
+               set -x
+       fi
+       
+       local script="${1}" operation="${2}" success="${3}"     
+       local sql="INSERT INTO ${METADATA_HISTORY}(script,operation,success,atTime) "
+       sql+="VALUES ('${script}','${operation}','${success}',now()) "
+       sql+="ON DUPLICATE KEY UPDATE operation=values(operation), success=values(success), atTime=values(atTime);"
+                               
+       ${MYSQL} --execute "${sql}"
+       return $?
+}
+
+
+#####################################################
+# execute sql script
+#####################################################
+
+function run_script
+{
+       if [[ ${DEBUG} == y ]]; then
+               echo "-- ${FUNCNAME[0]} $* --"
+               set -x
+       fi
+       
+       local operation="${1}" script="${2}" scriptPath="${3}"
+       
+       echo
+       echo "> ${operation} ${script}"
+       
+       ${MYSQL} --verbose < "${scriptPath}"
+       local rc=$?
+       if [[ ${rc} != 0 ]]; then
+               success="0"
+       else
+               success="1"
+       fi
+       
+       track_script "${script}" "${operation}" "${success}"
+       
+       return ${rc}
+}
+
+#####################################################
+# upgrade
+#####################################################
+
+function upgrade
+{
+       if [[ ${DEBUG} == y ]]; then
+               echo "-- ${FUNCNAME[0]} --"
+               set -x
+       fi
+       
+       local sqlName sqlFile schemaVersion upgradeSqls rc
+       
+       ${MYSQL} --execute "USE ${SCHEMA_DB}"
+       
+       echo "upgrade: ${CURRENT_RELEASE} -> ${TARGET_UPGRADE_RELEASE}"
+       
+       if [[ ${CURRENT_RELEASE} < ${TARGET_UPGRADE_RELEASE} ]]; then 
+               upgradeSqls=$(ls -v "${UPGRADE_DIR}"/*"${UPGRADE_SQL_SUFFIX}" 2> /dev/null)
+           for sqlFile in ${upgradeSqls}; do
+               sqlName=$(basename "${sqlFile}")
+                       schemaVersion="${sqlName%-*}"
+               if [ "${schemaVersion}" -gt "${CURRENT_RELEASE}" ] && \
+                       [ "${schemaVersion}" -le "${TARGET_UPGRADE_RELEASE}" ]; then
+                       run_script "upgrade" "${sqlName}" "${sqlFile}"
+                       rc=$?
+                               if [[ ${rc} != 0 ]]; then
+                               echo "${SCHEMA}: upgrade aborted at ${schemaVersion} by script ${sqlName}"
+                               set_current_release "${schemaVersion}"
+                               return ${rc}
+                       fi
+               fi
+           done
+       
+       set_current_release "${TARGET_UPGRADE_RELEASE}"
+       fi
+       
+       return 0
+}
+
+#####################################################
+# downgrade
+#####################################################
+
+function downgrade
+{
+       if [[ ${DEBUG} == y ]]; then
+               echo "-- ${FUNCNAME[0]} --"
+               set -x
+       fi
+       
+       local sqlName sqlFile schemaVersion downgradeSqls rc
+       
+       ${MYSQL} --execute "USE ${SCHEMA_DB}"
+       
+       echo "downgrade: ${CURRENT_RELEASE} -> ${TARGET_DOWNGRADE_RELEASE}"
+       
+       if [[ ${CURRENT_RELEASE} > ${TARGET_DOWNGRADE_RELEASE} ]]; then 
+               downgradeSqls=$(ls -v -r "${DOWNGRADE_DIR}"/*"${DOWNGRADE_SQL_SUFFIX}" 2> /dev/null)
+               for sqlFile in ${downgradeSqls}; do
+               sqlName=$(basename "${sqlFile}")
+                       schemaVersion="${sqlName%-*}"
+               if [ "${schemaVersion}" -le "${CURRENT_RELEASE}" ] && \
+                       [ "${schemaVersion}" -gt "${TARGET_DOWNGRADE_RELEASE}" ]; then
+                       run_script "downgrade" "${sqlName}" "${sqlFile}"
+                       rc=$?
+                               if [[ ${rc} != 0 ]]; then
+                               echo "${SCHEMA}: downgrade aborted at ${schemaVersion} by script ${sqlName}"
+                               set_current_release "${schemaVersion}"
+                               return ${rc}
+                       fi
+               fi
+           done
+       
+       set_current_release "${TARGET_DOWNGRADE_RELEASE}"
+       fi
+       
+       return 0
+}
+
+#####################################################
+# erase
+#####################################################
+
+function erase
+{
+       if [[ ${DEBUG} == y ]]; then
+               echo "-- ${FUNCNAME[0]} --"
+               set -x
+       fi
+       
+       local updateMetadata="UPDATE ${METADATA_TABLE} SET version='${ZERO_VERSION}';"
+       ${MYSQL} --execute "${updateMetadata}"
+       
+       local deleteHistory="DELETE FROM ${METADATA_HISTORY};"
+       ${MYSQL} --execute "${deleteHistory}"
+       
+       local dropDB="DROP DATABASE IF EXISTS ${SCHEMA_DB}";
+       ${MYSQL} --execute "${dropDB}"
+}
+
+#####################################################
+# report
+#####################################################
+
+function report
+{
+       if [[ ${DEBUG} == y ]]; then
+               echo "-- ${FUNCNAME[0]} --"
+               set -x
+       fi
+       
+       local versionSql="SELECT * FROM ${METADATA_TABLE} WHERE name='${SCHEMA}';"
+       ${MYSQL} --execute "${versionSql}"
+       
+       local historySql="SELECT * FROM ${METADATA_HISTORY} ORDER BY atTime ASC;"
+       ${MYSQL} --execute "${historySql}"
+       
+       okay
+}
+
+function okay
+{
+       if [[ ${DEBUG} == y ]]; then
+               echo "-- ${FUNCNAME[0]} --"
+               set -x
+       fi
+       
+       local rc=0
+       if is_upgrade; then
+               if [[ ${CURRENT_RELEASE} == "${TARGET_UPGRADE_RELEASE}" ]]; then
+                       echo "${SCHEMA}: OK @ ${CURRENT_RELEASE}"
+               else
+                       echo "${SCHEMA}: upgrade available: ${CURRENT_RELEASE} -> ${TARGET_UPGRADE_RELEASE}"
+                       rc=1
+               fi
+       else
+               if [[ ${CURRENT_RELEASE} == "${TARGET_DOWNGRADE_RELEASE}" ]]; then
+                       echo "${SCHEMA}: OK @ ${CURRENT_RELEASE}"
+               else
+                       echo "${SCHEMA}: downgrade available: ${CURRENT_RELEASE} -> ${TARGET_DOWNGRADE_RELEASE}"
+                       rc=1
+               fi
+       fi
+       
+       return ${rc}
+}
+
+#####################################################
+# MAIN
+#####################################################
+
+if [[ ${DEBUG} == y ]]; then
+       echo "-- $0 $* --"
+       set -x
+fi
+
+until [[ -z "$1" ]]; do
+       case $1 in
+               -s|--schema|--database) shift
+                                                               SCHEMA=$1
+                                                               ;;
+               -b|--basedir)   shift
+                                               MIGRATION_DIR=$1                
+                                               ;;
+               -t|--target)    shift
+                                               TARGET_RELEASE=$1               
+                                               ;;
+               -f|--from)      shift
+                                               CURRENT_RELEASE=$1              
+                                               ;;
+               -o|--operation) shift
+                                               OPERATION=$1            
+                                               ;;
+               *)                              usage
+                                               exit 1
+                                               ;;
+       esac
+       shift
+done
+
+case ${OPERATION} in
+       upgrade)        ;;
+       downgrade)      ;;
+       auto)   ;;
+       version)        ;;
+       erase)          ;;
+       report)         ;;
+       ok)                     ;;
+       *)                      echo "error: invalid operation provided"
+                               usage
+                               exit 1
+                               ;;
+esac
+
+if [[ -z ${SCHEMA} ]]; then
+       echo "error: a database name must be provided"
+       usage
+       exit 2
+fi
+
+source "${POLICY_HOME}"/etc/profile.d/base.conf
+
+if [[ -z ${SQL_HOST} ]] || [[ -z ${SQL_USER} ]] || [[ -z ${SQL_PASSWORD} ]]; then
+       echo "error: no database has been set up" 
+       exit 4
+fi
+
+MYSQL="mysql -u${SQL_USER} -p${SQL_PASSWORD} -h ${SQL_HOST}";
+if ! ${MYSQL} -h"${SQL_HOST}" --execute "show databases;" > /dev/null 2>&1; then
+       echo "error: No DB connectivity to ${SQL_HOST} for ${SQL_USER}"
+       exit 5
+fi
+
+if [[ ${SCHEMA} == ALL ]]; then
+       SCHEMA="*"
+fi
+
+SCHEMA_S=$(ls -d "${MIGRATION_DIR}"/${SCHEMA}/ 2> /dev/null)
+if [[ -z ${SCHEMA_S} ]]; then
+       echo "error: no databases available"
+       exit 0
+fi
+
+if ! ensure_metadata; then
+       echo "error: migration metadata not accessible"
+       exit 7
+fi
+
+rc=0
+for dbPath in ${SCHEMA_S}; do
+       SCHEMA=$(basename "${dbPath}")
+       SCHEMA_DB="\`${SCHEMA}\`"
+       UPGRADE_DIR="${MIGRATION_DIR}"/"${SCHEMA}"/sql
+       DOWNGRADE_DIR=${UPGRADE_DIR}
+       METADATA_HISTORY="${METADATA_DB}.\`${SCHEMA}_history\`"
+       
+       if is_upgrade && is_downgrade; then
+               echo "${SCHEMA}: failure: invalid configuration: ${UPGRADE_SQL_SUFFIX} and "\
+                       "${DOWNGRADE_SQL_SUFFIX} exist under ${DOWNGRADE_DIR}"
+               rc=1
+               continue
+       fi
+       
+       if [[ ${operation} == auto ]]; then
+               if is_upgrade; then
+                       operation=upgrade
+               else
+                       operation=downgrade
+               fi
+       fi
+       
+       if ! ensure_metadata_schema; then
+               echo "${SCHEMA}: failure: metadata not accessible for this schema"
+               continue
+       fi      
+       
+       if [[ -z ${TARGET_RELEASE} ]]; then
+               target_release
+       else
+               # user asked to override
+               TARGET_UPGRADE_RELEASE="${TARGET_RELEASE}"
+               TARGET_DOWNGRADE_RELEASE="${TARGET_RELEASE}"
+       fi
+       
+       if [[ -z ${CURRENT_RELEASE} ]]; then
+               if ! current_release; then
+                       echo "${SCHEMA}: failure: cannot obtain current release"
+                       continue
+               fi
+       else
+               if ! set_current_release "${CURRENT_RELEASE}"; then
+                       echo "${SCHEMA}: failure: cannot set current release"
+                       continue
+               fi
+       fi
+       
+       case ${OPERATION} in
+               upgrade)        if upgrade; then
+                                               echo "${SCHEMA}: OK: upgrade (${CURRENT_RELEASE})"
+                                       else
+                                               rc=1
+                                               echo "${SCHEMA}: failure: upgrade to release ${TARGET_UPGRADE_RELEASE} (${CURRENT_RELEASE})"
+                                       fi                      
+                                       ;;
+               downgrade)  if downgrade; then
+                                               echo "${SCHEMA}: OK: downgrade (${CURRENT_RELEASE})"
+                                       else
+                                               rc=1
+                                               echo "${SCHEMA}: failure: downgrade to release ${TARGET_DOWNGRADE_RELEASE} (${CURRENT_RELEASE})"
+                                       fi                                      
+                                       ;;
+               version)        echo "${SCHEMA}: ${CURRENT_RELEASE}"
+                                       ;;
+               erase)          erase 
+                                       ;;
+               report)         report
+                                       ;;
+               ok)             okay
+                                       ;;
+       esac
+
+done
+exit ${rc}
index 5bbac7a..057abeb 100644 (file)
 # Features Directory Layout:
 #
 # POLICY_HOME/
-#   â””── features/
-#        â””── <feature-name>*/
-#        Â Â Â  â””── [config]/
-#        Â Â Â  â”‚   â””── <config-file>+
-#        Â Â Â  â””── lib/
-#        Â Â Â  â”‚   â””── [dependencies]/
-#        Â Â Â  â”‚   â”‚   â””── <dependent-jar>+
-#        Â Â Â  â”‚   â””── feature/
-#        Â Â Â  â”‚       â””── <feature-jar>
-#        Â Â Â  â””── [db]/
-#        Â Â Â  â”‚   â””── <db-name>/+
-#        Â Â Â  â”‚       â””── sql/
-#        Â Â Â  â”‚           â””── <sql-scripts>*
-#        Â Â Â  â””── [install]
-#        Â Â Â Â     â””── [enable]
-#        Â Â Â Â     â””── [disable]
-#        Â Â Â Â     â””── [other-directories-or-files]
+#   L─ features/
+#        L─ <feature-name>*/
+#        Â Â Â  L─ [config]/
+#        Â Â Â  |   L─ <config-file>+
+#        Â Â Â  L─ lib/
+#        Â Â Â  |   L─ [dependencies]/
+#        Â Â Â  |   |   L─ <dependent-jar>+
+#        Â Â Â  â”‚   L─ feature/
+#        Â Â Â  â”‚       L─ <feature-jar>
+#        Â Â Â  L─ [db]/
+#        Â Â Â  â”‚   L─ <db-name>/+
+#        Â Â Â  â”‚       L─ sql/
+#        Â Â Â  â”‚           L─ <sql-scripts>*
+#        Â Â Â  L─ [install]
+#        Â Â Â Â     L─ [enable]
+#        Â Â Â Â     L─ [disable]
+#        Â Â Â Â     L─ [other-directories-or-files]
 #
 # notes:  [] = optional , * = 0 or more , + = 1 or more
 #   <feature-name> directory without "feature-" prefix.
@@ -61,7 +61,7 @@
 #                  a somewhat independent isolated unit of functionality,the <db-name>
 #                  database ideally isolates all its data.
 #   [db]/<db-name>/sql  directory with all the sql scripts.
-#   [db]/<db-name>/sql/<sql-scripts>  for this featuresql scripts
+#   [db]/<db-name>/sql/<sql-scripts>  for this feature sql scripts
 #                  upgrade scripts should be suffixed with ".upgrade.sql"
 #                  downgrade scripts should be suffixed with ".downgrade.sql"
 #   [install]      custom installation directory where custom enable or disable scripts
 #                  by the feature designer.
 #  
 # Operations:
-#   enable : enables 1) dependencies, 2) configuration, 3) database, and 4) feature 
-#   disable: disables 1) dependencies, 2) configuration, and 3) feature 
-#               * note: no operation on the DB.
+#   install: installs a feature
+#   uninstall: uninstalls a feature
+#   enable : enables 1) dependencies, 2) configuration, 3) database, 4) feature, 5) customization
+#   disable: disables 1) dependencies, 2) configuration, 3) database, 4) feature, 6) customization
 #   status : status of a feature
 #
+# 'enable' operation details:
+#  0. Validates current state before the operation is committed
+#  1. sets the symbolic link to the actual feature jar in pdp-d classpath ($POLICY_HOME/lib)
+#  2. sets symbolic links to feature dependencies in pdp-d classpath ($POLICY_HOME/lib)
+#  3. sets symbolic links to feature configuration in pdp-d configuration directory ($POLICY_HOME/config)
+#  4. sets symbolic links to feature upgrade scripts and removes links to downgrade scripts (if any) 
+#     in the pdp-d migration directory ($POLICY_HOME/etc/db/migration).
+#  5. cd to the feature 'install' directory an executes (if exists) the 'enable' script to allow for specific
+#     customizations for this feature.
+#
+# 'disable' operation details:
+#  0. Validates current state before the operation is committed
+#  1. removes the symbolic link to the actual feature jar in pdp-d classpath ($POLICY_HOME/lib)
+#  2. removes symbolic links to feature dependencies in pdp-d classpath ($POLICY_HOME/lib)
+#  3. removes symbolic links to feature configuration in pdp-d configuration directory ($POLICY_HOME/config)
+#  4. removes symbolic links to feature upgrade scripts and sets links to downgrade scripts (if any) 
+#     in the pdp-d migration directory ($POLICY_HOME/etc/db/migration).
+#  5. cd to the feature 'install' directory an executes (if exists) the 'disable' script to allow for specific
+#     customizations for this feature.
+# 
+# Notes for DB enabled features:
+#      A. Upgrade/Downgrade SQL File Name Format:  
+#                      <VERSION>-<pdp|feature-name>[-description](.upgrade|.downgrade).sql
+#   B. See related tooling: db-migrator and policy
+#
 # Example:
 #
 # POLICY_HOME/
-#   â””── features/
-#        â”œâ”€â”€ eelf/
-#        â”‚   â”œâ”€â”€ config/
-#        â”‚   â”‚   â”œâ”€â”€ logback-eelf.xml
-#        â”‚   â””── lib/
-#        â”‚ Â  â”‚   â””── dependencies/
-#        â”‚   â”‚   â”‚   â””── ONAP-Logging-1.1.0-SNAPSHOT.jar
-#        â”‚   â”‚   â”‚   â””── eelf-core-1.0.0.jar
-#        â”‚   â”‚   â””── feature/
-#        â”‚   â”‚       â””── feature-eelf-1.1.0-SNAPSHOT.jar
-#        â”‚ Â  â””── install/
-#        â”‚       â””── enable
-#        â”‚       â””── disable
-#        â””── healthcheck/
-#            â”œâ”€â”€ config/
-#            â”‚   â””── feature-healthcheck.properties
-#            â””── lib/
-#                â””── feature/
-#                    â””── feature-healthcheck-1.1.0-SNAPSHOT.jar
+#   L─ features/
+#        L── eelf/
+#        â”‚   L── config/
+#        â”‚   â”‚   L── logback-eelf.xml
+#        â”‚   L─ lib/
+#        â”‚ Â  â”‚   L─ dependencies/
+#        â”‚   â”‚   â”‚   L─ ONAP-Logging-1.1.0-SNAPSHOT.jar
+#        â”‚   â”‚   â”‚   L─ eelf-core-1.0.0.jar
+#        â”‚   â”‚   L─ feature/
+#        â”‚   â”‚       L─ feature-eelf-1.1.0-SNAPSHOT.jar
+#        â”‚ Â  L─ install/
+#        â”‚       L─ enable
+#        â”‚       L─ disable
+#        L─ healthcheck/
+#            L── config/
+#            â”‚   L─ feature-healthcheck.properties
+#            L─ lib/
+#                L─ feature/
+#                    L─ feature-healthcheck-1.1.0-SNAPSHOT.jar
 # #############################################################
 
 if [[ ${DEBUG} == y ]]; then
@@ -114,21 +140,21 @@ fi
 
 LIB=${POLICY_HOME}/lib
 CONFIG=${POLICY_HOME}/config
-DB=${POLICY_HOME}/etc/db
+DB=${POLICY_HOME}/etc/db/migration
 FEATURES=${POLICY_HOME}/features
 
 if [[ ! ( -d "${LIB}" && -x "${LIB}" ) ]]; then
-       echo "ERROR: no ${LIB} directory"
+       echo "error: no ${LIB} directory"
        exit 1
 fi
 
 if [[ ! ( -d "${CONFIG}" && -x "${CONFIG}" ) ]]; then
-       echo "ERROR: no ${CONFIG} directory"
+       echo "error: no ${CONFIG} directory"
        exit 2
 fi
 
 # ensure that the directory exists
-mkdir -p "${FEATURES}"
+mkdir -p "${FEATURES}" 2> /dev/null
 
 if [[ ! -d "${DB}" ]]; then
        mkdir -p "${DB}"
@@ -143,6 +169,9 @@ FEATURE_INSTALL="install"
 FEATURE_DB="db"
 FEATURE_SQL="sql"
 
+UPGRADE_SQL_SUFFIX=".upgrade.sql"
+DOWNGRADE_SQL_SUFFIX=".downgrade.sql"
+
 featureJars=$(find "${FEATURES}" -name "feature-*.jar" -type f -exec basename {} \; 2> /dev/null)
 
 # default field lengths
@@ -198,7 +227,7 @@ function usage
 function status
 {
        if [[ ${DEBUG} == y ]]; then
-               echo "-- ${FUNCNAME[0]} $@ --"
+               echo "-- ${FUNCNAME[0]} --"
                set -x
        fi
        
@@ -228,6 +257,7 @@ function status
                fi
                printf "${format}" "${name}" "${version}" "${status}"
        done
+       echo
 }
 
 # ##########################################################
@@ -238,7 +268,7 @@ function status
 function enableDepAnalysis ()
 {
        if [[ ${DEBUG} == y ]]; then
-               echo "-- ${FUNCNAME[0]} $@ --"
+               echo "-- ${FUNCNAME[0]} $* --"
                set -x
        fi
        
@@ -246,7 +276,7 @@ function enableDepAnalysis ()
        local featureDepJars featureDepJarPath depJarName multiVersionJars
        
        if [[ -z ${featureName} ]]; then
-               echo "WARN: no feature name"
+               echo "warning: no feature name"
                return 1
        fi
        
@@ -257,7 +287,7 @@ function enableDepAnalysis ()
                # it could be a base jar
 
                if [[ -f "${LIB}"/"${depJarName}" ]]; then
-                       echo "WARN: dependency ${depJarName} already in use"
+                       echo "warning: dependency ${depJarName} already in use"
                        continue
                fi
                
@@ -271,7 +301,7 @@ function enableDepAnalysis ()
 
                multiVersionJars=$(ls "${LIB}"/"${depJarName%%-[0-9]*.jar}"-*.jar 2> /dev/null)
                if [[ -n "${multiVersionJars}" ]]; then
-                       echo "WARN: other version of library ${depJarName} present: ${multiVersionJars}"
+                       echo "warning: other version of library ${depJarName} present: ${multiVersionJars}"
                        return 2
                fi
        done
@@ -285,7 +315,7 @@ function enableDepAnalysis ()
 function enableConfigAnalysis ()
 {
        if [[ ${DEBUG} == y ]]; then
-               echo "-- ${FUNCNAME[0]} $@ --"
+               echo "-- ${FUNCNAME[0]} $* --"
                set -x
        fi
        
@@ -293,7 +323,7 @@ function enableConfigAnalysis ()
        local featureConfigs configPath configFileName
        
        if [[ -z ${featureName} ]]; then
-               echo "WARN: no feature name"
+               echo "warning: no feature name"
                return 1
        fi
        
@@ -301,7 +331,7 @@ function enableConfigAnalysis ()
        for configPath in ${featureConfigs}; do
                configFileName=$(basename "${configPath}")
                if [[ -e "${LIB}"/"${configFileName}" ]]; then
-                       echo "ERROR: a config file of the same name is already in the base: ${configFileName}"
+                       echo "error: a config file of the same name is already in the base: ${configFileName}"
                        return 2
                fi
        done
@@ -315,7 +345,7 @@ function enableConfigAnalysis ()
 function enableDbAnalysis()
 {
        if [[ ${DEBUG} == y ]]; then
-               echo "-- ${FUNCNAME[0]} $@ --"
+               echo "-- ${FUNCNAME[0]} $* --"
                set -x
        fi
        
@@ -323,37 +353,28 @@ function enableDbAnalysis()
        local featureSqls
        
        if [[ -z ${featureName} ]]; then
-               echo "WARN: no feature name"
+               echo "warning: no feature name"
                return 1
        fi
        
-       featureSqls=$(ls "${FEATURES}"/"${featureName}"/"${FEATURE_DB}"/*/sql/*.upgrade.sql 2> /dev/null)
+       featureSqls=$(ls "${FEATURES}"/"${featureName}"/"${FEATURE_DB}"/*/${FEATURE_SQL}/*${UPGRADE_SQL_SUFFIX} 2> /dev/null)
        if [[ -z ${featureSqls} ]]; then
                return 0
        fi
        
        source "${POLICY_HOME}"/etc/profile.d/base.conf
        if [[ -z ${SQL_HOST} ]] || [[ -z ${SQL_USER} ]] || [[ -z ${SQL_PASSWORD} ]]; then
-               echo "ERROR: not existing configuration to contact the database"
+               echo "error: not existing configuration to contact the database"
                return 2
        fi
        
-       # check reachability
+       # check DB set up
        
-       if which mysqlshow; then
-               if ! mysqlshow -u"${SQL_USER}" -p"${SQL_PASSWORD}" -h"${SQL_HOST}" > /dev/null 2>&1; then
-                       echo "ERROR: No DB connectivity to ${SQL_HOST} for ${SQL_USER}"
-                       return 3
-               else
-                       echo "OK: DB connect to ${SQL_HOST} connectivity for ${SQL_USER}"
-               fi
-       else 
-               if ! ping -c2 -W2 "${SQL_HOST}"; then
-                       echo "ERROR: database ${SQL_HOST} not reachable"
-                       return 4
-               else
-                       echo "OK: ping ${SQL_HOST} connectivity"
-               fi
+       source "${POLICY_HOME}"/etc/profile.d/base.conf
+       
+       if [[ -z ${SQL_HOST} ]] || [[ -z ${SQL_USER} ]] || [[ -z ${SQL_PASSWORD} ]]; then
+               echo "error: database credentials do not exist" 
+               return 3
        fi
        
        return 0
@@ -367,7 +388,7 @@ function enableDbAnalysis()
 function enableFeatureDeps()
 {
        if [[ ${DEBUG} == y ]]; then
-               echo "-- ${FUNCNAME[0]} $@ --"
+               echo "-- ${FUNCNAME[0]} $* --"
                set -x
        fi
        
@@ -375,7 +396,7 @@ function enableFeatureDeps()
        local featureDeps featureDepPath depJarName
        
        if [[ -z ${featureName} ]]; then
-               echo "WARN: no feature name"
+               echo "warning: no feature name"
                return 1
        fi
        
@@ -396,7 +417,7 @@ function enableFeatureDeps()
 function enableFeatureConfig()
 {
        if [[ ${DEBUG} == y ]]; then
-               echo "-- ${FUNCNAME[0]} $@ --"
+               echo "-- ${FUNCNAME[0]} $* --"
                set -x
        fi
        
@@ -404,7 +425,7 @@ function enableFeatureConfig()
        local featureConfigs featureConfigPath
        
        if [[ -z ${featureName} ]]; then
-               echo "WARN: no feature name"
+               echo "warning: no feature name"
                return 1
        fi
        
@@ -422,7 +443,7 @@ function enableFeatureConfig()
 function enableFeatureDbSchema()
 {
        if [[ ${DEBUG} == y ]]; then
-               echo "-- ${FUNCNAME[0]} $@ --"
+               echo "-- ${FUNCNAME[0]} $* --"
                set -x
        fi
        
@@ -431,31 +452,46 @@ function enableFeatureDbSchema()
        local schemaName="$3"
        
        if [[ -z ${featureName} ]]; then
-               echo "WARN: no feature name"
+               echo "warning: no feature name"
                return 1
        fi
        
        if [[ -z ${featureDbPath} ]]; then
-               echo "WARN: no feature DB path"
+               echo "warning: ${featureName} contains no DB path"
                return 2
        fi
        
        if [[ -z ${schemaName} ]]; then
-               echo "WARN: no feature schema name"
+               echo "warning: feature ${featureName} contains no schema name"
                return 3
        fi
        
-       sqlUpgradeScripts=$(ls "${featureDbPath%/}"/sql/*.upgrade.sql 2> /dev/null)
-       if [[ -z "${sqlUpgradeScripts}" ]]; then
-               return 0
-       fi
-       
+       rc=0
+       sqlUpgradeScripts=$(ls "${featureDbPath%/}"/${FEATURE_SQL}/*${UPGRADE_SQL_SUFFIX} 2> /dev/null)
        for sqlUpgradeScript in ${sqlUpgradeScripts}; do
-               if [[ ! -d "${DB}"/"${schemaName}"/sql ]]; then
-                       mkdir -p "${DB}"/"${schemaName}"/sql 2> /dev/null
+               if [[ ! -d "${DB}"/"${schemaName}"/${FEATURE_SQL} ]]; then
+                       mkdir -p "${DB}"/"${schemaName}"/${FEATURE_SQL} 2> /dev/null
                fi              
-               ln -s -f "${sqlUpgradeScript}" "${DB}"/"${schemaName}"/sql/
+               ln -s -f "${sqlUpgradeScript}" "${DB}"/"${schemaName}"/${FEATURE_SQL}/
        done
+       
+       sqlDowngradeScripts=$(ls "${featureDbPath%/}"/${FEATURE_SQL}/*${DOWNGRADE_SQL_SUFFIX} 2> /dev/null)
+       for sqlDowngradeScript in ${sqlDowngradeScripts}; do
+               if [[ -d "${DB}"/"${schemaName}"/${FEATURE_SQL} ]]; then
+                       sqlName=$(basename "${sqlDowngradeScript}")
+                       rm -f "${DB}"/"${schemaName}"/"${FEATURE_SQL}"/"${sqlName}" 2> /dev/null
+               else
+                       echo "warning: feature ${featureName} only contains downgrade scripts"
+                       rc=4
+                       break
+               fi
+       done
+       
+       if [[ -n ${sqlUpgradeScripts} || -n ${sqlDowngradeScripts} ]]; then
+               DEBUG=${DEBUG} db-migrator -s "${schemaName}" -o ok
+       fi
+       
+       return ${rc}
 }
 
 # ##########################################################
@@ -466,21 +502,22 @@ function enableFeatureDbSchema()
 function enableFeatureDb()
 {
        if [[ ${DEBUG} == y ]]; then
-               echo "-- ${FUNCNAME[0]} $@ --"
+               echo "-- ${FUNCNAME[0]} $* --"
                set -x
        fi
        
        local featureName="$1"
-       local featureDbs featureDbPath schemaName
+       local featureDbs featureDbPath schemaName sqls
        
        if [[ -z ${featureName} ]]; then
-               echo "WARN: no feature name"
+               echo "warning: no feature name"
                return 1
        fi
        
        featureDbs=$(ls -d "${FEATURES}"/"${featureName}"/"${FEATURE_DB}"/*/ 2> /dev/null)
        for featureDbPath in ${featureDbs}; do
-               if [[ -z "$(ls "${featureDbPath%/}"/"${FEATURE_SQL}"/*.upgrade.sql 2> /dev/null)" ]]; then
+               sqls=$(ls "${featureDbPath%/}"/"${FEATURE_SQL}"/*.sql 2> /dev/null)
+               if [[ -z ${sqls} ]]; then
                        continue
                fi
                schemaName=$(basename "${featureDbPath%/}")
@@ -488,31 +525,40 @@ function enableFeatureDb()
        done
 }
 
-
 # ##########################################################
-# enableFeatureOp(featureName): 'enable' feature operation
-#   featureName: name of the feature
+# customize(featureName): 
+#      executes customized script for an operation.
+#
+# featureName - feature name
+# operation - operation, ie. 
+#  'enable', 'disable', 'install', or 'uninstall'
 # ##########################################################
-function enableFeatureOp()
+function customOpScript()
 {
        if [[ ${DEBUG} == y ]]; then
-               echo "-- ${FUNCNAME[0]} $@ --"
+               echo "-- ${FUNCNAME[0]} $* --"
                set -x
        fi
        
        local featureName="$1"
+       local operation="$2"
        
        if [[ -z ${featureName} ]]; then
-               echo "WARN: no feature name"
+               echo "warning: no feature name"
+               return 1
+       fi
+       
+       if [[ -z ${operation} ]]; then
+               echo "warning: ${featureName} : a custom operation script must be provided"
                return 1
        fi
        
-       enableScript="${FEATURES}"/"${featureName}"/"${FEATURE_INSTALL}"/enable
-       if [[ -f ${enableScript} ]]; then
+       local customScript="${FEATURES}"/"${featureName}"/"${FEATURE_INSTALL}"/"${operation}"
+       if [[ -f ${customScript} ]]; then
                (
                        cd "${FEATURES}"/"${featureName}"/"${FEATURE_INSTALL}"
-                       chmod u+x enable
-                       ./enable
+                       chmod u+x "${customScript}"
+                       ./"$(basename "${customScript}")"
                )
        fi
 }
@@ -525,7 +571,7 @@ function enableFeatureOp()
 function enableFeature()
 {
        if [[ $DEBUG == y ]]; then
-               echo "-- ${FUNCNAME[0]} $@ --"
+               echo "-- ${FUNCNAME[0]} $* --"
                set -x
        fi
        
@@ -533,12 +579,12 @@ function enableFeature()
        local featureJar="$2"
        
        if [[ -z ${featureName} ]]; then
-               echo "WARN: no feature name"
+               echo "warning: no feature name"
                return 1
        fi
        
        if [[ -z ${featureJar} ]]; then
-               echo "WARN: no feature jar"
+               echo "warning: no feature jar"
                return 2
        fi
        
@@ -572,7 +618,7 @@ function enableFeature()
 
        # run custom enable if any
 
-       enableFeatureOp "${featureName}"
+       customOpScript "${featureName}" "enable"
 }
 
 # ##########################################################
@@ -582,7 +628,7 @@ function enableFeature()
 function disableFeatureDeps()
 {
        if [[ ${DEBUG} == y ]]; then
-               echo "-- ${FUNCNAME[0]} $@ --"
+               echo "-- ${FUNCNAME[0]} $* --"
                set -x
        fi
        
@@ -592,7 +638,7 @@ function disableFeatureDeps()
        local depJarPath depJarName depJarRealPath
        
        if [[ -z ${featureName} ]]; then
-               echo "WARN: no feature name"
+               echo "warning: no feature name"
                return 1
        fi
        
@@ -646,7 +692,7 @@ function disableFeatureDeps()
 function disableFeatureConfig()
 {
        if [[ ${DEBUG} == y ]]; then
-               echo "-- ${FUNCNAME[0]} $@ --"
+               echo "-- ${FUNCNAME[0]} $* --"
                set -x
        fi
        
@@ -654,7 +700,7 @@ function disableFeatureConfig()
        local featureConfigs featureConfigPath
        
        if [[ -z ${featureName} ]]; then
-               echo "WARN: no feature name"
+               echo "warning: no feature name"
                return 1
        fi
        
@@ -666,61 +712,95 @@ function disableFeatureConfig()
 }
 
 # ##########################################################
-# disableFeatureDb(featureName):  
-#                              disables feature db configuration
+# disableFeatureDbSchema(featureName, featureDbPath, schemaName):  
+#                              disables feature db configuration for a schema
 #   featureName: name of the feature
 # ##########################################################
-function disableFeatureDb()
+function disableFeatureDbSchema()
 {
        if [[ ${DEBUG} == y ]]; then
-               echo "-- ${FUNCNAME[0]} $@ --"
+               echo "-- ${FUNCNAME[0]} $* --"
                set -x
        fi
-
-       local featureName="$1"
-       local featureSqls sqlDir schemaDir schemaName
+       
+       local featureName="$1" featureDbPath="$2" schemaName="$3"
+       local upgradeFeatureSqls downgradeFeatureSqls featureSql sqlDir sqlName schemaDir schemaName
        
        if [[ -z ${featureName} ]]; then
-               echo "WARN: no feature name"
+               echo "warning: no feature name"
                return 1
        fi
        
-       featureSqls=$(find "${FEATURES}"/"${featureName}"/"${FEATURE_DB}"/*/sql/*.upgrade.sql -type f -maxdepth 1 2> /dev/null)
-       for featureSql in ${featureSqls}; do
+       if [[ -z ${featureDbPath} ]]; then
+               echo "warning: ${featureName} contains no DB path"
+               return 2
+       fi
+       
+       if [[ -z ${schemaName} ]]; then
+               echo "warning: feature ${featureName} contains no schema name"
+               return 3
+       fi
+       
+       
+       if [[ -z ${featureName} ]]; then
+               echo "warning: no feature name"
+               return 1
+       fi
+       
+       upgradeFeatureSqls=$(find "${FEATURES}"/"${featureName}"/"${FEATURE_DB}"/"${schemaName}"/"${FEATURE_SQL}"/*"${UPGRADE_SQL_SUFFIX}" -type f -maxdepth 1 2> /dev/null)
+       for featureSql in ${upgradeFeatureSqls}; do
                sqlName=$(basename "${featureSql}")
                sqlDir=$(dirname "${featureSql}")
                schemaDir=$(dirname "${sqlDir}")
                schemaName=$(basename "${schemaDir}")
                rm -f "${DB}"/"${schemaName}"/"${FEATURE_SQL}"/"${sqlName}" 2> /dev/null
        done
+       
+       downgradeFeatureSqls=$(find "${FEATURES}"/"${featureName}"/"${FEATURE_DB}"/"${schemaName}"/"${FEATURE_SQL}"/*"${DOWNGRADE_SQL_SUFFIX}" -type f -maxdepth 1 2> /dev/null)
+       for featureSql in ${downgradeFeatureSqls}; do
+               sqlName=$(basename "${featureSql}")
+               sqlDir=$(dirname "${featureSql}")
+               schemaDir=$(dirname "${sqlDir}")
+               schemaName=$(basename "${schemaDir}")
+               if [[ ! -d "${DB}"/"${schemaName}"/${FEATURE_SQL} ]]; then
+                       mkdir -p "${DB}"/"${schemaName}"/${FEATURE_SQL} 2> /dev/null
+               fi              
+               ln -s -f "${featureSql}" "${DB}"/"${schemaName}"/${FEATURE_SQL}/
+       done
+       
+       if [[ -n ${sqlUpgradeScripts} || -n ${sqlDowngradeScripts} ]]; then
+               DEBUG=${DEBUG} db-migrator -s "${schemaName}" -o ok
+       fi
 }
 
 # ##########################################################
-# disableFeatureOp(featureName): 'enable' feature operation
+# disableFeatureDb(featureName):  
+#                              disables feature db configuration
 #   featureName: name of the feature
 # ##########################################################
-function disableFeatureOp()
+function disableFeatureDb()
 {
        if [[ ${DEBUG} == y ]]; then
-               echo "-- ${FUNCNAME[0]} $@ --"
+               echo "-- ${FUNCNAME[0]} $* --"
                set -x
        fi
-       
+
        local featureName="$1"
+       local featureDbPath featureDbs schemaName
        
        if [[ -z ${featureName} ]]; then
-               echo "WARN: no feature name"
+               echo "warning: no feature name"
                return 1
        fi
        
-       disableScript="${FEATURES}"/"${featureName}"/"${FEATURE_INSTALL}"/disable
-       if [[ -f ${disableScript} ]]; then
-               (
-                       cd "${FEATURES}"/"${featureName}"/"${FEATURE_INSTALL}"
-                       chmod u+x disable
-                       ./disable
-               )
-       fi
+       featureDbs=$(ls -d "${FEATURES}"/"${featureName}"/"${FEATURE_DB}"/*/ 2> /dev/null)
+       for featureDbPath in ${featureDbs}; do
+               if [[ -z "$(ls "${featureDbPath%/}"/"${FEATURE_SQL}"/*${UPGRADE_SQL_SUFFIX} 2> /dev/null)" ]]; then
+                       continue
+               fi
+               schemaName=$(basename "${featureDbPath%/}")
+               disableFeatureDbSchema "${featureName}" "${featureDbPath%/}" "${schemaName}"
+       done
 }
 
 # ##########################################################
@@ -730,14 +810,14 @@ function disableFeatureOp()
 function disableFeature()
 {
        if [[ ${DEBUG} == y ]]; then
-               echo "-- ${FUNCNAME[0]} $@ --"
+               echo "-- ${FUNCNAME[0]} $* --"
                set -x
        fi
        
        local featureName="$1"
        
        if [[ -z ${featureName} ]]; then
-               echo "WARN: no feature name"
+               echo "warning: no feature name"
                return
        fi
        
@@ -762,58 +842,48 @@ function disableFeature()
 
        # run custom disable if any
 
-       disableFeatureOp "${featureName}"
+       customOpScript "${featureName}" "disable"
 }
 
 ############################################################
-# configureComponent <config-file> <features-root-directory>
+# configureFeature <config-file> <features-root-directory>
 #
 # This was copied from 'policy-drools/docker-install.sh'
-# in the 'docker' repository.
+# in the 'docker' repository, and modified where needed.
 ############################################################
-function configure_component() {
+function configureFeature() 
+{
        if [[ $DEBUG == y ]]; then
-               echo "-- ${FUNCNAME[0]} $@ --"
+               echo "-- ${FUNCNAME[0]} $* --"
                set -x
        fi
 
-       local CONF_FILE COMPONENT_ROOT_DIR SED_LINE SED_FILES name value
-               
-       CONF_FILE=$1
-       COMPONENT_ROOT_DIR=$2
-       
-       SED_LINE="sed -i"
-       SED_LINE+=" -e 's!\${{POLICY_HOME}}!${POLICY_HOME}!g' "
-       SED_LINE+=" -e 's!\${{POLICY_USER}}!${POLICY_USER}!g' "
-       SED_LINE+=" -e 's!\${{POLICY_GROUP}}!${POLICY_GROUP}!g' "
-       SED_LINE+=" -e 's!\${{KEYSTORE_PASSWD}}!${KEYSTORE_PASSWD}!g' "
-       SED_LINE+=" -e 's!\${{JAVA_HOME}}!${JAVA_HOME}!g' "
+       local envConfig=$1 featureRoot=$2 
+       local sedLine="sed -i" 
+       local sedFiles="" nonBinaryFiles sedFile name value
                
        while read line || [ -n "${line}" ]; do
                if [[ -n ${line} ]] && [[ ${line:0:1} != \# ]]; then
-                       name=$(echo "${line%%=*}")
-                       value=$(echo "${line#*=}")
-                       # escape ampersand so that sed does not replace it with the search string
+                       name="${line%%=*}"
+                       value="${line#*=}"
                        value=$(echo "${value}" | sed -e 's/[\/&]/\\&/g')
                        if [[ -z ${name} ]] || [[ -z ${value} ]]; then
-                               echo "WARNING: ${line} missing name or value"
+                               echo "warning: ${line} missing name or value"
                        fi
-                       SED_LINE+=" -e 's/\${{${name}}}/${value}/g' "
+                       sedLine+=" -e 's/\${{${name}}}/${value}/g' "
                fi
-       done < "$CONF_FILE"
+       done < "${envConfig}"
        
-       SED_FILES=""
-       for sed_file in $(find "${COMPONENT_ROOT_DIR}" -path ${COMPONENT_ROOT_DIR}/backup -prune -o -name '*.xml' -o -name '*.sh' -o -name '*.properties' -o -name '*.json' -o -name '*.conf' -o -name '*.cfg' -o -name '*.template' -o -name '*.conf' -o -name '*.cron'); do
-               if fgrep -l '${{' ${sed_file} > /dev/null 2>&1; then
-                       SED_FILES+="${sed_file} "
+       nonBinaryFiles=$(find "${featureRoot}" -type f -exec grep -Iq . {} \; -print 2> /dev/null)
+       for sedFile in ${nonBinaryFiles}; do
+               if fgrep -l '${{' ${sedFile} > /dev/null 2>&1; then
+                       sedFiles+="${sedFile} "
                fi
        done
 
-       if [[ -z ${SED_FILES} ]]; then
-               echo "WARNING: no xml, sh, properties, or conf files to perform configuration expansion"
-       else
-               SED_LINE+=${SED_FILES}
-               eval "${SED_LINE}"
+       if [[ -n ${sedFiles} ]]; then
+               sedLine+=${sedFiles}
+               eval "${sedLine}"
        fi
 }
 
@@ -826,7 +896,7 @@ function configure_component() {
 function installFeatures
 {
        if [[ $DEBUG == y ]]; then
-               echo "-- ${FUNCNAME[0]} $@ --"
+               echo "-- ${FUNCNAME[0]} $* --"
                set -x
        fi
 
@@ -854,41 +924,43 @@ function installFeatures
                                # file. If there is more than one, choose the one with the
                                # highest version number.
                                name="${feature}"
-                               feature=$(ls -v feature-${name}-[0-9]*.zip 2>/dev/null|tail -1)
+                               feature=$(ls -v feature-"${name}"-[0-9]*.zip 2>/dev/null|tail -1)
                        fi
                        if [[ ! -f "${feature}" ]] ; then
                                # include the file name in the error message, unless we don't
                                # have one -- in this case, use the feature name
-                               echo "ERROR: feature file ${feature:-for ${name}} not found"
+                               echo "error: feature file ${feature:-for ${name}} not found"
                                continue
                        fi
                        if [[ -d "${FEATURES}/${name}" ]] ; then
-                               echo "ERROR: feature ${name} has already been installed"
+                               echo "error: feature ${name} has already been installed"
                                continue
                        fi
 
                        # extract contents of ZIP file in to feature directory
                        mkdir -p "${FEATURES}/${name}" > /dev/null 2>&1
-                       (cd "${FEATURES}/${name}"; jar xf ${SOURCE_DIR}/${feature})
+                       (cd "${FEATURES}/${name}"; jar xf "${SOURCE_DIR}"/"${feature}")
 
                        # if there is a configuration file available,
                        # use it to configure the feature
                        featureConf="${dir:+$dir/}feature-${name}.conf"
                        if [[ -r "${featureConf}" ]]; then
-                               configure_component "${featureConf}" "${FEATURES}"
+                               configureFeature "${featureConf}" "${FEATURES}"/"${name}"
                                cp "${featureConf}" "${POLICY_HOME}"/etc/profile.d
                                echo "feature ${name} has been installed (configuration present)"
                        else
                                echo "feature ${name} has been installed (no configuration present)"
                        fi
+                                       
+                       customOpScript "${featureName}" "install"
                done
                
                # check the current directory and the 'config' directory for a
                # 'base.conf' file -- use the first one that is found
-               for conf in base.conf ${POLICY_HOME}/config/base.conf ; do
+               for conf in base.conf ${POLICY_HOME}/config/base.conf ${POLICY_HOME}/etc/profile.d/base.conf; do
                        if [[ -f "${conf}" ]] ; then
                                echo "applying base configuration '${conf}' to features"
-                               configure_component "${conf}" "${FEATURES}"
+                               configureFeature "${conf}" "${FEATURES}"
                                break
                        fi
                done
@@ -898,19 +970,91 @@ function installFeatures
        fi
 }
 
+# ##########################################################
+# uninstallFeatureDb(featureName):  
+#                              uninstalls the feature db configuration
+#   featureName: name of the feature
+# ##########################################################
+function uninstallFeatureDb()
+{
+       if [[ ${DEBUG} == y ]]; then
+               echo "-- ${FUNCNAME[0]} $* --"
+               set -x
+       fi
+
+       local featureName="$1"
+       local featureSqls sqlDir sqlName schemaDir schemaName schemaNames leftSqls
+       
+       if [[ -z ${featureName} ]]; then
+               echo "warning: no feature name"
+               return 1
+       fi
+       
+       featureSqls=$(find "${FEATURES}"/"${featureName}"/"${FEATURE_DB}"/*/${FEATURE_SQL}/*.sql -type f -maxdepth 1 2> /dev/null)
+       for featureSql in ${featureSqls}; do
+               sqlName=$(basename "${featureSql}")
+               sqlDir=$(dirname "${featureSql}")
+               schemaDir=$(dirname "${sqlDir}")
+               schemaName=$(basename "${schemaDir}")
+               schemaNames+=${schemaName} 
+               rm -f "${DB}"/"${schemaName}"/"${FEATURE_SQL}"/"${sqlName}" 2> /dev/null
+       done
+       
+       for schemaName in ${schemaNames};
+       do
+               leftSqls=$(ls "${DB}"/"${schemaName}"/"${FEATURE_SQL}"/*.sql 2> /dev/null)
+               if [[ -z ${leftSqls} ]]; then
+                       if ! DEBUG=${DEBUG} db-migrator -s "${schemaName}" -o ok; then
+                               echo -n "warning: ${featureName}: ${schemaName}: database data is leftover. "
+                               echo -n "Consider cleaning left over data with 'db-migrator'."
+                       fi
+               fi              
+       done
+}
+
+############################################################
+# uninstallFeature <feature-name> ...
+############################################################
+function uninstallFeature
+{
+       if [[ $DEBUG == y ]]; then
+               echo "-- ${FUNCNAME[0]} $* --"
+               set -x
+       fi
+       
+       local featureName="$1"
+       
+       if [[ -z ${featureName} ]]; then
+               echo "warning: no feature name"
+               return
+       fi
+       
+       disableFeature "${featureName}"
+       uninstallFeatureDb "${featureName}"
+       customOpScript "${featureName}" "uninstall"
+       
+       if [[ -n ${FEATURES} && -n ${featureName} ]]; then
+               rm -rf "${FEATURES:-???}/${featureName}"
+       fi
+}
+
 ############################################################
 # uninstallFeatures <feature-name> ...
 ############################################################
 function uninstallFeatures
 {
+       if [[ $DEBUG == y ]]; then
+               echo "-- ${FUNCNAME[0]} --"
+               set -x
+       fi
+       
        local name
        local allFeatures=$'\n'$(cd ${FEATURES};ls)$'\n'
        for name in "$@" ; do
                # the following check takes care of potentially troublesome names
                # like '.', '..', and names containing '/'
                if [[ "${allFeatures}" =~ $'\n'${name}$'\n' ]] ; then
-                       disableFeature "${name}"
-                       rm -rf "${FEATURES}/${name}"
+                       uninstallFeature "${name}"
                else
                        echo "feature ${name} not found"
                fi
@@ -927,7 +1071,7 @@ case "$1" in
        enable)
        {
                if [[ -f "${POLICY_HOME}"/PID ]]; then
-                       echo "ERROR: enable: not allowed when policy is running .."
+                       echo "error: enable: not allowed when policy is running .."
                        echo
                        status
                        exit 10
@@ -946,7 +1090,7 @@ case "$1" in
                                # make sure there is only one feature jar
                                countFeatureJars=$(echo "${file}" | wc -w)
                                if [[ ${countFeatureJars} != 1 ]]; then
-                                       echo "WARNING: skipping ${name},  ${countFeatureJars} feature libraries found"
+                                       echo "warning: skipping ${name},  ${countFeatureJars} feature libraries found"
                                        continue
                                fi                      
                                
@@ -965,7 +1109,7 @@ case "$1" in
        disable)
        {
                if [[ -f "${POLICY_HOME}"/PID ]]; then
-                       echo "ERROR: disable: not allowed when policy is running .."
+                       echo "error: disable: not allowed when policy is running .."
                        echo
                        status
                        exit 11
@@ -1001,7 +1145,7 @@ case "$1" in
        uninstall)
        {
                if [[ -f "${POLICY_HOME}"/PID ]]; then
-                       echo "ERROR: uninstall: not allowed when policy is running .."
+                       echo "error: uninstall: not allowed when policy is running .."
                        echo
                        status
                        exit 12