From 5fee4488f065cce0f9a6bc6bc68f48a1d3b643f0 Mon Sep 17 00:00:00 2001 From: "waqas.ikram" Date: Mon, 22 Jul 2019 13:56:13 +0000 Subject: [PATCH] Adding time out handling in setup.sh script Change-Id: I410d597da6a1fe15d4b02794f56b3507563978c8 Issue-ID: SO-1955 Signed-off-by: waqas.ikram --- .../config/apply-workarounds.sh | 39 +++++----- plans/so/integration-etsi-testing/config/env | 1 + .../config/wait-for-workaround-job.sh | 84 ++++++++++++++++++++++ plans/so/integration-etsi-testing/setup.sh | 13 +++- 4 files changed, 117 insertions(+), 20 deletions(-) create mode 100755 plans/so/integration-etsi-testing/config/wait-for-workaround-job.sh diff --git a/plans/so/integration-etsi-testing/config/apply-workarounds.sh b/plans/so/integration-etsi-testing/config/apply-workarounds.sh index ff167bf7..4200a361 100755 --- a/plans/so/integration-etsi-testing/config/apply-workarounds.sh +++ b/plans/so/integration-etsi-testing/config/apply-workarounds.sh @@ -29,6 +29,7 @@ BUILDING_BLOCK_COUNT_QUERY="select count(*) from $BUIDLING_BLOCK_TABLE_NAME;" FLY_WAY_MIGRATION_QUERY="SELECT COUNT(*) FROM flyway_schema_history WHERE script LIKE '%R__MacroData%' AND installed_on IS NOT NULL;" SLEEP_TIME=5 TIME_OUT_DEFAULT_VALUE_SEC=1200 #20 mins +SCRIPT_NAME=$(basename $0) current_timestamp() { @@ -37,53 +38,53 @@ current_timestamp() wait_for_database_availability() { - echo "$(current_timestamp): Checking for database availability" + echo "$SCRIPT_NAME $(current_timestamp): Checking for database availability" until echo '\q' | mysql -h $DB_HOST -P $DB_PORT -uroot -p$MYSQL_ROOT_PASSWORD $CATALOG_DB; do - >&2 echo "$(current_timestamp): Database is unavailable - sleeping for ${SLEEP_TIME} seconds" + >&2 echo "$SCRIPT_NAME $(current_timestamp): Database is unavailable - sleeping for ${SLEEP_TIME} seconds" isTimeOut sleep ${SLEEP_TIME} done - echo "$(current_timestamp): Database is available now" + echo "$SCRIPT_NAME $(current_timestamp): Database is available now" } wait_container_to_create_table() { while [ $(mysql -h $DB_HOST -P $DB_PORT -uroot -p$MYSQL_ROOT_PASSWORD $CATALOG_DB -sse "$TABLE_EXISTS_QUERY") -eq "0" ] ; do - echo "$(current_timestamp): Waiting for so-catalog container to create tables - sleeping for ${SLEEP_TIME} seconds" + echo "$SCRIPT_NAME $(current_timestamp): Waiting for so-catalog container to create tables - sleeping for ${SLEEP_TIME} seconds" isTimeOut sleep ${SLEEP_TIME} done while [ $(mysql -h $DB_HOST -P $DB_PORT -uroot -p$MYSQL_ROOT_PASSWORD $CATALOG_DB -sse "$BUILDING_BLOCK_COUNT_QUERY") -eq "0" ] ; do - echo "$(current_timestamp): Waiting for so-catalog container to insert records in $BUIDLING_BLOCK_TABLE_NAME - sleeping for ${SLEEP_TIME} seconds" + echo "$SCRIPT_NAME $(current_timestamp): Waiting for so-catalog container to insert records in $BUIDLING_BLOCK_TABLE_NAME - sleeping for ${SLEEP_TIME} seconds" isTimeOut sleep ${SLEEP_TIME} done - echo "$(current_timestamp): $CATALOG_DB tables available now . . ." + echo "$SCRIPT_NAME $(current_timestamp): $CATALOG_DB tables available now . . ." } wait_for_flyway_migration_to_finish() { while [ $(mysql -h $DB_HOST -P $DB_PORT -uroot -p$MYSQL_ROOT_PASSWORD $CATALOG_DB -sse "$FLY_WAY_MIGRATION_QUERY") -ne "1" ] ; do - echo "$(current_timestamp): Waiting for flyway migration sql statement to finish with success - sleeping for ${SLEEP_TIME} seconds" + echo "$SCRIPT_NAME $(current_timestamp): Waiting for flyway migration sql statement to finish with success - sleeping for ${SLEEP_TIME} seconds" isTimeOut sleep ${SLEEP_TIME} done - echo "$(current_timestamp): flyway migration finished . . . " + echo "$SCRIPT_NAME $(current_timestamp): flyway migration finished . . . " } apply_workaround() { - echo "$(current_timestamp): Applying workaround . . ." + echo "$SCRIPT_NAME $(current_timestamp): Applying workaround . . ." wait_for_database_availability wait_container_to_create_table wait_for_flyway_migration_to_finish - echo "$(current_timestamp): Will insert data into $CATALOG_DB" + echo "$SCRIPT_NAME $(current_timestamp): Will insert data into $CATALOG_DB" mysql -h $DB_HOST -uroot -p$MYSQL_ROOT_PASSWORD $CATALOG_DB << EOF BEGIN; INSERT INTO $BUIDLING_BLOCK_TABLE_NAME (BUILDING_BLOCK_NAME,RESOURCE_TYPE,TARGET_ACTION) values ("EtsiVnfInstantiateBB", "VNF", "ACTIVATE"); @@ -109,36 +110,36 @@ mysql -h $DB_HOST -uroot -p$MYSQL_ROOT_PASSWORD $CATALOG_DB << EOF EOF if [ $? -ne 0 ]; then - echo "$(current_timestamp): Failed to execute workaround . . ." + echo "$SCRIPT_NAME $(current_timestamp): Failed to execute workaround . . ." exit 1 fi - echo "$(current_timestamp): Finished applying workaround . . ." + echo "$SCRIPT_NAME $(current_timestamp): Finished applying workaround . . ." } isTimeOut() { - if [ `date +%s` -gt $TIME_OUT_END_TIME_IN_MS ]; then - echo "$(current_timestamp): workaround script timed out . . ." + if [ `date +%s` -gt $TIME_OUT_END_TIME_IN_SECONDS ]; then + echo "$SCRIPT_NAME $(current_timestamp): workaround script timed out . . ." exit 1; fi } # main body if [ -z "$TIME_OUT_IN_SECONDS"]; then - echo "$(current_timestamp): TIME_OUT_IN_SECONDS attribute is empty will use default val: $TIME_OUT_DEFAULT_VALUE_SEC" + echo "$SCRIPT_NAME $(current_timestamp): TIME_OUT_IN_SECONDS attribute is empty will use default val: $TIME_OUT_DEFAULT_VALUE_SEC" TIME_OUT_IN_SECONDS=$TIME_OUT_DEFAULT_VALUE_SEC fi DIGITS_REGEX='^[0-9]+$' if ! [[ $TIME_OUT_IN_SECONDS =~ $DIGIT_REGEX ]] ; then - echo "$(current_timestamp): TIME_OUT_IN_SECONDS attribute Must be number: $TIME_OUT_IN_SECONDS, will use default val: $TIME_OUT_DEFAULT_VALUE_SEC" + echo "$SCRIPT_NAME $(current_timestamp): TIME_OUT_IN_SECONDS attribute Must be number: $TIME_OUT_IN_SECONDS, will use default val: $TIME_OUT_DEFAULT_VALUE_SEC" TIME_OUT_IN_SECONDS=$TIME_OUT_DEFAULT_VALUE_SEC fi -START_TIME_IN_MS=`date +%s` -TIME_OUT_END_TIME_IN_MS=$(($START_TIME_IN_MS+$TIME_OUT_IN_SECONDS)); -echo "$(current_timestamp): Workaround script will time out at `date -d @$TIME_OUT_END_TIME_IN_MS`" +START_TIME_IN_SECONDS=`date +%s` +TIME_OUT_END_TIME_IN_SECONDS=$(($START_TIME_IN_SECONDS+$TIME_OUT_IN_SECONDS)); +echo "$SCRIPT_NAME $(current_timestamp): Workaround script will time out at `date -d @$TIME_OUT_END_TIME_IN_SECONDS`" apply_workaround diff --git a/plans/so/integration-etsi-testing/config/env b/plans/so/integration-etsi-testing/config/env index f34f9c48..9b6dc557 100644 --- a/plans/so/integration-etsi-testing/config/env +++ b/plans/so/integration-etsi-testing/config/env @@ -1,2 +1,3 @@ NEXUS_DOCKER_REPO_MSO=nexus3.onap.org:10001 TAG=1.4.0-STAGING-latest +TIME_OUT_DEFAULT_VALUE_SEC=1200 diff --git a/plans/so/integration-etsi-testing/config/wait-for-workaround-job.sh b/plans/so/integration-etsi-testing/config/wait-for-workaround-job.sh new file mode 100755 index 00000000..ae6c2f57 --- /dev/null +++ b/plans/so/integration-etsi-testing/config/wait-for-workaround-job.sh @@ -0,0 +1,84 @@ + +# +# ============LICENSE_START======================================================= +# Copyright (C) 2019 Nordix Foundation. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# ============LICENSE_END========================================================= +# + +# @author Waqas Ikram (waqas.ikram@est.tech) + +SLEEP_TIME=5 +WORKAROUND_SUCCESSFUL_TEXT="Finished applying workaround" +WORKAROUND_FAILURE_TEXT="Failed to execute workaround" +WORKAROUND_TIME_OUT_TEXT="workaround script timed out" +WORKAROUND_CONTAINER_NAME=$(docker ps -aqf "name=workaround-config" --format "{{.Names}}") +SCRIPT_HOME="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +SCRIPT_NAME=$(basename $0) + +current_timestamp() +{ + date +"%Y-%m-%d %H:%M:%S" +} + +# main body +if [ -z $TIME_OUT_DEFAULT_VALUE_SEC ]; then + echo "$SCRIPT_NAME $(current_timestamp): ERROR: Undefined value for TIME_OUT_DEFAULT_VALUE_SEC attribute" + exit 1 +fi + +if [ -z $WORKAROUND_CONTAINER_NAME ]; then + echo "$SCRIPT_NAME $(current_timestamp): Unable to find docker container id " + exit 1 +fi + +START_TIME_IN_SECONDS=`date +%s` +TIME_OUT_END_TIME_IN_SECONDS=$(($START_TIME_IN_SECONDS+$TIME_OUT_DEFAULT_VALUE_SEC)); + + +echo echo "$SCRIPT_NAME $(current_timestamp): $SCRIPT_NAME script Start Time `date -d @$START_TIME_IN_SECONDS`" +echo echo "$SCRIPT_NAME $(current_timestamp): $SCRIPT_NAME will time out at `date -d @$TIME_OUT_END_TIME_IN_SECONDS`" + +while [ `date +%s` -lt "$TIME_OUT_END_TIME_IN_SECONDS" ]; do + echo "$(current_timestamp): Waiting for $WORKAROUND_CONTAINER_NAME to finish ..." + + result=$(docker logs $WORKAROUND_CONTAINER_NAME 2>&1 | grep -E "$WORKAROUND_SUCCESSFUL_TEXT|$WORKAROUND_FAILURE_TEXT|$WORKAROUND_TIME_OUT_TEXT") + if [ ! -z "$result" ]; then + echo "$SCRIPT_NAME $(current_timestamp): Found result: $result" + break; + fi + echo "$(current_timestamp): Sleeping for ${SLEEP_TIME} seconds" + sleep ${SLEEP_TIME} +done + +if [ -z "$result" ]; then + echo "$SCRIPT_NAME $(current_timestamp): ERROR: failed to apply workaround . . . " + echo "-------------- $WORKAROUND_CONTAINER_NAME logs -------------" + docker logs $WORKAROUND_CONTAINER_NAME + echo "------------------------------------------------------------" + exit 1 +fi + +if echo "$result" | grep -E "$WORKAROUND_FAILURE_TEXT|$WORKAROUND_TIME_OUT_TEXT"; then + echo "$SCRIPT_NAME $(current_timestamp): Work around script failed" + echo "-------------- $WORKAROUND_CONTAINER_NAME logs -------------" + docker logs $WORKAROUND_CONTAINER_NAME + echo "------------------------------------------------------------" + exit 1 +fi + +echo "$SCRIPT_NAME $(current_timestamp): Successfully applied workaround configuration . . ." +exit 0 diff --git a/plans/so/integration-etsi-testing/setup.sh b/plans/so/integration-etsi-testing/setup.sh index 75e75275..597bf2ea 100755 --- a/plans/so/integration-etsi-testing/setup.sh +++ b/plans/so/integration-etsi-testing/setup.sh @@ -39,6 +39,7 @@ MVN_VERSION="$MVN -v" MVN_SETTINGS_XML="$SCRIPT_HOME/settings.xml" MVN_CLEAN_INSTALL="$MVN clean install" SIMULATOR_MAVEN_PROJECT_POM="$SCRIPT_HOME/so-simulators/pom.xml" +WAIT_FOR_WORKAROUND_SCRIPT=$CONFIG_DIR/"wait-for-workaround-job.sh" echo "Running $SCRIPT_HOME/$SCRIPT_NAME ..." @@ -121,7 +122,6 @@ fi git clone http://gerrit.onap.org/r/so/docker-config.git $TEST_LAB_DIR_PATH - export TEST_LAB_DIR=$TEST_LAB_DIR_PATH export CONFIG_DIR_PATH=$CONFIG_DIR @@ -130,6 +130,17 @@ docker-compose -f $DOCKER_COMPOSE_FILE_PATH up -d echo "Sleeping for 3m" sleep 3m +echo "Will execute $WAIT_FOR_WORKAROUND_SCRIPT script" +$WAIT_FOR_WORKAROUND_SCRIPT + +if [ $? -ne 0 ]; then + echo "ERROR: $WAIT_FOR_WORKAROUND_SCRIPT failed" + echo "Will stop running docker containers . . ." + docker-compose -f $DOCKER_COMPOSE_FILE_PATH down + exit 1 +fi + + REPO_IP='127.0.0.1' ROBOT_VARIABLES="-v REPO_IP:${REPO_IP}" -- 2.16.6