Add retry logic and error handling in csit scripts

Db-migrator exits with error sometimes due to Postgres database being
restarted during initialization.
docker compose return code is captured to exit the flow before starting the
tests.

Issue-ID: POLICY-5557
Signed-off-by: rameshiyer27 <ramesh.murugan.iyer@est.tech>
Change-Id: I8ab831a69f4abea3b59045f06a71affcbaf15a2e

diff --git a/compose/config/db-migrator/init_pg.sh b/compose/config/db-migrator/init_pg.sh
index 057fef5..81aad86 100755 (executable)
--- a/compose/config/db-migrator/init_pg.sh
+++ b/compose/config/db-migrator/init_pg.sh
@@ -22,17 +22,28 @@ export SQL_USER=${PGSQL_USER}
 export SQL_PASSWORD=${PGSQL_PASSWORD}
 export SCRIPT_DIRECTORY=postgres
 
+MAX_RETRY=5
+RETRY_INTERVAL=10
+
 for schema in ${SQL_DB}; do
     echo "Initializing $schema..."
-    /opt/app/policy/bin/prepare_upgrade.sh ${schema}
+    rc=1
+    for i in $(seq 1 ${MAX_RETRY}); do
+        /opt/app/policy/bin/prepare_upgrade.sh ${schema}
 
-    /opt/app/policy/bin/db-migrator-pg -s ${schema} -o report
+        /opt/app/policy/bin/db-migrator-pg -s ${schema} -o report
 
-    /opt/app/policy/bin/db-migrator-pg -s ${schema} -o upgrade
-    rc=$?
+        /opt/app/policy/bin/db-migrator-pg -s ${schema} -o upgrade
+        rc=$?
 
-    /opt/app/policy/bin/db-migrator-pg -s ${schema} -o report
+        /opt/app/policy/bin/db-migrator-pg -s ${schema} -o report
 
+        if [ "$rc" = 0 ]; then
+            break
+        fi
+        echo "Schema $schema initialization failed (attempt ${i}/${MAX_RETRY}), retrying in ${RETRY_INTERVAL}s..."
+        sleep ${RETRY_INTERVAL}
+    done
     if [ "$rc" != 0 ]; then
         break
     fi

diff --git a/compose/start-compose.sh b/compose/start-compose.sh
index 2815f2a..6e6a04a 100755 (executable)
--- a/compose/start-compose.sh
+++ b/compose/start-compose.sh
@@ -75,14 +75,22 @@ fi
 if [ -n "$component" ]; then
   if [ "$grafana" = true ]; then
     docker compose up -d "${component}" postgres grafana --wait
+    COMPOSE_RC=$?
     echo "Prometheus server: http://localhost:${PROMETHEUS_PORT}"
     echo "Grafana server: http://localhost:${GRAFANA_PORT}"
   else
     docker compose up -d "${component}" postgres --wait
+    COMPOSE_RC=$?
   fi
 else
   export PROJECT=policy-api # policy-api has groups.json complete with all 3 pdps
   docker compose up -d --wait
+  COMPOSE_RC=$?
+fi
+
+if [ "${COMPOSE_RC}" -ne 0 ]; then
+  echo "ERROR: One or more containers failed to start (exit code: ${COMPOSE_RC}). Aborting."
+  exit ${COMPOSE_RC}
 fi
 
 cd "${WORKSPACE}"

diff --git a/csit/run-project-csit.sh b/csit/run-project-csit.sh
index 9b39b14..75c8ea1 100755 (executable)
--- a/csit/run-project-csit.sh
+++ b/csit/run-project-csit.sh
@@ -259,6 +259,11 @@ if [ "${TEARDOWN}" == "true" ]; then
 fi
 
 set_project_config
+if [ $? -ne 0 ]; then
+    echo "ERROR: Project setup failed. Aborting tests."
+    RC=1
+    exit ${RC}
+fi
 
 unset http_proxy https_proxy