Improve cluster installation and test scripts 98/139998/4
authorwaynedunican <wayne.dunican@est.tech>
Fri, 17 Jan 2025 08:52:45 +0000 (08:52 +0000)
committerWayne Dunican <wayne.dunican@est.tech>
Thu, 23 Jan 2025 10:21:32 +0000 (10:21 +0000)
Issue-ID: POLICY-5246
Change-Id: I18d1c0375bc656a46d0e7742dffaf45d2400cd3c
Signed-off-by: waynedunican <wayne.dunican@est.tech>
csit/resources/scripts/cluster_setup.sh [new file with mode: 0755]
csit/resources/scripts/config_setup.sh [new file with mode: 0755]
csit/resources/scripts/get-cluster-info.sh
csit/resources/scripts/robot_setup.sh [new file with mode: 0755]
csit/run-k8s-csit.sh
csit/run-s3p-tests.sh [new file with mode: 0755]
csit/start-s3p-tests.sh [deleted file]

diff --git a/csit/resources/scripts/cluster_setup.sh b/csit/resources/scripts/cluster_setup.sh
new file mode 100755 (executable)
index 0000000..6d72bac
--- /dev/null
@@ -0,0 +1,250 @@
+#!/bin/bash
+# ============LICENSE_START=======================================================
+#  Copyright (C) 2025 Nordix Foundation. All rights reserved.
+# ================================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+# ============LICENSE_END=========================================================
+
+WORKSPACE=$(git rev-parse --show-toplevel)
+export WORKSPACE
+
+export GERRIT_BRANCH=$(awk -F= '$1 == "defaultbranch" { print $2 }' "${WORKSPACE}"/.gitreview)
+
+# Source the shared config script
+source "$(dirname "$0")/config_setup.sh"
+
+KAFKA_DIR=${WORKSPACE}/helm/cp-kafka
+SET_VALUES=""
+
+ZK_CONTAINER="zookeeper-deployment"
+KAFKA_CONTAINER="kafka-deployment"
+
+function spin_microk8s_cluster() {
+    echo "Verify if Microk8s cluster is running.."
+    microk8s version
+    exitcode="${?}"
+
+    if [ "$exitcode" -ne 0 ]; then
+        echo "Microk8s cluster not available, Spinning up the cluster.."
+        sudo snap install microk8s --classic --channel=1.30/stable
+
+        if [ "${?}" -ne 0 ]; then
+            echo "Failed to install kubernetes cluster. Aborting.."
+            return 1
+        fi
+        echo "Microk8s cluster installed successfully"
+        sudo usermod -a -G microk8s $USER
+        echo "Enabling DNS and Storage plugins"
+        sudo microk8s.enable dns hostpath-storage
+        echo "Creating configuration file for Microk8s"
+        sudo mkdir -p $HOME/.kube
+        sudo chown -R $USER:$USER $HOME/.kube
+        sudo microk8s kubectl config view --raw >$HOME/.kube/config
+        sudo chmod 600 $HOME/.kube/config
+        echo "K8s installation completed"
+        echo "----------------------------------------"
+    else
+        echo "K8s cluster is already running"
+        echo "----------------------------------------"
+    fi
+
+    echo "Verify if kubectl is running.."
+    kubectl version
+    exitcode="${?}"
+
+    if [ "$exitcode" -ne 0 ]; then
+        echo "Kubectl not available, Installing.."
+        sudo snap install kubectl --classic --channel=1.30/stable
+
+        if [ "${?}" -ne 0 ]; then
+            echo "Failed to install Kubectl. Aborting.."
+            return 1
+        fi
+        echo "Kubectl installation completed"
+        echo "----------------------------------------"
+    else
+        echo "Kubectl is already running"
+        echo "----------------------------------------"
+        return 0
+    fi
+
+    echo "Verify if helm is running.."
+    helm version
+    exitcode="${?}"
+
+    if [ "$exitcode" -ne 0 ]; then
+        echo "Helm not available, Installing.."
+        sudo snap install helm --classic --channel=3.7
+
+        if [ "${?}" -ne 0 ]; then
+            echo "Failed to install Helm client. Aborting.."
+            return 1
+        fi
+        echo "Helm installation completed"
+        echo "----------------------------------------"
+    else
+        echo "Helm is already running"
+        echo "----------------------------------------"
+        return 0
+    fi
+}
+
+function install_kafka() {
+  echo "Installing Confluent kafka"
+  kubectl apply -f $KAFKA_DIR/zookeeper.yaml
+  kubectl apply -f $KAFKA_DIR/kafka.yaml
+  echo "----------------------------------------"
+}
+
+function uninstall_policy() {
+    echo "Removing the policy helm deployment"
+    helm uninstall csit-policy
+    helm uninstall prometheus
+    helm uninstall csit-robot
+    kubectl delete deploy $ZK_CONTAINER $KAFKA_CONTAINER
+    rm -rf ${WORKSPACE}/helm/policy/Chart.lock
+    if [ "$PROJECT" == "clamp" ] || [ "$PROJECT" == "policy-clamp" ]; then
+      helm uninstall policy-chartmuseum
+      helm repo remove chartmuseum-git policy-chartmuseum
+    fi
+    sudo rm -rf /dockerdata-nfs/mariadb-galera/
+    kubectl delete pvc --all
+    echo "Policy deployment deleted"
+    echo "Clean up docker"
+    docker image prune -f
+}
+
+function teardown_cluster() {
+    echo "Removing k8s cluster and k8s configuration file"
+    sudo snap remove microk8s;rm -rf $HOME/.kube/config
+    sudo snap remove helm;
+    sudo snap remove kubectl;
+    echo "MicroK8s Cluster removed"
+}
+
+function install_chartmuseum () {
+    echo "---------------------------------------------"
+    echo "Installing Chartmuseum helm repository..."
+    helm repo add chartmuseum-git https://chartmuseum.github.io/charts
+    helm repo update
+    helm install policy-chartmuseum chartmuseum-git/chartmuseum --set env.open.DISABLE_API=false --set service.type=NodePort --set service.nodePort=30208
+    helm plugin install https://github.com/chartmuseum/helm-push
+    echo "---------------------------------------------"
+}
+
+function get_pod_name() {
+  pods=$(kubectl get pods --no-headers -o custom-columns=':metadata.name' | grep $1)
+  read -rd '' -a pod_array <<< "$pods"
+  echo "${pod_array[@]}"
+}
+
+function wait_for_pods_running() {
+  local namespace="$1"
+  shift
+  local timeout_seconds="$1"
+  shift
+
+  IFS=',' read -ra pod_names <<< "$@"
+  shift
+
+  local pending_pods=("${pod_names[@]}")
+  local start_time
+  start_time=$(date +%s)
+
+  while [ ${#pending_pods[@]} -gt 0 ]; do
+    local current_time
+    current_time=$(date +%s)
+    local elapsed_time
+    elapsed_time=$((current_time - start_time))
+
+    if [ "$elapsed_time" -ge "$timeout_seconds" ]; then
+      echo "Timed out waiting for the pods to reach 'Running' state."
+      echo "Printing the current status of the deployment before exiting.."
+      kubectl get po;
+      kubectl describe pods;
+      echo "------------------------------------------------------------"
+      for pod in "${pending_pods[@]}"; do
+        echo "Logs of the pod $pod"
+        kubectl logs $pod
+        echo "---------------------------------------------------------"
+      done
+      exit 1
+    fi
+
+    local newly_running_pods=()
+
+    for pod_name_prefix in "${pending_pods[@]}"; do
+      local pod_names=$(get_pod_name "$pod_name_prefix")
+      IFS=' ' read -r -a pod_array <<< "$pod_names"
+      if [ "${#pod_array[@]}" -eq 0 ]; then
+             echo "*** Error: No pods found for the deployment $pod_name_prefix . Exiting ***"
+             return -1
+      fi
+      for pod in "${pod_array[@]}"; do
+         local pod_status
+         local pod_ready
+         pod_status=$(kubectl get pod "$pod" -n "$namespace" --no-headers -o custom-columns=STATUS:.status.phase 2>/dev/null)
+         pod_ready=$(kubectl get pod "$pod" -o jsonpath='{.status.containerStatuses[*].ready}')
+
+         if [ "$pod_status" == "Running" ] && { [ "$pod_ready" == "true" ] || [ "$pod_ready" == "true true" ]; }; then
+           echo "Pod '$pod' in namespace '$namespace' is now in 'Running' state and 'Readiness' is true"
+         else
+           newly_running_pods+=("$pod")
+           echo "Waiting for pod '$pod' in namespace '$namespace' to reach 'Running' and 'Ready' state..."
+         fi
+      done
+    done
+
+    pending_pods=("${newly_running_pods[@]}")
+
+    sleep 5
+  done
+
+  echo "All specified pods are in the 'Running and Ready' state. Exiting the function."
+}
+
+OPERATION="$1"
+PROJECT="$2"
+LOCALIMAGE="${3:-false}"
+
+if [ $OPERATION == "install" ]; then
+    spin_microk8s_cluster
+    if [ "${?}" -eq 0 ]; then
+        export KAFKA_CONTAINERS=($KAFKA_CONTAINER,$ZK_CONTAINER)
+        install_kafka
+        wait_for_pods_running default 300 $KAFKA_CONTAINERS
+        set_project_config "$PROJECT"
+        echo "Installing policy helm charts in the default namespace"
+        source ${WORKSPACE}/compose/get-k8s-versions.sh
+        if [ $LOCALIMAGE == "true" ]; then
+            echo "loading local image"
+            source ${WORKSPACE}/compose/get-versions.sh
+            ${WORKSPACE}/compose/loaddockerimage.sh
+        fi
+        cd ${WORKSPACE}/helm || exit
+        helm dependency build policy
+        helm install csit-policy policy ${SET_VALUES}
+        helm install prometheus prometheus
+        wait_for_pods_running default 900 ${READINESS_CONTAINERS[@]}
+        echo "Policy chart installation completed"
+        echo "-------------------------------------------"
+    fi
+elif [ $OPERATION == "uninstall" ]; then
+    uninstall_policy
+elif [ $OPERATION == "clean" ]; then
+    teardown_cluster
+else
+    echo "Invalid arguments provided. Usage: $0 [options..] {install {project_name} | uninstall | clean} {uselocalimage = true/false}"
+fi
diff --git a/csit/resources/scripts/config_setup.sh b/csit/resources/scripts/config_setup.sh
new file mode 100755 (executable)
index 0000000..1cdd260
--- /dev/null
@@ -0,0 +1,113 @@
+#!/bin/bash
+# ============LICENSE_START=======================================================
+#  Copyright (C) 2025 Nordix Foundation. All rights reserved.
+# ================================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+# ============LICENSE_END=========================================================
+
+export POLICY_CLAMP_ROBOT="policy-clamp-test.robot clamp-slas.robot"
+export POLICY_API_ROBOT="api-test.robot api-slas.robot"
+export POLICY_PAP_ROBOT="pap-test.robot pap-slas.robot"
+export POLICY_APEX_PDP_ROBOT="apex-pdp-test.robot apex-slas.robot"
+export POLICY_XACML_PDP_ROBOT="xacml-pdp-test.robot xacml-pdp-slas.robot"
+export POLICY_OPA_PDP_ROBOT="opa-pdp-test.robot"
+export POLICY_DROOLS_PDP_ROBOT="drools-pdp-test.robot"
+export POLICY_DISTRIBUTION_ROBOT="distribution-test.robot"
+
+export POLICY_API_CONTAINER="policy-api"
+export POLICY_PAP_CONTAINER="policy-pap"
+export POLICY_CLAMP_CONTAINER="policy-clamp-runtime-acm"
+export POLICY_APEX_CONTAINER="policy-apex-pdp"
+export POLICY_DROOLS_CONTAINER="policy-drools-pdp"
+export POLICY_XACML_CONTAINER="policy-xacml-pdp"
+export POLICY_OPA_CONTAINER="policy-opa-pdp"
+export POLICY_DISTRIBUTION_CONTAINER="policy-distribution"
+export POLICY_K8S_PPNT_CONTAINER="policy-clamp-ac-k8s-ppnt"
+export POLICY_HTTP_PPNT_CONTAINER="policy-clamp-ac-http-ppnt"
+export POLICY_SIM_PPNT_CONTAINER="policy-clamp-ac-sim-ppnt"
+export POLICY_PF_PPNT_CONTAINER="policy-clamp-ac-pf-ppnt"
+export JAEGER_CONTAINER="jaeger"
+
+function install_chartmuseum () {
+    echo "---------------------------------------------"
+    echo "Installing Chartmuseum helm repository..."
+    helm repo add chartmuseum-git https://chartmuseum.github.io/charts
+    helm repo update
+    helm install policy-chartmuseum chartmuseum-git/chartmuseum --set env.open.DISABLE_API=false --set service.type=NodePort --set service.nodePort=30208
+    helm plugin install https://github.com/chartmuseum/helm-push
+    echo "---------------------------------------------"
+}
+
+function set_project_config() {
+    echo "Setting project configuration for: $PROJECT"
+    case $PROJECT in
+    clamp | policy-clamp)
+        export ROBOT_FILE=$POLICY_CLAMP_ROBOT
+        export READINESS_CONTAINERS=($POLICY_CLAMP_CONTAINER,$POLICY_APEX_CONTAINER,$POLICY_PF_PPNT_CONTAINER,$POLICY_K8S_PPNT_CONTAINER,
+            $POLICY_HTTP_PPNT_CONTAINER,$POLICY_SIM_PPNT_CONTAINER,$JAEGER_CONTAINER)
+        export SET_VALUES="--set $POLICY_CLAMP_CONTAINER.enabled=true --set $POLICY_APEX_CONTAINER.enabled=true
+            --set $POLICY_PF_PPNT_CONTAINER.enabled=true --set $POLICY_K8S_PPNT_CONTAINER.enabled=true
+            --set $POLICY_HTTP_PPNT_CONTAINER.enabled=true --set $POLICY_SIM_PPNT_CONTAINER.enabled=true
+            --set $JAEGER_CONTAINER.enabled=true"
+        install_chartmuseum
+        ;;
+    api | policy-api)
+        export ROBOT_FILE=$POLICY_API_ROBOT
+        export READINESS_CONTAINERS=($POLICY_API_CONTAINER)
+        ;;
+    pap | policy-pap)
+        export ROBOT_FILE=$POLICY_PAP_ROBOT
+        export READINESS_CONTAINERS=($POLICY_APEX_CONTAINER,$POLICY_PAP_CONTAINER,$POLICY_API_CONTAINER,$POLICY_XACML_CONTAINER)
+        export SET_VALUES="--set $POLICY_APEX_CONTAINER.enabled=true --set $POLICY_XACML_CONTAINER.enabled=true"
+        ;;
+    apex-pdp | policy-apex-pdp)
+        export ROBOT_FILE=$POLICY_APEX_PDP_ROBOT
+        export READINESS_CONTAINERS=($POLICY_APEX_CONTAINER,$POLICY_API_CONTAINER,$POLICY_PAP_CONTAINER)
+        export SET_VALUES="--set $POLICY_APEX_CONTAINER.enabled=true"
+        ;;
+    xacml-pdp | policy-xacml-pdp)
+        export ROBOT_FILE=($POLICY_XACML_PDP_ROBOT)
+        export READINESS_CONTAINERS=($POLICY_API_CONTAINER,$POLICY_PAP_CONTAINER,$POLICY_XACML_CONTAINER)
+        export SET_VALUES="--set $POLICY_XACML_CONTAINER.enabled=true"
+        ;;
+    opa-pdp | policy-opa-pdp)
+        export ROBOT_FILE=($POLICY_OPA_PDP_ROBOT)
+        export READINESS_CONTAINERS=($POLICY_API_CONTAINER,$POLICY_PAP_CONTAINER,$POLICY_OPA_CONTAINER)
+        export SET_VALUES="--set $POLICY_OPA_CONTAINER.enabled=true"
+        ;;
+    drools-pdp | policy-drools-pdp)
+        export ROBOT_FILE=($POLICY_DROOLS_PDP_ROBOT)
+        export READINESS_CONTAINERS=($POLICY_DROOLS_CONTAINER)
+        export SET_VALUES="--set $POLICY_DROOLS_CONTAINER.enabled=true"
+        ;;
+    distribution | policy-distribution)
+        export ROBOT_FILE=($POLICY_DISTRIBUTION_ROBOT)
+        export READINESS_CONTAINERS=($POLICY_APEX_CONTAINER,$POLICY_API_CONTAINER,$POLICY_PAP_CONTAINER,$POLICY_DISTRIBUTION_CONTAINER)
+        export SET_VALUES="--set $POLICY_APEX_CONTAINER.enabled=true --set $POLICY_DISTRIBUTION_CONTAINER.enabled=true"
+        ;;
+    *)
+        echo "Unknown project supplied. Enabling all policy charts for the deployment"
+        export READINESS_CONTAINERS=($POLICY_APEX_CONTAINER,$POLICY_API_CONTAINER,$POLICY_PAP_CONTAINER,
+                    $POLICY_DISTRIBUTION_CONTAINER,$POLICY_DROOLS_CONTAINER,$POLICY_XACML_CONTAINER,$POLICY_OPA_CONTAINER,
+                    $POLICY_CLAMP_CONTAINER,$POLICY_PF_PPNT_CONTAINER,$POLICY_K8S_PPNT_CONTAINER,
+                    $POLICY_HTTP_PPNT_CONTAINER,$POLICY_SIM_PPNT_CONTAINER)
+        export SET_VALUES="--set $POLICY_APEX_CONTAINER.enabled=true --set $POLICY_XACML_CONTAINER.enabled=true
+            --set $POLICY_OPA_CONTAINER.enabled=true --set $POLICY_DISTRIBUTION_CONTAINER.enabled=true --set $POLICY_DROOLS_CONTAINER.enabled=true
+            --set $POLICY_CLAMP_CONTAINER.enabled=true --set $POLICY_PF_PPNT_CONTAINER.enabled=true
+            --set $POLICY_K8S_PPNT_CONTAINER.enabled=true --set $POLICY_HTTP_PPNT_CONTAINER.enabled=true
+            --set $POLICY_SIM_PPNT_CONTAINER.enabled=true"
+        ;;
+    esac
+}
index 75fe719..1252f3e 100755 (executable)
@@ -133,7 +133,38 @@ function expose_services() {
     patch_ports
 }
 
+# Port forward Kafka to handle traffic to/from JMeter
+function setup_kafka_connection() {
+  # Get the Kafka pod name
+  KAFKA_POD=$(kubectl get pods -l app=kafka -o jsonpath="{.items[0].metadata.name}")
+
+  # Set up port forwarding
+  kubectl port-forward pod/$KAFKA_POD 29092:29092 &
+  PF_PID=$!
+
+  # Wait for port forwarding to be established
+  sleep 5
+
+  KAFKA_POD_IP=$(kubectl get pod $KAFKA_POD -o jsonpath='{.status.podIP}')
+
+  # Update hosts file
+  echo "127.0.0.1 $KAFKA_POD" | sudo tee -a /etc/hosts
+
+  export KAFKA_HOST="127.0.0.1"
+  export KAFKA_PORT="29092"
+}
+
+function teardown_kafka_connection() {
+  kill $PF_PID
+  sudo sed -i "/$KAFKA_POD/d" /etc/hosts
+}
+
 ####MAIN###
-get_pod_names
-get_svc_names
-expose_services
+if [ "$1" = "teardown" ]; then
+  teardown_kafka_connection
+else
+  get_pod_names
+  get_svc_names
+  expose_services
+  setup_kafka_connection
+fi
\ No newline at end of file
diff --git a/csit/resources/scripts/robot_setup.sh b/csit/resources/scripts/robot_setup.sh
new file mode 100755 (executable)
index 0000000..f5ef2f3
--- /dev/null
@@ -0,0 +1,162 @@
+#!/bin/bash
+# ============LICENSE_START=======================================================
+#  Copyright (C) 2025 Nordix Foundation. All rights reserved.
+# ================================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+# ============LICENSE_END=========================================================
+
+WORKSPACE=$(git rev-parse --show-toplevel)
+export WORKSPACE
+
+export ROBOT_FILE=""
+
+PROJECT="$1"
+CSIT_SCRIPT="scripts/run-test.sh"
+ROBOT_DOCKER_IMAGE="policy-csit-robot"
+ROBOT_LOG_DIR=${WORKSPACE}/csit/archives
+
+# Source the shared config script
+source "$(dirname "$0")/config_setup.sh"
+
+DISTRIBUTION_CSAR=${WORKSPACE}/csit/resources/tests/data/csar
+DIST_TEMP_FOLDER=/tmp/distribution
+
+function clone_models() {
+    local retry_count=3
+    local success=false
+    cd tests
+    for ((i = 1; i <= retry_count; i++)); do
+        git clone "https://gerrit.onap.org/r/policy/models" && success=true && break
+        echo "Retrying git clone ($i/$retry_count)..."
+        sleep 5
+    done
+
+    cd ../
+    if [ "$success" = false ]; then
+        echo "Error: failed to clone policy-models repository after $retry_count attempts"
+        exit 1
+    fi
+
+    sed -e 's!Measurement_vGMUX!ADifferentValue!' \
+        tests/models/models-examples/src/main/resources/policies/vCPE.policy.monitoring.input.tosca.json \
+        >tests/models/models-examples/src/main/resources/policies/vCPE.policy.monitoring.input.tosca.v1_2.json
+
+    sed -e 's!"version": "1.0.0"!"version": "2.0.0"!' \
+        -e 's!"policy-version": 1!"policy-version": 2!' \
+        tests/models/models-examples/src/main/resources/policies/vCPE.policy.monitoring.input.tosca.json \
+        >tests/models/models-examples/src/main/resources/policies/vCPE.policy.monitoring.input.tosca.v2.json
+
+}
+
+function copy_csar_file() {
+    zip -F ${DISTRIBUTION_CSAR}/sample_csar_with_apex_policy.csar \
+        --out ${DISTRIBUTION_CSAR}/csar_temp.csar -q
+    sudo rm -rf "${DIST_TEMP_FOLDER}"
+    sudo mkdir "${DIST_TEMP_FOLDER}"
+    sudo cp ${DISTRIBUTION_CSAR}/csar_temp.csar ${DISTRIBUTION_CSAR}/temp.csar
+    sudo mv ${DISTRIBUTION_CSAR}/temp.csar ${DIST_TEMP_FOLDER}/sample_csar_with_apex_policy.csar
+}
+
+function build_robot_image() {
+    echo "Build docker image for robot framework"
+    cd ${WORKSPACE}/csit/resources || exit
+    clone_models
+    if [ "${PROJECT}" == "distribution" ] || [ "${PROJECT}" == "policy-distribution" ]; then
+        copy_csar_file
+    fi
+    echo "Build robot framework docker image"
+    sudo apt install gnupg2 pass -y
+    export DOCKERPW=docker
+    echo "$DOCKERPW" | docker login -u docker --password-stdin nexus3.onap.org:10001
+    docker build . --file Dockerfile \
+        --build-arg CSIT_SCRIPT="$CSIT_SCRIPT" \
+        --build-arg ROBOT_FILE="$ROBOT_FILE" \
+        --tag "${ROBOT_DOCKER_IMAGE}" --no-cache
+    echo "---------------------------------------------"
+}
+
+function push_acelement_chart() {
+    echo "Pushing acelement chart to the chartmuseum repo..."
+    helm repo add policy-chartmuseum http://localhost:30208
+    cd tests || exit
+    local retry_count=3
+    local success=false
+    for ((i = 1; i <= retry_count; i++)); do
+        git clone "https://gerrit.onap.org/r/policy/clamp" && success=true && break
+        echo "Retrying git clone ($i/$retry_count)..."
+        sleep 5
+    done
+
+    ACELEMENT_CHART=${WORKSPACE}/csit/resources/tests/clamp/examples/src/main/resources/clamp/acm/acelement-helm/acelement
+    helm cm-push $ACELEMENT_CHART policy-chartmuseum
+    helm repo update
+    rm -rf ${WORKSPACE}/csit/resources/tests/clamp/
+    echo "-------------------------------------------"
+}
+
+function print_robot_log() {
+    count_pods=0
+    while [[ ${count_pods} -eq 0 ]]; do
+        echo "Waiting for pods to come up..."
+        sleep 5
+        count_pods=$(kubectl get pods --output name | wc -l)
+    done
+    robotpod=$(kubectl get po | grep policy-csit)
+    podName=$(echo "$robotpod" | awk '{print $1}')
+    echo "The robot tests will begin once the policy components {${READINESS_CONTAINERS[*]}} are up and running..."
+    kubectl wait --for=jsonpath='{.status.phase}'=Running --timeout=18m pod/"$podName"
+    echo "Policy deployment status:"
+    kubectl get po
+    kubectl get all -A
+    echo "Robot Test logs:"
+    kubectl logs -f "$podName"
+}
+
+function start_csit() {
+    build_robot_image
+    if [ "${?}" -eq 0 ]; then
+        echo "Importing robot image into microk8s registry"
+        docker save -o policy-csit-robot.tar ${ROBOT_DOCKER_IMAGE}:latest
+        sudo microk8s ctr image import policy-csit-robot.tar
+        rm -rf ${WORKSPACE}/csit/resources/policy-csit-robot.tar
+        rm -rf ${WORKSPACE}/csit/resources/tests/models/
+        echo "---------------------------------------------"
+        if [ "$PROJECT" == "clamp" ] || [ "$PROJECT" == "policy-clamp" ]; then
+          POD_READY_STATUS="0/1"
+          while [[ ${POD_READY_STATUS} != "1/1" ]]; do
+            echo "Waiting for chartmuseum pod to come up..."
+            sleep 5
+            POD_READY_STATUS=$(kubectl get pods | grep -e "policy-chartmuseum" | awk '{print $2}')
+          done
+          push_acelement_chart
+        fi
+        echo "Installing Robot framework pod for running CSIT"
+        cd ${WORKSPACE}/helm || exit
+        mkdir -p ${ROBOT_LOG_DIR}
+        helm install csit-robot robot --set robot="$ROBOT_FILE" --set "readiness={$(echo ${READINESS_CONTAINERS} | sed 's/[{}]//g' | sed 's/,$//')}" --set robotLogDir=$ROBOT_LOG_DIR
+        print_robot_log
+    fi
+}
+
+if [ "$PROJECT" ]; then
+    set_project_config "$PROJECT"
+    export ROBOT_LOG_DIR=${WORKSPACE}/csit/archives/${PROJECT}
+    echo "CSIT will be invoked from $ROBOT_FILE"
+    echo "Readiness containers: ${READINESS_CONTAINERS[*]}"
+    echo "-------------------------------------------"
+    start_csit
+else
+    echo "No project supplied for running CSIT"
+fi
index 7d93fa8..2d5dfed 100755 (executable)
 # Runs CSITs in kubernetes.
 
 WORKSPACE=$(git rev-parse --show-toplevel)
-export WORKSPACE
 
-export GERRIT_BRANCH=$(awk -F= '$1 == "defaultbranch" { print $2 }' "${WORKSPACE}"/.gitreview)
-
-CSIT_SCRIPT="scripts/run-test.sh"
-ROBOT_DOCKER_IMAGE="policy-csit-robot"
-POLICY_CLAMP_ROBOT="policy-clamp-test.robot clamp-slas.robot"
-POLICY_API_ROBOT="api-test.robot api-slas.robot"
-POLICY_PAP_ROBOT="pap-test.robot pap-slas.robot"
-POLICY_APEX_PDP_ROBOT="apex-pdp-test.robot apex-slas.robot"
-POLICY_XACML_PDP_ROBOT="xacml-pdp-test.robot xacml-pdp-slas.robot"
-POLICY_OPA_PDP_ROBOT="opa-pdp-test.robot"
-POLICY_DROOLS_PDP_ROBOT="drools-pdp-test.robot"
-POLICY_DISTRIBUTION_ROBOT="distribution-test.robot"
-
-POLICY_API_CONTAINER="policy-api"
-POLICY_PAP_CONTAINER="policy-pap"
-POLICY_CLAMP_CONTAINER="policy-clamp-runtime-acm"
-POLICY_APEX_CONTAINER="policy-apex-pdp"
-POLICY_DROOLS_CONTAINER="policy-drools-pdp"
-POLICY_XACML_CONTAINER="policy-xacml-pdp"
-POLICY_OPA_CONTAINER="policy-opa-pdp"
-POLICY_DISTRIBUTION_CONTAINER="policy-distribution"
-POLICY_K8S_PPNT_CONTAINER="policy-clamp-ac-k8s-ppnt"
-POLICY_HTTP_PPNT_CONTAINER="policy-clamp-ac-http-ppnt"
-POLICY_SIM_PPNT_CONTAINER="policy-clamp-ac-sim-ppnt"
-POLICY_PF_PPNT_CONTAINER="policy-clamp-ac-pf-ppnt"
-JAEGER_CONTAINER="jaeger"
-KAFKA_CONTAINER="kafka-deployment"
-ZK_CONTAINER="zookeeper-deployment"
-KAFKA_DIR=${WORKSPACE}/helm/cp-kafka
-SET_VALUES=""
-
-DISTRIBUTION_CSAR=${WORKSPACE}/csit/resources/tests/data/csar
-DIST_TEMP_FOLDER=/tmp/distribution
-
-export PROJECT=""
-export ROBOT_FILE=""
-export ROBOT_LOG_DIR=${WORKSPACE}/csit/archives
-export READINESS_CONTAINERS=()
-
-
-function spin_microk8s_cluster() {
-    echo "Verify if Microk8s cluster is running.."
-    microk8s version
-    exitcode="${?}"
-
-    if [ "$exitcode" -ne 0 ]; then
-        echo "Microk8s cluster not available, Spinning up the cluster.."
-        sudo snap install microk8s --classic --channel=1.30/stable
-
-        if [ "${?}" -ne 0 ]; then
-            echo "Failed to install kubernetes cluster. Aborting.."
-            return 1
-        fi
-        echo "Microk8s cluster installed successfully"
-        sudo usermod -a -G microk8s $USER
-        echo "Enabling DNS and Storage plugins"
-        sudo microk8s.enable dns hostpath-storage
-        echo "Creating configuration file for Microk8s"
-        sudo mkdir -p $HOME/.kube
-        sudo chown -R $USER:$USER $HOME/.kube
-        sudo microk8s kubectl config view --raw >$HOME/.kube/config
-        sudo chmod 600 $HOME/.kube/config
-        echo "K8s installation completed"
-        echo "----------------------------------------"
-    else
-        echo "K8s cluster is already running"
-        echo "----------------------------------------"
-    fi
-
-    echo "Verify if kubectl is running.."
-    kubectl version
-    exitcode="${?}"
-
-    if [ "$exitcode" -ne 0 ]; then
-        echo "Kubectl not available, Installing.."
-        sudo snap install kubectl --classic --channel=1.30/stable
-
-        if [ "${?}" -ne 0 ]; then
-            echo "Failed to install Kubectl. Aborting.."
-            return 1
-        fi
-        echo "Kubectl installation completed"
-        echo "----------------------------------------"
-    else
-        echo "Kubectl is already running"
-        echo "----------------------------------------"
-        return 0
-    fi
-
-    echo "Verify if helm is running.."
-    helm version
-    exitcode="${?}"
-
-    if [ "$exitcode" -ne 0 ]; then
-        echo "Helm not available, Installing.."
-        sudo snap install helm --classic --channel=3.7
-
-        if [ "${?}" -ne 0 ]; then
-            echo "Failed to install Helm client. Aborting.."
-            return 1
-        fi
-        echo "Helm installation completed"
-        echo "----------------------------------------"
-    else
-        echo "Helm is already running"
-        echo "----------------------------------------"
-        return 0
-    fi
+function print_usage() {
+    echo "Usage: $0 [OPTIONS] OPERATION PROJECT"
+    echo ""
+    echo "OPTIONS:"
+    echo "  -c, --cluster-only    Install cluster only, without running robot tests"
+    echo "  -l, --local-image     Use local Docker image"
+    echo "  -h, --help            Display this help message"
+    echo ""
+    echo "OPERATION:"
+    echo "  install               Install the cluster and optionally run robot tests"
+    echo "  uninstall             Uninstall the policy deployment"
+    echo "  clean                 Teardown the cluster"
+    echo ""
+    echo "PROJECT:"
+    echo "  Specify the project name (e.g., clamp, api, pap, etc.)"
 }
 
-function install_kafka() {
-  echo "Installing Confluent kafka"
-  kubectl apply -f $KAFKA_DIR/zookeeper.yaml
-  kubectl apply -f $KAFKA_DIR/kafka.yaml
-  echo "----------------------------------------"
-}
-
-function uninstall_policy() {
-    echo "Removing the policy helm deployment"
-    helm uninstall csit-policy
-    helm uninstall prometheus
-    helm uninstall csit-robot
-    kubectl delete deploy $ZK_CONTAINER $KAFKA_CONTAINER
-    rm -rf ${WORKSPACE}/helm/policy/Chart.lock
-    if [ "$PROJECT" == "clamp" ] || [ "$PROJECT" == "policy-clamp" ]; then
-      helm uninstall policy-chartmuseum
-      helm repo remove chartmuseum-git policy-chartmuseum
-    fi
-    sudo rm -rf /dockerdata-nfs/mariadb-galera/
-    kubectl delete pvc --all
-    echo "Policy deployment deleted"
-    echo "Clean up docker"
-    docker image prune -f
-}
-
-function teardown_cluster() {
-    echo "Removing k8s cluster and k8s configuration file"
-    sudo snap remove microk8s;rm -rf $HOME/.kube/config
-    sudo snap remove helm;
-    sudo snap remove kubectl;
-    echo "MicroK8s Cluster removed"
-}
-
-function clone_models() {
-
-    # download models examples
-    git clone -b "${GERRIT_BRANCH}" --single-branch https://github.com/onap/policy-models.git "${WORKSPACE}"/csit/resources/tests/models
-
-    # create a couple of variations of the policy definitions
-    sed -e 's!Measurement_vGMUX!ADifferentValue!' \
-        tests/models/models-examples/src/main/resources/policies/vCPE.policy.monitoring.input.tosca.json \
-        >tests/models/models-examples/src/main/resources/policies/vCPE.policy.monitoring.input.tosca.v1_2.json
-
-    sed -e 's!"version": "1.0.0"!"version": "2.0.0"!' \
-        -e 's!"policy-version": 1!"policy-version": 2!' \
-        tests/models/models-examples/src/main/resources/policies/vCPE.policy.monitoring.input.tosca.json \
-        >tests/models/models-examples/src/main/resources/policies/vCPE.policy.monitoring.input.tosca.v2.json
-}
-
-function copy_csar_file() {
-    zip -F ${DISTRIBUTION_CSAR}/sample_csar_with_apex_policy.csar \
-        --out ${DISTRIBUTION_CSAR}/csar_temp.csar -q
-    # Remake temp directory
-    sudo rm -rf "${DIST_TEMP_FOLDER}"
-    sudo mkdir "${DIST_TEMP_FOLDER}"
-    sudo cp ${DISTRIBUTION_CSAR}/csar_temp.csar ${DISTRIBUTION_CSAR}/temp.csar
-    sudo mv ${DISTRIBUTION_CSAR}/temp.csar ${DIST_TEMP_FOLDER}/sample_csar_with_apex_policy.csar
-}
+CLUSTER_ONLY=false
+LOCAL_IMAGE=false
+
+# Parse command-line options
+while [[ $# -gt 0 ]]; do
+    case $1 in
+        -c|--cluster-only)
+            CLUSTER_ONLY=true
+            shift
+            ;;
+        -l|--local-image)
+            LOCAL_IMAGE=true
+            shift
+            ;;
+        -h|--help)
+            print_usage
+            exit 0
+            ;;
+        *)
+            break
+            ;;
+    esac
+done
 
-function build_robot_image() {
-    echo "Build docker image for robot framework"
-    cd ${WORKSPACE}/csit/resources || exit
-    clone_models
-    if [ "${PROJECT}" == "distribution" ] || [ "${PROJECT}" == "policy-distribution" ]; then
-        copy_csar_file
-    fi
-    echo "Build robot framework docker image"
-    docker login -u docker -p docker nexus3.onap.org:10001
-    docker build . --file Dockerfile \
-        --build-arg CSIT_SCRIPT="$CSIT_SCRIPT" \
-        --build-arg ROBOT_FILE="$ROBOT_FILE" \
-        --tag "${ROBOT_DOCKER_IMAGE}" --no-cache
-    echo "---------------------------------------------"
-}
+# Check for required arguments
+if [ $# -lt 2 ]; then
+    echo "Error: Insufficient arguments"
+    print_usage
+    exit 1
+fi
 
-function push_acelement_chart() {
-    echo "Pushing acelement chart to the chartmuseum repo..."
-    helm repo add policy-chartmuseum http://localhost:30208
+OPERATION=$1
+PROJECT=$2
 
-    # download clamp repo
-    git clone -b "${GERRIT_BRANCH}" --single-branch https://github.com/onap/policy-clamp.git "${WORKSPACE}"/csit/resources/tests/clamp
-    ACELEMENT_CHART=${WORKSPACE}/csit/resources/tests/clamp/examples/src/main/resources/clamp/acm/acelement-helm/acelement
-    helm cm-push $ACELEMENT_CHART policy-chartmuseum
-    helm repo update
-    rm -rf ${WORKSPACE}/csit/resources/tests/clamp/
-    echo "-------------------------------------------"
-}
-
-function print_robot_log() {
-    count_pods=0
-    while [[ ${count_pods} -eq 0 ]]; do
-        echo "Waiting for pods to come up..."
-        sleep 5
-        count_pods=$(kubectl get pods --output name | wc -l)
-    done
-    robotpod=$(kubectl get po | grep policy-csit)
-    podName=$(echo "$robotpod" | awk '{print $1}')
-    echo "The robot tests will begin once the policy components {${READINESS_CONTAINERS[*]}} are up and running..."
-    kubectl wait --for=jsonpath='{.status.phase}'=Running --timeout=18m pod/"$podName"
-    echo "Policy deployment status:"
-    kubectl get po
-    kubectl get all -A
-    echo "Robot Test logs:"
-    kubectl logs -f "$podName"
-}
+# Set local image flag
+if [ "$LOCAL_IMAGE" = true ]; then
+    LOCAL_IMAGE_ARG="true"
+else
+    LOCAL_IMAGE_ARG="false"
+fi
 
-function start_csit() {
-    build_robot_image
-    if [ "${?}" -eq 0 ]; then
-        echo "Importing robot image into microk8s registry"
-        docker save -o policy-csit-robot.tar ${ROBOT_DOCKER_IMAGE}:latest
-        sudo microk8s ctr image import policy-csit-robot.tar
-        rm -rf ${WORKSPACE}/csit/resources/policy-csit-robot.tar
-        rm -rf ${WORKSPACE}/csit/resources/tests/models/
-        echo "---------------------------------------------"
-        if [ "$PROJECT" == "clamp" ] || [ "$PROJECT" == "policy-clamp" ]; then
-          POD_READY_STATUS="0/1"
-          while [[ ${POD_READY_STATUS} != "1/1" ]]; do
-            echo "Waiting for chartmuseum pod to come up..."
-            sleep 5
-            POD_READY_STATUS=$(kubectl get pods | grep -e "policy-chartmuseum" | awk '{print $2}')
-          done
-          push_acelement_chart
+# Execute the appropriate script based on the operation
+case $OPERATION in
+    install)
+        ${WORKSPACE}/csit/resources/scripts/cluster_setup.sh install $PROJECT $LOCAL_IMAGE_ARG
+        if [ "$CLUSTER_ONLY" = false ]; then
+            ${WORKSPACE}/csit/resources/scripts/robot_setup.sh $PROJECT
         fi
-        echo "Installing Robot framework pod for running CSIT"
-        cd ${WORKSPACE}/helm
-        mkdir -p ${ROBOT_LOG_DIR}
-        helm install csit-robot robot --set robot="$ROBOT_FILE" --set "readiness={${READINESS_CONTAINERS[*]}}" --set robotLogDir=$ROBOT_LOG_DIR
-        print_robot_log
-    fi
-}
-
-function install_chartmuseum () {
-    echo "---------------------------------------------"
-    echo "Installing Chartmuseum helm repository..."
-    helm repo add chartmuseum-git https://chartmuseum.github.io/charts
-    helm repo update
-    helm install policy-chartmuseum chartmuseum-git/chartmuseum --set env.open.DISABLE_API=false --set service.type=NodePort --set service.nodePort=30208
-    helm plugin install https://github.com/chartmuseum/helm-push
-    echo "---------------------------------------------"
-}
-
-function set_project_config() {
-    echo "Setting project configuration for: $PROJECT"
-    case $PROJECT in
-
-    clamp | policy-clamp)
-        export ROBOT_FILE=$POLICY_CLAMP_ROBOT
-        export READINESS_CONTAINERS=($POLICY_CLAMP_CONTAINER,$POLICY_APEX_CONTAINER,$POLICY_PF_PPNT_CONTAINER,$POLICY_K8S_PPNT_CONTAINER,
-            $POLICY_HTTP_PPNT_CONTAINER,$POLICY_SIM_PPNT_CONTAINER,$JAEGER_CONTAINER)
-        export SET_VALUES="--set $POLICY_CLAMP_CONTAINER.enabled=true --set $POLICY_APEX_CONTAINER.enabled=true
-            --set $POLICY_PF_PPNT_CONTAINER.enabled=true --set $POLICY_K8S_PPNT_CONTAINER.enabled=true
-            --set $POLICY_HTTP_PPNT_CONTAINER.enabled=true --set $POLICY_SIM_PPNT_CONTAINER.enabled=true
-            --set $JAEGER_CONTAINER.enabled=true"
-        install_chartmuseum
         ;;
-
-    api | policy-api)
-        export ROBOT_FILE=$POLICY_API_ROBOT
-        export READINESS_CONTAINERS=($POLICY_API_CONTAINER)
+    uninstall)
+        ${WORKSPACE}/csit/resources/scripts/cluster_setup.sh uninstall
         ;;
-
-    pap | policy-pap)
-        export ROBOT_FILE=$POLICY_PAP_ROBOT
-        export READINESS_CONTAINERS=($POLICY_APEX_CONTAINER,$POLICY_PAP_CONTAINER,$POLICY_API_CONTAINER,$POLICY_XACML_CONTAINER)
-        export SET_VALUES="--set $POLICY_APEX_CONTAINER.enabled=true --set $POLICY_XACML_CONTAINER.enabled=true"
-        ;;
-
-    apex-pdp | policy-apex-pdp)
-        export ROBOT_FILE=$POLICY_APEX_PDP_ROBOT
-        export READINESS_CONTAINERS=($POLICY_APEX_CONTAINER,$POLICY_API_CONTAINER,$POLICY_PAP_CONTAINER)
-        export SET_VALUES="--set $POLICY_APEX_CONTAINER.enabled=true"
-        ;;
-
-    xacml-pdp | policy-xacml-pdp)
-        export ROBOT_FILE=($POLICY_XACML_PDP_ROBOT)
-        export READINESS_CONTAINERS=($POLICY_API_CONTAINER,$POLICY_PAP_CONTAINER,$POLICY_XACML_CONTAINER)
-        export SET_VALUES="--set $POLICY_XACML_CONTAINER.enabled=true"
-        ;;
-
-    opa-pdp | policy-opa-pdp)
-        export ROBOT_FILE=($POLICY_OPA_PDP_ROBOT)
-        export READINESS_CONTAINERS=($POLICY_API_CONTAINER,$POLICY_PAP_CONTAINER,$POLICY_OPA_CONTAINER)
-        export SET_VALUES="--set $POLICY_OPA_CONTAINER.enabled=true"
-        ;;
-
-    drools-pdp | policy-drools-pdp)
-        export ROBOT_FILE=($POLICY_DROOLS_PDP_ROBOT)
-        export READINESS_CONTAINERS=($POLICY_DROOLS_CONTAINER)
-        export SET_VALUES="--set $POLICY_DROOLS_CONTAINER.enabled=true"
-        ;;
-
-    distribution | policy-distribution)
-        export ROBOT_FILE=($POLICY_DISTRIBUTION_ROBOT)
-        export READINESS_CONTAINERS=($POLICY_APEX_CONTAINER,$POLICY_API_CONTAINER,$POLICY_PAP_CONTAINER,$POLICY_DISTRIBUTION_CONTAINER)
-        export SET_VALUES="--set $POLICY_APEX_CONTAINER.enabled=true --set $POLICY_DISTRIBUTION_CONTAINER.enabled=true"
+    clean)
+        ${WORKSPACE}/csit/resources/scripts/cluster_setup.sh clean
         ;;
-
     *)
-        echo "Unknown project supplied. Enabling all policy charts for the deployment"
-        export READINESS_CONTAINERS=($POLICY_APEX_CONTAINER,$POLICY_API_CONTAINER,$POLICY_PAP_CONTAINER,
-                    $POLICY_DISTRIBUTION_CONTAINER,$POLICY_DROOLS_CONTAINER,$POLICY_XACML_CONTAINER,$POLICY_OPA_CONTAINER,
-                    $POLICY_CLAMP_CONTAINER,$POLICY_PF_PPNT_CONTAINER,$POLICY_K8S_PPNT_CONTAINER,
-                    $POLICY_HTTP_PPNT_CONTAINER,$POLICY_SIM_PPNT_CONTAINER)
-        export SET_VALUES="--set $POLICY_APEX_CONTAINER.enabled=true --set $POLICY_XACML_CONTAINER.enabled=true
-            --set $POLICY_OPA_CONTAINER.enabled=true --set $POLICY_DISTRIBUTION_CONTAINER.enabled=true --set $POLICY_DROOLS_CONTAINER.enabled=true
-            --set $POLICY_CLAMP_CONTAINER.enabled=true --set $POLICY_PF_PPNT_CONTAINER.enabled=true
-            --set $POLICY_K8S_PPNT_CONTAINER.enabled=true --set $POLICY_HTTP_PPNT_CONTAINER.enabled=true
-            --set $POLICY_SIM_PPNT_CONTAINER.enabled=true"
+        echo "Error: Invalid operation"
+        print_usage
+        exit 1
         ;;
-    esac
-
-}
-
-function get_pod_name() {
-  pods=$(kubectl get pods --no-headers -o custom-columns=':metadata.name' | grep $1)
-  read -rd '' -a pod_array <<< "$pods"
-  echo "${pod_array[@]}"
-}
-
-function wait_for_pods_running() {
-  local namespace="$1"
-  shift
-  local timeout_seconds="$1"
-  shift
-
-  IFS=',' read -ra pod_names <<< "$@"
-  shift
-
-  local pending_pods=("${pod_names[@]}")
-  local start_time
-  start_time=$(date +%s)
-
-  while [ ${#pending_pods[@]} -gt 0 ]; do
-    local current_time
-    current_time=$(date +%s)
-    local elapsed_time
-    elapsed_time=$((current_time - start_time))
-
-    if [ "$elapsed_time" -ge "$timeout_seconds" ]; then
-      echo "Timed out waiting for the pods to reach 'Running' state."
-      echo "Printing the current status of the deployment before exiting.."
-      kubectl get po;
-      kubectl describe pods;
-      echo "------------------------------------------------------------"
-      for pod in "${pending_pods[@]}"; do
-        echo "Logs of the pod $pod"
-        kubectl logs $pod
-        echo "---------------------------------------------------------"
-      done
-      exit 1
-    fi
-
-    local newly_running_pods=()
-
-    for pod_name_prefix in "${pending_pods[@]}"; do
-      local pod_names=$(get_pod_name "$pod_name_prefix")
-      IFS=' ' read -r -a pod_array <<< "$pod_names"
-      if [ "${#pod_array[@]}" -eq 0 ]; then
-             echo "*** Error: No pods found for the deployment $pod_name_prefix . Exiting ***"
-             return -1
-      fi
-      for pod in "${pod_array[@]}"; do
-         local pod_status
-         local pod_ready
-         pod_status=$(kubectl get pod "$pod" -n "$namespace" --no-headers -o custom-columns=STATUS:.status.phase 2>/dev/null)
-         pod_ready=$(kubectl get pod "$pod" -o jsonpath='{.status.containerStatuses[*].ready}')
-
-         if [ "$pod_status" == "Running" ] && { [ "$pod_ready" == "true" ] || [ "$pod_ready" == "true true" ]; }; then
-           echo "Pod '$pod' in namespace '$namespace' is now in 'Running' state and 'Readiness' is true"
-         else
-           newly_running_pods+=("$pod")
-           echo "Waiting for pod '$pod' in namespace '$namespace' to reach 'Running' and 'Ready' state..."
-         fi
-
-      done
-    done
-
-    pending_pods=("${newly_running_pods[@]}")
-
-    sleep 5
-  done
-
-  echo "All specified pods are in the 'Running and Ready' state. Exiting the function."
-}
-
-OPERATION="$1"
-PROJECT="$2"
-if [ -z "$3" ]
-then
-    LOCALIMAGE="false"
-else
-    LOCALIMAGE="$3"
-fi
-
-
-if [ $OPERATION == "install" ]; then
-    spin_microk8s_cluster
-    if [ "${?}" -eq 0 ]; then
-        export KAFKA_CONTAINERS=($KAFKA_CONTAINER,$ZK_CONTAINER)
-        install_kafka
-        wait_for_pods_running default 300 $KAFKA_CONTAINERS
-        set_project_config
-        echo "Installing policy helm charts in the default namespace"
-        source ${WORKSPACE}/compose/get-k8s-versions.sh
-        if [ $LOCALIMAGE == "true" ]; then
-            echo "loading local image"
-            source ${WORKSPACE}/compose/get-versions.sh
-            ${WORKSPACE}/compose/loaddockerimage.sh
-        fi
-        cd ${WORKSPACE}/helm || exit
-        helm dependency build policy
-        helm install csit-policy policy ${SET_VALUES}
-        helm install prometheus prometheus
-        wait_for_pods_running default 900 ${READINESS_CONTAINERS[@]}
-        echo "Policy chart installation completed"
-        echo "-------------------------------------------"
-    fi
-
-    if [ "$PROJECT" ]; then
-        export ROBOT_LOG_DIR=${WORKSPACE}/csit/archives/${PROJECT}
-        echo "CSIT will be invoked from $ROBOT_FILE"
-        echo "Readiness containers: ${READINESS_CONTAINERS[*]}"
-        echo "-------------------------------------------"
-        start_csit
-    else
-        echo "No project supplied for running CSIT"
-    fi
-
-elif [ $OPERATION == "uninstall" ]; then
-    uninstall_policy
-
-elif [ $OPERATION == "clean" ]; then
-    teardown_cluster
-
-else
-    echo "Invalid arguments provided. Usage: $0 [options..] {install {project_name} | uninstall | clean} {uselocalimage = true/false}"
-fi
+esac
diff --git a/csit/run-s3p-tests.sh b/csit/run-s3p-tests.sh
new file mode 100755 (executable)
index 0000000..4dce3b7
--- /dev/null
@@ -0,0 +1,165 @@
+#!/bin/bash
+# ============LICENSE_START=======================================================
+#  Copyright (C) 2023-2025 Nordix Foundation. All rights reserved.
+# ================================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+# ============LICENSE_END=========================================================
+
+# This script will be used to automatically trigger the S3P
+# tests for policy components.
+
+script_start_time=$(date +%s)
+log_file="${TESTDIR:-$(pwd)}/s3p_test_log_$(date +%Y%m%d_%H%M%S).log"
+files_processed=0
+errors_encountered=0
+warnings_issued=0
+
+# Function to log messages
+log_message() {
+    local level="$1"
+    local message="$2"
+    echo "[$(date '+%Y-%m-%d %H:%M:%S')] [$level] $message" | tee -a "$log_file"
+}
+
+# Start Kubernetes
+function start_kubernetes() {
+    log_message "INFO" "Starting Kubernetes cluster for $PROJECT"
+    bash resources/scripts/cluster_setup.sh install $PROJECT
+    if [ $? -eq 0 ]; then
+        log_message "INFO" "Kubernetes cluster started successfully"
+    else
+        log_message "ERROR" "Failed to start Kubernetes cluster"
+        ((errors_encountered++))
+    fi
+    bash resources/scripts/get-cluster-info.sh
+}
+
+function install_jmeter() {
+    log_message "INFO" "Installing JMeter"
+    cd ${TESTDIR}/automate-s3p-test || { log_message "ERROR" "Failed to change directory"; ((errors_encountered++)); return 1; }
+
+    sudo apt-get update
+    sudo apt install curl -y
+    sudo apt install -y default-jdk
+
+    curl -O https://archive.apache.org/dist/jmeter/binaries/apache-jmeter-5.6.2.tgz
+    tar -xvf apache-jmeter-5.6.2.tgz
+    mv apache-jmeter-5.6.2 apache-jmeter
+
+    echo 'export JVM_ARGS="-Xms2g -Xmx4g"' > apache-jmeter/bin/setenv.sh
+    echo 'export HEAP="-Xms1G -Xmx2G -XX:MaxMetaspaceSize=512m"' >> apache-jmeter/bin/setenv.sh
+
+    rm -rf apache-jmeter/docs apache-jmeter/printable_docs
+
+    cd apache-jmeter/lib || { log_message "ERROR" "Failed to change directory"; ((errors_encountered++)); return 1; }
+    curl -O https://repo1.maven.org/maven2/kg/apc/cmdrunner/2.2.1/cmdrunner-2.2.1.jar
+    curl -O https://repo1.maven.org/maven2/org/apache/kafka/kafka-clients/3.9.0/kafka-clients-3.9.0.jar
+    curl -O https://repo1.maven.org/maven2/org/apache/kafka/kafka_2.13/3.9.0/kafka_2.13-3.9.0.jar
+
+    sudo cp -r ../../apache-jmeter /opt/
+
+    export JMETER_HOME="/opt/apache-jmeter"
+    export PATH="$JMETER_HOME/bin:$PATH"
+
+    log_message "INFO" "JMeter installation completed"
+    ((files_processed+=7))
+}
+
+function on_exit() {
+    local exit_status=$?
+    local end_time=$(date +%s)
+    local runtime=$((end_time - script_start_time))
+
+    log_message "INFO" "=============== Exit Report ==============="
+    log_message "INFO" "Script execution completed at $(date)"
+    log_message "INFO" "Exit status: $exit_status"
+    log_message "INFO" "Total runtime: $runtime seconds"
+    log_message "INFO" "Operations summary:"
+    log_message "INFO" "  - Files processed: $files_processed"
+    log_message "INFO" "  - Errors encountered: $errors_encountered"
+    log_message "INFO" "  - Warnings issued: $warnings_issued"
+    log_message "INFO" "Resource usage:"
+    ps -p $$ -o %cpu,%mem,etime >> "$log_file"
+    log_message "INFO" "Full log available at: $log_file"
+    log_message "INFO" "============================================"
+}
+
+function show_usage() {
+    echo "Usage: $0 [option] {test <jmx_file> | clean}"
+    echo "Options:"
+    echo "  test <jmx_file>  Start the environment and run the specified JMX test plan"
+    echo "  clean            Uninstall the environment and remove temporary folders"
+}
+
+function teardown() {
+    log_message "INFO" "Starting teardown process"
+
+    log_message "INFO" "Tearing down Kubernetes cluster"
+    bash resources/scripts/cluster_setup.sh uninstall
+
+    log_message "INFO" "Deleting created services"
+    microk8s kubectl get svc | awk '/svc/{system("microk8s kubectl delete svc " $1)}'
+
+    log_message "INFO" "Teardown process completed"
+}
+
+function main() {
+    PROJECT="$3"
+    case "$1" in
+        clean)
+            log_message "INFO" "Uninstalling environment and removing temp folders"
+            teardown
+            ;;
+        test)
+            if [ -z "$2" ]; then
+                log_message "ERROR" "JMX file not specified for test option"
+                show_usage
+                ((errors_encountered++))
+                exit 1
+            fi
+            log_message "INFO" "Starting K8s Environment"
+            start_kubernetes
+
+            log_message "INFO" "Installing JMeter"
+            install_jmeter
+
+            log_message "INFO" "Executing tests"
+            cd "${TESTDIR}/automate-s3p-test" || { log_message "ERROR" "Failed to change directory"; ((errors_encountered++)); exit 1; }
+            nohup jmeter -n -t "$2" -l s3pTestResults.jtl
+            if [ $? -eq 0 ]; then
+                log_message "INFO" "JMeter test completed successfully"
+                ((files_processed++))
+            else
+                log_message "ERROR" "JMeter test failed"
+                ((errors_encountered++))
+            fi
+            ;;
+        *)
+            log_message "WARNING" "Invalid option provided"
+            show_usage
+            ((warnings_issued++))
+            exit 1
+            ;;
+    esac
+}
+
+# Set TESTDIR if not already set
+TESTDIR=${TESTDIR:-$(pwd)}
+
+# Set up trap for exit
+trap on_exit EXIT
+
+# Call the main function with all script arguments
+main "$@"
diff --git a/csit/start-s3p-tests.sh b/csit/start-s3p-tests.sh
deleted file mode 100755 (executable)
index 4197460..0000000
+++ /dev/null
@@ -1,116 +0,0 @@
-#!/bin/bash
-# ============LICENSE_START=======================================================
-#  Copyright (C) 2023 Nordix Foundation. All rights reserved.
-# ================================================================================
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# SPDX-License-Identifier: Apache-2.0
-# ============LICENSE_END=========================================================
-
-# This script will be used to automatically trigger the S3P
-# tests for policy components.
-
-# Start Kubernetes
-function start_kubernetes() {
-  bash run-k8s-csit.sh install
-  bash resources/scripts/get-cluster-info.sh
-}
-
-function install_jmeter() {
-
-  #NOTE: $TESTDIR is set by the component triggering this script
-  cd ${TESTDIR}/automate-s3p-test
-
-  sudo apt-get update
-
-  # Install curl
-  sudo apt install curl -y
-
-  # Install JDK
-  sudo apt install -y default-jdk
-
-  # Install JMeter
-  curl -O https://archive.apache.org/dist/jmeter/binaries/apache-jmeter-5.6.2.tgz
-  tar -xvf apache-jmeter-5.6.2.tgz
-  mv apache-jmeter-5.6.2 apache-jmeter
-
-  # Define your desired heap size values
-  echo 'export JVM_ARGS="-Xms2g -Xmx4g"' > apache-jmeter/bin/setenv.sh
-  echo 'export HEAP="-Xms1G -Xmx2G -XX:MaxMetaspaceSize=512m"' >> apache-jmeter/bin/setenv.sh
-
-
-  # Remove unnecessary files
-  rm -rf apache-jmeter/docs apache-jmeter/printable_docs
-
-  # Install CMD Runner
-  cd apache-jmeter/lib
-  curl -O https://repo1.maven.org/maven2/kg/apc/cmdrunner/2.2.1/cmdrunner-2.2.1.jar
-
-  # Move JMeter to /opt
-  sudo cp -r ../../apache-jmeter /opt/
-
-  # Add JMeter Path Variable
-  export JMETER_HOME="/opt/apache-jmeter"
-  export PATH="$JMETER_HOME/bin:$PATH"
-}
-
-function on_exit() {
-  # TODO: Generate report
-  echo "Generating report..."
-}
-
-function teardown() {
-  echo "Removing temp directories.."
-
-  rm -r ${TESTDIR}/automate-s3p-test
-
-  echo "Removed directories"
-
-  echo "Tearing down kubernetes cluster..."
-  bash run-k8s-csit.sh uninstall
-
-  # DELETE created services
-  microk8s kubectl get svc | awk '/svc/{system("microk8s kubectl delete svc " $1)}'
-}
-
-#===MAIN===#
-
-if [ $1 == "run" ]
-then
-
-  echo "==========================="
-  echo "Starting K8s Environment"
-  echo "==========================="
-  start_kubernetes
-
-  echo "==========================="
-  echo "Installing JMeter"
-  echo "==========================="
-  install_jmeter
-
-  # Run the JMX test plan
-  echo "==========================="
-  echo "Executing tests"
-  echo "==========================="
-  cd ${TESTDIR}/automate-s3p-test || exit
-  nohup jmeter -n -t $2 -l s3pTestResults.jtl
-
-  # TODO: Generate report on on_exit()
-
-elif [ $1 == "uninstall" ]
-then
-  echo "Uninstalling environment and removing temp folders..."
-  teardown
-else
-  echo "Invalid arguments provided. Usage: $0 [option..] {run | uninstall}"
-fi