csit/run-k8s-csit.sh

   1 #!/bin/bash
   2 #
   3 # ============LICENSE_START====================================================
   4 #  Copyright (C) 2022-2024 Nordix Foundation.
   5 # =============================================================================
   6 # Licensed under the Apache License, Version 2.0 (the "License");
   7 # you may not use this file except in compliance with the License.
   8 # You may obtain a copy of the License at
   9 #
  10 #      http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 # Unless required by applicable law or agreed to in writing, software
  13 # distributed under the License is distributed on an "AS IS" BASIS,
  14 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15 # See the License for the specific language governing permissions and
  16 # limitations under the License.
  17 #
  18 # SPDX-License-Identifier: Apache-2.0
  19 # ============LICENSE_END======================================================
  20
  21 # This script spins up kubernetes cluster in Microk8s for deploying policy helm charts.
  22 # Runs CSITs in kubernetes.
  23
  24 WORKSPACE=$(git rev-parse --show-toplevel)
  25 export WORKSPACE
  26
  27 export GERRIT_BRANCH=$(awk -F= '$1 == "defaultbranch" { print $2 }' "${WORKSPACE}"/.gitreview)
  28
  29 CSIT_SCRIPT="scripts/run-test.sh"
  30 ROBOT_DOCKER_IMAGE="policy-csit-robot"
  31 POLICY_CLAMP_ROBOT="policy-clamp-test.robot"
  32 POLICY_API_ROBOT="api-test.robot api-slas.robot"
  33 POLICY_PAP_ROBOT="pap-test.robot pap-slas.robot"
  34 POLICY_APEX_PDP_ROBOT="apex-pdp-test.robot apex-slas.robot"
  35 POLICY_XACML_PDP_ROBOT="xacml-pdp-test.robot"
  36 POLICY_DROOLS_PDP_ROBOT="drools-pdp-test.robot"
  37 POLICY_DISTRIBUTION_ROBOT="distribution-test.robot"
  38
  39 POLICY_API_CONTAINER="policy-api"
  40 POLICY_PAP_CONTAINER="policy-pap"
  41 POLICY_CLAMP_CONTAINER="policy-clamp-runtime-acm"
  42 POLICY_APEX_CONTAINER="policy-apex-pdp"
  43 POLICY_DROOLS_CONTAINER="policy-drools-pdp"
  44 POLICY_XACML_CONTAINER="policy-xacml-pdp"
  45 POLICY_DISTRIBUTION_CONTAINER="policy-distribution"
  46 POLICY_K8S_PPNT_CONTAINER="policy-clamp-ac-k8s-ppnt"
  47 POLICY_HTTP_PPNT_CONTAINER="policy-clamp-ac-http-ppnt"
  48 POLICY_SIM_PPNT_CONTAINER="policy-clamp-ac-sim-ppnt"
  49 POLICY_PF_PPNT_CONTAINER="policy-clamp-ac-pf-ppnt"
  50 KAFKA_CONTAINER="kafka-deployment"
  51 ZK_CONTAINER="zookeeper-deployment"
  52 KAFKA_DIR=${WORKSPACE}/helm/cp-kafka
  53 SET_VALUES=""
  54
  55 DISTRIBUTION_CSAR=${WORKSPACE}/csit/resources/tests/data/csar
  56 DIST_TEMP_FOLDER=/tmp/distribution
  57
  58 export PROJECT=""
  59 export ROBOT_FILE=""
  60 export ROBOT_LOG_DIR=${WORKSPACE}/csit/archives
  61 export READINESS_CONTAINERS=()
  62
  63
  64 function spin_microk8s_cluster() {
  65     echo "Verify if Microk8s cluster is running.."
  66     microk8s version
  67     exitcode="${?}"
  68
  69     if [ "$exitcode" -ne 0 ]; then
  70         echo "Microk8s cluster not available, Spinning up the cluster.."
  71         sudo snap install microk8s --classic --channel=1.26/stable
  72
  73         if [ "${?}" -ne 0 ]; then
  74             echo "Failed to install kubernetes cluster. Aborting.."
  75             return 1
  76         fi
  77         echo "Microk8s cluster installed successfully"
  78         sudo usermod -a -G microk8s $USER
  79         echo "Enabling DNS and helm3 plugins"
  80         sudo microk8s.enable dns helm3 hostpath-storage
  81         echo "Creating configuration file for Microk8s"
  82         sudo mkdir -p $HOME/.kube
  83         sudo chown -R $USER:$USER $HOME/.kube
  84         sudo microk8s kubectl config view --raw >$HOME/.kube/config
  85         sudo chmod 600 $HOME/.kube/config
  86         echo "K8s installation completed"
  87         echo "----------------------------------------"
  88     else
  89         echo "K8s cluster is already running"
  90         echo "----------------------------------------"
  91     fi
  92
  93     echo "Verify if kubectl is running.."
  94     kubectl version
  95     exitcode="${?}"
  96
  97     if [ "$exitcode" -ne 0 ]; then
  98         echo "Kubectl not available, Spinning up the cluster.."
  99         sudo snap install kubectl --classic --channel=1.26/stable
 100
 101         if [ "${?}" -ne 0 ]; then
 102             echo "Failed to install Kubectl. Aborting.."
 103             return 1
 104         fi
 105         echo "Kubectl installation completed"
 106         echo "----------------------------------------"
 107     else
 108         echo "Kubectl is already running"
 109         echo "----------------------------------------"
 110         return 0
 111     fi
 112 }
 113
 114 function install_kafka() {
 115   echo "Installing Confluent kafka"
 116   kubectl apply -f $KAFKA_DIR/zookeeper.yaml
 117   kubectl apply -f $KAFKA_DIR/kafka.yaml
 118   echo "----------------------------------------"
 119 }
 120
 121 function uninstall_policy() {
 122     echo "Removing the policy helm deployment"
 123     sudo microk8s helm uninstall csit-policy
 124     sudo microk8s helm uninstall prometheus
 125     sudo microk8s helm uninstall csit-robot
 126     sudo kubectl delete deploy $ZK_CONTAINER $KAFKA_CONTAINER
 127     rm -rf ${WORKSPACE}/helm/policy/Chart.lock
 128     if [ "$PROJECT" == "clamp" ] || [ "$PROJECT" == "policy-clamp" ]; then
 129       sudo microk8s helm uninstall policy-chartmuseum
 130       sudo microk8s helm repo remove chartmuseum-git policy-chartmuseum
 131     fi
 132     sudo rm -rf /dockerdata-nfs/mariadb-galera/
 133     sudo microk8s kubectl delete pvc --all
 134     echo "Policy deployment deleted"
 135     echo "Clean up docker"
 136     docker image prune -f
 137 }
 138
 139 function teardown_cluster() {
 140     echo "Removing k8s cluster and k8s configuration file"
 141     sudo snap remove microk8s;rm -rf $HOME/.kube/config
 142     echo "MicroK8s Cluster removed"
 143 }
 144
 145 function build_robot_image() {
 146     echo "Build docker image for robot framework"
 147     cd ${WORKSPACE}/csit/resources || exit
 148     clone_models
 149     if [ "${PROJECT}" == "distribution" ] || [ "${PROJECT}" == "policy-distribution" ]; then
 150         copy_csar_file
 151     fi
 152     echo "Build robot framework docker image"
 153     docker login -u docker -p docker nexus3.onap.org:10001
 154     docker build . --file Dockerfile \
 155         --build-arg CSIT_SCRIPT="$CSIT_SCRIPT" \
 156         --build-arg ROBOT_FILE="$ROBOT_FILE" \
 157         --tag "${ROBOT_DOCKER_IMAGE}" --no-cache
 158     echo "---------------------------------------------"
 159 }
 160
 161 function start_csit() {
 162     build_robot_image
 163     if [ "${?}" -eq 0 ]; then
 164         echo "Importing robot image into microk8s registry"
 165         docker save -o policy-csit-robot.tar ${ROBOT_DOCKER_IMAGE}:latest
 166         sudo microk8s ctr image import policy-csit-robot.tar
 167         rm -rf ${WORKSPACE}/csit/resources/policy-csit-robot.tar
 168         rm -rf ${WORKSPACE}/csit/resources/tests/models/
 169         echo "---------------------------------------------"
 170         if [ "$PROJECT" == "clamp" ] || [ "$PROJECT" == "policy-clamp" ]; then
 171           POD_READY_STATUS="0/1"
 172           while [[ ${POD_READY_STATUS} != "1/1" ]]; do
 173             echo "Waiting for chartmuseum pod to come up..."
 174             sleep 5
 175             POD_READY_STATUS=$(sudo microk8s kubectl get pods | grep -e "policy-chartmuseum" | awk '{print $2}')
 176           done
 177           push_acelement_chart
 178         fi
 179         echo "Installing Robot framework pod for running CSIT"
 180         cd ${WORKSPACE}/helm
 181         mkdir -p ${ROBOT_LOG_DIR}
 182         sudo microk8s helm install csit-robot robot --set robot="$ROBOT_FILE" --set "readiness={${READINESS_CONTAINERS[*]}}" --set robotLogDir=$ROBOT_LOG_DIR
 183         print_robot_log
 184     fi
 185 }
 186
 187 function print_robot_log() {
 188     count_pods=0
 189     while [[ ${count_pods} -eq 0 ]]; do
 190         echo "Waiting for pods to come up..."
 191         sleep 5
 192         count_pods=$(sudo microk8s kubectl get pods --output name | wc -l)
 193     done
 194     robotpod=$(sudo microk8s kubectl get po | grep policy-csit)
 195     podName=$(echo "$robotpod" | awk '{print $1}')
 196     echo "The robot tests will begin once the policy components {${READINESS_CONTAINERS[*]}} are up and running..."
 197     sudo microk8s kubectl wait --for=jsonpath='{.status.phase}'=Running --timeout=18m pod/"$podName"
 198     echo "Policy deployment status:"
 199     sudo microk8s kubectl get po
 200     sudo microk8s kubectl get all -A
 201     echo "Robot Test logs:"
 202     sudo microk8s kubectl logs -f "$podName"
 203 }
 204
 205 function clone_models() {
 206
 207     # download models examples
 208     git clone -b "${GERRIT_BRANCH}" --single-branch https://github.com/onap/policy-models.git "${WORKSPACE}"/csit/resources/tests/models
 209
 210     # create a couple of variations of the policy definitions
 211     sed -e 's!Measurement_vGMUX!ADifferentValue!' \
 212         tests/models/models-examples/src/main/resources/policies/vCPE.policy.monitoring.input.tosca.json \
 213         >tests/models/models-examples/src/main/resources/policies/vCPE.policy.monitoring.input.tosca.v1_2.json
 214
 215     sed -e 's!"version": "1.0.0"!"version": "2.0.0"!' \
 216         -e 's!"policy-version": 1!"policy-version": 2!' \
 217         tests/models/models-examples/src/main/resources/policies/vCPE.policy.monitoring.input.tosca.json \
 218         >tests/models/models-examples/src/main/resources/policies/vCPE.policy.monitoring.input.tosca.v2.json
 219 }
 220
 221 function copy_csar_file() {
 222     zip -F ${DISTRIBUTION_CSAR}/sample_csar_with_apex_policy.csar \
 223         --out ${DISTRIBUTION_CSAR}/csar_temp.csar -q
 224     # Remake temp directory
 225     sudo rm -rf "${DIST_TEMP_FOLDER}"
 226     sudo mkdir "${DIST_TEMP_FOLDER}"
 227     sudo cp ${DISTRIBUTION_CSAR}/csar_temp.csar ${DISTRIBUTION_CSAR}/temp.csar
 228     sudo mv ${DISTRIBUTION_CSAR}/temp.csar ${DIST_TEMP_FOLDER}/sample_csar_with_apex_policy.csar
 229 }
 230
 231 function set_project_config() {
 232     echo "Setting project configuration for: $PROJECT"
 233     case $PROJECT in
 234
 235     clamp | policy-clamp)
 236         export ROBOT_FILE=$POLICY_CLAMP_ROBOT
 237         export READINESS_CONTAINERS=($POLICY_CLAMP_CONTAINER,$POLICY_APEX_CONTAINER,$POLICY_PF_PPNT_CONTAINER,$POLICY_K8S_PPNT_CONTAINER,
 238             $POLICY_HTTP_PPNT_CONTAINER,$POLICY_SIM_PPNT_CONTAINER)
 239         export SET_VALUES="--set $POLICY_CLAMP_CONTAINER.enabled=true --set $POLICY_APEX_CONTAINER.enabled=true
 240             --set $POLICY_PF_PPNT_CONTAINER.enabled=true --set $POLICY_K8S_PPNT_CONTAINER.enabled=true
 241             --set $POLICY_HTTP_PPNT_CONTAINER.enabled=true --set $POLICY_SIM_PPNT_CONTAINER.enabled=true"
 242         install_chartmuseum
 243         ;;
 244
 245     api | policy-api)
 246         export ROBOT_FILE=$POLICY_API_ROBOT
 247         export READINESS_CONTAINERS=($POLICY_API_CONTAINER)
 248         ;;
 249
 250     pap | policy-pap)
 251         export ROBOT_FILE=$POLICY_PAP_ROBOT
 252         export READINESS_CONTAINERS=($POLICY_APEX_CONTAINER,$POLICY_PAP_CONTAINER,$POLICY_API_CONTAINER,$POLICY_XACML_CONTAINER)
 253         export SET_VALUES="--set $POLICY_APEX_CONTAINER.enabled=true --set $POLICY_XACML_CONTAINER.enabled=true"
 254         ;;
 255
 256     apex-pdp | policy-apex-pdp)
 257         export ROBOT_FILE=$POLICY_APEX_PDP_ROBOT
 258         export READINESS_CONTAINERS=($POLICY_APEX_CONTAINER,$POLICY_API_CONTAINER,$POLICY_PAP_CONTAINER)
 259         export SET_VALUES="--set $POLICY_APEX_CONTAINER.enabled=true"
 260         ;;
 261
 262     xacml-pdp | policy-xacml-pdp)
 263         export ROBOT_FILE=($POLICY_XACML_PDP_ROBOT)
 264         export READINESS_CONTAINERS=($POLICY_API_CONTAINER,$POLICY_PAP_CONTAINER,$POLICY_XACML_CONTAINER)
 265         export SET_VALUES="--set $POLICY_XACML_CONTAINER.enabled=true"
 266         ;;
 267
 268     drools-pdp | policy-drools-pdp)
 269         export ROBOT_FILE=($POLICY_DROOLS_PDP_ROBOT)
 270         export READINESS_CONTAINERS=($POLICY_DROOLS_CONTAINER)
 271         export SET_VALUES="--set $POLICY_DROOLS_CONTAINER.enabled=true"
 272         ;;
 273
 274     distribution | policy-distribution)
 275         export ROBOT_FILE=($POLICY_DISTRIBUTION_ROBOT)
 276         export READINESS_CONTAINERS=($POLICY_APEX_CONTAINER,$POLICY_API_CONTAINER,$POLICY_PAP_CONTAINER,
 277             $POLICY_DISTRIBUTION_CONTAINER)
 278         export SET_VALUES="--set $POLICY_APEX_CONTAINER.enabled=true --set $POLICY_DISTRIBUTION_CONTAINER.enabled=true"
 279         ;;
 280
 281     *)
 282         echo "Unknown project supplied. Enabling all policy charts for the deployment"
 283         export READINESS_CONTAINERS=($POLICY_APEX_CONTAINER,$POLICY_API_CONTAINER,$POLICY_PAP_CONTAINER,
 284                     $POLICY_DISTRIBUTION_CONTAINER,$POLICY_DROOLS_CONTAINER,$POLICY_XACML_CONTAINER,
 285                     $POLICY_CLAMP_CONTAINER,$POLICY_PF_PPNT_CONTAINER,$POLICY_K8S_PPNT_CONTAINER,
 286                     $POLICY_HTTP_PPNT_CONTAINER,$POLICY_SIM_PPNT_CONTAINER)
 287         export SET_VALUES="--set $POLICY_APEX_CONTAINER.enabled=true --set $POLICY_XACML_CONTAINER.enabled=true
 288             --set $POLICY_DISTRIBUTION_CONTAINER.enabled=true --set $POLICY_DROOLS_CONTAINER.enabled=true
 289             --set $POLICY_CLAMP_CONTAINER.enabled=true --set $POLICY_PF_PPNT_CONTAINER.enabled=true
 290             --set $POLICY_K8S_PPNT_CONTAINER.enabled=true --set $POLICY_HTTP_PPNT_CONTAINER.enabled=true
 291             --set $POLICY_SIM_PPNT_CONTAINER.enabled=true"
 292         ;;
 293     esac
 294
 295 }
 296
 297 function install_chartmuseum () {
 298     echo "Installing Chartmuseum helm repository..."
 299     sudo microk8s helm repo add chartmuseum-git https://chartmuseum.github.io/charts
 300     sudo microk8s helm repo update
 301     sudo microk8s helm install policy-chartmuseum chartmuseum-git/chartmuseum --set env.open.DISABLE_API=false --set service.type=NodePort --set service.nodePort=30208
 302     sudo microk8s helm plugin install https://github.com/chartmuseum/helm-push
 303     echo "---------------------------------------------"
 304 }
 305
 306 function push_acelement_chart() {
 307     echo "Pushing acelement chart to the chartmuseum repo..."
 308     sudo microk8s helm repo add policy-chartmuseum http://localhost:30208
 309
 310     # download clamp repo
 311     git clone -b "${GERRIT_BRANCH}" --single-branch https://github.com/onap/policy-clamp.git "${WORKSPACE}"/csit/resources/tests/clamp
 312     ACELEMENT_CHART=${WORKSPACE}/csit/resources/tests/clamp/examples/src/main/resources/clamp/acm/acelement-helm/acelement
 313     sudo microk8s helm cm-push $ACELEMENT_CHART policy-chartmuseum
 314     sudo microk8s helm repo update
 315     rm -rf ${WORKSPACE}/csit/resources/tests/clamp/
 316     echo "-------------------------------------------"
 317 }
 318
 319 function get_pod_name() {
 320   microk8s kubectl get pods --no-headers -o custom-columns=':metadata.name' | grep $1
 321 }
 322
 323 wait_for_pods_running() {
 324   local namespace="$1"
 325   shift
 326   local timeout_seconds="$1"
 327   shift
 328
 329   IFS=',' read -ra pod_names <<< "$1"
 330   shift
 331
 332   local pending_pods=("${pod_names[@]}")
 333
 334   local start_time
 335   start_time=$(date +%s)
 336
 337   while [ ${#pending_pods[@]} -gt 0 ]; do
 338     local current_time
 339     current_time=$(date +%s)
 340     local elapsed_time
 341     elapsed_time=$((current_time - start_time))
 342
 343     if [ "$elapsed_time" -ge "$timeout_seconds" ]; then
 344       echo "Timed out waiting for all pods to reach 'Running' state."
 345       exit 1
 346     fi
 347
 348     local newly_running_pods=()
 349
 350     for pod_name_prefix in "${pending_pods[@]}"; do
 351       local pod_name=$(get_pod_name "$pod_name_prefix")
 352       local pod_status
 353       pod_status=$(kubectl get pod "$pod_name" -n "$namespace" --no-headers -o custom-columns=STATUS:.status.phase 2>/dev/null)
 354
 355       if [ "$pod_status" == "Running" ]; then
 356         echo "Pod '$pod_name' in namespace '$namespace' is now in 'Running' state."
 357       else
 358         newly_running_pods+=("$pod_name")
 359         echo "Waiting for pod '$pod_name' in namespace '$namespace' to reach 'Running' state..."
 360       fi
 361     done
 362
 363     pending_pods=("${newly_running_pods[@]}")
 364
 365     sleep 5
 366   done
 367
 368   echo "All specified pods are in the 'Running' state. Exiting the function."
 369 }
 370
 371
 372 OPERATION="$1"
 373 PROJECT="$2"
 374 if [ -z "$3" ]
 375 then
 376     LOCALIMAGE="false"
 377 else
 378     LOCALIMAGE="$3"
 379 fi
 380
 381
 382 if [ $OPERATION == "install" ]; then
 383     spin_microk8s_cluster
 384     if [ "${?}" -eq 0 ]; then
 385         export KAFKA_CONTAINERS=($KAFKA_CONTAINER,$ZK_CONTAINER)
 386         install_kafka
 387         wait_for_pods_running default 300 $KAFKA_CONTAINERS
 388         set_project_config
 389         echo "Installing policy helm charts in the default namespace"
 390         source ${WORKSPACE}/compose/get-k8s-versions.sh
 391         if [ $LOCALIMAGE == "true" ]; then
 392             echo "loading local image"
 393             source ${WORKSPACE}/compose/get-versions.sh
 394             ${WORKSPACE}/compose/loaddockerimage.sh
 395         fi
 396         cd ${WORKSPACE}/helm || exit
 397         sudo microk8s helm dependency build policy
 398         sudo microk8s helm install csit-policy policy ${SET_VALUES}
 399         sudo microk8s helm install prometheus prometheus
 400         wait_for_pods_running default 300 $READINESS_CONTAINERS
 401         echo "Policy chart installation completed"
 402         echo "-------------------------------------------"
 403     fi
 404
 405     if [ "$PROJECT" ]; then
 406         export ROBOT_LOG_DIR=${WORKSPACE}/csit/archives/${PROJECT}
 407         echo "CSIT will be invoked from $ROBOT_FILE"
 408         echo "Readiness containers: ${READINESS_CONTAINERS[*]}"
 409         echo "-------------------------------------------"
 410         start_csit
 411     else
 412         echo "No project supplied for running CSIT"
 413     fi
 414
 415 elif [ $OPERATION == "uninstall" ]; then
 416     uninstall_policy
 417
 418 elif [ $OPERATION == "clean" ]; then
 419     teardown_cluster
 420
 421 else
 422     echo "Invalid arguments provided. Usage: $0 [options..] {install {project_name} | uninstall | clean} {uselocalimage = true/false}"
 423 fi