tools/performance/cloud/cloud-based-performance-test.sh

   1 #!/usr/bin/env bash
   2 # ============LICENSE_START=======================================================
   3 # dcaegen2-collectors-veshv
   4 # ================================================================================
   5 # Copyright (C) 2019 NOKIA
   6 # ================================================================================
   7 # Licensed under the Apache License, Version 2.0 (the "License");
   8 # you may not use this file except in compliance with the License.
   9 # You may obtain a copy of the License at
  10 #
  11 #      http://www.apache.org/licenses/LICENSE-2.0
  12 #
  13 # Unless required by applicable law or agreed to in writing, software
  14 # distributed under the License is distributed on an "AS IS" BASIS,
  15 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16 # See the License for the specific language governing permissions and
  17 # limitations under the License.
  18 # ============LICENSE_END=========================================================
  19
  20 SCRIPT_DIRECTORY="$(pwd "$0")"
  21 CONTAINERS_COUNT=1
  22 COMPLETED_PRODUCERS_SUM=0
  23 LOAD_TEST="false"
  24 PROPERTIES_FILE=${SCRIPT_DIRECTORY}/test.properties
  25 PRODUCER_APPS_LABEL=hv-collector-producer
  26 CONSUMER_APPS_LABEL=hv-collector-kafka-consumer
  27 PROMETHEUS_CONF_LABEL=prometheus-server-conf
  28 PROMETHEUS_APPS_LABEL=hv-collector-prometheus
  29 GRAFANA_APPS_LABEL=hv-collector-grafana
  30 GRAFANA_DATASOURCE=grafana-datasources
  31 GRAFANA_DASHBOARD=grafana-dashboards
  32 GRAFANA_DASHBOARD_PROVIDERS=grafana-dashboards-providers
  33 ONAP_NAMESPACE=onap
  34 MAXIMUM_BACK_OFF_CHECK_ITERATIONS=30
  35 CHECK_NUMBER=0
  36 PRODUCERS_TO_RECREATE=0
  37 NAME_REASON_PATTERN="custom-columns=NAME:.metadata.name,REASON:.status.containerStatuses[].state.waiting.reason"
  38 HVVES_POD_NAME=$(kubectl -n ${ONAP_NAMESPACE} get pods --no-headers=true -o custom-columns=:metadata.name | grep hv-ves-collector)
  39 HVVES_CERT_PATH=/etc/ves-hv/ssl/
  40
  41 function clean() {
  42     echo "Cleaning up environment"
  43
  44     echo "Attempting to delete test parameters ConfigMap"
  45     kubectl delete configmap performance-test-config -n ${ONAP_NAMESPACE}
  46
  47     echo "Attempting to delete prometheus ConfigMap"
  48     kubectl delete configmap -l name=${PROMETHEUS_CONF_LABEL} -n ${ONAP_NAMESPACE}
  49
  50     echo "Attempting to delete prometheus deployment and service"
  51     kubectl delete service,deployments -l app=${PROMETHEUS_APPS_LABEL} -n ${ONAP_NAMESPACE}
  52
  53     echo "Attempting to delete grafana deployment and service"
  54     kubectl delete service,deployments -l app=${GRAFANA_APPS_LABEL} -n ${ONAP_NAMESPACE}
  55
  56     echo "Attempting to delete grafana ConfigMap(DASHBOARD)"
  57     kubectl delete configmap -l name=${GRAFANA_DASHBOARD} -n ${ONAP_NAMESPACE}
  58
  59     echo "Attempting to delete grafana ConfigMap(GRAFANA_DASHBOARD_PROVIDERS)"
  60     kubectl delete configmap -l name=${GRAFANA_DASHBOARD_PROVIDERS} -n ${ONAP_NAMESPACE}
  61
  62     echo "Attempting to delete grafana ConfigMap(GRAFANA_DATASOURCE)"
  63     kubectl delete configmap -l name=${GRAFANA_DATASOURCE} -n ${ONAP_NAMESPACE}
  64
  65     echo "Attempting to delete consumer deployments"
  66     kubectl delete deployments -l app=${CONSUMER_APPS_LABEL} -n ${ONAP_NAMESPACE}
  67
  68     echo "Attempting to delete producer pods"
  69     kubectl delete pods -l app=${PRODUCER_APPS_LABEL} -n ${ONAP_NAMESPACE}
  70
  71     echo "Attempting to delete client certs secret"
  72     kubectl delete secret cert -n ${ONAP_NAMESPACE}
  73
  74     echo "Attempting to turn off SSL"
  75     ./configure-consul.sh true
  76
  77     echo "Environment clean up finished!"
  78 }
  79
  80 function copy_certs_to_hvves() {
  81          cd ../../ssl
  82          for file in {trust.p12,trust.pass,server.p12,server.pass}
  83          do
  84        echo "Copying file: ${file}"
  85        kubectl cp ${file} ${ONAP_NAMESPACE}/${HVVES_POD_NAME}:${HVVES_CERT_PATH}
  86    done
  87 }
  88
  89 function create_producers() {
  90     echo "Recreating test properties ConfigMap from: $PROPERTIES_FILE"
  91     kubectl delete configmap performance-test-config -n ${ONAP_NAMESPACE}
  92     kubectl create configmap performance-test-config --from-env-file=${PROPERTIES_FILE} -n ${ONAP_NAMESPACE}
  93
  94     set -e
  95     for i in $(seq 1 ${CONTAINERS_COUNT});
  96     do
  97         echo "Creating ${i}/${CONTAINERS_COUNT} producer"
  98         kubectl create -f producer-pod.yaml -n ${ONAP_NAMESPACE}
  99     done
 100     echo "Producers created"
 101     set +e
 102 }
 103
 104 function generate_certs() {
 105     echo "Generation of certs"
 106     cd ../../ssl
 107     ./gen-certs.sh
 108 }
 109
 110 function handle_backoffs() {
 111         IMAGE_PULL_BACK_OFFS=$(kubectl get pods -l app=${PRODUCER_APPS_LABEL} -n ${ONAP_NAMESPACE} -o ${NAME_REASON_PATTERN} | grep -c "ImagePullBackOff \| ErrImagePull")
 112         if [[ ${IMAGE_PULL_BACK_OFFS} -gt 0 ]]; then
 113             CHECK_NUMBER=$((CHECK_NUMBER + 1))
 114             if [[ ${CHECK_NUMBER} -gt ${MAXIMUM_BACK_OFF_CHECK_ITERATIONS} ]]; then
 115                 echo "Error: Image pull problem"
 116                 exit 1
 117             fi
 118         fi
 119 }
 120
 121 function handle_key_interrupt() {
 122     trap SIGINT
 123     echo "Script interrupted, attempt to delete producers"
 124     echo "Wait with patience"
 125     COMPLETED_PRODUCERS_SUM=$(($(kubectl delete pods -l app=${PRODUCER_APPS_LABEL} -n ${ONAP_NAMESPACE} | grep producer | wc -l) + COMPLETED_PRODUCERS_SUM))
 126     echo "Total number of completed producers: ${COMPLETED_PRODUCERS_SUM}"
 127     exit 0
 128 }
 129
 130 function print_test_setup_info() {
 131     echo "Starting cloud based performance tests"
 132     echo "________________________________________"
 133     echo "Test configuration:"
 134     echo "Producer containers count: ${CONTAINERS_COUNT}"
 135     echo "Properties file path: ${PROPERTIES_FILE}"
 136     echo "________________________________________"
 137 }
 138
 139 function usage() {
 140     echo ""
 141     echo "Run cloud based HV-VES performance test"
 142     echo "Usage $0 gen_certs|setup|start|clean|help"
 143     echo "  gen_certs: generate certs in ../../ssl directory"
 144     echo "  setup    : set up ConfigMap and consumers"
 145     echo "  start    : create producers - start the performance test"
 146     echo "    Optional parameters:"
 147     echo "      --load      : should test keep defined containers number till script interruption (false)"
 148     echo "      --containers      : number of producer containers to create (1)"
 149     echo "      --properties-file : path to file with benchmark properties (./test.properties)"
 150     echo "  clean    : remove ConfigMap, HV-VES consumers and producers"
 151     echo "  help     : print usage"
 152     echo "Example invocations:"
 153     echo "./cloud-based-performance-test.sh gen_certs"
 154     echo "./cloud-based-performance-test.sh setup"
 155     echo "./cloud-based-performance-test.sh start"
 156     echo "./cloud-based-performance-test.sh start --containers 10"
 157     echo "./cloud-based-performance-test.sh start --load true --containers 10"
 158     echo "./cloud-based-performance-test.sh start --properties-file ~/other_test.properties"
 159     echo "./cloud-based-performance-test.sh clean"
 160     exit 1
 161 }
 162
 163 function setup_environment() {
 164     echo "Setting up environment"
 165     echo "Copying certs to hv-ves pod"
 166     copy_certs_to_hvves
 167
 168     echo "Creating secrets with clients cert"
 169     kubectl create secret generic cert --from-file=./client.p12 --from-file=./client.pass -n ${ONAP_NAMESPACE}
 170     cd ${SCRIPT_DIRECTORY}
 171
 172     echo "Turning on SSL"
 173     ./configure-consul.sh false
 174
 175     echo "Creating test properties ConfigMap from: $PROPERTIES_FILE"
 176     kubectl create configmap performance-test-config --from-env-file=${PROPERTIES_FILE} -n ${ONAP_NAMESPACE}
 177
 178     echo "Creating consumer deployment"
 179     kubectl apply -f consumer-deployment.yaml
 180
 181     echo "Creating ConfigMap for prometheus deployment"
 182     kubectl apply -f prometheus/prometheus-config-map.yaml
 183
 184     echo "Creating prometheus deployment"
 185     kubectl apply -f prometheus-deployment.yaml
 186
 187     echo "Creating ConfigMap for grafana connections dashboard"
 188     kubectl apply -f grafana/dashboards/connections.yaml
 189
 190     echo "Creating ConfigMap for grafana processing dashboard"
 191     kubectl apply -f grafana/dashboards/processing.yaml
 192
 193     echo "Creating ConfigMap for grafana grafana-performance-tests dashboard"
 194     kubectl apply -f grafana/dashboards/performance-tests.yaml
 195
 196     echo "Creating ConfigMap for grafana datasource"
 197     kubectl apply -f grafana/datasources/datasource.yaml
 198
 199     echo "Creating ConfigMap for grafana dashboards-providers"
 200     kubectl apply -f grafana/dashboards-providers/dashboards.yaml
 201
 202     echo "Creating grafana deployment"
 203     kubectl apply -f grafana-deployment.yaml
 204
 205     echo "Waiting for consumers to be running."
 206     while [[ $(kubectl get pods -l app=${CONSUMER_APPS_LABEL} -n ${ONAP_NAMESPACE} | grep -c "unhealthy\|starting") -ne 0 ]] ; do
 207         sleep 1
 208     done
 209     echo "Setting up environment finished!"
 210 }
 211
 212 function start_load_tests() {
 213     print_test_setup_info
 214
 215     echo "CTRL + C to stop/interrupt this script"
 216     create_producers
 217
 218     trap "handle_key_interrupt" INT
 219
 220     echo "Constant producer number keeper started working"
 221     while :; do
 222         PRODUCERS_TO_RECREATE=$((CONTAINERS_COUNT-$(kubectl get pods -l app=${PRODUCER_APPS_LABEL} -n ${ONAP_NAMESPACE} | grep -c "Running")))
 223         handle_backoffs
 224
 225         set -e
 226         for i in $(seq 1 ${PRODUCERS_TO_RECREATE});
 227         do
 228             echo "Recreating ${i}/${PRODUCERS_TO_RECREATE} producer"
 229             kubectl create -f producer-pod.yaml -n ${ONAP_NAMESPACE}
 230         done
 231         set +e
 232         COMPLETED_PRODUCERS_SUM=$((COMPLETED_PRODUCERS_SUM + PRODUCERS_TO_RECREATE))
 233         echo "Attempting to clear completed producers"
 234         kubectl delete pod --field-selector=status.phase==Succeeded -l app=${PRODUCER_APPS_LABEL} -n ${ONAP_NAMESPACE}
 235
 236         [[ ${CHECK_NUMBER} -gt ${MAXIMUM_BACK_OFF_CHECK_ITERATIONS} ]] && break
 237         sleep 1
 238     done
 239
 240     trap SIGINT
 241     exit 0
 242 }
 243
 244 function start_performance_test() {
 245     print_test_setup_info
 246
 247     create_producers
 248
 249     echo "Waiting for producers completion"
 250     while :; do
 251         COMPLETED_PRODUCERS=$(kubectl get pods -l app=${PRODUCER_APPS_LABEL} -n ${ONAP_NAMESPACE} | grep -c "Completed")
 252         handle_backoffs
 253
 254         [[ ${COMPLETED_PRODUCERS} -eq ${CONTAINERS_COUNT} || ${CHECK_NUMBER} -gt ${MAXIMUM_BACK_OFF_CHECK_ITERATIONS} ]] && break
 255         sleep 1
 256     done
 257
 258     echo "Attempting to delete producer pods"
 259     kubectl delete pods -l app=${PRODUCER_APPS_LABEL} -n ${ONAP_NAMESPACE}
 260     echo "Performance test finished"
 261     exit 0
 262 }
 263
 264 cd ${SCRIPT_DIRECTORY}
 265
 266 if [[ $# -eq 0 ]]; then
 267     usage
 268 else
 269     for arg in ${@}
 270     do
 271         case ${arg} in
 272             gen_certs)
 273             generate_certs
 274             ;;
 275             setup)
 276             setup_environment
 277             ;;
 278             start)
 279             shift 1
 280             while [[ $(($#)) -gt 0 ]]; do
 281                 case "${1}" in
 282                     --load)
 283                         LOAD_TEST=${2}
 284                         ;;
 285                     --containers)
 286                         CONTAINERS_COUNT=${2}
 287                         ;;
 288                     --properties-file)
 289                         PROPERTIES_FILE=${2}
 290                         ;;
 291                     *)
 292                         echo "Unknown option: ${1}"
 293                         usage
 294                         ;;
 295                 esac
 296                 shift 2
 297             done
 298             if [ ${LOAD_TEST} == "true" ] ; then
 299                 start_load_tests
 300             else
 301                 start_performance_test
 302             fi
 303             ;;
 304             clean)
 305             clean
 306             ;;
 307             help)
 308             usage
 309             ;;
 310             *)
 311             echo "Unknown action: ${arg}" >&2
 312             usage
 313             ;;
 314         esac
 315     done
 316 fi