tools/performance/cloud/cloud-based-performance-test.sh

   1 #!/usr/bin/env bash
   2 # ============LICENSE_START=======================================================
   3 # dcaegen2-collectors-veshv
   4 # ================================================================================
   5 # Copyright (C) 2019 NOKIA
   6 # ================================================================================
   7 # Licensed under the Apache License, Version 2.0 (the "License");
   8 # you may not use this file except in compliance with the License.
   9 # You may obtain a copy of the License at
  10 #
  11 #      http://www.apache.org/licenses/LICENSE-2.0
  12 #
  13 # Unless required by applicable law or agreed to in writing, software
  14 # distributed under the License is distributed on an "AS IS" BASIS,
  15 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16 # See the License for the specific language governing permissions and
  17 # limitations under the License.
  18 # ============LICENSE_END=========================================================
  19
  20 SCRIPT_DIRECTORY="$(pwd "$0")"
  21 CONTAINERS_COUNT=1
  22 LOAD_TEST="false"
  23 TEST_CONFIG_MAP=performance-test-config
  24 PROPERTIES_FILE=${SCRIPT_DIRECTORY}/test.properties
  25 PRODUCER_APPS_LABEL=hv-collector-producer
  26 CONSUMER_APPS_LABEL=hv-collector-kafka-consumer
  27 PROMETHEUS_CONF_LABEL=prometheus-server-conf
  28 PROMETHEUS_APPS_LABEL=hv-collector-prometheus
  29 GRAFANA_APPS_LABEL=hv-collector-grafana
  30 GRAFANA_DATASOURCE=grafana-datasources
  31 GRAFANA_DASHBOARD=grafana-dashboards
  32 GRAFANA_DASHBOARD_PROVIDERS=grafana-dashboards-providers
  33 ONAP_NAMESPACE=onap
  34 MAXIMUM_BACK_OFF_CHECK_ITERATIONS=30
  35 CHECK_NUMBER=0
  36 COMPLETED_PRODUCERS_SUM=0
  37 NAME_REASON_PATTERN="custom-columns=NAME:.metadata.name,REASON:.status.containerStatuses[].state.waiting.reason"
  38 HVVES_POD_NAME=$(kubectl -n ${ONAP_NAMESPACE} get pods --no-headers=true -o custom-columns=:metadata.name | grep hv-ves-collector)
  39 HVVES_CERT_PATH=/etc/ves-hv/ssl/
  40 KAFKA_RETENTION_TIME_MINUTES=60
  41 MILISECONDS_IN_MINUTE=60000
  42 CALC_RETENTION_TIME_IN_MS_CMD='expr $KAFKA_RETENTION_TIME_MINUTES \* $MILISECONDS_IN_MINUTE'
  43 KAFKA_ROUTER_0_POD_NAME=$(kubectl -n ${ONAP_NAMESPACE} get pods --no-headers=true -o custom-columns=:metadata.name | grep router-kafka-0)
  44 KAFKA_SET_TOPIC_RETENTION_TIME_CMD='kafka-topics.sh --zookeeper message-router-zookeeper:2181 --alter --topic HV_VES_PERF3GPP --config retention.ms='
  45 HIDE_OUTPUT='grep abc | grep 123'
  46
  47 function clean() {
  48     echo "Cleaning up environment"
  49
  50     echo "Attempting to delete test parameters ConfigMap"
  51     kubectl delete configmap ${TEST_CONFIG_MAP} -n ${ONAP_NAMESPACE}
  52
  53     echo "Attempting to delete prometheus ConfigMap"
  54     kubectl delete configmap -l name=${PROMETHEUS_CONF_LABEL} -n ${ONAP_NAMESPACE}
  55
  56     echo "Attempting to delete prometheus deployment and service"
  57     kubectl delete service,deployments -l app=${PROMETHEUS_APPS_LABEL} -n ${ONAP_NAMESPACE}
  58
  59     echo "Attempting to delete grafana deployment and service"
  60     kubectl delete service,deployments -l app=${GRAFANA_APPS_LABEL} -n ${ONAP_NAMESPACE}
  61
  62     echo "Attempting to delete grafana ConfigMap (DASHBOARD)"
  63     kubectl delete configmap -l name=${GRAFANA_DASHBOARD} -n ${ONAP_NAMESPACE}
  64
  65     echo "Attempting to delete grafana ConfigMap (GRAFANA_DASHBOARD_PROVIDERS)"
  66     kubectl delete configmap -l name=${GRAFANA_DASHBOARD_PROVIDERS} -n ${ONAP_NAMESPACE}
  67
  68     echo "Attempting to delete grafana ConfigMap (GRAFANA_DATASOURCE)"
  69     kubectl delete configmap -l name=${GRAFANA_DATASOURCE} -n ${ONAP_NAMESPACE}
  70
  71     echo "Attempting to delete consumer deployments"
  72     kubectl delete deployments -l app=${CONSUMER_APPS_LABEL} -n ${ONAP_NAMESPACE}
  73
  74     echo "Attempting to delete producer pods"
  75     kubectl delete pods -l app=${PRODUCER_APPS_LABEL} -n ${ONAP_NAMESPACE}
  76
  77     echo "Attempting to delete client certs secret"
  78     kubectl delete secret cert -n ${ONAP_NAMESPACE}
  79
  80     echo "Attempting to turn off SSL"
  81     ./configure-consul.sh true
  82
  83     echo "Environment clean up finished!"
  84 }
  85
  86 function copy_certs_to_hvves() {
  87          cd ../../ssl
  88          for file in {trust.p12,trust.pass,server.p12,server.pass}
  89          do
  90        echo "Copying file: ${file}"
  91        kubectl cp ${file} ${ONAP_NAMESPACE}/${HVVES_POD_NAME}:${HVVES_CERT_PATH}
  92    done
  93 }
  94
  95 function create_producers() {
  96     echo "Recreating test properties ConfigMap from: $PROPERTIES_FILE"
  97     kubectl delete configmap ${TEST_CONFIG_MAP} -n ${ONAP_NAMESPACE}
  98     kubectl create configmap ${TEST_CONFIG_MAP} --from-env-file=${PROPERTIES_FILE} -n ${ONAP_NAMESPACE}
  99
 100     set -e
 101     for i in $(seq 1 ${CONTAINERS_COUNT});
 102     do
 103         echo "Creating ${i}/${CONTAINERS_COUNT} producer"
 104         kubectl create -f producer-pod.yaml -n ${ONAP_NAMESPACE}
 105     done
 106     echo "Producers created"
 107     set +e
 108 }
 109
 110 function generate_certs() {
 111     echo "Generation of certs"
 112     cd ../../ssl
 113     ./gen-certs.sh
 114 }
 115
 116 function handle_backoffs() {
 117         if [[ ${IMAGE_PULL_BACK_OFFS} -gt 0 ]]; then
 118             CHECK_NUMBER=$((CHECK_NUMBER + 1))
 119             if [[ ${CHECK_NUMBER} -gt ${MAXIMUM_BACK_OFF_CHECK_ITERATIONS} ]]; then
 120                 echo "Error: Image pull problem"
 121                 exit 1
 122             fi
 123         fi
 124 }
 125
 126 function handle_key_interrupt() {
 127     trap SIGINT
 128     echo "Script interrupted, attempt to delete producers"
 129     kubectl delete pods -l app=${PRODUCER_APPS_LABEL} -n ${ONAP_NAMESPACE}
 130     exit 0
 131 }
 132
 133 function print_test_setup_info() {
 134     echo "Starting cloud based performance tests"
 135     echo "________________________________________"
 136     echo "Test configuration:"
 137     echo "Producer containers count: ${CONTAINERS_COUNT}"
 138     echo "Properties file path: ${PROPERTIES_FILE}"
 139     echo "Retention time of kafka messages in minutes: ${KAFKA_RETENTION_TIME_MINUTES}"
 140     echo "________________________________________"
 141 }
 142
 143 function usage() {
 144     echo ""
 145     echo "Run cloud based HV-VES performance test"
 146     echo "Usage $0 gen_certs|setup|start|clean|help"
 147     echo "  gen_certs: generate certs in ../../ssl directory"
 148     echo "  setup    : set up ConfigMap and consumers"
 149     echo "  start    : create producers - start the performance test"
 150     echo "    Optional parameters:"
 151     echo "      --load              : should test keep defined containers number till script interruption (false)"
 152     echo "      --containers        : number of producer containers to create (1)"
 153     echo "      --properties-file   : path to file with benchmark properties (./test.properties)"
 154     echo "      --retention-time-minutes : messages retention time on kafka in minutes - only for load tests (60)"
 155     echo "  clean    : remove ConfigMap, HV-VES consumers and producers"
 156     echo "  help     : print usage"
 157     echo "Example invocations:"
 158     echo "./cloud-based-performance-test.sh gen_certs"
 159     echo "./cloud-based-performance-test.sh setup"
 160     echo "./cloud-based-performance-test.sh start"
 161     echo "./cloud-based-performance-test.sh start --containers 10"
 162     echo "./cloud-based-performance-test.sh start --load true --containers 10"
 163     echo "./cloud-based-performance-test.sh start --load true --containers 10 --retention-time-minutes 50"
 164     echo "./cloud-based-performance-test.sh start --properties-file ~/other_test.properties"
 165     echo "./cloud-based-performance-test.sh clean"
 166     exit 1
 167 }
 168
 169 function setup_environment() {
 170     echo "Setting up environment"
 171
 172     echo "Copying certs to hv-ves pod"
 173     copy_certs_to_hvves
 174
 175     echo "Creating secrets with clients cert"
 176     kubectl create secret generic cert --from-file=./client.p12 --from-file=./client.pass -n ${ONAP_NAMESPACE}
 177     cd ${SCRIPT_DIRECTORY}
 178
 179     echo "Turning on SSL"
 180     ./configure-consul.sh false
 181
 182     echo "Creating test properties ConfigMap from: $PROPERTIES_FILE"
 183     kubectl create configmap ${TEST_CONFIG_MAP} --from-env-file=${PROPERTIES_FILE} -n ${ONAP_NAMESPACE}
 184
 185     echo "Creating consumer deployment"
 186     kubectl apply -f consumer-deployment.yaml
 187
 188     echo "Creating ConfigMap for prometheus deployment"
 189     kubectl apply -f prometheus/prometheus-config-map.yaml
 190
 191     echo "Creating prometheus deployment"
 192     kubectl apply -f prometheus-deployment.yaml
 193
 194     echo "Creating ConfigMap for grafana connections dashboard"
 195     kubectl apply -f grafana/dashboards/connections.yaml
 196
 197     echo "Creating ConfigMap for grafana processing dashboard"
 198     kubectl apply -f grafana/dashboards/processing.yaml
 199
 200     echo "Creating ConfigMap for grafana grafana-performance-tests dashboard"
 201     kubectl apply -f grafana/dashboards/performance-tests.yaml
 202
 203     echo "Creating ConfigMap for grafana datasource"
 204     kubectl apply -f grafana/datasources/datasource.yaml
 205
 206     echo "Creating ConfigMap for grafana dashboards-providers"
 207     kubectl apply -f grafana/dashboards-providers/dashboards.yaml
 208
 209     echo "Creating grafana deployment"
 210     kubectl apply -f grafana-deployment.yaml
 211
 212     echo "Waiting for consumers to be running."
 213     while [[ $(kubectl get pods -l app=${CONSUMER_APPS_LABEL} -n ${ONAP_NAMESPACE} | grep -c "unhealthy\|starting") -ne 0 ]] ; do
 214         sleep 1
 215     done
 216     echo "Setting up environment finished!"
 217 }
 218
 219 function start_load_tests() {
 220     print_test_setup_info
 221
 222     echo "Setting message retention time"
 223     kubectl exec -it ${KAFKA_ROUTER_0_POD_NAME} -n ${ONAP_NAMESPACE} -- ${KAFKA_SET_TOPIC_RETENTION_TIME_CMD}$(eval $CALC_RETENTION_TIME_IN_MS_CMD) | eval $HIDE_OUTPUT
 224
 225     echo "CTRL + C to stop/interrupt this script"
 226     create_producers
 227
 228     trap "handle_key_interrupt" INT
 229
 230     echo "Constant producer number keeper started working"
 231     while :; do
 232         COMPLETED_PRODUCERS=$(($(kubectl get pods -l app=${PRODUCER_APPS_LABEL} -n ${ONAP_NAMESPACE} | grep -c "Completed")-COMPLETED_PRODUCERS_SUM))
 233         IMAGE_PULL_BACK_OFFS=$(kubectl get pods -l app=${PRODUCER_APPS_LABEL} -n ${ONAP_NAMESPACE} -o ${NAME_REASON_PATTERN} | grep -c "ImagePullBackOff \| ErrImagePull")
 234
 235         handle_backoffs
 236
 237         set -e
 238         for i in $(seq 1 ${COMPLETED_PRODUCERS});
 239         do
 240             echo "Recreating ${i}/${COMPLETED_PRODUCERS} producer"
 241             kubectl create -f producer-pod.yaml -n ${ONAP_NAMESPACE}
 242             COMPLETED_PRODUCERS_SUM=$((COMPLETED_PRODUCERS_SUM + 1))
 243         done
 244         set +e
 245         [[ ${CHECK_NUMBER} -gt ${MAXIMUM_BACK_OFF_CHECK_ITERATIONS} ]] && break
 246         sleep 2
 247     done
 248
 249     trap SIGINT
 250     exit 0
 251 }
 252
 253 function start_performance_test() {
 254     print_test_setup_info
 255
 256     create_producers
 257
 258     echo "Waiting for producers completion"
 259     while :; do
 260         COMPLETED_PRODUCERS=$(kubectl get pods -l app=${PRODUCER_APPS_LABEL} -n ${ONAP_NAMESPACE} | grep -c "Completed")
 261         IMAGE_PULL_BACK_OFFS=$(kubectl get pods -l app=${PRODUCER_APPS_LABEL} -n ${ONAP_NAMESPACE} -o ${NAME_REASON_PATTERN} | grep -c "ImagePullBackOff \| ErrImagePull")
 262
 263         handle_backoffs
 264
 265         [[ ${COMPLETED_PRODUCERS} -eq ${CONTAINERS_COUNT} || ${CHECK_NUMBER} -gt ${MAXIMUM_BACK_OFF_CHECK_ITERATIONS} ]] && break
 266         sleep 1
 267     done
 268
 269     echo "Attempting to delete producer pods"
 270     kubectl delete pods -l app=${PRODUCER_APPS_LABEL} -n ${ONAP_NAMESPACE}
 271     echo "Performance test finished"
 272     exit 0
 273 }
 274
 275 cd ${SCRIPT_DIRECTORY}
 276
 277 if [[ $# -eq 0 ]]; then
 278     usage
 279 else
 280     for arg in ${@}
 281     do
 282         case ${arg} in
 283             gen_certs)
 284             generate_certs
 285             ;;
 286             setup)
 287             setup_environment
 288             ;;
 289             start)
 290             shift 1
 291             while [[ $(($#)) -gt 0 ]]; do
 292                 case "${1}" in
 293                     --load)
 294                         LOAD_TEST=${2}
 295                         ;;
 296                     --containers)
 297                         CONTAINERS_COUNT=${2}
 298                         ;;
 299                     --properties-file)
 300                         PROPERTIES_FILE=${2}
 301                         ;;
 302                     --retention-time-minutes)
 303                         KAFKA_RETENTION_TIME_MINUTES=${2}
 304                         ;;
 305                     *)
 306                         echo "Unknown option: ${1}"
 307                         usage
 308                         ;;
 309                 esac
 310                 shift 2
 311             done
 312             if [ ${LOAD_TEST} == "true" ] ; then
 313                 start_load_tests
 314             else
 315                 start_performance_test
 316             fi
 317             ;;
 318             clean)
 319             clean
 320             ;;
 321             help)
 322             usage
 323             ;;
 324             *)
 325             echo "Unknown action: ${arg}" >&2
 326             usage
 327             ;;
 328         esac
 329     done
 330 fi