tools/performance/cloud/cloud-based-performance-test.sh

   1 #!/usr/bin/env bash
   2 # ============LICENSE_START=======================================================
   3 # dcaegen2-collectors-veshv
   4 # ================================================================================
   5 # Copyright (C) 2019-2020 NOKIA
   6 # ================================================================================
   7 # Licensed under the Apache License, Version 2.0 (the "License");
   8 # you may not use this file except in compliance with the License.
   9 # You may obtain a copy of the License at
  10 #
  11 #      http://www.apache.org/licenses/LICENSE-2.0
  12 #
  13 # Unless required by applicable law or agreed to in writing, software
  14 # distributed under the License is distributed on an "AS IS" BASIS,
  15 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16 # See the License for the specific language governing permissions and
  17 # limitations under the License.
  18 # ============LICENSE_END=========================================================
  19
  20 SCRIPT_DIRECTORY="$(pwd "$0")"
  21 CONTAINERS_COUNT=1
  22 COMPLETED_PRODUCERS_SUM=0
  23 LOAD_TEST="false"
  24 TEST_CONFIG_MAP=performance-test-config
  25 PROPERTIES_FILE=${SCRIPT_DIRECTORY}/test.properties
  26 PRODUCER_APPS_LABEL=hv-collector-producer
  27 CONSUMER_APPS_LABEL=hv-collector-kafka-consumer
  28 PROMETHEUS_CONF_LABEL=prometheus-server-conf
  29 PROMETHEUS_APPS_LABEL=hv-collector-prometheus
  30 GRAFANA_APPS_LABEL=hv-collector-grafana
  31 GRAFANA_DATASOURCE=grafana-datasources
  32 GRAFANA_DASHBOARD=grafana-dashboards
  33 GRAFANA_DASHBOARD_PROVIDERS=grafana-dashboards-providers
  34 ONAP_NAMESPACE=onap
  35 MAXIMUM_BACK_OFF_CHECK_ITERATIONS=30
  36 CHECK_NUMBER=0
  37 PRODUCERS_TO_RECREATE=0
  38 NAME_REASON_PATTERN="custom-columns=NAME:.metadata.name,REASON:.status.containerStatuses[].state.waiting.reason"
  39 HVVES_POD_NAME=$(kubectl -n ${ONAP_NAMESPACE} get pods --no-headers=true -o custom-columns=:metadata.name | grep hv-ves-collector)
  40 HVVES_CERT_PATH=/etc/ves-hv/ssl/server
  41 KAFKA_RETENTION_TIME_MINUTES=60
  42 MILISECONDS_IN_MINUTE=60000
  43 CALC_RETENTION_TIME_IN_MS_CMD='expr $KAFKA_RETENTION_TIME_MINUTES \* $MILISECONDS_IN_MINUTE'
  44 KAFKA_ROUTER_0_POD_NAME=$(kubectl -n ${ONAP_NAMESPACE} get pods --no-headers=true -o custom-columns=:metadata.name | grep router-kafka-0)
  45 KAFKA_SET_TOPIC_RETENTION_TIME_CMD='kafka-topics --zookeeper message-router-zookeeper:2181 --alter --topic HV_VES_PERF3GPP --config retention.ms='
  46 HIDE_OUTPUT='grep abc | grep 123'
  47
  48 function clean() {
  49     echo "Cleaning up environment"
  50
  51     echo "Attempting to delete test parameters ConfigMap"
  52     kubectl delete configmap ${TEST_CONFIG_MAP} -n ${ONAP_NAMESPACE}
  53
  54     echo "Attempting to delete prometheus ConfigMap"
  55     kubectl delete configmap -l name=${PROMETHEUS_CONF_LABEL} -n ${ONAP_NAMESPACE}
  56
  57     echo "Attempting to delete prometheus deployment and service"
  58     kubectl delete service,deployments -l app=${PROMETHEUS_APPS_LABEL} -n ${ONAP_NAMESPACE}
  59
  60     echo "Attempting to delete grafana deployment and service"
  61     kubectl delete service,deployments -l app=${GRAFANA_APPS_LABEL} -n ${ONAP_NAMESPACE}
  62
  63     echo "Attempting to delete grafana ConfigMap (DASHBOARD)"
  64     kubectl delete configmap -l name=${GRAFANA_DASHBOARD} -n ${ONAP_NAMESPACE}
  65
  66     echo "Attempting to delete grafana ConfigMap (GRAFANA_DASHBOARD_PROVIDERS)"
  67     kubectl delete configmap -l name=${GRAFANA_DASHBOARD_PROVIDERS} -n ${ONAP_NAMESPACE}
  68
  69     echo "Attempting to delete grafana ConfigMap (GRAFANA_DATASOURCE)"
  70     kubectl delete configmap -l name=${GRAFANA_DATASOURCE} -n ${ONAP_NAMESPACE}
  71
  72     echo "Attempting to delete consumer deployments"
  73     kubectl delete deployments -l app=${CONSUMER_APPS_LABEL} -n ${ONAP_NAMESPACE}
  74
  75     echo "Attempting to delete producer pods"
  76     kubectl delete pods -l app=${PRODUCER_APPS_LABEL} -n ${ONAP_NAMESPACE}
  77
  78     echo "Attempting to delete client certs secret"
  79     kubectl delete secret cert -n ${ONAP_NAMESPACE}
  80
  81     echo "Attempting to turn off SSL"
  82     ./configure-consul.sh true
  83
  84     echo "Environment clean up finished!"
  85 }
  86
  87 function copy_certs_to_hvves() {
  88     cd ../../ssl
  89     echo "Attempting to create certs directory in HV-VES"
  90     kubectl exec -n ${ONAP_NAMESPACE} ${HVVES_POD_NAME} 'mkdir' ${HVVES_CERT_PATH}
  91     for file in {trust.p12,trust.pass,server.p12,server.pass}
  92     do
  93         echo "Copying file: ${file}"
  94         kubectl cp ${file} ${ONAP_NAMESPACE}/${HVVES_POD_NAME}:${HVVES_CERT_PATH}
  95     done
  96 }
  97
  98 function set_kafka_retention_time() {
  99     echo "Setting message retention time"
 100     kubectl exec -it ${KAFKA_ROUTER_0_POD_NAME} -n ${ONAP_NAMESPACE} -- ${KAFKA_SET_TOPIC_RETENTION_TIME_CMD}$(eval $CALC_RETENTION_TIME_IN_MS_CMD) | eval $HIDE_OUTPUT
 101 }
 102
 103 function create_producers() {
 104     echo "Recreating test properties ConfigMap from: $PROPERTIES_FILE"
 105     kubectl delete configmap ${TEST_CONFIG_MAP} -n ${ONAP_NAMESPACE}
 106     kubectl create configmap ${TEST_CONFIG_MAP} --from-env-file=${PROPERTIES_FILE} -n ${ONAP_NAMESPACE}
 107
 108     set -e
 109     for i in $(seq 1 ${CONTAINERS_COUNT});
 110     do
 111         echo "Creating ${i}/${CONTAINERS_COUNT} producer"
 112         kubectl create -f producer-pod.yaml -n ${ONAP_NAMESPACE}
 113     done
 114     echo "Producers created"
 115     set +e
 116 }
 117
 118 function generate_certs() {
 119     echo "Generation of certs"
 120     cd ../../ssl
 121     ./gen-certs.sh
 122 }
 123
 124 function handle_backoffs() {
 125     IMAGE_PULL_BACK_OFFS=$(kubectl get pods -l app=${PRODUCER_APPS_LABEL} -n ${ONAP_NAMESPACE} -o ${NAME_REASON_PATTERN} | grep -c "ImagePullBackOff \| ErrImagePull")
 126     if [[ ${IMAGE_PULL_BACK_OFFS} -gt 0 ]]; then
 127         CHECK_NUMBER=$((CHECK_NUMBER + 1))
 128         if [[ ${CHECK_NUMBER} -gt ${MAXIMUM_BACK_OFF_CHECK_ITERATIONS} ]]; then
 129             echo "Error: Image pull problem"
 130             exit 1
 131         fi
 132     fi
 133 }
 134
 135 function handle_key_interrupt() {
 136     trap SIGINT
 137     echo "Script interrupted, attempt to delete producers"
 138     echo "Wait with patience"
 139     COMPLETED_PRODUCERS_SUM=$(($(kubectl delete pods -l app=${PRODUCER_APPS_LABEL} -n ${ONAP_NAMESPACE} | grep producer | wc -l) + COMPLETED_PRODUCERS_SUM))
 140     echo "Total number of completed producers: ${COMPLETED_PRODUCERS_SUM}"
 141     exit 0
 142 }
 143
 144 function print_test_setup_info() {
 145     echo "Starting cloud based performance tests"
 146     echo "________________________________________"
 147     echo "Test configuration:"
 148     echo "Producer containers count: ${CONTAINERS_COUNT}"
 149     echo "Properties file path: ${PROPERTIES_FILE}"
 150     echo "Retention time of kafka messages in minutes: ${KAFKA_RETENTION_TIME_MINUTES}"
 151     echo "________________________________________"
 152 }
 153
 154 function usage() {
 155     echo ""
 156     echo "Run cloud based HV-VES performance test"
 157     echo "Usage $0 gen_certs|setup|start|clean|help"
 158     echo "  gen_certs: generate certs in ../../ssl directory"
 159     echo "  setup    : set up ConfigMap and consumers"
 160     echo "  start    : create producers - start the performance test"
 161     echo "    Optional parameters:"
 162     echo "      --load              : should test keep defined containers number till script interruption (false)"
 163     echo "      --containers        : number of producer containers to create (1)"
 164     echo "      --properties-file   : path to file with benchmark properties (./test.properties)"
 165     echo "      --retention-time-minutes : messages retention time on kafka in minutes (60)"
 166     echo "  clean    : remove ConfigMap, HV-VES consumers and producers"
 167     echo "  help     : print usage"
 168     echo "Example invocations:"
 169     echo "./cloud-based-performance-test.sh gen_certs"
 170     echo "./cloud-based-performance-test.sh setup"
 171     echo "./cloud-based-performance-test.sh start"
 172     echo "./cloud-based-performance-test.sh start --containers 10"
 173     echo "./cloud-based-performance-test.sh start --load true --containers 10"
 174     echo "./cloud-based-performance-test.sh start --load true --containers 10 --retention-time-minutes 50"
 175     echo "./cloud-based-performance-test.sh start --properties-file ~/other_test.properties"
 176     echo "./cloud-based-performance-test.sh clean"
 177     exit 1
 178 }
 179
 180 function setup_environment() {
 181     echo "Setting up environment"
 182
 183     echo "Copying certs to hv-ves pod"
 184     copy_certs_to_hvves
 185
 186     echo "Creating secrets with clients cert"
 187     kubectl create secret generic cert --from-file=./client.p12 --from-file=./client.pass -n ${ONAP_NAMESPACE}
 188     cd ${SCRIPT_DIRECTORY}
 189
 190     echo "Turning on SSL"
 191     ./configure-consul.sh false
 192
 193     echo "Creating test properties ConfigMap from: $PROPERTIES_FILE"
 194     kubectl create configmap ${TEST_CONFIG_MAP} --from-env-file=${PROPERTIES_FILE} -n ${ONAP_NAMESPACE}
 195
 196     echo "Creating consumer deployment"
 197     kubectl apply -f consumer-deployment.yaml
 198
 199     echo "Creating ConfigMap for prometheus deployment"
 200     kubectl apply -f prometheus/prometheus-config-map.yaml
 201
 202     echo "Creating prometheus deployment"
 203     kubectl apply -f prometheus-deployment.yaml
 204
 205     echo "Creating ConfigMap for grafana connections dashboard"
 206     kubectl apply -f grafana/dashboards/connections.yaml
 207
 208     echo "Creating ConfigMap for grafana processing dashboard"
 209     kubectl apply -f grafana/dashboards/processing.yaml
 210
 211     echo "Creating ConfigMap for grafana grafana-performance-tests dashboard"
 212     kubectl apply -f grafana/dashboards/performance-tests.yaml
 213
 214     echo "Creating ConfigMap for grafana grafana-kafka-and-producers dashboard"
 215     kubectl apply -f grafana/dashboards/kafka-and-producers.yaml
 216
 217     echo "Creating ConfigMap for grafana grafana-k8s-metrics dashboard"
 218     kubectl apply -f grafana/dashboards/k8s-metrics.yaml
 219
 220     echo "Creating ConfigMap for grafana datasource"
 221     kubectl apply -f grafana/datasources/datasource.yaml
 222
 223     echo "Creating ConfigMap for grafana dashboards-providers"
 224     kubectl apply -f grafana/dashboards-providers/dashboards.yaml
 225
 226     echo "Creating grafana deployment"
 227     kubectl apply -f grafana-deployment.yaml
 228
 229     echo "Waiting for consumers to be running."
 230     while [[ $(kubectl get pods -l app=${CONSUMER_APPS_LABEL} -n ${ONAP_NAMESPACE} | grep -c "unhealthy\|starting") -ne 0 ]] ; do
 231         sleep 1
 232     done
 233     echo "Setting up environment finished!"
 234 }
 235
 236 function start_load_tests() {
 237     print_test_setup_info
 238
 239     set_kafka_retention_time
 240
 241     echo "CTRL + C to stop/interrupt this script"
 242     create_producers
 243
 244     trap "handle_key_interrupt" INT
 245
 246     echo "Constant producer number keeper started working"
 247     while :; do
 248         PRODUCERS_TO_RECREATE=$((CONTAINERS_COUNT-$(kubectl get pods -l app=${PRODUCER_APPS_LABEL} -n ${ONAP_NAMESPACE} | grep -c "Running")))
 249         handle_backoffs
 250
 251         set -e
 252         for i in $(seq 1 ${PRODUCERS_TO_RECREATE});
 253         do
 254             echo "Recreating ${i}/${PRODUCERS_TO_RECREATE} producer"
 255             kubectl create -f producer-pod.yaml -n ${ONAP_NAMESPACE}
 256         done
 257         set +e
 258         COMPLETED_PRODUCERS_SUM=$((COMPLETED_PRODUCERS_SUM + PRODUCERS_TO_RECREATE))
 259         echo "Attempting to clear completed producers"
 260         kubectl delete pod --field-selector=status.phase==Succeeded -l app=${PRODUCER_APPS_LABEL} -n ${ONAP_NAMESPACE}
 261
 262         [[ ${CHECK_NUMBER} -gt ${MAXIMUM_BACK_OFF_CHECK_ITERATIONS} ]] && break
 263         sleep 1
 264     done
 265
 266     trap SIGINT
 267     exit 0
 268 }
 269
 270 function start_performance_test() {
 271     print_test_setup_info
 272
 273     set_kafka_retention_time
 274
 275     create_producers
 276
 277     echo "Waiting for producers completion"
 278     while :; do
 279         COMPLETED_PRODUCERS=$(kubectl get pods -l app=${PRODUCER_APPS_LABEL} -n ${ONAP_NAMESPACE} | grep -c "Completed")
 280         handle_backoffs
 281
 282         [[ ${COMPLETED_PRODUCERS} -eq ${CONTAINERS_COUNT} || ${CHECK_NUMBER} -gt ${MAXIMUM_BACK_OFF_CHECK_ITERATIONS} ]] && break
 283         sleep 1
 284     done
 285
 286     echo "Attempting to delete producer pods"
 287     kubectl delete pods -l app=${PRODUCER_APPS_LABEL} -n ${ONAP_NAMESPACE}
 288     echo "Performance test finished"
 289     exit 0
 290 }
 291
 292 cd ${SCRIPT_DIRECTORY}
 293
 294 if [[ $# -eq 0 ]]; then
 295     usage
 296 else
 297     for arg in ${@}
 298     do
 299         case ${arg} in
 300             gen_certs)
 301             generate_certs
 302             ;;
 303             setup)
 304             setup_environment
 305             ;;
 306             start)
 307             shift 1
 308             while [[ $(($#)) -gt 0 ]]; do
 309                 case "${1}" in
 310                     --load)
 311                         LOAD_TEST=${2}
 312                         ;;
 313                     --containers)
 314                         CONTAINERS_COUNT=${2}
 315                         ;;
 316                     --properties-file)
 317                         PROPERTIES_FILE=${2}
 318                         ;;
 319                     --retention-time-minutes)
 320                         KAFKA_RETENTION_TIME_MINUTES=${2}
 321                         ;;
 322                     *)
 323                         echo "Unknown option: ${1}"
 324                         usage
 325                         ;;
 326                 esac
 327                 shift 2
 328             done
 329             if [ ${LOAD_TEST} == "true" ] ; then
 330                 start_load_tests
 331             else
 332                 start_performance_test
 333             fi
 334             ;;
 335             clean)
 336             clean
 337             ;;
 338             help)
 339             usage
 340             ;;
 341             *)
 342             echo "Unknown action: ${arg}" >&2
 343             usage
 344             ;;
 345         esac
 346     done
 347 fi