tools/performance/cloud/cloud-based-performance-test.sh

   1 #!/usr/bin/env bash
   2 # ============LICENSE_START=======================================================
   3 # dcaegen2-collectors-veshv
   4 # ================================================================================
   5 # Copyright (C) 2019-2020 NOKIA
   6 # ================================================================================
   7 # Licensed under the Apache License, Version 2.0 (the "License");
   8 # you may not use this file except in compliance with the License.
   9 # You may obtain a copy of the License at
  10 #
  11 #      http://www.apache.org/licenses/LICENSE-2.0
  12 #
  13 # Unless required by applicable law or agreed to in writing, software
  14 # distributed under the License is distributed on an "AS IS" BASIS,
  15 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16 # See the License for the specific language governing permissions and
  17 # limitations under the License.
  18 # ============LICENSE_END=========================================================
  19
  20 SCRIPT_DIRECTORY="$(pwd "$0")"
  21 PRODUCERS_COUNT=1
  22 TEST_CONFIG_MAP=performance-test-config
  23 PROPERTIES_FILE=${SCRIPT_DIRECTORY}/test.properties
  24 PRODUCER_APPS_LABEL=hv-collector-producer
  25 PRODUCER_APPS_DEPLOYMENT=hv-collector-producer-deployment
  26 PRODUCER_SERVICE=hv-collector-producer
  27 PRODUCER_PROXY_APPS_LABEL=hv-collector-producer-proxy
  28 CONSUMER_APPS_LABEL=hv-collector-kafka-consumer
  29 PROMETHEUS_CONF_LABEL=prometheus-server-conf
  30 PROMETHEUS_APPS_LABEL=hv-collector-prometheus
  31 GRAFANA_APPS_LABEL=hv-collector-grafana
  32 GRAFANA_DATASOURCE=grafana-datasources
  33 GRAFANA_DASHBOARDS=grafana-dashboards
  34 GRAFANA_DASHBOARD_PROVIDERS=grafana-dashboards-providers
  35 ONAP_NAMESPACE=onap
  36 NAME_REASON_PATTERN="custom-columns=NAME:.metadata.name,REASON:.status.containerStatuses[].state.waiting.reason"
  37 HVVES_POD_NAME=$(kubectl -n ${ONAP_NAMESPACE} get pods --no-headers=true -o custom-columns=:metadata.name | grep hv-ves-collector)
  38 HVVES_CERT_PATH=/etc/ves-hv/ssl/server
  39 KAFKA_RETENTION_TIME_MINUTES=60
  40 MILISECONDS_IN_MINUTE=60000
  41 CALC_RETENTION_TIME_IN_MS_CMD='expr $KAFKA_RETENTION_TIME_MINUTES \* $MILISECONDS_IN_MINUTE'
  42 KAFKA_ROUTER_0_POD_NAME=$(kubectl -n ${ONAP_NAMESPACE} get pods --no-headers=true -o custom-columns=:metadata.name | grep router-kafka-0)
  43 KAFKA_SET_TOPIC_RETENTION_TIME_CMD='kafka-topics --zookeeper message-router-zookeeper:2181 --alter --topic HV_VES_PERF3GPP --config retention.ms='
  44 HIDE_OUTPUT='grep abc | grep 123'
  45 CONTENT_TYPE_HEADER='Content-Type: application/json'
  46 REQUEST_ENDPOINT="/"
  47 REQUEST_JSON_DATA="{}"
  48 PRODUCER_INTERNAL_PORT=8080
  49
  50 function clean() {
  51     echo "Cleaning up environment"
  52
  53     echo "Attempting to delete test parameters ConfigMap"
  54     kubectl delete configmap ${TEST_CONFIG_MAP} -n ${ONAP_NAMESPACE}
  55
  56     echo "Attempting to delete prometheus ConfigMap"
  57     kubectl delete configmap -l name=${PROMETHEUS_CONF_LABEL} -n ${ONAP_NAMESPACE}
  58
  59     echo "Attempting to delete prometheus deployment and service"
  60     kubectl delete service,deployments -l app=${PROMETHEUS_APPS_LABEL} -n ${ONAP_NAMESPACE}
  61
  62     echo "Attempting to delete grafana deployment and service"
  63     kubectl delete service,deployments -l app=${GRAFANA_APPS_LABEL} -n ${ONAP_NAMESPACE}
  64
  65     echo "Attempting to delete grafana ConfigMap (DASHBOARDS)"
  66     kubectl delete configmap ${GRAFANA_DASHBOARDS} -n ${ONAP_NAMESPACE}
  67
  68     echo "Attempting to delete grafana ConfigMap (GRAFANA_DASHBOARD_PROVIDERS)"
  69     kubectl delete configmap -l name=${GRAFANA_DASHBOARD_PROVIDERS} -n ${ONAP_NAMESPACE}
  70
  71     echo "Attempting to delete grafana ConfigMap (GRAFANA_DATASOURCE)"
  72     kubectl delete configmap -l name=${GRAFANA_DATASOURCE} -n ${ONAP_NAMESPACE}
  73
  74     echo "Attempting to delete consumer deployments"
  75     kubectl delete deployments -l app=${CONSUMER_APPS_LABEL} -n ${ONAP_NAMESPACE}
  76
  77     echo "Attempting to delete client certs secret"
  78     kubectl delete secret cert -n ${ONAP_NAMESPACE}
  79
  80     echo "Attempting to turn off SSL"
  81     ./configure-consul.sh true
  82
  83     echo "Environment clean up finished!"
  84 }
  85
  86 function clean_all() {
  87     clean
  88     echo "Attepting to delete producer pods"
  89     kubectl delete service,deployments -l app=${PRODUCER_APPS_LABEL} -n ${ONAP_NAMESPACE}
  90     echo "Attepting to delete producer-proxy pod"
  91     kubectl delete deployments -l app=${PRODUCER_PROXY_APPS_LABEL} -n ${ONAP_NAMESPACE}
  92
  93     echo "Producers deleted"
  94 }
  95
  96 function copy_certs_to_hvves() {
  97     cd ../../ssl
  98     echo "Attempting to create certs directory in HV-VES"
  99     kubectl exec -n ${ONAP_NAMESPACE} ${HVVES_POD_NAME} 'mkdir' ${HVVES_CERT_PATH}
 100     for file in {trust.p12,trust.pass,server.p12,server.pass}
 101     do
 102         echo "Copying file: ${file}"
 103         kubectl cp ${file} ${ONAP_NAMESPACE}/${HVVES_POD_NAME}:${HVVES_CERT_PATH}
 104     done
 105 }
 106
 107 function set_kafka_retention_time() {
 108     echo "Setting message retention time"
 109     kubectl exec -it ${KAFKA_ROUTER_0_POD_NAME} -n ${ONAP_NAMESPACE} -- ${KAFKA_SET_TOPIC_RETENTION_TIME_CMD}$(eval $CALC_RETENTION_TIME_IN_MS_CMD) | eval $HIDE_OUTPUT
 110 }
 111
 112 function generate_certs() {
 113     echo "Generation of certs"
 114     cd ../../ssl
 115     ./gen-certs.sh
 116 }
 117
 118 function print_test_setup_info() {
 119     RUNNING_PRODUCERS=$(kubectl -n ${ONAP_NAMESPACE} get pods -l app=${PRODUCER_APPS_LABEL} | grep -c "Running")
 120     echo "Starting cloud based performance tests"
 121     echo "________________________________________"
 122     echo "Test configuration:"
 123     echo "Running producers: ${RUNNING_PRODUCERS}"
 124     echo "Sending scheme:"
 125     echo ${REQUEST_JSON_DATA}
 126     echo "Properties file path: ${PROPERTIES_FILE}"
 127     echo "Retention time of kafka messages in minutes: ${KAFKA_RETENTION_TIME_MINUTES}"
 128     echo "________________________________________"
 129 }
 130
 131 function usage() {
 132     echo ""
 133     echo "Run cloud based HV-VES performance test"
 134     echo "Usage $0 gen_certs|setup|send_config|start_interval|start_instant|stop|reset_producers|clean|help"
 135     echo "  gen_certs :    generate certs in ../../ssl directory"
 136     echo "  setup     :    set up ConfigMaps and consumers"
 137     echo "  setup_all     :    set up ConfigMaps consumers and producers"
 138     echo "  send_config : send producers configuration (message interval and payload), located in producers-config/producer-config.json to each producer"
 139     echo "  start_interval    : start interval mode, config file is located in producers-config/interval-config.json"
 140     echo "     Optional parameters:"
 141     echo "        --producers               : number of producers in deployment (10)"
 142     echo "        --retention-time-minutes  : messages retention time on kafka in minutes (60)"
 143     echo "  start_instant     : start instant mode, config file is located in producers-config/instant-config.json"
 144     echo "     Optional parameters:"
 145     echo "        --producers               : number of producers in deployment (10)"
 146     echo "        --retention-time-minutes  : messages retention time on kafka in minutes (60)"
 147     echo "  scale_producers   : scale producer deployment to number provide in argument"
 148     echo "  stop      : stop all producers"
 149     echo "  reset_producers   : reset all metrics on each producer"
 150     echo "  clean     : remove ConfigMap, HV-VES consumers"
 151     echo "  clean_all     : remove ConfigMap, HV-VES consumers and producers"
 152     echo "  help      : print usage"
 153     echo "Example invocations:"
 154     echo "./cloud-based-performance-test.sh gen_certs"
 155     echo "./cloud-based-performance-test.sh setup"
 156     echo "./cloud-based-performance-test.sh setup_all"
 157     echo "./cloud-based-performance-test.sh send_config"
 158     echo "./cloud-based-performance-test.sh start_interval"
 159     echo "./cloud-based-performance-test.sh start_interval --producers 8"
 160     echo "./cloud-based-performance-test.sh start_instant"
 161     echo "./cloud-based-performance-test.sh scale_producers 8"
 162     echo "./cloud-based-performance-test.sh clean"
 163     echo "./cloud-based-performance-test.sh clean_all"
 164     exit 1
 165 }
 166
 167 function setup_environment() {
 168     echo "Setting up environment"
 169
 170     echo "Copying certs to hv-ves pod"
 171     copy_certs_to_hvves
 172
 173     echo "Creating secrets with clients cert"
 174     kubectl create secret generic cert --from-file=./client.p12 --from-file=./client.pass -n ${ONAP_NAMESPACE}
 175     cd ${SCRIPT_DIRECTORY}
 176
 177     echo "Turning on SSL"
 178     ./configure-consul.sh false
 179
 180     echo "Creating test properties ConfigMap from: $PROPERTIES_FILE"
 181     kubectl create configmap ${TEST_CONFIG_MAP} --from-env-file=${PROPERTIES_FILE} -n ${ONAP_NAMESPACE}
 182
 183     echo "Creating consumer deployment"
 184     kubectl apply -f consumer-deployment.yaml
 185
 186     echo "Creating ConfigMap for prometheus deployment"
 187     kubectl apply -f prometheus/prometheus-config-map.yaml
 188
 189     echo "Creating prometheus deployment"
 190     kubectl apply -f prometheus-deployment.yaml
 191
 192     echo "Creating ConfigMap for grafana connections dashboard"
 193     kubectl create configmap ${GRAFANA_DASHBOARDS} -n ${ONAP_NAMESPACE} --from-file grafana/dashboards/
 194
 195     echo "Creating ConfigMap for grafana datasource"
 196     kubectl apply -f grafana/datasources/datasource.yaml
 197
 198     echo "Creating ConfigMap for grafana dashboards-providers"
 199     kubectl apply -f grafana/dashboards-providers/dashboards.yaml
 200
 201     echo "Creating grafana deployment"
 202     kubectl apply -f grafana-deployment.yaml
 203
 204     echo "Waiting for consumers to be running."
 205     while [[ $(kubectl get pods -l app=${CONSUMER_APPS_LABEL} -n ${ONAP_NAMESPACE} | grep -c "unhealthy\|starting") -ne 0 ]] ; do
 206         sleep 1
 207     done
 208     echo "Setting up environment finished!"
 209 }
 210
 211 function setup_all() {
 212     setup_environment
 213     echo "Creating producer deployment"
 214     kubectl apply -f producer-deployment.yaml
 215
 216     echo "Creating producer-proxy pod"
 217     kubectl apply -f producer-proxy.yaml
 218
 219     echo "Creating producers finished"
 220 }
 221
 222 function scale_producer_deployment() {
 223   echo "Scaling prodcuer deployment to ${PRODUCERS_COUNT}"
 224   kubectl scale --replicas=${PRODUCERS_COUNT} deployment ${PRODUCER_APPS_DEPLOYMENT} -n ${ONAP_NAMESPACE}
 225   RUNNING_PRODUCERS=""
 226   while [ "${RUNNING_PRODUCERS}" != "${PRODUCERS_COUNT}" ]; do
 227       RUNNING_PRODUCERS=$(kubectl -n ${ONAP_NAMESPACE} get pods -l app=${PRODUCER_APPS_LABEL} | grep -c "Running")
 228       sleep 1s
 229     done
 230   echo "Producers are ready"
 231 }
 232
 233 function set_producers_array_internal_IP() {
 234   PRODUCER_IP_ARRAY=$(kubectl -n ${ONAP_NAMESPACE} get endpoints ${PRODUCER_SERVICE} -o jsonpath="{.subsets[*].addresses[*].ip}")
 235 }
 236
 237 function send_post_request_to_each_producer() {
 238   set_producers_array_internal_IP
 239   PROXY_POD=$(kubectl -n ${ONAP_NAMESPACE} get pods -l app=${PRODUCER_PROXY_APPS_LABEL} -o name)
 240   echo "Sending POST request to each producer, endpoint: ${REQUEST_ENDPOINT}"
 241   REQUEST_ARRAY=""
 242   for item in ${PRODUCER_IP_ARRAY[*]}
 243     do
 244     URL="${item}:${PRODUCER_INTERNAL_PORT}${REQUEST_ENDPOINT}"
 245     echo ${URL}
 246     REQUEST_ARRAY="curl -H '${CONTENT_TYPE_HEADER}' -X POST -d '${REQUEST_JSON_DATA}' ${URL} ; ${REQUEST_ARRAY}"
 247     done
 248     kubectl -n onap exec -it ${PROXY_POD} -- /bin/sh -c "${REQUEST_ARRAY}"
 249   echo "Request was send to each producer"
 250 }
 251
 252 function send_get_request_to_each_producer() {
 253   set_producers_array_internal_IP
 254   PROXY_POD=$(kubectl -n ${ONAP_NAMESPACE} get pods -l app=${PRODUCER_PROXY_APPS_LABEL} -o name)
 255   echo "Sending GET request to each producer, endpoint: ${REQUEST_ENDPOINT}"
 256   REQUEST_ARRAY=""
 257   for item in ${PRODUCER_IP_ARRAY[*]}
 258     do
 259     URL="${item}:${PRODUCER_INTERNAL_PORT}${REQUEST_ENDPOINT}"
 260     echo ${URL}
 261     REQUEST_ARRAY="curl -X GET ${URL} ; ${REQUEST_ARRAY}"
 262     done
 263     kubectl -n onap exec -it ${PROXY_POD} -- /bin/sh -c "${REQUEST_ARRAY}"
 264   echo "Request was send to each producer"
 265 }
 266
 267 function send_configuration() {
 268   REQUEST_ENDPOINT="/configuration"
 269   REQUEST_JSON_DATA=$(cat producers-config/producer-config.json)
 270   echo "Sending producer configuration: "
 271   echo ${REQUEST_JSON_DATA}
 272   send_post_request_to_each_producer
 273   echo "Configuration was send to each producer pod"
 274   exit 0
 275 }
 276
 277 function start_interval_mode() {
 278   set_kafka_retention_time
 279   REQUEST_ENDPOINT="/interval"
 280   REQUEST_JSON_DATA=$(cat producers-config/interval-config.json)
 281   print_test_setup_info
 282   echo "Sending start interval command to producer pods"
 283   send_post_request_to_each_producer
 284   echo "Command was send to each producer pod"
 285   exit 0
 286 }
 287
 288 function start_instant_mode() {
 289   set_kafka_retention_time
 290   REQUEST_ENDPOINT="/instant"
 291   REQUEST_JSON_DATA=$(cat producers-config/instant-config.json)
 292   print_test_setup_info
 293
 294   echo "Sending start instant command to producer pods"
 295   send_post_request_to_each_producer
 296   echo "Command was send to each producer pod"
 297   exit 0
 298 }
 299 function stop_producer_pods() {
 300   REQUEST_ENDPOINT="/cancel"
 301
 302   echo "Sending stop command"
 303   send_get_request_to_each_producer
 304   echo "Stop command was send to each producer pod"
 305   exit 0
 306 }
 307 function reset_producer_pods() {
 308   REQUEST_ENDPOINT="/monitoring/prometheus/reset"
 309
 310   echo "Sending reset command"
 311   send_get_request_to_each_producer
 312   echo "Reset command was send to each producer pod"
 313   exit 0
 314 }
 315
 316
 317 cd ${SCRIPT_DIRECTORY}
 318
 319 if [[ $# -eq 0 ]]; then
 320     usage
 321 else
 322     for arg in ${@}
 323     do
 324         case ${arg} in
 325             gen_certs)
 326             generate_certs
 327             ;;
 328             setup)
 329             setup_environment
 330             ;;
 331             setup_all)
 332             setup_all
 333             ;;
 334             start)
 335             echo "Option start is deprecated. Last support commit: 3e7de0deb033e485d519c74feaffecc02e7e9dc7"
 336             ;;
 337             clean)
 338             clean
 339             ;;
 340             clean_all)
 341             clean_all
 342             ;;
 343             send_config)
 344             send_configuration
 345             ;;
 346             start_interval)
 347             shift 1
 348             while [[ $(($#)) -gt 0 ]]; do
 349                 case "${1}" in
 350                     --producers)
 351                         PRODUCERS_COUNT=${2}
 352                         scale_producer_deployment
 353                         send_configuration
 354                         ;;
 355                     --retention-time-minutes)
 356                         KAFKA_RETENTION_TIME_MINUTES=${2}
 357                         ;;
 358                     *)
 359                         echo "Unknown option: ${1}"
 360                         usage
 361                         ;;
 362                 esac
 363                 shift 2
 364             done
 365             start_interval_mode
 366             ;;
 367             start_instant)
 368             shift 1
 369             while [[ $(($#)) -gt 0 ]]; do
 370                 case "${1}" in
 371                     --producers)
 372                         PRODUCERS_COUNT=${2}
 373                         scale_producer_deployment
 374                         send_configuration
 375                         ;;
 376                     --retention-time-minutes)
 377                         KAFKA_RETENTION_TIME_MINUTES=${2}
 378                         ;;
 379                     *)
 380                         echo "Unknown option: ${1}"
 381                         usage
 382                         ;;
 383                 esac
 384                 shift 2
 385             done
 386             start_instant_mode
 387             ;;
 388             stop)
 389             stop_producer_pods
 390             ;;
 391             reset_producers)
 392             reset_producer_pods
 393             ;;
 394             scale_producers)
 395             shift 1
 396             PRODUCERS_COUNT=${1}
 397             scale_producer_deployment
 398             exit 0
 399             ;;
 400             help)
 401             usage
 402             ;;
 403             *)
 404             echo "Unknown action: ${arg}" >&2
 405             usage
 406             ;;
 407         esac
 408     done
 409 fi