tools/performance/cloud/cloud-based-performance-test.sh

   1 #!/usr/bin/env bash
   2 # ============LICENSE_START=======================================================
   3 # dcaegen2-collectors-veshv
   4 # ================================================================================
   5 # Copyright (C) 2019 NOKIA
   6 # ================================================================================
   7 # Licensed under the Apache License, Version 2.0 (the "License");
   8 # you may not use this file except in compliance with the License.
   9 # You may obtain a copy of the License at
  10 #
  11 #      http://www.apache.org/licenses/LICENSE-2.0
  12 #
  13 # Unless required by applicable law or agreed to in writing, software
  14 # distributed under the License is distributed on an "AS IS" BASIS,
  15 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16 # See the License for the specific language governing permissions and
  17 # limitations under the License.
  18 # ============LICENSE_END=========================================================
  19
  20 SCRIPT_DIRECTORY="$(pwd "$0")"
  21 CONTAINERS_COUNT=1
  22 PROPERTIES_FILE=${SCRIPT_DIRECTORY}/test.properties
  23 CONFIG_MAP_NAME=performance-test-config
  24 PRODUCER_APPS_LABEL=hv-collector-producer
  25 CONSUMER_APPS_LABEL=hv-collector-kafka-consumer
  26 PROMETHEUS_CONF_LABEL=prometheus-server-conf
  27 PROMETHEUS_APPS_LABEL=hv-collector-prometheus
  28 GRAFANA_APPS_LABEL=hv-collector-grafana
  29 GRAFANA_DATASOURCE=grafana-datasources
  30 GRAFANA_DASHBOARD=grafana-dashboards
  31 GRAFANA_DASHBOARD_PROVIDERS=grafana-dashboards-providers
  32 ONAP_NAMESPACE=onap
  33 MAXIMUM_BACK_OFF_CHECK_ITERATIONS=30
  34 CHECK_NUMBER=0
  35 NAME_REASON_PATTERN="custom-columns=NAME:.metadata.name,REASON:.status.containerStatuses[].state.waiting.reason"
  36
  37 function clean() {
  38     echo "Cleaning up environment"
  39
  40     echo "Attempting to delete test parameters ConfigMap"
  41     kubectl delete configmap ${CONFIG_MAP_NAME} -n ${ONAP_NAMESPACE}
  42
  43     echo "Attempting to delete prometheus ConfigMap"
  44     kubectl delete configmap -l name=${PROMETHEUS_CONF_LABEL} -n ${ONAP_NAMESPACE}
  45
  46     echo "Attempting to delete prometheus deployment and service"
  47     kubectl delete service,deployments -l app=${PROMETHEUS_APPS_LABEL} -n ${ONAP_NAMESPACE}
  48
  49     echo "Attempting to delete grafana deployment and service"
  50     kubectl delete service,deployments -l app=${GRAFANA_APPS_LABEL} -n ${ONAP_NAMESPACE}
  51
  52     echo "Attempting to delete grafana ConfigMap(DASHBOARD)"
  53     kubectl delete configmap -l name=${GRAFANA_DASHBOARD} -n ${ONAP_NAMESPACE}
  54
  55     echo "Attempting to delete grafana ConfigMap(GRAFANA_DASHBOARD_PROVIDERS)"
  56     kubectl delete configmap -l name=${GRAFANA_DASHBOARD_PROVIDERS} -n ${ONAP_NAMESPACE}
  57
  58     echo "Attempting to delete grafana ConfigMap(GRAFANA_DATASOURCE)"
  59     kubectl delete configmap -l name=${GRAFANA_DATASOURCE} -n ${ONAP_NAMESPACE}
  60
  61     echo "Attempting to delete consumer deployments"
  62     kubectl delete deployments -l app=${CONSUMER_APPS_LABEL} -n ${ONAP_NAMESPACE}
  63
  64     echo "Attempting to delete producer pods"
  65     kubectl delete pods -l app=${PRODUCER_APPS_LABEL} -n ${ONAP_NAMESPACE}
  66
  67     echo "Environment clean up finished!"
  68 }
  69
  70 function create_producers() {
  71     set -e
  72     for i in $(seq 1 ${CONTAINERS_COUNT});
  73     do
  74         echo "Creating ${i}/${CONTAINERS_COUNT} producer"
  75         kubectl create -f producer-pod.yaml -n ${ONAP_NAMESPACE}
  76     done
  77     echo "Producers created"
  78     set +e
  79 }
  80
  81 function usage() {
  82     echo ""
  83     echo "Run cloud based HV-VES performance test"
  84     echo "Usage $0 setup|start|clean|help"
  85     echo "  setup    : set up ConfigMap and consumers"
  86     echo "  start    : create producers - start the performance test"
  87     echo "    Optional parameters:"
  88     echo "      --containers      : number of producer containers to create (1)"
  89     echo "      --properties-file : path to file with benchmark properties (./test.properties)"
  90     echo "  clean    : remove ConfigMap, HV-VES consumers and producers"
  91     echo "  help     : print usage"
  92     echo "Example invocations:"
  93     echo "./cloud-based-performance-test.sh setup"
  94     echo "./cloud-based-performance-test.sh start"
  95     echo "./cloud-based-performance-test.sh start --containers 10"
  96     echo "./cloud-based-performance-test.sh start --properties-file ~/other_test.properties"
  97     echo "./cloud-based-performance-test.sh clean"
  98     exit 1
  99 }
 100
 101 function setup_environment() {
 102     echo "Setting up environment"
 103     echo "Creating test properties ConfigMap from: $PROPERTIES_FILE"
 104     kubectl create configmap ${CONFIG_MAP_NAME} --from-env-file=${PROPERTIES_FILE} -n ${ONAP_NAMESPACE}
 105
 106     echo "Creating consumer deployment"
 107     kubectl apply -f consumer-deployment.yaml
 108
 109     echo "Creating ConfigMap for prometheus deployment"
 110     kubectl apply -f prometheus-config-map.yaml
 111
 112     echo "Creating prometheus deployment"
 113     kubectl apply -f prometheus-deployment.yaml
 114
 115     echo "Creating ConfigMap for grafana connections dashboard"
 116     kubectl apply -f grafana/dashboards/connections.yaml
 117
 118     echo "Creating ConfigMap for grafana processing dashboard"
 119     kubectl apply -f grafana/dashboards/processing.yaml
 120
 121     echo "Creating ConfigMap for grafana grafana-performance-tests dashboard"
 122     kubectl apply -f grafana/dashboards/performance-tests.yaml
 123
 124     echo "Creating ConfigMap for grafana datasource"
 125     kubectl apply -f grafana/datasources/datasource.yaml
 126
 127     echo "Creating ConfigMap for grafana dashboards-providers"
 128     kubectl apply -f grafana/dashboards-providers/dashboards.yaml
 129
 130     echo "Creating grafana deployment"
 131     kubectl apply -f grafana-deployment.yaml
 132
 133     echo "Waiting for consumers to be running."
 134     while [[ $(kubectl get pods -l app=${CONSUMER_APPS_LABEL} -n ${ONAP_NAMESPACE} | grep -c "unhealthy\|starting") -ne 0 ]] ; do
 135         sleep 1
 136     done
 137     echo "Setting up environment finished!"
 138 }
 139
 140 function start_performance_test() {
 141     echo "Starting cloud based performance tests"
 142     echo "________________________________________"
 143     echo "Test configuration:"
 144     echo "Producer containers count: ${CONTAINERS_COUNT}"
 145     echo "Properties file path: ${PROPERTIES_FILE}"
 146     echo "________________________________________"
 147
 148     create_producers
 149
 150     echo "Waiting for producers completion"
 151     while :; do
 152         COMPLETED_PRODUCERS=$(kubectl get pods -l app=${PRODUCER_APPS_LABEL} -n ${ONAP_NAMESPACE} | grep -c "Completed")
 153         IMAGE_PULL_BACK_OFFS=$(kubectl get pods -l app=${PRODUCER_APPS_LABEL} -n ${ONAP_NAMESPACE} -o ${NAME_REASON_PATTERN} | grep -c "ImagePullBackOff \| ErrImagePull")
 154
 155         if [[ ${IMAGE_PULL_BACK_OFFS} -gt 0 ]]; then
 156             CHECK_NUMBER=$((CHECK_NUMBER + 1))
 157             if [[ ${CHECK_NUMBER} -gt ${MAXIMUM_BACK_OFF_CHECK_ITERATIONS} ]]; then
 158                 echo "Error: Image pull problem"
 159                 exit 1
 160             fi
 161         fi
 162
 163         [[ ${COMPLETED_PRODUCERS} -eq ${CONTAINERS_COUNT} || ${CHECK_NUMBER} -gt ${MAXIMUM_BACK_OFF_CHECK_ITERATIONS} ]] && break
 164         sleep 1
 165     done
 166
 167     echo "Attempting to delete producer pods"
 168     kubectl delete pods -l app=${PRODUCER_APPS_LABEL} -n ${ONAP_NAMESPACE}
 169     echo "Performance test finished"
 170     exit 0
 171 }
 172
 173 cd ${SCRIPT_DIRECTORY}
 174
 175 if [[ $# -eq 0 ]]; then
 176     usage
 177 else
 178     for arg in ${@}
 179     do
 180         case ${arg} in
 181             setup)
 182             setup_environment
 183             ;;
 184             start)
 185             shift 1
 186             while [[ $(($#)) -gt 0 ]]; do
 187                 case "${1}" in
 188                     --containers)
 189                         CONTAINERS_COUNT=${2}
 190                         ;;
 191                     --properties-file)
 192                         PROPERTIES_FILE=${2}
 193                         ;;
 194                     *)
 195                         echo "Unknown option: ${1}"
 196                         usage
 197                         ;;
 198                 esac
 199                 shift 2
 200             done
 201             start_performance_test
 202             ;;
 203             clean)
 204             clean
 205             ;;
 206             help)
 207             usage
 208             ;;
 209             *)
 210             echo "Unknown action: ${arg}" >&2
 211             usage
 212             ;;
 213         esac
 214     done
 215 fi