Merge "Reorganize deletion of completed producers"

diff --combined tools/performance/cloud/cloud-based-performance-test.sh
index 6c27b69,8e2b384..1b5f1f9
--- 1/tools/performance/cloud/cloud-based-performance-test.sh
--- 2/tools/performance/cloud/cloud-based-performance-test.sh
+++ b/tools/performance/cloud/cloud-based-performance-test.sh
@@@ -19,8 -19,8 +19,9 @@@
  
  SCRIPT_DIRECTORY="$(pwd "$0")"
  CONTAINERS_COUNT=1
+ COMPLETED_PRODUCERS_SUM=0
  LOAD_TEST="false"
 +TEST_CONFIG_MAP=performance-test-config
  PROPERTIES_FILE=${SCRIPT_DIRECTORY}/test.properties
  PRODUCER_APPS_LABEL=hv-collector-producer
  CONSUMER_APPS_LABEL=hv-collector-kafka-consumer
@@@ -33,22 -33,16 +34,22 @@@ GRAFANA_DASHBOARD_PROVIDERS=grafana-das
  ONAP_NAMESPACE=onap
  MAXIMUM_BACK_OFF_CHECK_ITERATIONS=30
  CHECK_NUMBER=0
- COMPLETED_PRODUCERS_SUM=0
+ PRODUCERS_TO_RECREATE=0
  NAME_REASON_PATTERN="custom-columns=NAME:.metadata.name,REASON:.status.containerStatuses[].state.waiting.reason"
  HVVES_POD_NAME=$(kubectl -n ${ONAP_NAMESPACE} get pods --no-headers=true -o custom-columns=:metadata.name | grep hv-ves-collector)
  HVVES_CERT_PATH=/etc/ves-hv/ssl/
 +KAFKA_RETENTION_TIME_MINUTES=60
 +MILISECONDS_IN_MINUTE=60000
 +CALC_RETENTION_TIME_IN_MS_CMD='expr $KAFKA_RETENTION_TIME_MINUTES \* $MILISECONDS_IN_MINUTE'
 +KAFKA_ROUTER_0_POD_NAME=$(kubectl -n ${ONAP_NAMESPACE} get pods --no-headers=true -o custom-columns=:metadata.name | grep router-kafka-0)
 +KAFKA_SET_TOPIC_RETENTION_TIME_CMD='kafka-topics.sh --zookeeper message-router-zookeeper:2181 --alter --topic HV_VES_PERF3GPP --config retention.ms='
 +HIDE_OUTPUT='grep abc | grep 123'
  
  function clean() {
      echo "Cleaning up environment"
  
      echo "Attempting to delete test parameters ConfigMap"
 -    kubectl delete configmap performance-test-config -n ${ONAP_NAMESPACE}
 +    kubectl delete configmap ${TEST_CONFIG_MAP} -n ${ONAP_NAMESPACE}
  
      echo "Attempting to delete prometheus ConfigMap"
      kubectl delete configmap -l name=${PROMETHEUS_CONF_LABEL} -n ${ONAP_NAMESPACE}
@@@ -59,13 -53,13 +60,13 @@@
      echo "Attempting to delete grafana deployment and service"
      kubectl delete service,deployments -l app=${GRAFANA_APPS_LABEL} -n ${ONAP_NAMESPACE}
  
 -    echo "Attempting to delete grafana ConfigMap(DASHBOARD)"
 +    echo "Attempting to delete grafana ConfigMap (DASHBOARD)"
      kubectl delete configmap -l name=${GRAFANA_DASHBOARD} -n ${ONAP_NAMESPACE}
  
 -    echo "Attempting to delete grafana ConfigMap(GRAFANA_DASHBOARD_PROVIDERS)"
 +    echo "Attempting to delete grafana ConfigMap (GRAFANA_DASHBOARD_PROVIDERS)"
      kubectl delete configmap -l name=${GRAFANA_DASHBOARD_PROVIDERS} -n ${ONAP_NAMESPACE}
  
 -    echo "Attempting to delete grafana ConfigMap(GRAFANA_DATASOURCE)"
 +    echo "Attempting to delete grafana ConfigMap (GRAFANA_DATASOURCE)"
      kubectl delete configmap -l name=${GRAFANA_DATASOURCE} -n ${ONAP_NAMESPACE}
  
      echo "Attempting to delete consumer deployments"
@@@ -94,8 -88,8 +95,8 @@@ function copy_certs_to_hvves() 
  
  function create_producers() {
      echo "Recreating test properties ConfigMap from: $PROPERTIES_FILE"
 -    kubectl delete configmap performance-test-config -n ${ONAP_NAMESPACE}
 -    kubectl create configmap performance-test-config --from-env-file=${PROPERTIES_FILE} -n ${ONAP_NAMESPACE}
 +    kubectl delete configmap ${TEST_CONFIG_MAP} -n ${ONAP_NAMESPACE}
 +    kubectl create configmap ${TEST_CONFIG_MAP} --from-env-file=${PROPERTIES_FILE} -n ${ONAP_NAMESPACE}
  
      set -e
      for i in $(seq 1 ${CONTAINERS_COUNT});
@@@ -114,6 -108,7 +115,7 @@@ function generate_certs() 
  }
  
  function handle_backoffs() {
+         IMAGE_PULL_BACK_OFFS=$(kubectl get pods -l app=${PRODUCER_APPS_LABEL} -n ${ONAP_NAMESPACE} -o ${NAME_REASON_PATTERN} | grep -c "ImagePullBackOff \| ErrImagePull")
          if [[ ${IMAGE_PULL_BACK_OFFS} -gt 0 ]]; then
              CHECK_NUMBER=$((CHECK_NUMBER + 1))
              if [[ ${CHECK_NUMBER} -gt ${MAXIMUM_BACK_OFF_CHECK_ITERATIONS} ]]; then
@@@ -126,7 -121,9 +128,9 @@@
  function handle_key_interrupt() {
      trap SIGINT
      echo "Script interrupted, attempt to delete producers"
-     kubectl delete pods -l app=${PRODUCER_APPS_LABEL} -n ${ONAP_NAMESPACE}
+     echo "Wait with patience"
+     COMPLETED_PRODUCERS_SUM=$(($(kubectl delete pods -l app=${PRODUCER_APPS_LABEL} -n ${ONAP_NAMESPACE} | grep producer | wc -l) + COMPLETED_PRODUCERS_SUM))
+     echo "Total number of completed producers: ${COMPLETED_PRODUCERS_SUM}"
      exit 0
  }
  
@@@ -136,7 -133,6 +140,7 @@@ function print_test_setup_info() 
      echo "Test configuration:"
      echo "Producer containers count: ${CONTAINERS_COUNT}"
      echo "Properties file path: ${PROPERTIES_FILE}"
 +    echo "Retention time of kafka messages in minutes: ${KAFKA_RETENTION_TIME_MINUTES}"
      echo "________________________________________"
  }
  
@@@ -148,10 -144,9 +152,10 @@@ function usage() 
      echo "  setup    : set up ConfigMap and consumers"
      echo "  start    : create producers - start the performance test"
      echo "    Optional parameters:"
 -    echo "      --load      : should test keep defined containers number till script interruption (false)"
 -    echo "      --containers      : number of producer containers to create (1)"
 -    echo "      --properties-file : path to file with benchmark properties (./test.properties)"
 +    echo "      --load              : should test keep defined containers number till script interruption (false)"
 +    echo "      --containers        : number of producer containers to create (1)"
 +    echo "      --properties-file   : path to file with benchmark properties (./test.properties)"
 +    echo "      --retention-time-minutes : messages retention time on kafka in minutes - only for load tests (60)"
      echo "  clean    : remove ConfigMap, HV-VES consumers and producers"
      echo "  help     : print usage"
      echo "Example invocations:"
@@@ -160,7 -155,6 +164,7 @@@
      echo "./cloud-based-performance-test.sh start"
      echo "./cloud-based-performance-test.sh start --containers 10"
      echo "./cloud-based-performance-test.sh start --load true --containers 10"
 +    echo "./cloud-based-performance-test.sh start --load true --containers 10 --retention-time-minutes 50"
      echo "./cloud-based-performance-test.sh start --properties-file ~/other_test.properties"
      echo "./cloud-based-performance-test.sh clean"
      exit 1
@@@ -168,7 -162,6 +172,7 @@@
  
  function setup_environment() {
      echo "Setting up environment"
 +
      echo "Copying certs to hv-ves pod"
      copy_certs_to_hvves
  
@@@ -180,7 -173,7 +184,7 @@@
      ./configure-consul.sh false
  
      echo "Creating test properties ConfigMap from: $PROPERTIES_FILE"
 -    kubectl create configmap performance-test-config --from-env-file=${PROPERTIES_FILE} -n ${ONAP_NAMESPACE}
 +    kubectl create configmap ${TEST_CONFIG_MAP} --from-env-file=${PROPERTIES_FILE} -n ${ONAP_NAMESPACE}
  
      echo "Creating consumer deployment"
      kubectl apply -f consumer-deployment.yaml
@@@ -219,9 -212,6 +223,9 @@@
  function start_load_tests() {
      print_test_setup_info
  
 +    echo "Setting message retention time"
 +    kubectl exec -it ${KAFKA_ROUTER_0_POD_NAME} -n ${ONAP_NAMESPACE} -- ${KAFKA_SET_TOPIC_RETENTION_TIME_CMD}$(eval $CALC_RETENTION_TIME_IN_MS_CMD) | eval $HIDE_OUTPUT
 +
      echo "CTRL + C to stop/interrupt this script"
      create_producers
  
@@@ -229,21 -219,22 +233,22 @@@
  
      echo "Constant producer number keeper started working"
      while :; do
-         COMPLETED_PRODUCERS=$(($(kubectl get pods -l app=${PRODUCER_APPS_LABEL} -n ${ONAP_NAMESPACE} | grep -c "Completed")-COMPLETED_PRODUCERS_SUM))
-         IMAGE_PULL_BACK_OFFS=$(kubectl get pods -l app=${PRODUCER_APPS_LABEL} -n ${ONAP_NAMESPACE} -o ${NAME_REASON_PATTERN} | grep -c "ImagePullBackOff \| ErrImagePull")
- 
+         PRODUCERS_TO_RECREATE=$((CONTAINERS_COUNT-$(kubectl get pods -l app=${PRODUCER_APPS_LABEL} -n ${ONAP_NAMESPACE} | grep -c "Running")))
          handle_backoffs
  
          set -e
-         for i in $(seq 1 ${COMPLETED_PRODUCERS});
+         for i in $(seq 1 ${PRODUCERS_TO_RECREATE});
          do
-             echo "Recreating ${i}/${COMPLETED_PRODUCERS} producer"
+             echo "Recreating ${i}/${PRODUCERS_TO_RECREATE} producer"
              kubectl create -f producer-pod.yaml -n ${ONAP_NAMESPACE}
-             COMPLETED_PRODUCERS_SUM=$((COMPLETED_PRODUCERS_SUM + 1))
          done
          set +e
+         COMPLETED_PRODUCERS_SUM=$((COMPLETED_PRODUCERS_SUM + PRODUCERS_TO_RECREATE))
+         echo "Attempting to clear completed producers"
+         kubectl delete pod --field-selector=status.phase==Succeeded -l app=${PRODUCER_APPS_LABEL} -n ${ONAP_NAMESPACE}
+ 
          [[ ${CHECK_NUMBER} -gt ${MAXIMUM_BACK_OFF_CHECK_ITERATIONS} ]] && break
-         sleep 2
+         sleep 1
      done
  
      trap SIGINT
@@@ -258,8 -249,6 +263,6 @@@ function start_performance_test() 
      echo "Waiting for producers completion"
      while :; do
          COMPLETED_PRODUCERS=$(kubectl get pods -l app=${PRODUCER_APPS_LABEL} -n ${ONAP_NAMESPACE} | grep -c "Completed")
-         IMAGE_PULL_BACK_OFFS=$(kubectl get pods -l app=${PRODUCER_APPS_LABEL} -n ${ONAP_NAMESPACE} -o ${NAME_REASON_PATTERN} | grep -c "ImagePullBackOff \| ErrImagePull")
- 
          handle_backoffs
          
          [[ ${COMPLETED_PRODUCERS} -eq ${CONTAINERS_COUNT} || ${CHECK_NUMBER} -gt ${MAXIMUM_BACK_OFF_CHECK_ITERATIONS} ]] && break
@@@ -299,9 -288,6 +302,9 @@@ els
                      --properties-file)
                          PROPERTIES_FILE=${2}
                          ;;
 +                    --retention-time-minutes)
 +                        KAFKA_RETENTION_TIME_MINUTES=${2}
 +                        ;;
                      *)
                          echo "Unknown option: ${1}"
                          usage