Merge "Add metric for processing without routing"

diff --git a/INFO.yaml b/INFO.yaml
index 6176c0f..b095763 100644 (file)
--- a/INFO.yaml
+++ b/INFO.yaml
@@ -28,8 +28,7 @@ meetings:
       repeats: 'weekly'
       time: '13:00 UTC (DST), 15:30 UTC (post DST)'
 repositories:
-    - 'dcaegen2-collectors-hv-ves'
-    - 'dcaegen2-services-prh'
+    - 'dcaegen2/collectors/hv-ves'
 committers:
     - <<: *onap_dcaegen2_ptl
     - name: 'Lusheng Ji'
@@ -62,11 +61,6 @@ committers:
       company: 'ATT'
       id: 'jflucas'
       timezone: 'America/New_York'
-    - name: 'Piotr Jaszczyk'
-      email: 'piotr.jaszczyk@nokia.com'
-      company: 'Nokia'
-      id: 'jaszczur'
-      timezone: 'Europe/Warsaw'
     - name: 'Piotr Wielebski'
       email: 'piotr.wielebski@nokia.com'
       company: 'Nokia'
@@ -74,11 +68,16 @@ committers:
       timezone: 'Europe/Warsaw'
 tsc:
     approval: 'https://lists.onap.org/pipermail/onap-tsc'
-    changes: 
+    changes:
         - type: 'Removal'
           name: 'Przemyslaw Wasala'
           link: 'https://lists.onap.org/g/onap-tsc/message/4248'
         - type: 'Addition'
           name: 'Piotr Jaszczyk'
+          link: 'https://lists.onap.org/g/onap-tsc/message/4259'
+        - type: 'Addition'
           name: 'Piotr Wielebski'
           link: 'https://lists.onap.org/g/onap-tsc/message/4259'
+        - type: 'Removal'
+          name: 'Piotr Jaszczyk'
+          link: 'https://lists.onap.org/g/onap-tsc/message/5539'

diff --git a/tools/performance/cloud/cloud-based-performance-test.sh b/tools/performance/cloud/cloud-based-performance-test.sh
index 283c130..6c27b69 100755 (executable)
--- a/tools/performance/cloud/cloud-based-performance-test.sh
+++ b/tools/performance/cloud/cloud-based-performance-test.sh
@@ -37,6 +37,12 @@ COMPLETED_PRODUCERS_SUM=0
 NAME_REASON_PATTERN="custom-columns=NAME:.metadata.name,REASON:.status.containerStatuses[].state.waiting.reason"
 HVVES_POD_NAME=$(kubectl -n ${ONAP_NAMESPACE} get pods --no-headers=true -o custom-columns=:metadata.name | grep hv-ves-collector)
 HVVES_CERT_PATH=/etc/ves-hv/ssl/
+KAFKA_RETENTION_TIME_MINUTES=60
+MILISECONDS_IN_MINUTE=60000
+CALC_RETENTION_TIME_IN_MS_CMD='expr $KAFKA_RETENTION_TIME_MINUTES \* $MILISECONDS_IN_MINUTE'
+KAFKA_ROUTER_0_POD_NAME=$(kubectl -n ${ONAP_NAMESPACE} get pods --no-headers=true -o custom-columns=:metadata.name | grep router-kafka-0)
+KAFKA_SET_TOPIC_RETENTION_TIME_CMD='kafka-topics.sh --zookeeper message-router-zookeeper:2181 --alter --topic HV_VES_PERF3GPP --config retention.ms='
+HIDE_OUTPUT='grep abc | grep 123'
 
 function clean() {
     echo "Cleaning up environment"
@@ -130,6 +136,7 @@ function print_test_setup_info() {
     echo "Test configuration:"
     echo "Producer containers count: ${CONTAINERS_COUNT}"
     echo "Properties file path: ${PROPERTIES_FILE}"
+    echo "Retention time of kafka messages in minutes: ${KAFKA_RETENTION_TIME_MINUTES}"
     echo "________________________________________"
 }
 
@@ -141,9 +148,10 @@ function usage() {
     echo "  setup    : set up ConfigMap and consumers"
     echo "  start    : create producers - start the performance test"
     echo "    Optional parameters:"
-    echo "      --load            : should test keep defined containers number till script interruption (false)"
-    echo "      --containers      : number of producer containers to create (1)"
-    echo "      --properties-file : path to file with benchmark properties (./test.properties)"
+    echo "      --load              : should test keep defined containers number till script interruption (false)"
+    echo "      --containers        : number of producer containers to create (1)"
+    echo "      --properties-file   : path to file with benchmark properties (./test.properties)"
+    echo "      --retention-time-minutes : messages retention time on kafka in minutes - only for load tests (60)"
     echo "  clean    : remove ConfigMap, HV-VES consumers and producers"
     echo "  help     : print usage"
     echo "Example invocations:"
@@ -152,6 +160,7 @@ function usage() {
     echo "./cloud-based-performance-test.sh start"
     echo "./cloud-based-performance-test.sh start --containers 10"
     echo "./cloud-based-performance-test.sh start --load true --containers 10"
+    echo "./cloud-based-performance-test.sh start --load true --containers 10 --retention-time-minutes 50"
     echo "./cloud-based-performance-test.sh start --properties-file ~/other_test.properties"
     echo "./cloud-based-performance-test.sh clean"
     exit 1
@@ -210,6 +219,9 @@ function setup_environment() {
 function start_load_tests() {
     print_test_setup_info
 
+    echo "Setting message retention time"
+    kubectl exec -it ${KAFKA_ROUTER_0_POD_NAME} -n ${ONAP_NAMESPACE} -- ${KAFKA_SET_TOPIC_RETENTION_TIME_CMD}$(eval $CALC_RETENTION_TIME_IN_MS_CMD) | eval $HIDE_OUTPUT
+
     echo "CTRL + C to stop/interrupt this script"
     create_producers
 
@@ -287,6 +299,9 @@ else
                     --properties-file)
                         PROPERTIES_FILE=${2}
                         ;;
+                    --retention-time-minutes)
+                        KAFKA_RETENTION_TIME_MINUTES=${2}
+                        ;;
                     *)
                         echo "Unknown option: ${1}"
                         usage

diff --git a/tools/performance/cloud/consumer-deployment.yaml b/tools/performance/cloud/consumer-deployment.yaml
index 199bf72..179f246 100755 (executable)
--- a/tools/performance/cloud/consumer-deployment.yaml
+++ b/tools/performance/cloud/consumer-deployment.yaml
@@ -42,7 +42,7 @@ spec:
     spec:
       containers:
         - name: kafka-consumer-counting
-          image: nexus3.dyn.nesc.nokia.net:10001/onap/org.onap.dcaegen2.collectors.hv-ves.hv-collector-kafka-consumer:1.3.0
+          image: nexus3.dyn.nesc.nokia.net:10001/onap/org.onap.dcaegen2.collectors.hv-ves.hv-collector-kafka-consumer:latest
           ports:
             - containerPort: 8080
           env:
@@ -86,7 +86,7 @@ spec:
     spec:
       containers:
         - name: kafka-processing-consumer
-          image: nexus3.dyn.nesc.nokia.net:10001/onap/org.onap.dcaegen2.collectors.hv-ves.hv-collector-kafka-consumer:1.3.0
+          image: nexus3.dyn.nesc.nokia.net:10001/onap/org.onap.dcaegen2.collectors.hv-ves.hv-collector-kafka-consumer:latest
           ports:
             - containerPort: 8080
           env:

diff --git a/tools/performance/cloud/reboot-test-environment.sh b/tools/performance/cloud/reboot-test-environment.sh
new file mode 100755 (executable)
index 0000000..853df14
--- /dev/null
+++ b/tools/performance/cloud/reboot-test-environment.sh
@@ -0,0 +1,116 @@
+#!/usr/bin/env bash
+# ============LICENSE_START=======================================================
+# dcaegen2-collectors-veshv
+# ================================================================================
+# Copyright (C) 2019 NOKIA
+# ================================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============LICENSE_END=========================================================
+
+ONAP_NAMESPACE=onap
+HVVES_POD_NAME=$(kubectl -n ${ONAP_NAMESPACE} get pods --no-headers=true -o custom-columns=:metadata.name | grep hv-ves-collector)
+HVVES_CONTAINER_NAME=dep-dcae-hv-ves-collector
+HV_VES_IMAGE="nexus3.dyn.nesc.nokia.net:10001/onap/org.onap.dcaegen2.collectors.hv-ves.hv-collector-main:latest"
+KAFKA_ROUTER_0_POD_NAME=$(kubectl -n ${ONAP_NAMESPACE} get pods --no-headers=true -o custom-columns=:metadata.name | grep router-kafka-0)
+KAFKA_TOPIC_RESET_CMD='kafka-topics.sh --delete  --zookeeper message-router-zookeeper:2181  --topic HV_VES_PERF3GPP'
+HIDE_OUTPUT='grep abc | grep 123'
+
+function usage() {
+    echo ""
+    echo "Reebot test environment for performance tests"
+    echo "Usage $0 reboot|help"
+    echo "  reboot: reboots the test environment"
+    echo "  help  : print usage"
+    echo "Example invocation:"
+    echo "./reboot-test-environment.sh reboot"
+    echo ""
+}
+
+function rebootEnvironment(){
+    ./cloud-based-performance-test.sh clean
+
+    redeployPod
+
+    waitForPodRedeployment
+
+    echo "Updating HV-VES image"
+    kubectl patch pod ${HVVES_POD_NAME} -n ${ONAP_NAMESPACE} --type='json' -p='[{"op": "replace", "path": "/spec/containers/0/image", "value":"'${HV_VES_IMAGE}'"}]'
+
+    echo "Deleting Kafka topic"
+    kubectl exec -it ${KAFKA_ROUTER_0_POD_NAME} -n ${ONAP_NAMESPACE} -- ${KAFKA_TOPIC_RESET_CMD} | eval $HIDE_OUTPUT
+
+    waitForHvVesRunning
+
+    ./cloud-based-performance-test.sh setup
+}
+
+
+function redeployPod(){
+    kubectl scale --replicas=0 deploy ${HVVES_CONTAINER_NAME} -n ${ONAP_NAMESPACE}
+    waitForPodTermination
+    kubectl scale --replicas=1 deploy ${HVVES_CONTAINER_NAME} -n ${ONAP_NAMESPACE}
+    sleep 10s
+}
+
+function waitForPodTermination(){
+    echo "Waiting for pod termination..."
+
+    while [ "${HVVES_POD_NAME}" != "" ] ; do
+      HVVES_POD_NAME=$(kubectl -n ${ONAP_NAMESPACE} get pods --no-headers=true -o custom-columns=:metadata.name | grep hv-ves-collector)
+      sleep 1s
+    done
+}
+
+function waitForPodRedeployment(){
+    HVVES_POD_NAME=""
+    echo "Waiting for pod redeploy..."
+
+    while [ "${HVVES_POD_NAME}" = "" ] ; do
+      HVVES_POD_NAME=$(kubectl -n ${ONAP_NAMESPACE} get pods --no-headers=true -o custom-columns=:metadata.name | grep hv-ves-collector)
+      sleep 1s
+    done
+}
+
+function waitForHvVesRunning(){
+    echo "Waiting for HV-VES running..."
+    POD_STATUS="";
+    while [ "${POD_STATUS}" != "2/2" ]; do
+      POD_STATUS=$(kubectl get pod ${HVVES_POD_NAME} -n ${ONAP_NAMESPACE} --no-headers=true | awk '{print $2}')
+      sleep 1s
+    done
+}
+
+function loadHvVesPodName(){
+    HVVES_POD_NAME=$(kubectl -n ${ONAP_NAMESPACE} get pods --no-headers=true -o custom-columns=:metadata.name | grep hv-ves-collector)
+}
+
+
+if [[ $# -eq 0 ]]; then
+    usage
+else
+    for arg in ${@}
+    do
+        case ${arg} in
+            reboot)
+            rebootEnvironment
+            ;;
+            help)
+            usage
+            ;;
+            *)
+            echo "Unknown action: ${arg}" >&2
+            usage
+            ;;
+        esac
+    done
+fi
\ No newline at end of file