Adjust KPI threshold and rate as per latest KPI agreement

- CM Handle ID searches now can take 2.6 sec
- CM Handle searches now can take 24 sec
- Updated 'blue' lines to reflect KPI
- Updated 'green' lines to reflect current average (i.e. we expect measurements to be 'around' this)
- Experimental: increased cm notification forwarding by using more VUs
- Add timer metric to be able to check cm notification forwarding rate
- Temporary: removed redirecting of warning and errors to null device

Issue-ID: CPS-2793
Change-Id: I560985d4f9ce7a4c8aa27b3ae089a43a8a76b28a
Signed-off-by: ToineSiebelink <toine.siebelink@est.tech>

diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/cmnotificationsubscription/cmavc/CmAvcEventConsumer.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/cmnotificationsubscription/cmavc/CmAvcEventConsumer.java
index eca8380..c8aec3e 100644 (file)
--- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/cmnotificationsubscription/cmavc/CmAvcEventConsumer.java
+++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/cmnotificationsubscription/cmavc/CmAvcEventConsumer.java
@@ -21,6 +21,7 @@
 package org.onap.cps.ncmp.impl.cmnotificationsubscription.cmavc;
 
 import io.cloudevents.CloudEvent;
+import io.micrometer.core.annotation.Timed;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.kafka.clients.consumer.ConsumerRecord;
@@ -31,7 +32,7 @@ import org.springframework.kafka.annotation.KafkaListener;
 import org.springframework.stereotype.Component;
 
 /**
- * Listener for AVC events based on Cm Subscriptions.
+ * Listener for AVC events based on CM Subscriptions.
  */
 @Component
 @Slf4j
@@ -53,6 +54,7 @@ public class CmAvcEventConsumer {
      */
     @KafkaListener(topics = "${app.dmi.cm-events.topic}",
             containerFactory = "cloudEventConcurrentKafkaListenerContainerFactory")
+    @Timed(value = "cps.ncmp.cmnotifications.consumeandforward", description = "Time taken to forward CM AVC events")
     public void consumeAndForward(
             final ConsumerRecord<String, CloudEvent> cmAvcEventAsConsumerRecord) {
         final CloudEvent outgoingAvcEvent = cmAvcEventAsConsumerRecord.value();

diff --git a/docs/test_ScrapeMetrics.py b/docs/test_ScrapeMetrics.py
index cbcc92d..f59e4fc 100644 (file)
--- a/docs/test_ScrapeMetrics.py
+++ b/docs/test_ScrapeMetrics.py
@@ -91,7 +91,7 @@ class TestScrapeMetrics(unittest.TestCase):
         with open(metrics_file, 'r') as f:
             lines = f.readlines()
 
-        expected_number_of_metrics = 56
+        expected_number_of_metrics = 57
         expected_number_of_lines = expected_number_of_metrics + 1  # Header
         self.assertEqual(len(lines), expected_number_of_lines, f"metrics.csv does not have {expected_number_of_lines} lines.")
 
@@ -101,4 +101,4 @@ if __name__ == '__main__':
     script_dir = os.path.dirname(os.path.abspath(__file__))
     if script_dir not in sys.path:
         sys.path.insert(0, script_dir)
-    unittest.main()
\ No newline at end of file
+    unittest.main()

diff --git a/k6-tests/ncmp/common/utils.js b/k6-tests/ncmp/common/utils.js
index ea77aae..9406c1c 100644 (file)
--- a/k6-tests/ncmp/common/utils.js
+++ b/k6-tests/ncmp/common/utils.js
@@ -115,21 +115,21 @@ export function makeCustomSummaryReport(testResults, scenarioConfig) {
     const summaryCsvLines = [
         '#,Test Name,Unit,Fs Requirement,Current Expectation,Actual',
         makeSummaryCsvLine('0', 'HTTP request failures for all tests', 'rate of failed requests', 'http_req_failed', 0, testResults, scenarioConfig),
-        makeSummaryCsvLine('1', 'Registration of CM-handles', 'CM-handles/second', 'cmhandles_created_per_second', 90, testResults, scenarioConfig),
-        makeSummaryCsvLine('2', 'De-registration of CM-handles', 'CM-handles/second', 'cmhandles_deleted_per_second', 140, testResults, scenarioConfig),
-        makeSummaryCsvLine('3a', 'CM-handle ID search with No filter', 'milliseconds', 'id_search_nofilter_duration', 300, testResults, scenarioConfig),
-        makeSummaryCsvLine('3b', 'CM-handle ID search with Module filter', 'milliseconds', 'id_search_module_duration', 120, testResults, scenarioConfig),
-        makeSummaryCsvLine('3c', 'CM-handle ID search with Property filter', 'milliseconds', 'id_search_property_duration', 750, testResults, scenarioConfig),
-        makeSummaryCsvLine('3d', 'CM-handle ID search with Cps Path filter', 'milliseconds', 'id_search_cpspath_duration', 750, testResults, scenarioConfig),
-        makeSummaryCsvLine('3e', 'CM-handle ID search with Trust Level filter', 'milliseconds', 'id_search_trustlevel_duration', 3000, testResults, scenarioConfig),
-        makeSummaryCsvLine('4a', 'CM-handle search with No filter', 'milliseconds', 'cm_search_nofilter_duration', 3000, testResults, scenarioConfig),
-        makeSummaryCsvLine('4b', 'CM-handle search with Module filter', 'milliseconds', 'cm_search_module_duration', 4000, testResults, scenarioConfig),
-        makeSummaryCsvLine('4c', 'CM-handle search with Property filter', 'milliseconds', 'cm_search_property_duration', 4500, testResults, scenarioConfig),
-        makeSummaryCsvLine('4d', 'CM-handle search with Cps Path filter', 'milliseconds', 'cm_search_cpspath_duration', 4500, testResults, scenarioConfig),
-        makeSummaryCsvLine('4e', 'CM-handle search with Trust Level filter', 'milliseconds', 'cm_search_trustlevel_duration', 7000, testResults, scenarioConfig),
-        makeSummaryCsvLine('5b', 'NCMP overhead for Synchronous single CM-handle pass-through read with alternate id', 'milliseconds', 'ncmp_overhead_passthrough_read_alt_id', 40, testResults, scenarioConfig),
-        makeSummaryCsvLine('6b', 'NCMP overhead for Synchronous single CM-handle pass-through write with alternate id', 'milliseconds', 'ncmp_overhead_passthrough_write_alt_id', 40, testResults, scenarioConfig),
-        makeSummaryCsvLine('7', 'Legacy batch read operation', 'events/second', 'legacy_batch_read_cmhandles_per_second', 300, testResults, scenarioConfig),
+        makeSummaryCsvLine('1', 'Registration of CM-handles', 'CM-handles/second', 'cmhandles_created_per_second', 100, testResults, scenarioConfig),
+        makeSummaryCsvLine('2', 'De-registration of CM-handles', 'CM-handles/second', 'cmhandles_deleted_per_second', 180, testResults, scenarioConfig),
+        makeSummaryCsvLine('3a', 'CM-handle ID search with No filter', 'milliseconds', 'id_search_nofilter_duration', 550, testResults, scenarioConfig),
+        makeSummaryCsvLine('3b', 'CM-handle ID search with Module filter', 'milliseconds', 'id_search_module_duration', 2300, testResults, scenarioConfig),
+        makeSummaryCsvLine('3c', 'CM-handle ID search with Property filter', 'milliseconds', 'id_search_property_duration', 1450, testResults, scenarioConfig),
+        makeSummaryCsvLine('3d', 'CM-handle ID search with Cps Path filter', 'milliseconds', 'id_search_cpspath_duration', 1500, testResults, scenarioConfig),
+        makeSummaryCsvLine('3e', 'CM-handle ID search with Trust Level filter', 'milliseconds', 'id_search_trustlevel_duration', 1600, testResults, scenarioConfig),
+        makeSummaryCsvLine('4a', 'CM-handle search with No filter', 'milliseconds', 'cm_search_nofilter_duration', 18000, testResults, scenarioConfig),
+        makeSummaryCsvLine('4b', 'CM-handle search with Module filter', 'milliseconds', 'cm_search_module_duration', 18000, testResults, scenarioConfig),
+        makeSummaryCsvLine('4c', 'CM-handle search with Property filter', 'milliseconds', 'cm_search_property_duration', 18000, testResults, scenarioConfig),
+        makeSummaryCsvLine('4d', 'CM-handle search with Cps Path filter', 'milliseconds', 'cm_search_cpspath_duration', 18000, testResults, scenarioConfig),
+        makeSummaryCsvLine('4e', 'CM-handle search with Trust Level filter', 'milliseconds', 'cm_search_trustlevel_duration', 18000, testResults, scenarioConfig),
+        makeSummaryCsvLine('5b', 'NCMP overhead for Synchronous single CM-handle pass-through read with alternate id', 'milliseconds', 'ncmp_overhead_passthrough_read_alt_id', 18, testResults, scenarioConfig),
+        makeSummaryCsvLine('6b', 'NCMP overhead for Synchronous single CM-handle pass-through write with alternate id', 'milliseconds', 'ncmp_overhead_passthrough_write_alt_id', 18, testResults, scenarioConfig),
+        makeSummaryCsvLine('7', 'Legacy batch read operation', 'events/second', 'legacy_batch_read_cmhandles_per_second', 1750, testResults, scenarioConfig),
     ];
     return summaryCsvLines.join('\n') + '\n';
 }

diff --git a/k6-tests/ncmp/config/kpi.json b/k6-tests/ncmp/config/kpi.json
index aa93091..0616fd8 100644 (file)
--- a/k6-tests/ncmp/config/kpi.json
+++ b/k6-tests/ncmp/config/kpi.json
@@ -12,7 +12,8 @@
       "rate": 25,
       "timeUnit": "1s",
       "duration": "15m",
-      "preAllocatedVUs": 10,
+      "preAllocatedVUs": 5,
+      "maxVUs": 25,
       "startTime": "200ms"
     },
     "passthrough_write_alt_id_scenario": {
@@ -21,97 +22,98 @@
       "rate": 13,
       "timeUnit": "1s",
       "duration": "15m",
-      "preAllocatedVUs": 10,
+      "preAllocatedVUs": 5,
+      "maxVUs": 25,
       "startTime": "600ms"
     },
     "cm_handle_id_search_nofilter_scenario": {
       "executor": "constant-arrival-rate",
       "exec": "cmHandleIdSearchNoFilterScenario",
       "rate": 1,
-      "timeUnit": "2s",
+      "timeUnit": "2600ms",
       "duration": "15m",
-      "preAllocatedVUs": 1,
+      "preAllocatedVUs": 2,
       "startTime": "0ms"
     },
     "cm_handle_id_search_module_scenario": {
       "executor": "constant-arrival-rate",
       "exec": "cmHandleIdSearchModuleScenario",
       "rate": 1,
-      "timeUnit": "2s",
+      "timeUnit": "2600ms",
       "duration": "15m",
-      "preAllocatedVUs": 1,
+      "preAllocatedVUs": 2,
       "startTime": "400ms"
     },
     "cm_handle_id_search_property_scenario": {
       "executor": "constant-arrival-rate",
       "exec": "cmHandleIdSearchPropertyScenario",
       "rate": 1,
-      "timeUnit": "2s",
+      "timeUnit": "2600ms",
       "duration": "15m",
-      "preAllocatedVUs": 1,
+      "preAllocatedVUs": 2,
       "startTime": "800ms"
     },
     "cm_handle_id_search_cpspath_scenario": {
       "executor": "constant-arrival-rate",
       "exec": "cmHandleIdSearchCpsPathScenario",
       "rate": 1,
-      "timeUnit": "2s",
+      "timeUnit": "2600ms",
       "duration": "15m",
-      "preAllocatedVUs": 1,
+      "preAllocatedVUs": 2,
       "startTime": "1200ms"
     },
     "cm_handle_id_search_trustlevel_scenario": {
       "executor": "constant-arrival-rate",
       "exec": "cmHandleIdSearchTrustLevelScenario",
       "rate": 1,
-      "timeUnit": "2s",
+      "timeUnit": "2600ms",
       "duration": "15m",
-      "preAllocatedVUs": 1,
+      "preAllocatedVUs": 2,
       "startTime": "1600ms"
     },
     "cm_handle_search_nofilter_scenario": {
       "executor": "constant-arrival-rate",
       "exec": "cmHandleSearchNoFilterScenario",
       "rate": 1,
-      "timeUnit": "15s",
+      "timeUnit": "24s",
       "duration": "15m",
-      "preAllocatedVUs": 1,
+      "preAllocatedVUs": 2,
       "startTime": "0s"
     },
     "cm_handle_search_module_scenario": {
       "executor": "constant-arrival-rate",
       "exec": "cmHandleSearchModuleScenario",
       "rate": 1,
-      "timeUnit": "15s",
+      "timeUnit": "24s",
       "duration": "15m",
-      "preAllocatedVUs": 1,
+      "preAllocatedVUs": 2,
       "startTime": "3s"
     },
     "cm_handle_search_property_scenario": {
       "executor": "constant-arrival-rate",
       "exec": "cmHandleSearchPropertyScenario",
       "rate": 1,
-      "timeUnit": "15s",
+      "timeUnit": "24s",
       "duration": "15m",
-      "preAllocatedVUs": 1,
+      "preAllocatedVUs": 2,
       "startTime": "6s"
     },
     "cm_handle_search_cpspath_scenario": {
       "executor": "constant-arrival-rate",
       "exec": "cmHandleSearchCpsPathScenario",
       "rate": 1,
-      "timeUnit": "15s",
+      "timeUnit": "24s",
       "duration": "15m",
-      "preAllocatedVUs": 1,
+      "preAllocatedVUs": 2,
       "startTime": "9s"
     },
     "cm_handle_search_trustlevel_scenario": {
       "executor": "constant-arrival-rate",
       "exec": "cmHandleSearchTrustLevelScenario",
       "rate": 1,
-      "timeUnit": "15s",
+      "timeUnit": "24s",
       "duration": "15m",
-      "preAllocatedVUs": 1,
+      "preAllocatedVUs": 2,
       "startTime": "12s"
     },
     "legacy_batch_produce_scenario": {
@@ -131,8 +133,8 @@
       "rate": 500,
       "timeUnit": "1s",
       "duration": "15m",
-      "preAllocatedVUs": 10,
-      "maxVUs": 10,
+      "preAllocatedVUs": 100,
+      "maxVUs": 300,
       "exec": "produceAvcEventsScenario",
       "gracefulStop": "10s"
     }
@@ -143,16 +145,16 @@
     "cmhandles_deleted_per_second": ["avg >= 22"],
     "ncmp_overhead_passthrough_read_alt_id": ["avg <= 40"],
     "ncmp_overhead_passthrough_write_alt_id": ["avg <= 40"],
-    "id_search_nofilter_duration": ["avg <= 2000"],
-    "id_search_module_duration": ["avg <= 2000"],
-    "id_search_property_duration": ["avg <= 2000"],
-    "id_search_cpspath_duration": ["avg <= 2000"],
-    "id_search_trustlevel_duration": ["avg <= 2000"],
-    "cm_search_nofilter_duration": ["avg <= 15000"],
-    "cm_search_module_duration": ["avg <= 15000"],
-    "cm_search_property_duration": ["avg <= 15000"],
-    "cm_search_cpspath_duration": ["avg <= 15000"],
-    "cm_search_trustlevel_duration": ["avg <= 15000"],
+    "id_search_nofilter_duration": ["avg <= 2600"],
+    "id_search_module_duration": ["avg <= 2600"],
+    "id_search_property_duration": ["avg <= 2600"],
+    "id_search_cpspath_duration": ["avg <= 2600"],
+    "id_search_trustlevel_duration": ["avg <= 2600"],
+    "cm_search_nofilter_duration": ["avg <= 24000"],
+    "cm_search_module_duration": ["avg <= 24000"],
+    "cm_search_property_duration": ["avg <= 24000"],
+    "cm_search_cpspath_duration": ["avg <= 24000"],
+    "cm_search_trustlevel_duration": ["avg <= 24000"],
     "legacy_batch_read_cmhandles_per_second": ["avg >= 150"]
   }
 }

diff --git a/k6-tests/ncmp/run-all-tests.sh b/k6-tests/ncmp/run-all-tests.sh
index 3769570..1b9332f 100755 (executable)
--- a/k6-tests/ncmp/run-all-tests.sh
+++ b/k6-tests/ncmp/run-all-tests.sh
@@ -22,7 +22,7 @@ testProfile=$1
 summaryFile="${testProfile}Summary.csv"
 echo "Running $testProfile performance tests..."
 
-k6 run ncmp-test-runner.js --quiet -e TEST_PROFILE="$testProfile"  > "$summaryFile" 2>/dev/null || ((number_of_failures++))
+k6 run ncmp-test-runner.js --quiet -e TEST_PROFILE="$testProfile"  > "$summaryFile" || ((number_of_failures++))
 
 if [ -f "$summaryFile" ]; then