From: ToineSiebelink Date: Wed, 30 Apr 2025 13:25:29 +0000 (+0100) Subject: Adjust KPI threshold and rate as per latest KPI agreement X-Git-Tag: 3.6.3~36^2 X-Git-Url: https://gerrit.onap.org/r/gitweb?a=commitdiff_plain;h=refs%2Fchanges%2F34%2F140834%2F1;p=cps.git Adjust KPI threshold and rate as per latest KPI agreement - CM Handle ID searches now can take 2.6 sec - CM Handle searches now can take 24 sec - Updated 'blue' lines to reflect KPI - Updated 'green' lines to reflect current average (i.e. we expect measurements to be 'around' this) - Experimental: increased cm notification forwarding by using more VUs - Add timer metric to be able to check cm notification forwarding rate - Temporary: removed redirecting of warning and errors to null device Issue-ID: CPS-2793 Change-Id: I560985d4f9ce7a4c8aa27b3ae089a43a8a76b28a Signed-off-by: ToineSiebelink --- diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/cmnotificationsubscription/cmavc/CmAvcEventConsumer.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/cmnotificationsubscription/cmavc/CmAvcEventConsumer.java index eca8380756..c8aec3e9e4 100644 --- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/cmnotificationsubscription/cmavc/CmAvcEventConsumer.java +++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/impl/cmnotificationsubscription/cmavc/CmAvcEventConsumer.java @@ -21,6 +21,7 @@ package org.onap.cps.ncmp.impl.cmnotificationsubscription.cmavc; import io.cloudevents.CloudEvent; +import io.micrometer.core.annotation.Timed; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.clients.consumer.ConsumerRecord; @@ -31,7 +32,7 @@ import org.springframework.kafka.annotation.KafkaListener; import org.springframework.stereotype.Component; /** - * Listener for AVC events based on Cm Subscriptions. + * Listener for AVC events based on CM Subscriptions. */ @Component @Slf4j @@ -53,6 +54,7 @@ public class CmAvcEventConsumer { */ @KafkaListener(topics = "${app.dmi.cm-events.topic}", containerFactory = "cloudEventConcurrentKafkaListenerContainerFactory") + @Timed(value = "cps.ncmp.cmnotifications.consumeandforward", description = "Time taken to forward CM AVC events") public void consumeAndForward( final ConsumerRecord cmAvcEventAsConsumerRecord) { final CloudEvent outgoingAvcEvent = cmAvcEventAsConsumerRecord.value(); diff --git a/docs/test_ScrapeMetrics.py b/docs/test_ScrapeMetrics.py index cbcc92d2a1..f59e4fc062 100644 --- a/docs/test_ScrapeMetrics.py +++ b/docs/test_ScrapeMetrics.py @@ -91,7 +91,7 @@ class TestScrapeMetrics(unittest.TestCase): with open(metrics_file, 'r') as f: lines = f.readlines() - expected_number_of_metrics = 56 + expected_number_of_metrics = 57 expected_number_of_lines = expected_number_of_metrics + 1 # Header self.assertEqual(len(lines), expected_number_of_lines, f"metrics.csv does not have {expected_number_of_lines} lines.") @@ -101,4 +101,4 @@ if __name__ == '__main__': script_dir = os.path.dirname(os.path.abspath(__file__)) if script_dir not in sys.path: sys.path.insert(0, script_dir) - unittest.main() \ No newline at end of file + unittest.main() diff --git a/k6-tests/ncmp/common/utils.js b/k6-tests/ncmp/common/utils.js index ea77aae176..9406c1cd3e 100644 --- a/k6-tests/ncmp/common/utils.js +++ b/k6-tests/ncmp/common/utils.js @@ -115,21 +115,21 @@ export function makeCustomSummaryReport(testResults, scenarioConfig) { const summaryCsvLines = [ '#,Test Name,Unit,Fs Requirement,Current Expectation,Actual', makeSummaryCsvLine('0', 'HTTP request failures for all tests', 'rate of failed requests', 'http_req_failed', 0, testResults, scenarioConfig), - makeSummaryCsvLine('1', 'Registration of CM-handles', 'CM-handles/second', 'cmhandles_created_per_second', 90, testResults, scenarioConfig), - makeSummaryCsvLine('2', 'De-registration of CM-handles', 'CM-handles/second', 'cmhandles_deleted_per_second', 140, testResults, scenarioConfig), - makeSummaryCsvLine('3a', 'CM-handle ID search with No filter', 'milliseconds', 'id_search_nofilter_duration', 300, testResults, scenarioConfig), - makeSummaryCsvLine('3b', 'CM-handle ID search with Module filter', 'milliseconds', 'id_search_module_duration', 120, testResults, scenarioConfig), - makeSummaryCsvLine('3c', 'CM-handle ID search with Property filter', 'milliseconds', 'id_search_property_duration', 750, testResults, scenarioConfig), - makeSummaryCsvLine('3d', 'CM-handle ID search with Cps Path filter', 'milliseconds', 'id_search_cpspath_duration', 750, testResults, scenarioConfig), - makeSummaryCsvLine('3e', 'CM-handle ID search with Trust Level filter', 'milliseconds', 'id_search_trustlevel_duration', 3000, testResults, scenarioConfig), - makeSummaryCsvLine('4a', 'CM-handle search with No filter', 'milliseconds', 'cm_search_nofilter_duration', 3000, testResults, scenarioConfig), - makeSummaryCsvLine('4b', 'CM-handle search with Module filter', 'milliseconds', 'cm_search_module_duration', 4000, testResults, scenarioConfig), - makeSummaryCsvLine('4c', 'CM-handle search with Property filter', 'milliseconds', 'cm_search_property_duration', 4500, testResults, scenarioConfig), - makeSummaryCsvLine('4d', 'CM-handle search with Cps Path filter', 'milliseconds', 'cm_search_cpspath_duration', 4500, testResults, scenarioConfig), - makeSummaryCsvLine('4e', 'CM-handle search with Trust Level filter', 'milliseconds', 'cm_search_trustlevel_duration', 7000, testResults, scenarioConfig), - makeSummaryCsvLine('5b', 'NCMP overhead for Synchronous single CM-handle pass-through read with alternate id', 'milliseconds', 'ncmp_overhead_passthrough_read_alt_id', 40, testResults, scenarioConfig), - makeSummaryCsvLine('6b', 'NCMP overhead for Synchronous single CM-handle pass-through write with alternate id', 'milliseconds', 'ncmp_overhead_passthrough_write_alt_id', 40, testResults, scenarioConfig), - makeSummaryCsvLine('7', 'Legacy batch read operation', 'events/second', 'legacy_batch_read_cmhandles_per_second', 300, testResults, scenarioConfig), + makeSummaryCsvLine('1', 'Registration of CM-handles', 'CM-handles/second', 'cmhandles_created_per_second', 100, testResults, scenarioConfig), + makeSummaryCsvLine('2', 'De-registration of CM-handles', 'CM-handles/second', 'cmhandles_deleted_per_second', 180, testResults, scenarioConfig), + makeSummaryCsvLine('3a', 'CM-handle ID search with No filter', 'milliseconds', 'id_search_nofilter_duration', 550, testResults, scenarioConfig), + makeSummaryCsvLine('3b', 'CM-handle ID search with Module filter', 'milliseconds', 'id_search_module_duration', 2300, testResults, scenarioConfig), + makeSummaryCsvLine('3c', 'CM-handle ID search with Property filter', 'milliseconds', 'id_search_property_duration', 1450, testResults, scenarioConfig), + makeSummaryCsvLine('3d', 'CM-handle ID search with Cps Path filter', 'milliseconds', 'id_search_cpspath_duration', 1500, testResults, scenarioConfig), + makeSummaryCsvLine('3e', 'CM-handle ID search with Trust Level filter', 'milliseconds', 'id_search_trustlevel_duration', 1600, testResults, scenarioConfig), + makeSummaryCsvLine('4a', 'CM-handle search with No filter', 'milliseconds', 'cm_search_nofilter_duration', 18000, testResults, scenarioConfig), + makeSummaryCsvLine('4b', 'CM-handle search with Module filter', 'milliseconds', 'cm_search_module_duration', 18000, testResults, scenarioConfig), + makeSummaryCsvLine('4c', 'CM-handle search with Property filter', 'milliseconds', 'cm_search_property_duration', 18000, testResults, scenarioConfig), + makeSummaryCsvLine('4d', 'CM-handle search with Cps Path filter', 'milliseconds', 'cm_search_cpspath_duration', 18000, testResults, scenarioConfig), + makeSummaryCsvLine('4e', 'CM-handle search with Trust Level filter', 'milliseconds', 'cm_search_trustlevel_duration', 18000, testResults, scenarioConfig), + makeSummaryCsvLine('5b', 'NCMP overhead for Synchronous single CM-handle pass-through read with alternate id', 'milliseconds', 'ncmp_overhead_passthrough_read_alt_id', 18, testResults, scenarioConfig), + makeSummaryCsvLine('6b', 'NCMP overhead for Synchronous single CM-handle pass-through write with alternate id', 'milliseconds', 'ncmp_overhead_passthrough_write_alt_id', 18, testResults, scenarioConfig), + makeSummaryCsvLine('7', 'Legacy batch read operation', 'events/second', 'legacy_batch_read_cmhandles_per_second', 1750, testResults, scenarioConfig), ]; return summaryCsvLines.join('\n') + '\n'; } diff --git a/k6-tests/ncmp/config/kpi.json b/k6-tests/ncmp/config/kpi.json index aa93091b67..0616fd876b 100644 --- a/k6-tests/ncmp/config/kpi.json +++ b/k6-tests/ncmp/config/kpi.json @@ -12,7 +12,8 @@ "rate": 25, "timeUnit": "1s", "duration": "15m", - "preAllocatedVUs": 10, + "preAllocatedVUs": 5, + "maxVUs": 25, "startTime": "200ms" }, "passthrough_write_alt_id_scenario": { @@ -21,97 +22,98 @@ "rate": 13, "timeUnit": "1s", "duration": "15m", - "preAllocatedVUs": 10, + "preAllocatedVUs": 5, + "maxVUs": 25, "startTime": "600ms" }, "cm_handle_id_search_nofilter_scenario": { "executor": "constant-arrival-rate", "exec": "cmHandleIdSearchNoFilterScenario", "rate": 1, - "timeUnit": "2s", + "timeUnit": "2600ms", "duration": "15m", - "preAllocatedVUs": 1, + "preAllocatedVUs": 2, "startTime": "0ms" }, "cm_handle_id_search_module_scenario": { "executor": "constant-arrival-rate", "exec": "cmHandleIdSearchModuleScenario", "rate": 1, - "timeUnit": "2s", + "timeUnit": "2600ms", "duration": "15m", - "preAllocatedVUs": 1, + "preAllocatedVUs": 2, "startTime": "400ms" }, "cm_handle_id_search_property_scenario": { "executor": "constant-arrival-rate", "exec": "cmHandleIdSearchPropertyScenario", "rate": 1, - "timeUnit": "2s", + "timeUnit": "2600ms", "duration": "15m", - "preAllocatedVUs": 1, + "preAllocatedVUs": 2, "startTime": "800ms" }, "cm_handle_id_search_cpspath_scenario": { "executor": "constant-arrival-rate", "exec": "cmHandleIdSearchCpsPathScenario", "rate": 1, - "timeUnit": "2s", + "timeUnit": "2600ms", "duration": "15m", - "preAllocatedVUs": 1, + "preAllocatedVUs": 2, "startTime": "1200ms" }, "cm_handle_id_search_trustlevel_scenario": { "executor": "constant-arrival-rate", "exec": "cmHandleIdSearchTrustLevelScenario", "rate": 1, - "timeUnit": "2s", + "timeUnit": "2600ms", "duration": "15m", - "preAllocatedVUs": 1, + "preAllocatedVUs": 2, "startTime": "1600ms" }, "cm_handle_search_nofilter_scenario": { "executor": "constant-arrival-rate", "exec": "cmHandleSearchNoFilterScenario", "rate": 1, - "timeUnit": "15s", + "timeUnit": "24s", "duration": "15m", - "preAllocatedVUs": 1, + "preAllocatedVUs": 2, "startTime": "0s" }, "cm_handle_search_module_scenario": { "executor": "constant-arrival-rate", "exec": "cmHandleSearchModuleScenario", "rate": 1, - "timeUnit": "15s", + "timeUnit": "24s", "duration": "15m", - "preAllocatedVUs": 1, + "preAllocatedVUs": 2, "startTime": "3s" }, "cm_handle_search_property_scenario": { "executor": "constant-arrival-rate", "exec": "cmHandleSearchPropertyScenario", "rate": 1, - "timeUnit": "15s", + "timeUnit": "24s", "duration": "15m", - "preAllocatedVUs": 1, + "preAllocatedVUs": 2, "startTime": "6s" }, "cm_handle_search_cpspath_scenario": { "executor": "constant-arrival-rate", "exec": "cmHandleSearchCpsPathScenario", "rate": 1, - "timeUnit": "15s", + "timeUnit": "24s", "duration": "15m", - "preAllocatedVUs": 1, + "preAllocatedVUs": 2, "startTime": "9s" }, "cm_handle_search_trustlevel_scenario": { "executor": "constant-arrival-rate", "exec": "cmHandleSearchTrustLevelScenario", "rate": 1, - "timeUnit": "15s", + "timeUnit": "24s", "duration": "15m", - "preAllocatedVUs": 1, + "preAllocatedVUs": 2, "startTime": "12s" }, "legacy_batch_produce_scenario": { @@ -131,8 +133,8 @@ "rate": 500, "timeUnit": "1s", "duration": "15m", - "preAllocatedVUs": 10, - "maxVUs": 10, + "preAllocatedVUs": 100, + "maxVUs": 300, "exec": "produceAvcEventsScenario", "gracefulStop": "10s" } @@ -143,16 +145,16 @@ "cmhandles_deleted_per_second": ["avg >= 22"], "ncmp_overhead_passthrough_read_alt_id": ["avg <= 40"], "ncmp_overhead_passthrough_write_alt_id": ["avg <= 40"], - "id_search_nofilter_duration": ["avg <= 2000"], - "id_search_module_duration": ["avg <= 2000"], - "id_search_property_duration": ["avg <= 2000"], - "id_search_cpspath_duration": ["avg <= 2000"], - "id_search_trustlevel_duration": ["avg <= 2000"], - "cm_search_nofilter_duration": ["avg <= 15000"], - "cm_search_module_duration": ["avg <= 15000"], - "cm_search_property_duration": ["avg <= 15000"], - "cm_search_cpspath_duration": ["avg <= 15000"], - "cm_search_trustlevel_duration": ["avg <= 15000"], + "id_search_nofilter_duration": ["avg <= 2600"], + "id_search_module_duration": ["avg <= 2600"], + "id_search_property_duration": ["avg <= 2600"], + "id_search_cpspath_duration": ["avg <= 2600"], + "id_search_trustlevel_duration": ["avg <= 2600"], + "cm_search_nofilter_duration": ["avg <= 24000"], + "cm_search_module_duration": ["avg <= 24000"], + "cm_search_property_duration": ["avg <= 24000"], + "cm_search_cpspath_duration": ["avg <= 24000"], + "cm_search_trustlevel_duration": ["avg <= 24000"], "legacy_batch_read_cmhandles_per_second": ["avg >= 150"] } } diff --git a/k6-tests/ncmp/run-all-tests.sh b/k6-tests/ncmp/run-all-tests.sh index 3769570ead..1b9332f834 100755 --- a/k6-tests/ncmp/run-all-tests.sh +++ b/k6-tests/ncmp/run-all-tests.sh @@ -22,7 +22,7 @@ testProfile=$1 summaryFile="${testProfile}Summary.csv" echo "Running $testProfile performance tests..." -k6 run ncmp-test-runner.js --quiet -e TEST_PROFILE="$testProfile" > "$summaryFile" 2>/dev/null || ((number_of_failures++)) +k6 run ncmp-test-runner.js --quiet -e TEST_PROFILE="$testProfile" > "$summaryFile" || ((number_of_failures++)) if [ -f "$summaryFile" ]; then